From 91a8e868cb661eeadb7316ac36c212945c8de8ee Mon Sep 17 00:00:00 2001 From: Sarah Sunday <3252602+ssunday@users.noreply.github.com> Date: Thu, 31 Oct 2024 13:33:44 -0500 Subject: [PATCH 1/4] [README] Improve queue config docs --- README.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9b7704d9..f794406e 100644 --- a/README.md +++ b/README.md @@ -414,7 +414,19 @@ queues: default: 'https://my-queue-url.amazon.aws' ``` +Or (note the key must be a symbol): + +```ruby +Aws::Rails::SqsActiveJob.configure do |config| + config.queues = { + :default => 'https://my-queue-url.amazon.aws', + ENV.fetch('JOB_QUEUE_NAME').to_sym => 'https://my-other-queue-url.amazon.aws', + } +end +``` + To queue a job, you can just use standard ActiveJob methods: + ```ruby # To queue for immediate processing YourJob.perform_later(args) @@ -523,14 +535,14 @@ require_relative 'config/environment' # load rails ### Elastic Beanstalk workers: processing activejobs using worker environments Another option for processing jobs without managing the worker process is hosting the application in a scalable -[Elastic Beanstalk worker environment](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features-managing-env-tiers.html). -[Configure the worker](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features-managing-env-tiers.html#using-features-managing-env-tiers-worker-settings) -to read from the correct SQS queue that you want to process jobs from and set the +[Elastic Beanstalk worker environment](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features-managing-env-tiers.html). +[Configure the worker](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features-managing-env-tiers.html#using-features-managing-env-tiers-worker-settings) +to read from the correct SQS queue that you want to process jobs from and set the ```AWS_PROCESS_BEANSTALK_WORKER_REQUESTS``` environment variable to `true` in the worker environment configuration. Note that this will NOT start the poller. Instead the [SQS Daemon](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features-managing-env-tiers.html#worker-daemon) -running on the worker sends messages as a POST request to `http://localhost/`. The middleware will forward each -request and parameters to their appropriate jobs. The middleware will only process requests from the SQS daemon +running on the worker sends messages as a POST request to `http://localhost/`. The middleware will forward each +request and parameters to their appropriate jobs. The middleware will only process requests from the SQS daemon and will pass on others and so will not interfere with other routes in your application. To add the middleware on application startup, set the ```AWS_PROCESS_BEANSTALK_WORKER_REQUESTS``` environment variable to true From 5f6a4d34809d62bdf50f96e5dc090b1ff57247a4 Mon Sep 17 00:00:00 2001 From: Sarah Sunday <3252602+ssunday@users.noreply.github.com> Date: Tue, 5 Nov 2024 12:32:16 -0600 Subject: [PATCH 2/4] [SQS] Use remote ip and remote addr to check IP matching --- .../ebs_sqs_active_job_middleware.rb | 15 +- .../ebs_sqs_active_job_middleware_spec.rb | 372 +++++++++++------- 2 files changed, 245 insertions(+), 142 deletions(-) diff --git a/lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb b/lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb index f5394046..655de0cf 100644 --- a/lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb +++ b/lib/aws/rails/middleware/ebs_sqs_active_job_middleware.rb @@ -24,7 +24,7 @@ def call(env) # Only accept requests from this user agent if it is from localhost or a docker host in case of forgery. unless request.local? || sent_from_docker_host?(request) - @logger.warn("SQSD request detected from untrusted address #{request.ip}; returning 403 forbidden.") + @logger.warn('SQSD request detected from untrusted address; returning 403 forbidden.') return FORBIDDEN_RESPONSE end @@ -81,7 +81,7 @@ def periodic_task?(request) end def sent_from_docker_host?(request) - app_runs_in_docker_container? && default_gw_ips.include?(request.ip) + app_runs_in_docker_container? && ip_originates_from_docker_host?(request) end def app_runs_in_docker_container? @@ -96,7 +96,16 @@ def in_docker_container_with_cgroup2? File.exist?('/proc/self/mountinfo') && File.read('/proc/self/mountinfo') =~ %r{/docker/containers/} end - def default_gw_ips + def ip_originates_from_docker_host?(request) + default_docker_ips.include?(request.remote_ip) || + default_docker_ips.include?(request.remote_addr) + end + + def default_docker_ips + @default_docker_ips ||= build_default_docker_ips + end + + def build_default_docker_ips default_gw_ips = ['172.17.0.1'] if File.exist?('/proc/net/route') diff --git a/spec/aws/rails/middleware/ebs_sqs_active_job_middleware_spec.rb b/spec/aws/rails/middleware/ebs_sqs_active_job_middleware_spec.rb index 5f5f52ec..0839a9c5 100644 --- a/spec/aws/rails/middleware/ebs_sqs_active_job_middleware_spec.rb +++ b/spec/aws/rails/middleware/ebs_sqs_active_job_middleware_spec.rb @@ -4,193 +4,287 @@ require_relative 'elastic_beanstalk_job' require_relative 'elastic_beanstalk_periodic_task' -module Aws - module Rails - describe EbsSqsActiveJobMiddleware do - # Simple mock Rack app that always returns 200 - let(:mock_rack_app) { ->(_) { [200, { 'Content-Type' => 'text/plain' }, ['OK']] } } +describe Aws::Rails::EbsSqsActiveJobMiddleware do + subject(:response) do + mock_rack_env = create_mock_env + test_middleware = described_class.new(mock_rack_app) + test_middleware.call(mock_rack_env) + end - let(:logger) { double(error: nil, debug: nil, warn: nil) } + # Simple mock Rack app that always returns 200 + let(:mock_rack_app) { ->(_) { [200, { 'Content-Type' => 'text/plain' }, ['OK']] } } - it 'passes request through if user-agent is not SQS Daemon' do - mock_rack_env = create_mock_env('127.0.0.1', 'not-aws-sqsd') + let(:logger) { double(error: nil, debug: nil, warn: nil) } + let(:user_agent) { 'aws-sqsd/1.1' } + let(:remote_ip) { '127.0.0.1' } + let(:remote_addr) { nil } + let(:is_periodic_task) { nil } + let(:period_task_name) { 'ElasticBeanstalkPeriodicTask' } - test_middleware = EbsSqsActiveJobMiddleware.new(mock_rack_app) - response = test_middleware.call(mock_rack_env) + before do + allow(File).to receive(:exist?).and_call_original + allow(File).to receive(:open).and_call_original + end - expect(response[0]).to eq(200) - expect(response[2]).to eq(['OK']) - end + shared_examples_for 'passes request through' do + it 'passes request' do + expect(response[0]).to eq(200) + expect(response[2]).to eq(['OK']) + end + end - it 'returns forbidden when called from untrusted source' do - mock_rack_env = create_mock_env('1.2.3.4', 'aws-sqsd/1.1') + shared_examples_for 'runs job' do + it 'invokes job' do + expect(response[0]).to eq(200) + expect(response[2]).to eq(['Successfully ran job ElasticBeanstalkJob.']) + end - test_middleware = EbsSqsActiveJobMiddleware.new(mock_rack_app) - response = test_middleware.call(mock_rack_env) + it 'returns internal server error if job name cannot be resolved' do + # Stub execute call to avoid invoking Active Job callbacks + # Local testing indicates this failure results in a NameError + allow(ActiveJob::Base).to receive(:execute).and_raise(NameError) - expect(response[0]).to eq(403) - end + expect(response[0]).to eq(500) + end + + context 'when user-agent is not sqs daemon' do + let(:user_agent) { 'not-aws-sqsd' } - it 'successfully invokes job when passed through request body' do - # Stub execute call to avoid invoking Active Job callbacks - expect(ActiveJob::Base).to receive(:execute).and_return(nil) - mock_rack_env = create_mock_env('::1', 'aws-sqsd/1.1') + include_examples 'passes request through' + end - test_middleware = EbsSqsActiveJobMiddleware.new(mock_rack_app) - response = test_middleware.call(mock_rack_env) + context 'when periodic task' do + let(:is_periodic_task) { true } + it 'successfully invokes periodic task when passed through custom header' do expect(response[0]).to eq(200) expect(response[1]['Content-Type']).to eq('text/plain') - expect(response[2]).to eq(['Successfully ran job ElasticBeanstalkJob.']) + expect(response[2]).to eq(['Successfully ran periodic task ElasticBeanstalkPeriodicTask.']) end - it 'returns internal server error if job name cannot be resolved' do - # Stub execute call to avoid invoking Active Job callbacks - # Local testing indicates this failure results in a NameError - allow(ActiveJob::Base).to receive(:execute).and_raise(NameError) - mock_rack_env = create_mock_env('::1', 'aws-sqsd/1.1') + context 'when unknown periodic task name' do + let(:period_task_name) { 'NonExistentTask' } - test_middleware = EbsSqsActiveJobMiddleware.new(mock_rack_app) - response = test_middleware.call(mock_rack_env) - - expect(response[0]).to eq(500) + it 'returns internal server error' do + expect(response[0]).to eq(500) + end end + end + end - it 'successfully invokes periodic task when passed through custom header' do - mock_rack_env = create_mock_env('127.0.0.1', 'aws-sqsd/1.1', is_periodic_task: true) - test_middleware = EbsSqsActiveJobMiddleware.new(mock_rack_app) + shared_examples_for 'is forbidden' do + it 'passes request' do + expect(response[0]).to eq(403) + end - expect_any_instance_of(ElasticBeanstalkPeriodicTask).to receive(:perform_now) - response = test_middleware.call(mock_rack_env) + context 'when user-agent is not sqs daemon' do + let(:user_agent) { 'not-aws-sqsd' } - expect(response[0]).to eq(200) - expect(response[1]['Content-Type']).to eq('text/plain') - expect(response[2]).to eq(['Successfully ran periodic task ElasticBeanstalkPeriodicTask.']) - end + include_examples 'passes request through' + end + end - it 'returns internal server error if periodic task cannot be resolved' do - mock_rack_env = create_mock_env('127.0.0.1', 'aws-sqsd/1.1', is_periodic_task: true) - mock_rack_env['HTTP_X_AWS_SQSD_TASKNAME'] = 'NonExistentTask' + context 'when local IP' do + let(:remote_ip) { '127.0.0.1' } - test_middleware = EbsSqsActiveJobMiddleware.new(mock_rack_app) - response = test_middleware.call(mock_rack_env) + include_examples 'runs job' + end - expect(response[0]).to eq(500) - end + context 'when ::1 IP' do + let(:remote_ip) { '::1' } - it 'successfully invokes job when docker default gateway ip is changed' do - mock_rack_env = create_mock_env('192.168.176.1', 'aws-sqsd/1.1', is_periodic_task: false) - test_middleware = EbsSqsActiveJobMiddleware.new(mock_rack_app) + include_examples 'runs job' + end - proc_net_route = <<~CONTENT - Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT - eth0\t00000000\t01B0A8C0\t0003\t0\t0\t0\t00000000\t0\t0\t0 - eth0\t00B0A8C0\t00000000\t0001\t0\t0\t0\t00F0FFFF\t0\t0\t0 - CONTENT + context 'when non-local IP' do + let(:remote_ip) { '1.2.3.4' } - allow(File).to receive(:exist?).and_call_original - allow(File).to receive(:open).and_call_original + include_examples 'is forbidden' + end - expect(File).to receive(:exist?).with('/proc/net/route').and_return(true) - expect(File).to receive(:open).with('/proc/net/route').and_return(StringIO.new(proc_net_route)) - expect(test_middleware).to receive(:app_runs_in_docker_container?).and_return(true) + shared_examples_for 'is valid in either cgroup1 or cgroup2' do + context 'when not in a docker container' do + before { stub_runs_in_neither_docker_container } - response = test_middleware.call(mock_rack_env) + include_examples 'is forbidden' + end - expect(response[0]).to eq(200) - expect(response[1]['Content-Type']).to eq('text/plain') - expect(response[2]).to eq(['Successfully ran job ElasticBeanstalkJob.']) - end + context 'when docker container cgroup1' do + before { stub_runs_in_docker_container_cgroup1 } - it 'successfully invokes job when /proc/net/route does not exist' do - mock_rack_env = create_mock_env('172.17.0.1', 'aws-sqsd/1.1', is_periodic_task: false) - test_middleware = EbsSqsActiveJobMiddleware.new(mock_rack_app) + include_examples 'runs job' + end - allow(File).to receive(:exist?).and_call_original + context 'when docker container cgroup2' do + before { stub_runs_in_docker_container_cgroup2 } - expect(File).to receive(:exist?).with('/proc/net/route').and_return(false) - expect(test_middleware).to receive(:app_runs_in_docker_container?).and_return(true) + include_examples 'runs job' + end + end - response = test_middleware.call(mock_rack_env) + shared_examples_for 'is invalid in either cgroup1 or cgroup2' do + context 'when not in a docker container' do + before { stub_runs_in_neither_docker_container } - expect(response[0]).to eq(200) - expect(response[1]['Content-Type']).to eq('text/plain') - expect(response[2]).to eq(['Successfully ran job ElasticBeanstalkJob.']) - end + include_examples 'is forbidden' + end - it 'successfully invokes job in docker container with cgroup1' do - mock_rack_env = create_mock_env('172.17.0.1', 'aws-sqsd/1.1', is_periodic_task: false) - test_middleware = EbsSqsActiveJobMiddleware.new(mock_rack_app) + context 'when docker container cgroup1' do + before { stub_runs_in_docker_container_cgroup1 } - proc_1_cgroup = <<~CONTENT - 13:rdma:/docker/d59538e9b3d3aa6012f08587c13199cbad3f882ecaa9637905971df18ab89757 - 12:hugetlb:/docker/d59538e9b3d3aa6012f08587c13199cbad3f882ecaa9637905971df18ab89757 - 11:memory:/docker/d59538e9b3d3aa6012f08587c13199cbad3f882ecaa9637905971df18ab89757 - 10:devices:/docker/d59538e9b3d3aa6012f08587c13199cbad3f882ecaa9637905971df18ab89757 - 9:blkio:/docker/d59538e9b3d3aa6012f08587c13199cbad3f882ecaa9637905971df18ab89757 - CONTENT + include_examples 'is forbidden' + end - allow(File).to receive(:exist?).and_call_original - allow(File).to receive(:read).and_call_original + context 'when docker container cgroup2' do + before { stub_runs_in_docker_container_cgroup2 } - expect(File).to receive(:exist?).with('/proc/1/cgroup').and_return(true) - expect(File).to receive(:read).with('/proc/1/cgroup').and_return(proc_1_cgroup) + include_examples 'is forbidden' + end + end - response = test_middleware.call(mock_rack_env) + context 'when remote ip is invalid, but remote_addr is docker gw' do + let(:remote_addr) { '172.17.0.1' } + let(:remote_ip) { '192.168.176.1' } - expect(response[0]).to eq(200) - expect(response[1]['Content-Type']).to eq('text/plain') - expect(response[2]).to eq(['Successfully ran job ElasticBeanstalkJob.']) - end + include_examples 'is valid in either cgroup1 or cgroup2' - it 'successfully invokes job in docker container with cgroup2' do - mock_rack_env = create_mock_env('172.17.0.1', 'aws-sqsd/1.1', is_periodic_task: false) - test_middleware = EbsSqsActiveJobMiddleware.new(mock_rack_app) + it 'successfully invokes job when /proc/net/route does not exist' do + expect(File).to receive(:exist?).with('/proc/net/route').and_return(false) - proc_1_cgroup = <<~CONTENT - 0::/ - CONTENT + stub_runs_in_docker_container_cgroup2 - proc_self_mountinfo = <<~CONTENT - 355 354 0:21 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw,nsdelegate - 356 352 0:74 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw - 357 352 0:79 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k - 358 350 8:16 /var/lib/docker/containers/69e3febd00ac4720d2ea58c935574776285f6a0016d2aa30b0c280a81c385e69/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/sdb rw,discard,errors=remount-ro,data=ordered - 359 350 8:16 /var/lib/docker/containers/69e3febd00ac4720d2ea58c935574776285f6a0016d2aa30b0c280a81c385e69/hostname /etc/hostname rw,relatime - ext4 /dev/sdb rw,discard,errors=remount-ro,data=ordered - 360 350 8:16 /var/lib/docker/containers/69e3febd00ac4720d2ea58c935574776285f6a0016d2aa30b0c280a81c385e69/hosts /etc/hosts rw,relatime - ext4 /dev/sdb rw,discard,errors=remount-ro,data=ordered - 316 352 0:77 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 - CONTENT + expect(response[0]).to eq(200) + expect(response[1]['Content-Type']).to eq('text/plain') + expect(response[2]).to eq(['Successfully ran job ElasticBeanstalkJob.']) + end + end - allow(File).to receive(:exist?).and_call_original - allow(File).to receive(:read).and_call_original + context 'when remote addr is non-standard ip but in /proc/net/route' do + let(:remote_addr) { '192.168.176.1' } - expect(File).to receive(:exist?).with('/proc/1/cgroup').and_return(true) - expect(File).to receive(:read).with('/proc/1/cgroup').and_return(proc_1_cgroup) - expect(File).to receive(:exist?).with('/proc/self/mountinfo').and_return(true) - expect(File).to receive(:read).with('/proc/self/mountinfo').and_return(proc_self_mountinfo) + before do + proc_net_route = <<~CONTENT + Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT + eth0\t00000000\t01B0A8C0\t0003\t0\t0\t0\t00000000\t0\t0\t0 + eth0\t00B0A8C0\t00000000\t0001\t0\t0\t0\t00F0FFFF\t0\t0\t0 + CONTENT - response = test_middleware.call(mock_rack_env) + allow(File).to receive(:exist?).with('/proc/net/route').and_return(true) + allow(File).to receive(:open).with('/proc/net/route').and_return(StringIO.new(proc_net_route)) + end - expect(response[0]).to eq(200) - expect(response[1]['Content-Type']).to eq('text/plain') - expect(response[2]).to eq(['Successfully ran job ElasticBeanstalkJob.']) - end + include_examples 'is valid in either cgroup1 or cgroup2' + end - # Create a minimal mock Rack environment hash to test just what we need - def create_mock_env(source_ip, user_agent, is_periodic_task: false) - mock_env = { - 'REMOTE_ADDR' => source_ip, - 'HTTP_USER_AGENT' => user_agent - } - - if is_periodic_task - mock_env['HTTP_X_AWS_SQSD_TASKNAME'] = 'ElasticBeanstalkPeriodicTask' - else - mock_env['rack.input'] = StringIO.new('{"job_class": "ElasticBeanstalkJob"}') - end + context 'when remote ip is non-standard ip but in /proc/net/route' do + let(:remote_ip) { '192.168.176.1' } - mock_env - end + before do + proc_net_route = <<~CONTENT + Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT + eth0\t00000000\t01B0A8C0\t0003\t0\t0\t0\t00000000\t0\t0\t0 + eth0\t00B0A8C0\t00000000\t0001\t0\t0\t0\t00F0FFFF\t0\t0\t0 + CONTENT + + allow(File).to receive(:exist?).with('/proc/net/route').and_return(true) + allow(File).to receive(:open).with('/proc/net/route').and_return(StringIO.new(proc_net_route)) end + + include_examples 'is valid in either cgroup1 or cgroup2' + end + + context 'when remote addr is non-standard ip but not in /proc/net/route' do + let(:remote_addr) { '192.168.176.1' } + + before do + proc_net_route = <<~CONTENT + Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT + CONTENT + + allow(File).to receive(:exist?).with('/proc/net/route').and_return(true) + allow(File).to receive(:open).with('/proc/net/route').and_return(StringIO.new(proc_net_route)) + end + + include_examples 'is invalid in either cgroup1 or cgroup2' + end + + context 'when remote ip is default docker gw' do + let(:remote_ip) { '172.17.0.1' } + + include_examples 'is valid in either cgroup1 or cgroup2' + end + + context 'when remote addr is default docker gw' do + let(:remote_addr) { '172.17.0.1' } + + include_examples 'is valid in either cgroup1 or cgroup2' + end + + def stub_runs_in_neither_docker_container + proc_1_cgroup = <<~CONTENT + 0::/ + CONTENT + + proc_self_mountinfo = <<~CONTENT + 355 354 0:21 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw,nsdelegate + 356 352 0:74 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw + 357 352 0:79 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k + 316 352 0:77 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 + CONTENT + + allow(File).to receive(:exist?).with('/proc/1/cgroup').and_return(true) + allow(File).to receive(:read).with('/proc/1/cgroup').and_return(proc_1_cgroup) + allow(File).to receive(:exist?).with('/proc/self/mountinfo').and_return(true) + allow(File).to receive(:read).with('/proc/self/mountinfo').and_return(proc_self_mountinfo) + end + + def stub_runs_in_docker_container_cgroup1 + proc_1_cgroup = <<~CONTENT + 13:rdma:/docker/d59538e9b3d3aa6012f08587c13199cbad3f882ecaa9637905971df18ab89757 + 12:hugetlb:/docker/d59538e9b3d3aa6012f08587c13199cbad3f882ecaa9637905971df18ab89757 + 11:memory:/docker/d59538e9b3d3aa6012f08587c13199cbad3f882ecaa9637905971df18ab89757 + 10:devices:/docker/d59538e9b3d3aa6012f08587c13199cbad3f882ecaa9637905971df18ab89757 + 9:blkio:/docker/d59538e9b3d3aa6012f08587c13199cbad3f882ecaa9637905971df18ab89757 + CONTENT + allow(File).to receive(:exist?).with('/proc/1/cgroup').and_return(true) + allow(File).to receive(:read).with('/proc/1/cgroup').and_return(proc_1_cgroup) + end + + def stub_runs_in_docker_container_cgroup2 + proc_1_cgroup = <<~CONTENT + 0::/ + CONTENT + + proc_self_mountinfo = <<~CONTENT + 355 354 0:21 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw,nsdelegate + 356 352 0:74 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw + 357 352 0:79 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k + 358 350 8:16 /var/lib/docker/containers/69e3febd00ac4720d2ea58c935574776285f6a0016d2aa30b0c280a81c385e69/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/sdb rw,discard,errors=remount-ro,data=ordered + 359 350 8:16 /var/lib/docker/containers/69e3febd00ac4720d2ea58c935574776285f6a0016d2aa30b0c280a81c385e69/hostname /etc/hostname rw,relatime - ext4 /dev/sdb rw,discard,errors=remount-ro,data=ordered + 360 350 8:16 /var/lib/docker/containers/69e3febd00ac4720d2ea58c935574776285f6a0016d2aa30b0c280a81c385e69/hosts /etc/hosts rw,relatime - ext4 /dev/sdb rw,discard,errors=remount-ro,data=ordered + 316 352 0:77 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 + CONTENT + + allow(File).to receive(:exist?).with('/proc/1/cgroup').and_return(true) + allow(File).to receive(:read).with('/proc/1/cgroup').and_return(proc_1_cgroup) + allow(File).to receive(:exist?).with('/proc/self/mountinfo').and_return(true) + allow(File).to receive(:read).with('/proc/self/mountinfo').and_return(proc_self_mountinfo) + end + + # Create a minimal mock Rack environment hash to test just what we need + def create_mock_env + mock_env = { + 'HTTP_X_FORWARDED_FOR' => remote_ip, + 'REMOTE_ADDR' => remote_addr || remote_ip, + 'HTTP_USER_AGENT' => user_agent + } + + if is_periodic_task + mock_env['HTTP_X_AWS_SQSD_TASKNAME'] = period_task_name + else + mock_env['rack.input'] = StringIO.new('{"job_class": "ElasticBeanstalkJob"}') + end + + mock_env end end From 0f9ce787a5b5240b382b8d55d9284931f4fca146 Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Fri, 15 Nov 2024 15:58:22 -0500 Subject: [PATCH 3/4] Minor refactor and fix tests --- .../middleware/elastic_beanstalk_sqsd.rb | 25 +++++++++++-------- .../middleware/elastic_beanstalk_sqsd_spec.rb | 2 +- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/aws/rails/middleware/elastic_beanstalk_sqsd.rb b/lib/aws/rails/middleware/elastic_beanstalk_sqsd.rb index dc36ca72..f06b6060 100644 --- a/lib/aws/rails/middleware/elastic_beanstalk_sqsd.rb +++ b/lib/aws/rails/middleware/elastic_beanstalk_sqsd.rb @@ -5,18 +5,13 @@ module Rails module Middleware # Middleware to handle requests from the SQS Daemon present on Elastic Beanstalk worker environments. class ElasticBeanstalkSQSD - INTERNAL_ERROR_MESSAGE = 'Failed to execute job - see Rails log for more details.' - INTERNAL_ERROR_RESPONSE = [500, { 'Content-Type' => 'text/plain' }, [INTERNAL_ERROR_MESSAGE]].freeze - FORBIDDEN_MESSAGE = 'Request with aws-sqsd user agent was made from untrusted address.' - FORBIDDEN_RESPONSE = [403, { 'Content-Type' => 'text/plain' }, [FORBIDDEN_MESSAGE]].freeze - def initialize(app) @app = app @logger = ::Rails.logger end def call(env) - request = ActionDispatch::Request.new(env) + request = ::ActionDispatch::Request.new(env) # Pass through unless user agent is the SQS Daemon return @app.call(env) unless from_sqs_daemon?(request) @@ -26,7 +21,7 @@ def call(env) # Only accept requests from this user agent if it is from localhost or a docker host in case of forgery. unless request.local? || sent_from_docker_host?(request) @logger.warn('SQSD request detected from untrusted address; returning 403 forbidden.') - return FORBIDDEN_RESPONSE + return forbidden_response end # Execute job or periodic task based on HTTP request context @@ -42,11 +37,11 @@ def execute_job(request) @logger.debug("Executing job: #{job_name}") begin - ActiveJob::Base.execute(job) + ::ActiveJob::Base.execute(job) rescue NoMethodError, NameError => e @logger.error("Job #{job_name} could not resolve to a class that inherits from Active Job.") @logger.error("Error: #{e}") - return INTERNAL_ERROR_RESPONSE + return internal_error_response end [200, { 'Content-Type' => 'text/plain' }, ["Successfully ran job #{job_name}."]] @@ -63,12 +58,22 @@ def execute_periodic_task(request) rescue NoMethodError, NameError => e @logger.error("Periodic task #{job_name} could not resolve to an Active Job class - check the spelling in cron.yaml.") @logger.error("Error: #{e}.") - return INTERNAL_ERROR_RESPONSE + return internal_error_response end [200, { 'Content-Type' => 'text/plain' }, ["Successfully ran periodic task #{job_name}."]] end + def internal_error_response + message = 'Failed to execute job - see Rails log for more details.' + [500, { 'Content-Type' => 'text/plain' }, [message]] + end + + def forbidden_response + message = 'Request with aws-sqsd user agent was made from untrusted address.' + [403, { 'Content-Type' => 'text/plain' }, [message]] + end + # The beanstalk worker SQS Daemon sets a specific User-Agent headers that begins with 'aws-sqsd'. def from_sqs_daemon?(request) current_user_agent = request.headers['User-Agent'] diff --git a/spec/aws/rails/middleware/elastic_beanstalk_sqsd_spec.rb b/spec/aws/rails/middleware/elastic_beanstalk_sqsd_spec.rb index 4bacf053..40f76d23 100644 --- a/spec/aws/rails/middleware/elastic_beanstalk_sqsd_spec.rb +++ b/spec/aws/rails/middleware/elastic_beanstalk_sqsd_spec.rb @@ -43,7 +43,7 @@ module Middleware it 'returns internal server error if job name cannot be resolved' do # Stub execute call to avoid invoking Active Job callbacks # Local testing indicates this failure results in a NameError - allow(ActiveJob::Base).to receive(:execute).and_raise(NameError) + allow(::ActiveJob::Base).to receive(:execute).and_raise(NameError) expect(response[0]).to eq(500) end From 66a5e4aa69cbe221037a9fa73155320ac321450b Mon Sep 17 00:00:00 2001 From: Matt Muller Date: Fri, 15 Nov 2024 16:02:09 -0500 Subject: [PATCH 4/4] Rubocop --- lib/aws/rails/middleware/elastic_beanstalk_sqsd.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/aws/rails/middleware/elastic_beanstalk_sqsd.rb b/lib/aws/rails/middleware/elastic_beanstalk_sqsd.rb index f06b6060..bd8506f5 100644 --- a/lib/aws/rails/middleware/elastic_beanstalk_sqsd.rb +++ b/lib/aws/rails/middleware/elastic_beanstalk_sqsd.rb @@ -38,7 +38,7 @@ def execute_job(request) begin ::ActiveJob::Base.execute(job) - rescue NoMethodError, NameError => e + rescue NameError => e @logger.error("Job #{job_name} could not resolve to a class that inherits from Active Job.") @logger.error("Error: #{e}") return internal_error_response @@ -55,7 +55,7 @@ def execute_periodic_task(request) begin job = job_name.constantize.new job.perform_now - rescue NoMethodError, NameError => e + rescue NameError => e @logger.error("Periodic task #{job_name} could not resolve to an Active Job class - check the spelling in cron.yaml.") @logger.error("Error: #{e}.") return internal_error_response @@ -118,11 +118,10 @@ def build_default_docker_ips File.open('/proc/net/route').each_line do |line| fields = line.strip.split next if fields.size != 11 - # Destination == 0.0.0.0 and Flags & RTF_GATEWAY != 0 - if fields[1] == '00000000' && fields[3].hex.anybits?(0x2) - default_gw_ips << IPAddr.new_ntoh([fields[2].hex].pack('L')).to_s - end + next unless fields[1] == '00000000' && fields[3].hex.anybits?(0x2) + + default_gw_ips << IPAddr.new_ntoh([fields[2].hex].pack('L')).to_s end end