This repository has been archived by the owner on Jul 8, 2021. It is now read-only.
forked from cylc/cylc-flow
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
task remote init: fix strange tempfile issue
Users are experiencing issues on one of the platforms we deloy Cylc where remote submission fails intermittently with a file not found error complaining that a temporary file cannot be found. (The file is used for piping service files via STDIN (to the SSH) command.) In this change, we pass the temporary file handle directly instead of passing the name of the temporary file to the process pool. This appears to fix the issue. (Note: Before cylc#2590, we were unable to pass the file handle because the context needs to be serialised for the multiprocessing.Pool. This is no longer a requirement in our own subprocess pool, so it is now safer to pass the handle.) Add basic unit tests for running command with STDIN in subprocess pool. Rename SuiteProcPool to SubProcPool.
- Loading branch information
1 parent
c3d669e
commit 430b880
Showing
8 changed files
with
162 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| #!/usr/bin/env python2 | ||
|
|
||
| # THIS FILE IS PART OF THE CYLC SUITE ENGINE. | ||
| # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. | ||
| # | ||
| # This program is free software: you can redistribute it and/or modify | ||
| # it under the terms of the GNU General Public License as published by | ||
| # the Free Software Foundation, either version 3 of the License, or | ||
| # (at your option) any later version. | ||
| # | ||
| # This program is distributed in the hope that it will be useful, | ||
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| # GNU General Public License for more details. | ||
| # | ||
| # You should have received a copy of the GNU General Public License | ||
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| from tempfile import NamedTemporaryFile, TemporaryFile | ||
| import unittest | ||
|
|
||
| from cylc.subprocctx import SubProcContext | ||
| from cylc.subprocpool import SubProcPool | ||
|
|
||
|
|
||
| class TestSubProcPool(unittest.TestCase): | ||
|
|
||
| def test_run_command_returns_0(self): | ||
| """Test basic usage, command returns 0""" | ||
| ctx = SubProcContext('truth', ['true']) | ||
| SubProcPool.run_command(ctx) | ||
| self.assertEqual(ctx.err, '') | ||
| self.assertEqual(ctx.out, '') | ||
| self.assertEqual(ctx.ret_code, 0) | ||
|
|
||
| def test_run_command_returns_1(self): | ||
| """Test basic usage, command returns 1""" | ||
| ctx = SubProcContext('lies', ['false']) | ||
| SubProcPool.run_command(ctx) | ||
| self.assertEqual(ctx.err, '') | ||
| self.assertEqual(ctx.out, '') | ||
| self.assertEqual(ctx.ret_code, 1) | ||
|
|
||
| def test_run_command_writes_to_out(self): | ||
| """Test basic usage, command writes to STDOUT""" | ||
| ctx = SubProcContext('parrot', ['echo', 'pirate', 'urrrr']) | ||
| SubProcPool.run_command(ctx) | ||
| self.assertEqual(ctx.err, '') | ||
| self.assertEqual(ctx.out, 'pirate urrrr\n') | ||
| self.assertEqual(ctx.ret_code, 0) | ||
|
|
||
| def test_run_command_writes_to_err(self): | ||
| """Test basic usage, command writes to STDERR""" | ||
| ctx = SubProcContext( | ||
| 'parrot2', ['bash', '-c', 'echo pirate errrr >&2']) | ||
| SubProcPool.run_command(ctx) | ||
| self.assertEqual(ctx.err, 'pirate errrr\n') | ||
| self.assertEqual(ctx.out, '') | ||
| self.assertEqual(ctx.ret_code, 0) | ||
|
|
||
| def test_run_command_with_stdin_from_str(self): | ||
| """Test STDIN from string""" | ||
| ctx = SubProcContext('meow', ['cat'], stdin_str='catches mice.\n') | ||
| SubProcPool.run_command(ctx) | ||
| self.assertEqual(ctx.err, '') | ||
| self.assertEqual(ctx.out, 'catches mice.\n') | ||
| self.assertEqual(ctx.ret_code, 0) | ||
|
|
||
| def test_run_command_with_stdin_from_handle(self): | ||
| """Test STDIN from a single opened file handle""" | ||
| handle = TemporaryFile() | ||
| handle.write('catches mice.\n'.encode('UTF-8')) | ||
| handle.seek(0) | ||
| ctx = SubProcContext('meow', ['cat'], stdin_files=[handle]) | ||
| SubProcPool.run_command(ctx) | ||
| self.assertEqual(ctx.err, '') | ||
| self.assertEqual(ctx.out, 'catches mice.\n') | ||
| self.assertEqual(ctx.ret_code, 0) | ||
| handle.close() | ||
|
|
||
| def test_run_command_with_stdin_from_path(self): | ||
| """Test STDIN from a single file path""" | ||
| handle = NamedTemporaryFile() | ||
| handle.write('catches mice.\n'.encode('UTF-8')) | ||
| handle.seek(0) | ||
| ctx = SubProcContext('meow', ['cat'], stdin_files=[handle.name]) | ||
| SubProcPool.run_command(ctx) | ||
| self.assertEqual(ctx.err, '') | ||
| self.assertEqual(ctx.out, 'catches mice.\n') | ||
| self.assertEqual(ctx.ret_code, 0) | ||
| handle.close() | ||
|
|
||
| def test_run_command_with_stdin_from_handles(self): | ||
| """Test STDIN from multiple file handles""" | ||
| handles = [] | ||
| for txt in ['catches mice.\n', 'eat fish.\n']: | ||
| handle = TemporaryFile() | ||
| handle.write(txt.encode('UTF-8')) | ||
| handle.seek(0) | ||
| handles.append(handle) | ||
| ctx = SubProcContext('meow', ['cat'], stdin_files=handles) | ||
| SubProcPool.run_command(ctx) | ||
| self.assertEqual(ctx.err, '') | ||
| self.assertEqual(ctx.out, 'catches mice.\neat fish.\n') | ||
| self.assertEqual(ctx.ret_code, 0) | ||
| for handle in handles: | ||
| handle.close() | ||
|
|
||
| def test_run_command_with_stdin_from_paths(self): | ||
| """Test STDIN from multiple file paths""" | ||
| handles = [] | ||
| for txt in ['catches mice.\n', 'eat fish.\n']: | ||
| handle = NamedTemporaryFile() | ||
| handle.write(txt.encode('UTF-8')) | ||
| handle.seek(0) | ||
| handles.append(handle) | ||
| ctx = SubProcContext( | ||
| 'meow', ['cat'], stdin_files=[handle.name for handle in handles]) | ||
| SubProcPool.run_command(ctx) | ||
| self.assertEqual(ctx.err, '') | ||
| self.assertEqual(ctx.out, 'catches mice.\neat fish.\n') | ||
| self.assertEqual(ctx.ret_code, 0) | ||
| for handle in handles: | ||
| handle.close() | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| unittest.main() |