Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Process monitor async write buffering #2186

Merged
merged 1 commit into from
Aug 23, 2017
Merged

Conversation

SeanTAllen
Copy link
Member

Based on work originally done by @lispmeister. I brought it up to date
and fixed problematic test failures.

Add write buffering and asio events for writes to STDIN of the child
We now register an asio event on the STDIN file descriptor.
We write as much as possible. If we can't write everything we store the
ByteSeq in a list together with an offset and set the _stdin_writeable
flag to false. Once the OS signals STDIN is writeable again we try to
write the chunks stored in the list starting with the head chunk.

Closes #945

@SeanTAllen SeanTAllen added the changelog - added Automatically add "Added" CHANGELOG entry on merge label Aug 22, 2017
@SeanTAllen
Copy link
Member Author

Here is the updated version of the test script designed to verify #945 is fixed:

updated test script for latest ponyc:

// original credit to @lispmeister
use "process"
use "files"
use "options"

actor Main
  let _env: Env

  new create(env: Env) =>
    _env = env
    var lines: (I64 | None) = None
    var required_args_are_present: Bool = true
    // parse commandline
    try
      var options = Options(_env.args)
      options
      .add("lines", "l", I64Argument)
      for option in options do
        match option
        | ("lines", let arg: I64) => lines = arg
        else
          env.err.print("stdin-process-test: unknown argument")
          env.err.print("stdin-process-test: usage: [--lines=2000]")
        end
      end

      // check lines value
      if lines is None then
        env.err.print("dagon: Must supply required '--lines' argument")
        required_args_are_present = false
      elseif (lines as I64) < 0 then
          env.err.print("stdin-process-test: lines can't be negative")
          required_args_are_present = false
      end

      if not required_args_are_present then
        env.err.print("stdin-process-test: error parsing arguments. Bailing out!")
        return
      end

      // create a notifier
      let client = ProcessClient(_env)
      let notifier: ProcessNotify iso = consume client
      // define the binary to run
      try
        let path = FilePath(env.root as AmbientAuth, "/usr/bin/wc")?
        // define the arguments; first arg is always the binary name
        let args: Array[String] iso = recover Array[String](2) end
        args.push("wc")
        args.push("-l")
        // define the environment variable for the execution
        let vars: Array[String] iso = recover Array[String](2) end
        vars.push("HOME=/")
        vars.push("PATH=/bin")
        // create a ProcessMonitor and spawn the child process
        let auth = _env.root as AmbientAuth
        let pm: ProcessMonitor = ProcessMonitor(auth, consume notifier, path,
        consume args, consume vars)
        // write to STDIN of the child process
        var i:I64 = 0
        while i < (lines as I64) do
          pm.write("..............................\n")
          i = i + 1
        end
        pm.done_writing() // closing stdin
      else
        env.err.print("stdin-process-test: Could not create FilePath!")
      end
    else
      env.err.print("stdin-process-test: error parsing arguments")
    end


// define a client that implements the ProcessNotify interface
class ProcessClient is ProcessNotify
  let _env: Env

  new iso create(env: Env) =>
    _env = env

  fun ref stdout(process: ProcessMonitor ref, data: Array[U8] iso) =>
    let out = String.from_array(consume data)
    _env.out.print("STDOUT: " + out)

  fun ref stderr(process: ProcessMonitor ref, data: Array[U8] iso) =>
    let err = String.from_array(consume data)
    _env.out.print("STDERR: " + err)

  fun ref failed(process: ProcessMonitor ref, err: ProcessError) =>
    match err
    | ExecveError   => _env.out.print("ProcessError: ExecveError")
    | PipeError     => _env.out.print("ProcessError: PipeError")
    | ForkError     => _env.out.print("ProcessError: ForkError")
    | WaitpidError  => _env.out.print("ProcessError: WaitpidError")
    | WriteError    => _env.out.print("ProcessError: WriteError")
    | KillError     => _env.out.print("ProcessError: KillError")
    | CapError      => _env.out.print("ProcessError: CapError")
    | Unsupported   => _env.out.print("ProcessError: Unsupported")
    else
      _env.out.print("Unknown ProcessError!")
    end

  fun ref dispose(process: ProcessMonitor ref, child_exit_code: I32) =>
    let code: I32 = consume child_exit_code
    _env.out.print("Child exit code: " + code.string())

To trigger error run with command line option like --lines=100000. basically a large value that will trigger a write error.

This was referenced Aug 22, 2017
@SeanTAllen
Copy link
Member Author

SeanTAllen commented Aug 22, 2017

Found a flaw in the test logic. Fixing.

@SeanTAllen SeanTAllen force-pushed the process_monitor_buffering branch 2 times, most recently from 793534a to 883e3ba Compare August 22, 2017 04:09
@SeanTAllen
Copy link
Member Author

This seems to have some timeout issues that are environmental. I can't reproduce it locally. But I'm going to keep trying to verify.

@SeanTAllen SeanTAllen added the do not merge This PR should not be merged at this time label Aug 22, 2017
@SeanTAllen SeanTAllen force-pushed the process_monitor_buffering branch 3 times, most recently from 315a08e to be22376 Compare August 22, 2017 05:22
Based on work originally done by @lispmeister. I brought it up to date
and fixed problematic test failures.

Add write buffering and asio events for writes to STDIN of the child
We now register an asio event on the STDIN file descriptor.
We write as much as possible. If we can't write everything we store the
ByteSeq in a list together with an offset and set the _stdin_writeable
flag to false. Once the OS signals STDIN is writeable again we try to
write the chunks stored in the list starting with the head chunk.

Closes #945
@SeanTAllen SeanTAllen force-pushed the process_monitor_buffering branch from be22376 to e9d56ae Compare August 22, 2017 10:52
@SeanTAllen SeanTAllen removed the do not merge This PR should not be merged at this time label Aug 22, 2017
@sylvanc sylvanc merged commit a19991c into master Aug 23, 2017
ponylang-main added a commit that referenced this pull request Aug 23, 2017
@SeanTAllen SeanTAllen deleted the process_monitor_buffering branch June 7, 2018 20:19
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
changelog - added Automatically add "Added" CHANGELOG entry on merge
Projects
None yet
Development

Successfully merging this pull request may close these issues.

STDIN processing bug in process package
2 participants