qwenvl2_robot_recorder.yml

nodes:
  - id: arm # Run on car
    # build: wget -nc https://github.com/dora-rs/gosim-2024/releases/download/v0.0.1/robot # cargo install --git https://github.com/lyon-dora/chongyou-car
    path: dora-gen72
    _unstable_deploy:
      machine: ec7
    inputs:
      movec: arm-interpolation/movec
      movej: arm-interpolation/movej
      claw: arm-interpolation/claw
      save: arm-interpolation/save
      go_to: arm-interpolation/go_to
    env:
      ROBOT_IP: 192.168.1.18

  - id: mickrobot_chassis # Run on car
    # build: wget -nc https://github.com/dora-rs/gosim-2024/releases/download/v0.0.1/robot # cargo install --git https://github.com/lyon-dora/chongyou-car
    path: robot
    _unstable_deploy:
      machine: ec7
    inputs:
      text: dora-qwenvl-recorder/text

  - id: keyboard-listener # Run on car
    # build: pip install dora-keyboard
    path: dora-keyboard
    inputs:
      tick: dora/timer/millis/10
    outputs:
      - char
    env:
      DISPLAY: :1

  - id: key-interpolation
    path: key_interpolation.py
    inputs:
      keyboard: keyboard-listener/char
    outputs:
      - text

  - id: arm-interpolation
    path: arm_interpolation.py
    inputs:
      keyboard: dora-qwenvl-recorder/text
    outputs:
      - movec
      - movej
      - claw
      - save
      - go_to
      - prompt

  - id: camera1 # Run on car
    # build: pip install opencv-video-capture
    path: opencv-video-capture
    _unstable_deploy:
      machine: ec7
    inputs:
      tick: dora/timer/millis/50
    outputs:
      - image
    env:
      CAPTURE_PATH: 0
      IMAGE_WIDTH: 640
      IMAGE_HEIGHT: 480
      ENCODING: jpeg
      FLIP: BOTH

  - id: camera2 # Run on car
    # build: pip install opencv-video-capture
    path: opencv-video-capture
    _unstable_deploy:
      machine: ec7
    inputs:
      tick: dora/timer/millis/50
    outputs:
      - image
    env:
      CAPTURE_PATH: 2
      IMAGE_WIDTH: 640
      IMAGE_HEIGHT: 480
      ENCODING: jpeg

  - id: plot
    # build: |
    # wget -nc https://github.com/dora-rs/gosim-2024/releases/download/v0.0.1/rerun
    # wget -nc https://github.com/dora-rs/gosim-2024/releases/download/v0.0.1/dora-rerun
    path: dora-rerun
    inputs:
      image_1:
        source: camera1/image
        queue_size: 1
      image_2:
        source: camera2/image
        queue_size: 1
      textlog_vlm: dora-qwenvl-recorder/text
    env:
      IMAGE_WIDTH: 640
      IMAGE_HEIGHT: 480
      RERUN_MEMORY_LIMIT: 25%
      README: |
        To control the robot, use the following keys:
        - w: move forward
        - s: move backward
        - a: move left
        - d: move right
        - q: stop
        - r: open gripper
        - t: close gripper
        - h: move arm left
        - k: move arm right
        - y: move arm down
        - i: move arm up
        - u: move arm forward
        - j: move arm backward
        - v: go home
        - b: save home

  - id: dora-qwenvl-recorder
    path: llama-factory-recorder
    inputs:
      image_1:
        source: camera1/image
        queue_size: 1
      image_2:
        source: camera2/image
        queue_size: 1
      ground_truth: key-interpolation/text
    outputs:
      - text
    env:
      DEFAULT_QUESTION: Respond with arm left, arm right, arm up, arm down, arm forward, arm backward, open, close, or go home to grab the bottle
      LLAMA_FACTORY_ROOT_PATH: /home/peter/Documents/work/LLaMA-Factory