first commit

README.md

@@ -1 +1,15 @@
-# PyKinect
+# PyKinect
+
+This is the python package to use Kinect (remotely). It has two parts: server and client.
+
+## Server Installation
+- Follow [this guide](https://github.com/microsoft/Azure-Kinect-Sensor-SDK) to install the Azure Kinect SDK (K4A).
+- On Ubuntu, you’ll need to set up a udev rule to use the Kinect camera without `sudo`. Follow [these instructions](https://github.com/microsoft/Azure-Kinect-Sensor-SDK/blob/develop/docs/usage.md#linux-device-setup).
+- Make sure you can run `k4aviewer` from the terminal without `sudo`.
+- ```pip install open3d aiohttp imageio```.
+- ```python run_server.py --port PORT```
+- (optional) Modify `kinect_config.json`. For further instructions, please refer to [this guide](http://www.open3d.org/docs/latest/tutorial/Basic/azure_kinect.html).
+
+## Client Usage
+- Option1: browse. Open a browse and visit `IP:PORT/view`. You will see both color image and depth image.
+- Option2: python. `KinectClient` can get cam_intr and RGBD image. Please refer to `run_client.py` for more details.

cameras.py

@@ -0,0 +1,74 @@
+import json
+import time
+
+import numpy as np
+import open3d
+import os
+
+
+def get_intr(config, device, align_depth_to_color, record_filename, intr_json_filename):
+    # get intrinsic
+    recorder = open3d.io.AzureKinectRecorder(config, device)
+    recorder.init_sensor()
+    recorder.open_record(record_filename)
+    recorder.record_frame(True, align_depth_to_color)
+    recorder.close_record()
+    reader = open3d.io.AzureKinectMKVReader()
+    reader.open(record_filename)
+    metadata = reader.get_metadata()
+    open3d.io.write_azure_kinect_mkv_metadata(intr_json_filename, metadata)
+
+
+class Kinect(object):
+    def __init__(self, device, config=None, align_depth_to_color=True, depth_scale_offset=None):
+        if config is not None:
+            config = open3d.io.read_azure_kinect_sensor_config(config)
+        else:
+            print('[Warning] No Kinect Config!')
+            config = open3d.io.AzureKinectSensorConfig()
+        self.align_depth_to_color = align_depth_to_color
+
+        # get intr
+        record_filename = 'intr_record.mkv'
+        intr_json_filename = 'intrinsic.json'
+        get_intr(config, device, self.align_depth_to_color, record_filename, intr_json_filename)
+        self.color_intr = np.array(json.load(open('intrinsic.json'))['intrinsic_matrix']).reshape([3, 3]).T
+        os.remove(record_filename)
+        os.remove(intr_json_filename)
+
+        self.sensor = open3d.io.AzureKinectSensor(config)
+        if not self.sensor.connect(device):
+            raise RuntimeError('Failed to connect to sensor')
+        self.depth_scale_offset = 1 if depth_scale_offset is None else np.loadtxt(depth_scale_offset)
+
+
+    def get_rgbd(self):
+        while True:
+            rgbd = self.sensor.capture_frame(self.align_depth_to_color)
+            if rgbd is None:
+                continue
+            color_img = np.asarray(rgbd.color)
+            depth_img = np.asarray(rgbd.depth).astype(np.float) / 1000.
+            return color_img, depth_img
+
+
+    def get_avg_depth(self, n, imsize):
+        self.max_depth = 5
+        ds = np.empty((imsize[0], imsize[1], n), dtype=np.float64)
+        for i in range(n):
+            _, depth_img = self.get_rgbd()
+            ds[:, :, i] = depth_img
+            time.sleep(0.001)
+        ds[ds == 0] = np.nan
+        d = np.nanmedian(ds, axis=2)
+        d[np.isnan(d)] = self.max_depth
+        d[d == 0] = self.max_depth
+        d[d > self.max_depth] = self.max_depth
+        return d
+
+    def get_camera_data(self, n=50):
+        color_img, depth_img = self.get_rgbd()
+        if n > 1:
+            depth_img = self.get_avg_depth(n, color_img.shape)
+        depth_img *= self.depth_scale_offset
+        return color_img, depth_img

kinect_config.json

@@ -0,0 +1,11 @@
+{
+    "camera_fps" : "K4A_FRAMES_PER_SECOND_30",
+    "color_format" : "K4A_IMAGE_FORMAT_COLOR_MJPG",
+    "color_resolution" : "K4A_COLOR_RESOLUTION_720P",
+    "depth_delay_off_color_usec" : "0",
+    "depth_mode" : "K4A_DEPTH_MODE_NFOV_UNBINNED",
+    "disable_streaming_indicator" : "false",
+    "subordinate_delay_off_master_usec" : "0",
+    "synchronized_images_only" : "false",
+    "wired_sync_mode" : "K4A_WIRED_SYNC_MODE_STANDALONE"
+}

run_client.py

@@ -0,0 +1,26 @@
+import requests
+import pickle
+import argparse
+
+class KinectClient:
+    def __init__(self, ip, port):
+        self.ip = ip
+        self.port = port
+
+    def get_intr(self):
+        return pickle.loads(requests.get(f'http://{self.ip}:{self.port}/intr').content)
+
+    def get_camera_data(self, n=1):
+        return pickle.loads(requests.get(f'http://{self.ip}:{self.port}/pickle/{n}').content)
+
+if __name__=='__main__':
+    parser = argparse.ArgumentParser("Kinect Server")
+    parser.add_argument("--ip", type=str, default='0.0.0.0', help="ip")
+    parser.add_argument("--port", type=int, default=8080, help="port")
+    args = parser.parse_args()
+
+    kinect = KinectClient(args.ip, args.port)
+    intr = kinect.get_intr()
+    camera_data = kinect.get_camera_data(n=10)
+    print(intr)
+    print(camera_data['color_img'].shape, camera_data['depth_img'].shape)

run_server.py

@@ -0,0 +1,62 @@
+import argparse
+import pickle
+from io import BytesIO
+
+import imageio
+import numpy as np
+from aiohttp import web
+
+from cameras import Kinect
+
+camera = Kinect(
+    device=0,
+    align_depth_to_color=True,
+    config='kinect_config.json',
+    depth_scale_offset=None
+)
+
+async def view_handle(request):
+    img_buf = BytesIO()
+    color_img, depth_img = camera.get_camera_data(n=1)
+    color_img = color_img / 255.0
+
+    depth_img -= np.min(depth_img)
+    depth_img /= np.max(depth_img)
+    img = np.concatenate([color_img, np.repeat(depth_img[:, :, np.newaxis], 3, axis=2)], axis=0)
+    imageio.imwrite(img_buf, img, format='jpeg')
+
+    return web.Response(body=img_buf.getbuffer(), content_type='image/jpeg')
+
+
+async def pickle_handle(request):
+    n = int(request.match_info.get('avg', 1))
+    print(n)
+    img_buf = BytesIO()
+    color_img, depth_img = camera.get_camera_data(n=n)
+    cam_intr = camera.color_intr
+
+    enc = pickle.dumps({
+        'color_img':color_img,
+        'depth_img':depth_img
+    })
+    return web.Response(body=enc)
+
+
+async def intr_handle(request):
+    enc = pickle.dumps(camera.color_intr)
+    return web.Response(body=enc)
+
+
+app = web.Application()
+app.add_routes([web.get('/', old_handle),
+                web.get('/view', view_handle),
+                web.get('/intr', intr_handle),
+                web.get('/pickle', pickle_handle),
+                web.get('/pickle/{avg}', pickle_handle)])
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser("Kinect Server")
+    parser.add_argument("--port", type=int, default=8080, help="port")
+    args = parser.parse_args()
+
+    web.run_app(app, port=args.port)