From 0fb06aca7d02463917863a5eafe258ee27d45746 Mon Sep 17 00:00:00 2001 From: "Katz, Matan" Date: Tue, 20 Aug 2019 14:46:01 +0300 Subject: [PATCH 1/2] android pointcloud: - missing pointcloud filter API added. - 3D view added to the camera app. --- src/android/jni/frame.cpp | 23 +++ .../intel/realsense/librealsense/GLFrame.java | 8 + .../realsense/librealsense/GLMotionFrame.java | 12 ++ .../realsense/librealsense/GLPointsFrame.java | 133 +++++++++++++ .../realsense/librealsense/GLRenderer.java | 181 +++++++++++++++--- .../librealsense/GLRsSurfaceView.java | 28 ++- .../intel/realsense/librealsense/Points.java | 30 +++ .../intel/realsense/camera/InfoActivity.java | 5 + .../realsense/camera/PlaybackActivity.java | 101 +++++++++- .../realsense/camera/PresetsActivity.java | 2 +- .../realsense/camera/PreviewActivity.java | 119 ++++++------ .../realsense/camera/RecordingActivity.java | 9 +- .../realsense/camera/SettingsActivity.java | 39 ++-- .../com/intel/realsense/camera/Streamer.java | 12 +- .../res/layout-land/activity_playback.xml | 55 ++++++ .../main/res/layout-land/activity_preview.xml | 16 ++ .../src/main/res/layout/activity_playback.xml | 32 ++++ .../src/main/res/layout/activity_preview.xml | 15 ++ .../camera/src/main/res/values/strings.xml | 1 + 19 files changed, 705 insertions(+), 116 deletions(-) create mode 100644 wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLPointsFrame.java create mode 100644 wrappers/android/tools/camera/src/main/res/layout-land/activity_playback.xml diff --git a/src/android/jni/frame.cpp b/src/android/jni/frame.cpp index 1830de15fe..4f4b628a08 100644 --- a/src/android/jni/frame.cpp +++ b/src/android/jni/frame.cpp @@ -46,6 +46,29 @@ Java_com_intel_realsense_librealsense_Frame_nGetData(JNIEnv *env, jclass type, j handle_error(env, e); } +extern "C" +JNIEXPORT void JNICALL +Java_com_intel_realsense_librealsense_Points_nGetData(JNIEnv *env, jclass type, jlong handle, + jfloatArray data_) { + jsize length = env->GetArrayLength(data_); + rs2_error *e = NULL; + env->SetFloatArrayRegion(data_, 0, length, static_cast(rs2_get_frame_data( + reinterpret_cast(handle), &e))); + handle_error(env, e); +} + +extern "C" +JNIEXPORT void JNICALL +Java_com_intel_realsense_librealsense_Points_nGetTextureCoordinates(JNIEnv *env, jclass type, + jlong handle, + jfloatArray data_) { + jsize length = env->GetArrayLength(data_); + rs2_error *e = NULL; + env->SetFloatArrayRegion(data_, 0, length, reinterpret_cast(rs2_get_frame_texture_coordinates( + reinterpret_cast(handle), &e))); + handle_error(env, e); +} + extern "C" JNIEXPORT jboolean JNICALL Java_com_intel_realsense_librealsense_Frame_nIsFrameExtendableTo(JNIEnv *env, jclass type, jlong handle, jint extension) { diff --git a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLFrame.java b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLFrame.java index d644b242e6..f1d103876d 100644 --- a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLFrame.java +++ b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLFrame.java @@ -23,4 +23,12 @@ protected static void setViewport(Rect r) { GLES10.glMatrixMode(GLES10.GL_PROJECTION); GLES10.glOrthof(0, r.width(), r.height(), 0, -1, +1); } + + public String getLabel() { + if(mFrame == null) + return ""; + try(StreamProfile sp = mFrame.getProfile()){ + return sp.getType() + " - " + sp.getFormat(); + } + } } diff --git a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLMotionFrame.java b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLMotionFrame.java index d26b0653db..ac4e08c7f3 100644 --- a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLMotionFrame.java +++ b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLMotionFrame.java @@ -188,6 +188,18 @@ public synchronized void draw(Rect rect) GLES10.glPopMatrix(); } + @Override + public synchronized String getLabel() { + MotionFrame mf = mFrame.as(Extension.MOTION_FRAME); + Float3 d = mf.getMotionData(); + try(StreamProfile sp = mFrame.getProfile()){ + return sp.getType().name() + + " [ X: " + String.format("%+.2f", d.x) + + ", Y: " + String.format("%+.2f", d.y) + + ", Z: " + String.format("%+.2f", d.z) + " ]"; + } + } + @Override public synchronized void close() { if(mFrame != null) diff --git a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLPointsFrame.java b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLPointsFrame.java new file mode 100644 index 0000000000..64e7377438 --- /dev/null +++ b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLPointsFrame.java @@ -0,0 +1,133 @@ +package com.intel.realsense.librealsense; + +import android.graphics.Rect; +import android.opengl.GLES10; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.IntBuffer; + +public class GLPointsFrame extends GLFrame { + private Frame mTexture; + private IntBuffer mGlTexture; + private float mDeltaX = 0; + private float mDeltaY = 0; + private float mRotationFactor = 0.1f; + + public synchronized void setTextureFrame(Frame frame) { + if(mTexture != null) + mTexture.close(); + mTexture = frame.clone(); + } + public int getTexture() { return mGlTexture.array()[0]; } + + private void drawPoints(float[] verArray, byte[] colorArray) + { + if(colorArray != null){ + ByteBuffer tex = ByteBuffer.allocateDirect(colorArray.length); + tex.order(ByteOrder.nativeOrder()); + tex.put(colorArray); + tex.position(0); + + GLES10.glEnableClientState(GLES10.GL_COLOR_ARRAY); + GLES10.glColorPointer(4, GLES10.GL_UNSIGNED_BYTE, 0, tex); + } + + ByteBuffer ver = ByteBuffer.allocateDirect(verArray.length * 4); + ver.order(ByteOrder.nativeOrder()); + ver.asFloatBuffer().put(verArray); + + GLES10.glEnableClientState(GLES10.GL_VERTEX_ARRAY); + GLES10.glVertexPointer(3, GLES10.GL_FLOAT, 0, ver); + GLES10.glDrawArrays(GLES10.GL_POINTS,0,verArray.length / 3); + + GLES10.glDisableClientState(GLES10.GL_VERTEX_ARRAY); + if(colorArray != null) + GLES10.glDisableClientState(GLES10.GL_COLOR_ARRAY); + } + + @Override + public synchronized void draw(Rect rect) + { + if (mFrame == null || !(mFrame.is(Extension.POINTS))) + return; + + setViewport(rect); + + GLES10.glMatrixMode(GLES10.GL_PROJECTION); + GLES10.glPushMatrix(); + GLES10.glLoadIdentity(); + + GLES10.glOrthof(-1f, 1f, -1f, 1f, -7f, 0f); + + GLES10.glRotatef(180, 0.0f, 0.0f, 1.0f); + GLES10.glRotatef(-mDeltaY * mRotationFactor, 1.0f, 0.0f, 0.0f); + GLES10.glRotatef(-mDeltaX * mRotationFactor, 0.0f, 1.0f, 0.0f); + + Points points = mFrame.as(Extension.POINTS); + float[] data = points.getVertices(); + byte[] tex = mTexture == null ? createTexture(data, 1.2f) : createTexture(points); + drawPoints(data, tex); + + GLES10.glMatrixMode(GLES10.GL_PROJECTION); + GLES10.glPopMatrix(); + } + + //fallback to gray scale if no texture is available + private byte[] createTexture(float[] data, float maxRange){ + int size = data.length / 3; + byte[] texture = new byte[size * 4]; + for(int i = 0; i < size; i++){ + float z = data[i * 3 + 2]; + byte val = (byte) (z / maxRange * 255); + for(int j = 0; j < 3; j++) + texture[i*4+j] = val; + texture[i*4+3] = (byte) 255; + } + return texture; + } + + public byte[] createTexture(Points points){ + if(mTexture == null) + return null; + + VideoFrame vf = mTexture.as(Extension.VIDEO_FRAME); + int textureSize = vf.getWidth() * vf.getHeight(); + int pointCount = points.getCount(); + float[] pointsData = points.getVertices(); + byte[] textureData = new byte[textureSize * 3]; + byte[] texture = new byte[pointCount * 4]; + float[] textureCoordinates = points.getTextureCoordinates(); + + vf.getData(textureData); + int w = vf.getWidth(); + int h = vf.getHeight(); + for(int i = 0; i < pointCount; i++){ + if(pointsData[i * 3 + 2] == 0) + continue; + float x = (float) Math.round(textureCoordinates[2 * i] * w); + float y = (float) Math.round(textureCoordinates[2 * i + 1] * h); + if(x <= 0 || y <= 0 || x >= w || y >= h) + continue; + long texIndex = (long) (x + y * w); + for(int j = 0; j < 3; j++) + texture[i*4+j] = textureData[(int) (texIndex * 3 + j)]; + texture[i*4+3] = (byte) 255; + } + return texture; + } + + @Override + public synchronized void close() { + if(mFrame != null) + mFrame.close(); + if(mGlTexture != null) + GLES10.glDeleteTextures(1, mGlTexture); + mGlTexture = null; + } + + public synchronized void rotate(float deltaX, float deltaY) { + mDeltaX += deltaX; + mDeltaY += deltaY; + } +} diff --git a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLRenderer.java b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLRenderer.java index 38c9dc4608..839c95a2ab 100644 --- a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLRenderer.java +++ b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLRenderer.java @@ -4,8 +4,11 @@ import android.graphics.Rect; import android.opengl.GLES10; import android.opengl.GLSurfaceView; +import android.util.Pair; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.microedition.khronos.egl.EGLConfig; @@ -17,39 +20,113 @@ public class GLRenderer implements GLSurfaceView.Renderer{ private int mWindowHeight = 0; private int mWindowWidth = 0; private boolean mIsDirty = true; + private float mDeltaX = 0; + private float mDeltaY = 0; + private Frame mPointsTexture; + private boolean mHasColorRbg8 = false; + private Colorizer mColorizer = new Colorizer(); + private Map mPointcloud = null; + private boolean mHasColorizedDepth = false; - public Map getRectangles() { + public Map> getRectangles() { return calcRectangles(); } + private boolean showPoints(){ + return mPointcloud != null; + } + + private List createProcessingPipe(){ + List rv = new ArrayList<>(); + if(!mHasColorizedDepth && !showPoints()) + rv.add(mColorizer); + if(showPoints()){ + if(mHasColorRbg8) + rv.add(mPointcloud.get(StreamType.COLOR)); + else + rv.add(mPointcloud.get(StreamType.DEPTH)); + } + return rv; + } + + private FrameSet applyFilters(FrameSet frameSet, List filters){ + frameSet = frameSet.clone(); + for(FilterInterface f : filters){ + FrameSet newSet = frameSet.applyFilter(f); + frameSet.close(); + frameSet = newSet; + } + return frameSet; + } + public void upload(FrameSet frameSet) { + mHasColorRbg8 = mHasColorizedDepth = false; frameSet.foreach(new FrameCallback() { @Override public void onFrame(Frame f) { - addFrame(f); + getTexture(f); } }); - frameSet.foreach(new FrameCallback() { - @Override - public void onFrame(Frame f) { - upload(f); + + List filters = createProcessingPipe(); + try(FrameSet processed = applyFilters(frameSet, filters)){ + choosePointsTexture(processed); + processed.foreach(new FrameCallback() { + @Override + public void onFrame(Frame f) { + addFrame(f); + } + }); + processed.foreach(new FrameCallback() { + @Override + public void onFrame(Frame f) { + upload(f); + } + }); + } + } + + private void choosePointsTexture(FrameSet frameSet){ + if(!showPoints()) + return; + if(mHasColorRbg8) + mPointsTexture = frameSet.first(StreamType.COLOR, StreamFormat.RGB8); + else{ + try (Frame d = frameSet.first(StreamType.DEPTH, StreamFormat.Z16)) { + if(d != null) + mPointsTexture = mColorizer.process(d); } - }); + } + } + + private void getTexture(Frame f){ + try(StreamProfile sp = f.getProfile()){ + if(sp.getType() == StreamType.COLOR && sp.getFormat() == StreamFormat.RGB8) { + mHasColorRbg8 = true; + } + if(sp.getType() == StreamType.DEPTH && sp.getFormat() == StreamFormat.RGB8) { + mHasColorizedDepth = true; + } + } } private void addFrame(Frame f){ if(!isFormatSupported(f.getProfile().getFormat())) return; - int uid = f.getProfile().getUniqueId(); - if(!mFrames.containsKey(uid)){ - synchronized (mFrames) { - if(f.is(Extension.VIDEO_FRAME)) - mFrames.put(uid, new GLVideoFrame()); - if(f.is(Extension.MOTION_FRAME)) - mFrames.put(uid, new GLMotionFrame()); + try(StreamProfile sp = f.getProfile()){ + int uid = sp.getUniqueId(); + if(!mFrames.containsKey(uid)){ + synchronized (mFrames) { + if(f.is(Extension.VIDEO_FRAME) && !showPoints()) + mFrames.put(uid, new GLVideoFrame()); + if(f.is(Extension.MOTION_FRAME) && !showPoints()) + mFrames.put(uid, new GLMotionFrame()); + if(f.is(Extension.POINTS)) + mFrames.put(uid, new GLPointsFrame()); + } + mIsDirty = true; } - mIsDirty = true; } } @@ -57,24 +134,40 @@ public void upload(Frame f) { if(f == null) return; - if(!isFormatSupported(f.getProfile().getFormat())) - return; + try(StreamProfile sp = f.getProfile()){ + if(!isFormatSupported(sp.getFormat())) + return; - addFrame(f); - int uid = f.getProfile().getUniqueId(); + addFrame(f); + int uid = sp.getUniqueId(); - mFrames.get(uid).setFrame(f); + GLFrame curr = mFrames.get(uid); + if(curr == null) + return; + curr.setFrame(f); + + if(mPointsTexture != null && curr instanceof GLPointsFrame){ + ((GLPointsFrame) curr).setTextureFrame(mPointsTexture); + mPointsTexture.close(); + mPointsTexture = null; + } + } } public void clear() { synchronized (mFrames) { mFrames.clear(); + mIsDirty = true; + mDeltaX = 0; + mDeltaY = 0; + mPointcloud = null; + if(mPointsTexture != null) mPointsTexture.close(); + mPointsTexture = null; } - mIsDirty = true; } - private Map calcRectangles(){ - Map rv = new HashMap<>(); + private Map> calcRectangles(){ + Map> rv = new HashMap<>(); int i = 0; for (Map.Entry entry : mFrames.entrySet()){ @@ -84,8 +177,7 @@ private Map calcRectangles(){ Point pos = mWindowWidth > mWindowHeight ? new Point(i++ * size.x, 0) : new Point(0, i++ * size.y); -// new Point(0, (mWindowHeight-size.y) - i++ * size.y); - rv.put(entry.getKey(), new Rect(pos.x, pos.y, pos.x + size.x, pos.y + size.y)); + rv.put(entry.getKey(), new Pair<>(entry.getValue().getLabel(), new Rect(pos.x, pos.y, pos.x + size.x, pos.y + size.y))); } return rv; } @@ -114,16 +206,21 @@ public void onDrawFrame(GL10 gl) { if (mFrames.size() == 0) return; - Map rects = calcRectangles(); + Map> rects = calcRectangles(); for(Integer uid : mFrames.keySet()){ GLFrame fl = mFrames.get(uid); - Rect r = rects.get(uid); + Rect r = rects.get(uid).second; if(mWindowHeight > mWindowWidth){// TODO: remove, w/a for misaligned labels int newTop = mWindowHeight - r.height() - r.top; r = new Rect(r.left, newTop, r.right, newTop + r.height()); } fl.draw(r); + if(fl instanceof GLPointsFrame){ + ((GLPointsFrame)fl).rotate(mDeltaX, mDeltaY); + mDeltaX = 0; + mDeltaY = 0; + } } } } @@ -133,8 +230,34 @@ private boolean isFormatSupported(StreamFormat format) { case RGB8: case RGBA8: case Y8: - case MOTION_XYZ32F: return true; + case MOTION_XYZ32F: + case XYZ32F: return true; default: return false; } } -} + + public void onTouchEvent(float dx, float dy) { + synchronized (mFrames) { + mDeltaX = dx; + mDeltaY = dy; + } + } + + public void showPointcloud(boolean showPoints) { + if(showPoints){ + if(mPointcloud != null) + return; + mPointcloud = new HashMap<>(); + mPointcloud.put(StreamType.COLOR, new Pointcloud(StreamType.COLOR)); + mPointcloud.put(StreamType.DEPTH, new Pointcloud(StreamType.DEPTH)); + } + else { + if(mPointcloud == null) + return; + for(Pointcloud pc : mPointcloud.values()){ + pc.close(); + } + mPointcloud = null; + } + } +} \ No newline at end of file diff --git a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLRsSurfaceView.java b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLRsSurfaceView.java index 9ee9cb928b..16fa16809e 100644 --- a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLRsSurfaceView.java +++ b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/GLRsSurfaceView.java @@ -4,12 +4,16 @@ import android.graphics.Rect; import android.opengl.GLSurfaceView; import android.util.AttributeSet; +import android.util.Pair; +import android.view.MotionEvent; import java.util.Map; public class GLRsSurfaceView extends GLSurfaceView { private final GLRenderer mRenderer; + private float mPreviousX = 0; + private float mPreviousY = 0; public GLRsSurfaceView(Context context) { super(context); @@ -23,7 +27,7 @@ public GLRsSurfaceView(Context context, AttributeSet attrs) { setRenderer(mRenderer); } - public Map getRectangles() { + public Map> getRectangles() { return mRenderer.getRectangles(); } @@ -38,4 +42,26 @@ public void upload(Frame frame) { public void clear() { mRenderer.clear(); } + + @Override + public boolean onTouchEvent(MotionEvent e) { + float x = e.getX(); + float y = e.getY(); + + switch (e.getAction()) { + case MotionEvent.ACTION_MOVE: + + float dx = x - mPreviousX; + float dy = y - mPreviousY; + mRenderer.onTouchEvent(dx, dy); + } + + mPreviousX = x; + mPreviousY = y; + return true; + } + + public void showPointcloud(boolean showPoints) { + mRenderer.showPointcloud(showPoints); + } } diff --git a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/Points.java b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/Points.java index 227d6f1344..6df7f343e0 100644 --- a/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/Points.java +++ b/wrappers/android/librealsense/src/main/java/com/intel/realsense/librealsense/Points.java @@ -1,18 +1,48 @@ package com.intel.realsense.librealsense; public class Points extends Frame { + private float[] mData; + private float[] mTextureCoordinates; + protected Points(long handle) { super(handle); mOwner = false; } + public float[] getVertices(){ + if(mData == null){ + mData = new float[getCount() * 3]; + getData(mData); + } + return mData; + } + + public float[] getTextureCoordinates(){ + if(mTextureCoordinates == null){ + mTextureCoordinates = new float[getCount() * 2]; + getTextureCoordinates(mTextureCoordinates); + } + return mTextureCoordinates; + } + + public void getTextureCoordinates(float[] data) { + nGetTextureCoordinates(mHandle, data); + } + + public void getData(float[] data) { + nGetData(mHandle, data); + } + public int getCount(){ return nGetCount(mHandle); } + public void exportToPly(String filePath, VideoFrame texture) { nExportToPly(mHandle, filePath, texture.getHandle()); } private static native int nGetCount(long handle); + private static native void nGetData(long handle, float[] data); + private static native void nGetTextureCoordinates(long handle, float[] data); private static native void nExportToPly(long handle, String filePath, long textureHandle); } diff --git a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/InfoActivity.java b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/InfoActivity.java index 63d7d87d30..785578e5b1 100644 --- a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/InfoActivity.java +++ b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/InfoActivity.java @@ -2,6 +2,7 @@ import android.os.Bundle; import android.support.v7.app.AppCompatActivity; +import android.util.Log; import android.widget.ArrayAdapter; import android.widget.ListView; import android.widget.TextView; @@ -39,6 +40,10 @@ protected void onResume() { message.setText("Device info:"); try(final Device device = devices.createDevice(0)){ + if(device == null){ + Log.e(TAG, "failed to create device"); + return; + } for(CameraInfo ci : CameraInfo.values()){ if(device.supportsInfo(ci)) infoMap.put(ci, device.getInfo(ci)); diff --git a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PlaybackActivity.java b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PlaybackActivity.java index 10568d4985..634bdce931 100644 --- a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PlaybackActivity.java +++ b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PlaybackActivity.java @@ -1,16 +1,26 @@ package com.intel.realsense.camera; +import android.content.Context; import android.content.Intent; +import android.content.SharedPreferences; +import android.graphics.Color; +import android.graphics.Rect; import android.os.Bundle; import android.support.v7.app.AppCompatActivity; +import android.util.Pair; +import android.view.View; +import android.view.ViewGroup; import android.view.WindowManager; +import android.widget.RelativeLayout; +import android.widget.TextView; -import com.intel.realsense.librealsense.Colorizer; import com.intel.realsense.librealsense.Config; import com.intel.realsense.librealsense.FrameSet; import com.intel.realsense.librealsense.GLRsSurfaceView; import java.io.File; +import java.util.HashMap; +import java.util.Map; public class PlaybackActivity extends AppCompatActivity { private static final String TAG = "librs camera pb"; @@ -21,7 +31,11 @@ public class PlaybackActivity extends AppCompatActivity { private String mFilePath; private Streamer mStreamer; private GLRsSurfaceView mGLSurfaceView; - private Colorizer mColorizer = new Colorizer(); + + private Map mLabels; + + private boolean mShow3D = false; + private TextView m3dButton; @Override protected void onCreate(Bundle savedInstanceState) { @@ -30,6 +44,25 @@ protected void onCreate(Bundle savedInstanceState) { getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); mGLSurfaceView = findViewById(R.id.playbackGlSurfaceView); + m3dButton = findViewById(R.id.playback_3d_button); + m3dButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + mGLSurfaceView.setVisibility(View.GONE); + mGLSurfaceView.clear(); + clearLables(); + mShow3D = !mShow3D; + m3dButton.setTextColor(mShow3D ? Color.YELLOW : Color.WHITE); + mGLSurfaceView.setVisibility(View.VISIBLE); + SharedPreferences sharedPref = getSharedPreferences(getString(R.string.app_settings), Context.MODE_PRIVATE); + SharedPreferences.Editor editor = sharedPref.edit(); + editor.putBoolean(getString(R.string.show_3d), mShow3D); + editor.commit(); + } + }); + SharedPreferences sharedPref = getSharedPreferences(getString(R.string.app_settings), Context.MODE_PRIVATE); + mShow3D = sharedPref.getBoolean(getString(R.string.show_3d), false); + m3dButton.setTextColor(mShow3D ? Color.YELLOW : Color.WHITE); } @Override @@ -51,9 +84,10 @@ public void config(Config config) { @Override public void onFrameset(FrameSet frameSet) { - try (FrameSet processed = frameSet.applyFilter(mColorizer)) { - mGLSurfaceView.upload(processed); - } + mGLSurfaceView.showPointcloud(mShow3D); + mGLSurfaceView.upload(frameSet); + Map> rects = mGLSurfaceView.getRectangles(); + printLables(rects); } }); try { @@ -69,6 +103,8 @@ public void onFrameset(FrameSet frameSet) { protected void onPause() { super.onPause(); + clearLables(); + if(mStreamer != null) mStreamer.stop(); } @@ -99,4 +135,59 @@ protected void onRestoreInstanceState(final Bundle savedInstanceState) { super.onRestoreInstanceState(savedInstanceState); mFilePath = savedInstanceState.getString(FILE_PATH_KEY); } + + private synchronized Map createLabels(Map> rects){ + if(rects == null) + return null; + mLabels = new HashMap<>(); + + final RelativeLayout rl = findViewById(R.id.labels_layout); + for(Map.Entry> e : rects.entrySet()){ + TextView tv = new TextView(getApplicationContext()); + ViewGroup.LayoutParams lp = new RelativeLayout.LayoutParams( + ViewGroup.LayoutParams.WRAP_CONTENT, + ViewGroup.LayoutParams.WRAP_CONTENT); + tv.setLayoutParams(lp); + tv.setTextColor(Color.parseColor("#ffffff")); + tv.setTextSize(14); + rl.addView(tv); + mLabels.put(e.getKey(), tv); + } + return mLabels; + } + + private void printLables(final Map> rects){ + if(rects == null) + return; + final Map lables = new HashMap<>(); + if(mLabels == null) + mLabels = createLabels(rects); + for(Map.Entry> e : rects.entrySet()){ + lables.put(e.getKey(), e.getValue().first); + } + + runOnUiThread(new Runnable() { + @Override + public void run() { + for(Map.Entry e : mLabels.entrySet()){ + Integer uid = e.getKey(); + if(rects.get(uid) == null) + continue; + Rect r = rects.get(uid).second; + TextView tv = e.getValue(); + tv.setX(r.left); + tv.setY(r.top); + tv.setText(lables.get(uid)); + } + } + }); + } + + private void clearLables(){ + if(mLabels != null){ + for(Map.Entry label : mLabels.entrySet()) + label.getValue().setVisibility(View.GONE); + mLabels = null; + } + } } diff --git a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PresetsActivity.java b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PresetsActivity.java index 7638e47bb2..72a1be1459 100644 --- a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PresetsActivity.java +++ b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PresetsActivity.java @@ -65,7 +65,7 @@ public void onItemClick(AdapterView parent, final View view, finish(); } try(Device device = devices.createDevice(0)){ - if(!device.isInAdvancedMode()){ + if(device == null || !device.isInAdvancedMode()){ Log.e(TAG, "failed to set preset, device not in advanced mode"); finish(); } diff --git a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PreviewActivity.java b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PreviewActivity.java index e81962fd6c..eb0dd49ee2 100644 --- a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PreviewActivity.java +++ b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/PreviewActivity.java @@ -1,13 +1,15 @@ package com.intel.realsense.camera; +import android.content.Context; import android.content.Intent; +import android.content.SharedPreferences; import android.graphics.Color; import android.graphics.Rect; import android.os.Bundle; -import android.renderscript.Float3; import android.support.design.widget.FloatingActionButton; import android.support.v7.app.AppCompatActivity; import android.util.Log; +import android.util.Pair; import android.view.View; import android.view.ViewGroup; import android.view.WindowManager; @@ -15,15 +17,9 @@ import android.widget.TextView; import android.widget.Toast; -import com.intel.realsense.librealsense.Colorizer; import com.intel.realsense.librealsense.Config; -import com.intel.realsense.librealsense.Extension; -import com.intel.realsense.librealsense.Frame; -import com.intel.realsense.librealsense.FrameCallback; import com.intel.realsense.librealsense.FrameSet; import com.intel.realsense.librealsense.GLRsSurfaceView; -import com.intel.realsense.librealsense.MotionFrame; -import com.intel.realsense.librealsense.StreamProfile; import java.util.HashMap; import java.util.Map; @@ -36,16 +32,16 @@ public class PreviewActivity extends AppCompatActivity { private TextView mPlaybackButton; private TextView mSettingsButton; private TextView mStatisticsButton; + private TextView m3dButton; private TextView mStatsView; private Map mLabels; private Streamer mStreamer; - private Colorizer mColorizer = new Colorizer(); - private StreamingStats mStreamingStats; private boolean statsToggle = false; + private boolean mShow3D = false; public synchronized void toggleStats(){ statsToggle = !statsToggle; @@ -73,6 +69,7 @@ protected void onCreate(Bundle savedInstanceState) { mPlaybackButton = findViewById(R.id.preview_playback_button); mSettingsButton = findViewById(R.id.preview_settings_button); mStatisticsButton = findViewById(R.id.preview_stats_button); + m3dButton = findViewById(R.id.preview_3d_button); mStartRecordFab.setOnClickListener(new View.OnClickListener() { @Override @@ -96,66 +93,70 @@ public void onClick(View view) { startActivity(intent); } }); + m3dButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + mGLSurfaceView.setVisibility(View.GONE); + mGLSurfaceView.clear(); + clearLables(); + mShow3D = !mShow3D; + m3dButton.setTextColor(mShow3D ? Color.YELLOW : Color.WHITE); + mGLSurfaceView.setVisibility(View.VISIBLE); + SharedPreferences sharedPref = getSharedPreferences(getString(R.string.app_settings), Context.MODE_PRIVATE); + SharedPreferences.Editor editor = sharedPref.edit(); + editor.putBoolean(getString(R.string.show_3d), mShow3D); + editor.commit(); + } + }); mStatisticsButton.setOnClickListener(new View.OnClickListener() { @Override public void onClick(View view) { toggleStats(); } }); + SharedPreferences sharedPref = getSharedPreferences(getString(R.string.app_settings), Context.MODE_PRIVATE); + mShow3D = sharedPref.getBoolean(getString(R.string.show_3d), false); + m3dButton.setTextColor(mShow3D ? Color.YELLOW : Color.WHITE); } - private synchronized Map createLabels(FrameSet frameSet){ + private synchronized Map createLabels(Map> rects){ + if(rects == null) + return null; mLabels = new HashMap<>(); final RelativeLayout rl = findViewById(R.id.labels_layout); - - frameSet.foreach(new FrameCallback() { - @Override - public void onFrame(Frame f) { - TextView tv = new TextView(getApplicationContext()); - ViewGroup.LayoutParams lp = new RelativeLayout.LayoutParams( - ViewGroup.LayoutParams.WRAP_CONTENT, - ViewGroup.LayoutParams.WRAP_CONTENT); - tv.setLayoutParams(lp); - tv.setTextColor(Color.parseColor("#ffffff")); - tv.setTextSize(14); - rl.addView(tv); - StreamProfile p = f.getProfile(); - mLabels.put(p.getUniqueId(), tv); - } - }); + for(Map.Entry> e : rects.entrySet()){ + TextView tv = new TextView(getApplicationContext()); + ViewGroup.LayoutParams lp = new RelativeLayout.LayoutParams( + ViewGroup.LayoutParams.WRAP_CONTENT, + ViewGroup.LayoutParams.WRAP_CONTENT); + tv.setLayoutParams(lp); + tv.setTextColor(Color.parseColor("#ffffff")); + tv.setTextSize(14); + rl.addView(tv); + mLabels.put(e.getKey(), tv); + } return mLabels; } - private void printLables(FrameSet frames, final Map rects){ + private void printLables(final Map> rects){ + if(rects == null) + return; final Map lables = new HashMap<>(); if(mLabels == null) - mLabels = createLabels(frames); - frames.foreach(new FrameCallback() { - @Override - public void onFrame(Frame f) { - StreamProfile p = f.getProfile(); - if(f.is(Extension.MOTION_FRAME)) { - MotionFrame mf = f.as(Extension.MOTION_FRAME); - Float3 d = mf.getMotionData(); - lables.put(p.getUniqueId(),p.getType().name() + - " [ X: " + String.format("%+.2f", d.x) + - ", Y: " + String.format("%+.2f", d.y) + - ", Z: " + String.format("%+.2f", d.z) + " ]"); - } - else{ - lables.put(p.getUniqueId(), p.getType().name()); - } - } - }); + mLabels = createLabels(rects); + for(Map.Entry> e : rects.entrySet()){ + lables.put(e.getKey(), e.getValue().first); + } + runOnUiThread(new Runnable() { @Override public void run() { for(Map.Entry e : mLabels.entrySet()){ Integer uid = e.getKey(); - Rect r = rects.get(uid); - if(r == null) + if(rects.get(uid) == null) continue; + Rect r = rects.get(uid).second; TextView tv = e.getValue(); tv.setX(r.left); tv.setY(r.top); @@ -165,6 +166,14 @@ public void run() { }); } + private void clearLables(){ + if(mLabels != null){ + for(Map.Entry label : mLabels.entrySet()) + label.getValue().setVisibility(View.GONE); + mLabels = null; + } + } + @Override protected void onResume() { super.onResume(); @@ -180,6 +189,7 @@ public void config(Config config) { public void onFrameset(FrameSet frameSet) { mStreamingStats.onFrameset(frameSet); if(statsToggle){ + clearLables(); runOnUiThread(new Runnable() { @Override public void run() { @@ -188,11 +198,10 @@ public void run() { }); } else{ - try (FrameSet processed = frameSet.applyFilter(mColorizer)) { - mGLSurfaceView.upload(processed); - Map rects = mGLSurfaceView.getRectangles(); - printLables(processed, rects); - } + mGLSurfaceView.showPointcloud(mShow3D); + mGLSurfaceView.upload(frameSet); + Map> rects = mGLSurfaceView.getRectangles(); + printLables(rects); } } }); @@ -215,11 +224,7 @@ public void run() { protected void onPause() { super.onPause(); - for(Map.Entry e : mLabels.entrySet()){ - TextView tv = e.getValue(); - tv.setVisibility(View.GONE); - } - mLabels = null; + clearLables(); if(mStreamer != null) mStreamer.stop(); } diff --git a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/RecordingActivity.java b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/RecordingActivity.java index b7f4ee7dd0..b8d65a6c86 100644 --- a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/RecordingActivity.java +++ b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/RecordingActivity.java @@ -12,7 +12,6 @@ import android.view.View; import android.view.WindowManager; -import com.intel.realsense.librealsense.Colorizer; import com.intel.realsense.librealsense.Config; import com.intel.realsense.librealsense.FrameSet; import com.intel.realsense.librealsense.GLRsSurfaceView; @@ -27,7 +26,6 @@ public class RecordingActivity extends AppCompatActivity { private Streamer mStreamer; private GLRsSurfaceView mGLSurfaceView; - private Colorizer mColorizer = new Colorizer(); private boolean mPermissionsGrunted = false; @@ -82,9 +80,7 @@ public void config(Config config) { @Override public void onFrameset(FrameSet frameSet) { - try (FrameSet processed = frameSet.applyFilter(mColorizer)) { - mGLSurfaceView.upload(processed); - } + mGLSurfaceView.upload(frameSet); } }); try { @@ -105,6 +101,9 @@ protected void onPause() { } private String getFilePath(){ + File rsFolder = new File(Environment.getExternalStorageDirectory().getAbsolutePath() + + File.separator + getString(R.string.realsense_folder)); + rsFolder.mkdir(); File folder = new File(Environment.getExternalStorageDirectory().getAbsolutePath() + File.separator + getString(R.string.realsense_folder) + File.separator + "video"); folder.mkdir(); diff --git a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/SettingsActivity.java b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/SettingsActivity.java index 39b1f87a3a..61b45ec501 100644 --- a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/SettingsActivity.java +++ b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/SettingsActivity.java @@ -17,13 +17,11 @@ import com.intel.realsense.librealsense.Device; import com.intel.realsense.librealsense.DeviceList; import com.intel.realsense.librealsense.Extension; -import com.intel.realsense.librealsense.MotionStreamProfile; import com.intel.realsense.librealsense.RsContext; import com.intel.realsense.librealsense.Sensor; import com.intel.realsense.librealsense.StreamProfile; import com.intel.realsense.librealsense.StreamType; import com.intel.realsense.librealsense.Updatable; -import com.intel.realsense.librealsense.VideoStreamProfile; import java.io.File; import java.util.ArrayList; @@ -56,20 +54,29 @@ protected void onCreate(Bundle savedInstanceState) { protected void onResume() { super.onResume(); - RsContext ctx = new RsContext(); - try(DeviceList devices = ctx.queryDevices()) { - if (devices.getDeviceCount() == 0) { - throw new Exception("Failed to detect a connected device"); + int tries = 3; + for(int i = 0; i < tries; i++){ + RsContext ctx = new RsContext(); + try(DeviceList devices = ctx.queryDevices()) { + if (devices.getDeviceCount() == 0) { + Thread.sleep(500); + continue; + } + _device = devices.createDevice(0); + loadInfosList(); + loadSettingsList(_device); + StreamProfileSelector[] profilesList = createSettingList(_device); + loadStreamList(_device, profilesList); + return; + } catch(Exception e){ + Log.e(TAG, "failed to load settings, error: " + e.getMessage()); } - _device = devices.createDevice(0); - loadInfosList(); - loadSettingsList(_device); - StreamProfileSelector[] profilesList = createSettingList(_device); - loadStreamList(_device, profilesList); - } catch(Exception e){ - Log.e(TAG, "failed to load settings, error: " + e.getMessage()); - Toast.makeText(this, "Failed to load settings", Toast.LENGTH_LONG).show(); } + Log.e(TAG, "failed to load settings"); + Toast.makeText(this, "Failed to load settings", Toast.LENGTH_LONG).show(); + Intent intent = new Intent(this, DetachedActivity.class); + startActivity(intent); + finish(); } @Override protected void onPause() { @@ -98,7 +105,7 @@ private void loadSettingsList(final Device device){ if(device.is(Extension.UPDATABLE)){ settingsMap.put(INDEX_UPDATE,"Firmware update"); Updatable fwud = device.as(Extension.UPDATABLE); - if(fwud.supportsInfo(CameraInfo.CAMERA_LOCKED) && fwud.getInfo(CameraInfo.CAMERA_LOCKED).equals("NO")) + if(fwud != null && fwud.supportsInfo(CameraInfo.CAMERA_LOCKED) && fwud.getInfo(CameraInfo.CAMERA_LOCKED).equals("NO")) settingsMap.put(INDEX_UPDATE_UNSIGNED,"Firmware update (unsigned)"); } @@ -179,7 +186,7 @@ public static Map> createProfilesMap(Device device) } private void loadStreamList(Device device, StreamProfileSelector[] lines){ - if(lines == null) + if(device == null || lines == null) return; if(!device.supportsInfo(CameraInfo.PRODUCT_ID)) throw new RuntimeException("try to config unknown device"); diff --git a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/Streamer.java b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/Streamer.java index febf7ba933..472d2b28c3 100644 --- a/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/Streamer.java +++ b/wrappers/android/tools/camera/src/main/java/com/intel/realsense/camera/Streamer.java @@ -23,6 +23,8 @@ public class Streamer { private static final String TAG = "librs camera streamer"; + private static final int DEFAULT_TIMEOUT = 3000; + private static final int L500_TIMEOUT = 15000; interface Listener{ void config(Config config); @@ -63,11 +65,13 @@ private int getFirstFrameTimeout() { RsContext ctx = new RsContext(); try(DeviceList devices = ctx.queryDevices()) { if (devices.getDeviceCount() == 0) { - return 0; + return DEFAULT_TIMEOUT; } try (Device device = devices.createDevice(0)) { + if(device == null) + return DEFAULT_TIMEOUT; ProductLine pl = ProductLine.valueOf(device.getInfo(CameraInfo.PRODUCT_LINE)); - return pl == ProductLine.L500 ? 15000 : 3000; + return pl == ProductLine.L500 ? L500_TIMEOUT : DEFAULT_TIMEOUT; } } } @@ -82,6 +86,10 @@ private void configStream(Config config){ return; } try (Device device = devices.createDevice(0)) { + if(device == null){ + Log.e(TAG, "failed to create device"); + return; + } pid = device.getInfo(CameraInfo.PRODUCT_ID); profilesMap = SettingsActivity.createProfilesMap(device); diff --git a/wrappers/android/tools/camera/src/main/res/layout-land/activity_playback.xml b/wrappers/android/tools/camera/src/main/res/layout-land/activity_playback.xml new file mode 100644 index 0000000000..7a3047c8cb --- /dev/null +++ b/wrappers/android/tools/camera/src/main/res/layout-land/activity_playback.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/wrappers/android/tools/camera/src/main/res/layout-land/activity_preview.xml b/wrappers/android/tools/camera/src/main/res/layout-land/activity_preview.xml index 29b890adf8..9d9c08d4e8 100644 --- a/wrappers/android/tools/camera/src/main/res/layout-land/activity_preview.xml +++ b/wrappers/android/tools/camera/src/main/res/layout-land/activity_preview.xml @@ -30,6 +30,7 @@ android:visibility="gone"/> + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/wrappers/android/tools/camera/src/main/res/layout/activity_preview.xml b/wrappers/android/tools/camera/src/main/res/layout/activity_preview.xml index c1d0e01ff6..2437140df5 100644 --- a/wrappers/android/tools/camera/src/main/res/layout/activity_preview.xml +++ b/wrappers/android/tools/camera/src/main/res/layout/activity_preview.xml @@ -85,6 +85,21 @@ android:layout_height="1dp" android:layout_weight="1" /> + + + + firmware_update_file_path browse_folder realsense + show_3d From e2422242de92d48a1f3aab8a4c60d03c9bb1e1a1 Mon Sep 17 00:00:00 2001 From: "Katz, Matan" Date: Tue, 20 Aug 2019 15:03:46 +0300 Subject: [PATCH 2/2] android uvc - change bulk transfer frame read to usb request --- src/android/usb_host/android_uvc.cpp | 161 ++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 6 deletions(-) diff --git a/src/android/usb_host/android_uvc.cpp b/src/android/usb_host/android_uvc.cpp index 09e4444668..ff28962169 100644 --- a/src/android/usb_host/android_uvc.cpp +++ b/src/android/usb_host/android_uvc.cpp @@ -12,15 +12,20 @@ #include "../../libuvc/utlist.h" #include "../../usb/usb-types.h" #include "../../usb/usb-device.h" +#include "../../usbhost/messenger-usbhost.h" #include #include #include #include +#define USE_URB true #define DEQUEUE_TIMEOUT 50 +#define STREAMING_WATCHER_TIMEOUT 1000 #define STREAMING_BULK_TRANSFER_TIMEOUT 1000 -#define UVC_PAYLOAD_HEADER_LENGTH 256 +#define UVC_PAYLOAD_MAX_HEADER_LENGTH 256 + +using namespace librealsense; // Data structures for Backend-Frontend queue: struct frame; @@ -831,7 +836,7 @@ void usbhost_uvc_process_bulk_payload(frame_ptr fp, size_t payload_len, frames_q queue.enqueue(std::move(fp)); } -void stream_thread(usbhost_uvc_stream_context *strctx) { +void stream_thread_bulk(usbhost_uvc_stream_context *strctx) { auto inf = strctx->stream->stream_if->interface; auto dev = strctx->stream->devh->device; auto messenger = dev->open(); @@ -844,7 +849,7 @@ void stream_thread(usbhost_uvc_stream_context *strctx) { std::atomic_bool keep_sending_callbacks(true); frames_queue queue; - uint32_t read_buff_length = UVC_PAYLOAD_HEADER_LENGTH + strctx->stream->cur_ctrl.dwMaxVideoFrameSize; + uint32_t read_buff_length = UVC_PAYLOAD_MAX_HEADER_LENGTH + strctx->stream->cur_ctrl.dwMaxVideoFrameSize; LOG_INFO("endpoint " << (int)read_ep->get_address() << " read buffer size: " << read_buff_length); // Get all pointers from archive and initialize their content @@ -918,6 +923,152 @@ void stream_thread(usbhost_uvc_stream_context *strctx) { LOG_DEBUG("Transfer thread stopped for endpoint address: " << ep); }; +void stream_thread_urb(usbhost_uvc_stream_context *strctx) { + auto inf = strctx->stream->stream_if->interface; + auto dev = strctx->stream->devh->device; + auto messenger = std::static_pointer_cast(dev->open()); + + auto read_ep = std::static_pointer_cast + (inf->first_endpoint(platform::RS2_USB_ENDPOINT_DIRECTION_READ)); + + uint32_t reset_ep_timeout = 100; + messenger->reset_endpoint(read_ep, reset_ep_timeout); + + frames_archive archive; + std::atomic_bool keep_sending_callbacks(true); + frames_queue queue; + + uint32_t read_buff_length = UVC_PAYLOAD_MAX_HEADER_LENGTH + strctx->stream->cur_ctrl.dwMaxVideoFrameSize; + LOG_INFO("endpoint " << (int)read_ep->get_address() << " read buffer size: " << read_buff_length); + + // Get all pointers from archive and initialize their content + std::vector frames; + for (auto i = 0; i < frames_archive::CAPACITY; i++) { + auto ptr = archive.allocate(); + ptr->pixels.resize(read_buff_length, 0); + ptr->owner = &archive; + frames.push_back(ptr); + } + + for (auto ptr : frames) { + archive.deallocate(ptr); + } + + std::thread t([&]() { + while (keep_sending_callbacks) { + frame_ptr fp(nullptr, [](frame *) {}); + if (queue.dequeue(&fp, DEQUEUE_TIMEOUT)) { + strctx->stream->user_cb(&fp->fo, strctx->stream->user_ptr); + } + } + }); + + frame_ptr fp = frame_ptr(archive.allocate(), &cleanup_frame); + + auto desc = read_ep->get_descriptor(); + std::shared_ptr request = std::shared_ptr(usb_request_new(dev->get_handle(), + &desc), [](usb_request* req) {usb_request_free(req);}); + if(!request) + { + LOG_ERROR("invalid USB request"); + return; + } + + bool fatal_error = false; + uint32_t frame_count = 0; + int active_request_count = 0; + uint32_t zero_response = 0; + auto callback = std::make_shared([&](usb_request* r) + { + if(r) + { + active_request_count--; + if(r->actual_length == 0){ + LOG_WARNING("read_ep: " << r->endpoint << ", zero response"); + zero_response++; + if(zero_response > 10) + { + messenger->reset_endpoint(read_ep, reset_ep_timeout); + zero_response = 0; + } + } + if(r->actual_length >= strctx->stream->cur_ctrl.dwMaxVideoFrameSize){ + zero_response = 0; + frame_count++; + usbhost_uvc_process_bulk_payload(std::move(fp), r->actual_length, queue); + } + } + + if(!strctx->stream->running){ + return; + } + + fp = frame_ptr(archive.allocate(), &cleanup_frame); + if(!fp) + return; + request->buffer = fp->pixels.data(); + request->buffer_length = fp->pixels.size(); + + auto sts = messenger->submit_request(request); + switch(sts) + { + case platform::RS2_USB_STATUS_SUCCESS: + active_request_count++; + return; + case platform::RS2_USB_STATUS_OVERFLOW: + case platform::RS2_USB_STATUS_NO_DEVICE: + case platform::RS2_USB_STATUS_NO_MEM: + fatal_error = true; + return; + default: + messenger->reset_endpoint(read_ep, reset_ep_timeout); + break; + } + }); + + request->client_data = callback.get(); + + while(!fatal_error && strctx->stream->running) + { + if(frame_count == 0){ + if(active_request_count > 0) + { + messenger->cancel_request(request); + active_request_count--; + } + callback->callback(nullptr); + } + frame_count = 0; + std::this_thread::sleep_for(std::chrono::milliseconds(STREAMING_WATCHER_TIMEOUT)); + } + + if(active_request_count > 0) + { + messenger->cancel_request(request); + active_request_count--; + } + LOG_INFO("read endpoint: " << (int)read_ep->get_address() <<", active requests count: " << active_request_count); + + callback->cancel(); + messenger->reset_endpoint(read_ep, reset_ep_timeout); + free(strctx); + queue.clear(); + archive.stop_allocation(); + fp.reset(); + archive.wait_until_empty(); + keep_sending_callbacks = false; + + LOG_INFO("start join stream_thread"); + t.join(); + LOG_INFO("Transfer thread stopped for endpoint address: " << (int)read_ep->get_address()); +}; + +void stream_thread(usbhost_uvc_stream_context *strctx){ + if(USE_URB) + stream_thread_urb(strctx); + else + stream_thread_bulk(strctx); +} uvc_error_t usbhost_uvc_stream_start( usbhost_uvc_stream_handle_t *strmh, usbhost_uvc_frame_callback_t *cb, @@ -959,8 +1110,6 @@ uvc_error_t usbhost_uvc_stream_start( usbhost_uvc_stream_context *streamctx = new usbhost_uvc_stream_context; - //printf("Starting stream on EP = 0x%X, interface 0x%X\n", format_desc->parent->bEndpointAddress, iface); - streamctx->stream = strmh; streamctx->endpoint = format_desc->parent->bEndpointAddress; streamctx->iface = iface; @@ -1001,7 +1150,7 @@ void usbhost_uvc_stream_close(usbhost_uvc_stream_handle_t *strmh) { } uvc_error_t usbhost_uvc_stream_open_ctrl(usbhost_uvc_device *devh, usbhost_uvc_stream_handle_t **strmhp, - uvc_stream_ctrl_t *ctrl); + uvc_stream_ctrl_t *ctrl); uvc_error_t usbhost_start_streaming(usbhost_uvc_device *devh, uvc_stream_ctrl_t *ctrl, usbhost_uvc_frame_callback_t *cb, void *user_ptr,