package com.rockchip.gpadc.demo; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.List; import android.app.Activity; import android.content.SharedPreferences; import android.content.res.Resources; import android.graphics.Bitmap; import android.graphics.Canvas; import android.graphics.Paint; import android.graphics.PorterDuff; import android.graphics.PorterDuffXfermode; import android.graphics.RectF; import android.graphics.SurfaceTexture; import android.graphics.Typeface; import android.hardware.Camera; import android.os.Bundle; import android.os.Handler; import android.os.Message; import android.util.Log; import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.WindowManager; import android.widget.ImageView; import android.widget.TextView; import com.rockchip.gpadc.demo.rga.RGA; import com.rockchip.gpadc.demo.yolo.InferenceWrapper; import com.rockchip.gpadc.demo.yolo.PostProcess; import static com.rockchip.gpadc.demo.rga.HALDefine.CAMERA_PREVIEW_HEIGHT; import static com.rockchip.gpadc.demo.rga.HALDefine.CAMERA_PREVIEW_WIDTH; import static com.rockchip.gpadc.demo.rga.HALDefine.RK_FORMAT_RGBA_8888; import static com.rockchip.gpadc.demo.rga.HALDefine.RK_FORMAT_YCrCb_420_SP; import static com.rockchip.gpadc.demo.rga.HALDefine.IM_HAL_TRANSFORM_FLIP_H; import static com.rockchip.gpadc.demo.yolo.PostProcess.INPUT_CHANNEL; import static com.rockchip.gpadc.demo.yolo.PostProcess.INPUT_SIZE; import static java.lang.Thread.sleep; public class MainActivity extends Activity implements Camera.PreviewCallback { private final String TAG = "rkyolo"; private static final int MAGIC_TEXTURE_ID = 10; TSurfaceHolderCallback mSurfaceHolderCallback = null; private Camera mCamera0 = null; private SurfaceView mSurfaceView = null; public SurfaceTexture mSurfaceTexture = null; private SurfaceHolder mSurfaceHolder = null; public int flip = -1; // for CAMERA_FACING_BACK(camera comes with RK3588 using this mode), // we do not need flip, using -1, or we need using // IM_HAL_TRANSFORM_FLIP_H private boolean mIsCameraOpened = false; private int mCameraId = -1; private int BUFFER_SIZE0 = CAMERA_PREVIEW_WIDTH * CAMERA_PREVIEW_HEIGHT * 3 / 2; // NV21 private byte[][] mPreviewData0 = new byte[][]{new byte[BUFFER_SIZE0], new byte[BUFFER_SIZE0],new byte[BUFFER_SIZE0]}; public byte textureBuffer[]; // for inference private String mModelName = "yolov5s.rknn"; private InferenceWrapper mInferenceWrapper; private String fileDirPath; // file dir to store model cache private ImageBufferQueue mImageBufferQueue; // intermedia between camera thread and inference thread private InferenceResult mInferenceResult = new InferenceResult(); // detection result private int mWidth; //surface width private int mHeight; //surface height private volatile boolean mStopInference = false; //draw result private TextView mFpsNum1; private TextView mFpsNum2; private TextView mFpsNum3; private TextView mFpsNum4; private ImageView mTrackResultView; private Bitmap mTrackResultBitmap = null; private Canvas mTrackResultCanvas = null; private Paint mTrackResultPaint = null; private Paint mTrackResultTextPaint = null; private PorterDuffXfermode mPorterDuffXfermodeClear; private PorterDuffXfermode mPorterDuffXfermodeSRC; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_main); getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); mFpsNum1 = (TextView) findViewById(R.id.fps_num1); mFpsNum2 = (TextView) findViewById(R.id.fps_num2); mFpsNum3 = (TextView) findViewById(R.id.fps_num3); mFpsNum4 = (TextView) findViewById(R.id.fps_num4); mTrackResultView = (ImageView) findViewById(R.id.canvasView); fileDirPath = getCacheDir().getAbsolutePath(); createFile(mModelName, R.raw.yolov5s); try { mInferenceResult.init(getAssets()); } catch (IOException e) { e.printStackTrace(); } mInferenceWrapper = new InferenceWrapper(); } @Override protected void onDestroy() { Log.d(TAG, "onDestroy"); destroyPreviewView(); super.onDestroy(); } @Override protected void onPause() { Log.d(TAG, "onPause"); stopTrack(); stopCamera(); destroyPreviewView(); super.onPause(); } @Override protected void onResume() { Log.d(TAG, "onResume"); createPreviewView(); super.onResume(); } private boolean createPreviewView() { mSurfaceView = findViewById(R.id.surfaceViewCamera1); mSurfaceHolder = mSurfaceView.getHolder(); // mSurfaceView.setZOrderMediaOverlay(true); mSurfaceTexture = new SurfaceTexture(MAGIC_TEXTURE_ID); mSurfaceHolderCallback = new TSurfaceHolderCallback(); mSurfaceHolder.addCallback(mSurfaceHolderCallback); return true; } private void destroyPreviewView() { if (mSurfaceHolder != null) { mSurfaceHolder.removeCallback(mSurfaceHolderCallback); mSurfaceHolderCallback = null; mSurfaceHolder = null; } } @Override public void onPreviewFrame(byte[] data, Camera camera) { mCamera0.addCallbackBuffer(data); ImageBufferQueue.ImageBuffer imageBuffer = mImageBufferQueue.getFreeBuffer(); if (imageBuffer != null) { // arraycopy(data, 0, imageBuffer.mData, 0, imageBuffer.mBufferSize); RGA.colorConvert(data, RK_FORMAT_YCrCb_420_SP, imageBuffer.mImage, RK_FORMAT_RGBA_8888, CAMERA_PREVIEW_WIDTH, CAMERA_PREVIEW_HEIGHT, this.flip); mImageBufferQueue.postBuffer(imageBuffer); } } private class TSurfaceHolderCallback implements SurfaceHolder.Callback { @Override public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { Log.d(TAG, "surfaceChanged"); mWidth = width; mHeight = height; textureBuffer=new byte[CAMERA_PREVIEW_WIDTH * CAMERA_PREVIEW_HEIGHT * 4]; } @Override public void surfaceCreated(SurfaceHolder holder) { Log.d(TAG, "surfaceCreated"); startCamera(); startTrack(); } @Override public void surfaceDestroyed(SurfaceHolder holder) { Log.d(TAG, "surfaceDestroyed"); stopTrack(); stopCamera(); } } private boolean startCamera() { if (mIsCameraOpened) { return true; } //(Camera.CameraInfo.CAMERA_FACING_BACK); int num = Camera.getNumberOfCameras(); if (num > 2) mCameraId = 2; else mCameraId = 0; Log.d(TAG, "mCameraId = " + mCameraId); Camera.CameraInfo camInfo = new Camera.CameraInfo(); try { Camera.getCameraInfo(mCameraId, camInfo); if (mCameraId != -1) { mCamera0 = Camera.open(mCameraId); } else { mCamera0 = Camera.open(); } Log.d(TAG, "mCamera0 = " + mCamera0); Log.d(TAG, "camera facing: " + camInfo.facing); if (Camera.CameraInfo.CAMERA_FACING_FRONT == camInfo.facing) { this.flip = IM_HAL_TRANSFORM_FLIP_H; } } catch (RuntimeException e) { Log.w(TAG, "Unable to open camera!"); return false; } setCameraParameters(); try { mCamera0.setPreviewDisplay(mSurfaceHolder); // mCamera0.setPreviewTexture(mSurfaceTexture); mCamera0.setDisplayOrientation(0); //================================ for (byte[] buffer : mPreviewData0) mCamera0.addCallbackBuffer(buffer); mCamera0.setPreviewCallbackWithBuffer(this); //================================== mCamera0.startPreview(); } catch (Exception e) { mCamera0.release(); return false; } mIsCameraOpened = true; return true; } private void stopCamera() { if (mIsCameraOpened) { mCamera0.setPreviewCallback(null); mCamera0.stopPreview(); mCamera0.release(); mCamera0 = null; mIsCameraOpened = false; } } private void setCameraParameters() { Camera.Parameters parameters; boolean checkWH = false; parameters = mCamera0.getParameters(); List sizes = parameters.getSupportedPreviewSizes(); for (int i = 0; i < sizes.size(); i++) { Camera.Size size = sizes.get(i); if ( (size.width == CAMERA_PREVIEW_WIDTH) && (size.height == CAMERA_PREVIEW_HEIGHT)) { checkWH = true; } Log.v(TAG, "Camera Supported Preview Size = " + size.width + "x" + size.height); } if (!checkWH) { Log.e(TAG, "Camera don't support this preview Size = " + CAMERA_PREVIEW_WIDTH + "x" + CAMERA_PREVIEW_HEIGHT); } else { Log.w(TAG, "Use preview Size = " + CAMERA_PREVIEW_WIDTH + "x" + CAMERA_PREVIEW_HEIGHT); } // For camera comes with RK3588, this width and height is available. But for other camera, // we may need adjust them. parameters.setPreviewSize(CAMERA_PREVIEW_WIDTH, CAMERA_PREVIEW_HEIGHT); if (parameters.isZoomSupported()) { parameters.setZoom(0); } mCamera0.setParameters(parameters); Log.i(TAG, "mCamera0 set parameters success."); } private void startTrack() { mInferenceResult.reset(); mImageBufferQueue = new ImageBufferQueue(3, CAMERA_PREVIEW_WIDTH, CAMERA_PREVIEW_HEIGHT); mStopInference = false; mInferenceThread = new Thread(mInferenceRunnable); mInferenceThread.start(); } private void stopTrack() { mStopInference = true; try { mInferenceThread.join(); } catch (InterruptedException e) { e.printStackTrace(); } if (mImageBufferQueue != null) { mImageBufferQueue.release(); mImageBufferQueue = null; } } private Thread mInferenceThread; private Runnable mInferenceRunnable = new Runnable() { public void run() { int count = 0; long oldTime = System.currentTimeMillis(); long currentTime; updateMainUI(1, 0); String paramPath = fileDirPath + "/" + mModelName; try { mInferenceWrapper.initModel(CAMERA_PREVIEW_HEIGHT, CAMERA_PREVIEW_WIDTH, INPUT_CHANNEL, paramPath); } catch (Exception e) { e.printStackTrace(); System.exit(1); } while (!mStopInference) { ImageBufferQueue.ImageBuffer buffer = mImageBufferQueue.getReadyBuffer(); if (buffer == null) { try { // Log.w(TAG, "buffer is null."); sleep(10); } catch (InterruptedException e) { e.printStackTrace(); } continue; } InferenceResult.OutputBuffer outputs = mInferenceWrapper.run(buffer.mImage); mInferenceResult.setResult(outputs); mImageBufferQueue.releaseBuffer(buffer); if (++count >= 30) { currentTime = System.currentTimeMillis(); float fps = count * 1000.f / (currentTime - oldTime); // Log.d(TAG, "current fps = " + fps); oldTime = currentTime; count = 0; updateMainUI(0, fps); } updateMainUI(1, 0); } mInferenceWrapper.deinit(); } }; private void createFile(String fileName, int id) { String filePath = fileDirPath + "/" + fileName; try { File dir = new File(fileDirPath); if (!dir.exists()) { dir.mkdirs(); } // 目录存在,则将apk中raw中的需要的文档复制到该目录下 File file = new File(filePath); if (!file.exists() || isFirstRun()) { InputStream ins = getResources().openRawResource(id);// 通过raw得到数据资源 FileOutputStream fos = new FileOutputStream(file); byte[] buffer = new byte[8192]; int count = 0; while ((count = ins.read(buffer)) > 0) { fos.write(buffer, 0, count); } fos.close(); ins.close(); Log.d(TAG, "Create " + filePath); } } catch (Exception e) { e.printStackTrace(); } } private boolean isFirstRun() { SharedPreferences sharedPreferences = getSharedPreferences("setting", MODE_PRIVATE); boolean isFirstRun = sharedPreferences.getBoolean("isFirstRun", true); SharedPreferences.Editor editor = sharedPreferences.edit(); if (isFirstRun) { editor.putBoolean("isFirstRun", false); editor.commit(); } return isFirstRun; } // UI线程,用于更新处理结果 private Handler mHandler = new Handler() { public void handleMessage(Message msg) { if (msg.what == 0) { float fps = (float) msg.obj; DecimalFormat decimalFormat = new DecimalFormat("00.00"); String fpsStr = decimalFormat.format(fps); mFpsNum1.setText(String.valueOf(fpsStr.charAt(0))); mFpsNum2.setText(String.valueOf(fpsStr.charAt(1))); mFpsNum3.setText(String.valueOf(fpsStr.charAt(3))); mFpsNum4.setText(String.valueOf(fpsStr.charAt(4))); } else { showTrackSelectResults(); } } }; private void updateMainUI(int type, Object data) { Message msg = mHandler.obtainMessage(); msg.what = type; msg.obj = data; mHandler.sendMessage(msg); } public static int sp2px(float spValue) { Resources r = Resources.getSystem(); final float scale = r.getDisplayMetrics().scaledDensity; return (int) (spValue * scale + 0.5f); } private void showTrackSelectResults() { int width = CAMERA_PREVIEW_WIDTH; int height = CAMERA_PREVIEW_HEIGHT; if (mTrackResultBitmap == null) { mTrackResultBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); mTrackResultCanvas = new Canvas(mTrackResultBitmap); //用于画线 mTrackResultPaint = new Paint(); mTrackResultPaint.setColor(0xff06ebff); mTrackResultPaint.setStrokeJoin(Paint.Join.ROUND); mTrackResultPaint.setStrokeCap(Paint.Cap.ROUND); mTrackResultPaint.setStrokeWidth(4); mTrackResultPaint.setStyle(Paint.Style.STROKE); mTrackResultPaint.setTextAlign(Paint.Align.LEFT); mTrackResultPaint.setTextSize(sp2px(10)); mTrackResultPaint.setTypeface(Typeface.SANS_SERIF); mTrackResultPaint.setFakeBoldText(false); //用于文字 mTrackResultTextPaint = new Paint(); mTrackResultTextPaint.setColor(0xff06ebff); mTrackResultTextPaint.setStrokeWidth(2); mTrackResultTextPaint.setTextAlign(Paint.Align.LEFT); mTrackResultTextPaint.setTextSize(sp2px(12)); mTrackResultTextPaint.setTypeface(Typeface.SANS_SERIF); mTrackResultTextPaint.setFakeBoldText(false); mPorterDuffXfermodeClear = new PorterDuffXfermode(PorterDuff.Mode.CLEAR); mPorterDuffXfermodeSRC = new PorterDuffXfermode(PorterDuff.Mode.SRC); } // clear canvas mTrackResultPaint.setXfermode(mPorterDuffXfermodeClear); mTrackResultCanvas.drawPaint(mTrackResultPaint); mTrackResultPaint.setXfermode(mPorterDuffXfermodeSRC); //detect result ArrayList recognitions = mInferenceResult.getResult(mInferenceWrapper); for (int i=0; i