// Copyright 2019 Fuzhou Rockchip Electronics Co., Ltd. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include #include #include "st_task_handler.h" #include "logger/log.h" #include "parse/ai_results_parse.h" #include "RTAIDetectResults.h" #define MAX_KEEP_FACE_COUNT 24 #define FACE_FIRST_SEND_DELAY 8 #define FACE_SAME_SEND_DELAY 30 #ifdef LOG_TAG #undef LOG_TAG #endif #define LOG_TAG "STTaskHandler" namespace rockchip { namespace aiserver { static std::map mFaceMap; static int64_t getNowUs() { struct timeval tv; gettimeofday(&tv, NULL); return tv.tv_sec * 1000000.0 + tv.tv_usec; } STTaskHandler::STTaskHandler() { mShmNNcontroller = new ShmNNController(); } STTaskHandler::~STTaskHandler() { if (mShmNNcontroller != nullptr) { delete mShmNNcontroller; mShmNNcontroller = nullptr; } std::lock_guard lock(mOpMutex); std::map::iterator it; for (it = mFaceMap.begin(); it != mFaceMap.end();) { MattingFaceHolder *holder = it->second; it = mFaceMap.erase(it); LOG_INFO("clear matting face[%d]\n", holder->faceID); freeMattingFaceInfo(holder); } } int32_t STTaskHandler::processAIData(RTMediaBuffer *buffer) { if (buffer == NULL) return 0; void *nnResult = getAIDetectResults(buffer); if (nnResult != NULL) { postNNData(nnResult); } else { LOG_INFO("processAIData not find ai results\n"); } buffer->release(); return 0; } int32_t STTaskHandler::processAIMatting(RTMediaBuffer *buffer) { RtMetaData *extraInfo = RT_NULL; void *nnResult = RT_NULL; extraInfo = buffer->getMetaData(); if (RT_NULL != extraInfo) { extraInfo->findPointer("aimatting_out_result", &nnResult); if (RT_NULL != nnResult) { void* imgData = buffer->getData(); postAIMattingData(nnResult, imgData); } } buffer->release(); return 0; } int32_t STTaskHandler::processAIFeature(RTMediaBuffer *buffer) { RtMetaData *extraInfo = RT_NULL; void *nnResult = RT_NULL; const char *uuid = RT_NULL; INT32 retryAI = 0; INT32 hasFeature = 0; extraInfo = buffer->getMetaData(); if (RT_NULL != extraInfo) { if (!extraInfo->findCString("stream_uuid", &uuid)) { uuid = "feature_extract"; } nnResult = getAIDetectResults(buffer); if (NULL == nnResult) { LOG_INFO("ststst processAIFeature not find ai results\n"); goto __FAILED; } auto aiResult = (STDetectResult *)(nnResult); LOG_INFO("anlyse result(face:%d body:%d hand=%d)\n", \ aiResult->faceCount, aiResult->bodyCount, aiResult->handCount); for (INT32 idx = 0; idx < aiResult->faceCount; idx++) { if(aiResult->faces[idx].featureLen > 0) { hasFeature = 1; } } if (hasFeature) { extraInfo->setInt32("ai_retry", retryAI); postFeatureData(nnResult, uuid); } else { if (!extraInfo->findInt32("ai_retry", &retryAI)) { retryAI = 8; } if (retryAI > 0) { retryAI--; extraInfo->setInt32("ai_retry", retryAI); LOG_INFO("retry to recognize image(buf=%p,cnt=%d)\n", buffer, retryAI); return -1; } } } __FAILED: LOG_INFO("aifeature callback(buf=%p,uuid=%s)\n", buffer, uuid); if (hasFeature == 0) { postEmptyFeatureData(uuid); } buffer->release(); return 0; } void STTaskHandler::postNNData(void *nnResult) { std::lock_guard lock(mOpMutex); KKMessage kkMsg; KKAIData *mNNData = kkMsg.mutable_kkaidata(); auto stRes = (STDetectResult *)(nnResult); if (stRes->faceCount <= 0 && stRes->handCount <= 0 && stRes->bodyCount <= 0) { return; } pushDetectInfo(mNNData, stRes, 1); mNNData->set_width(1280); mNNData->set_height(720); mNNData->set_function("KK_AI_DATA"); kkMsg.set_msg_type(1); kkMsg.set_msg_name("KK_AI_DATA"); std::string sendbuf; kkMsg.SerializeToString(&sendbuf); mShmNNcontroller->send(sendbuf); } void STTaskHandler::pushDetectInfo(KKAIData *mNNData, STDetectResult *detectResult, int32_t size) { int32_t i,j; FaceData *facedetect; LOG_DEBUG("bodycnt=%d handcnt=%d\n", detectResult->bodyCount, detectResult->handCount); if (detectResult->faceCount > 0) { for (i = 0;ifaceCount;i++) { facedetect = mNNData->add_facedata(); facedetect->set_id(detectResult->faces[i].id); RectF *rectf = facedetect->mutable_rect(); rectf->set_left(detectResult->faces[i].rect.left); rectf->set_right(detectResult->faces[i].rect.right); rectf->set_top(detectResult->faces[i].rect.top); rectf->set_bottom(detectResult->faces[i].rect.bottom); AngleF *anglef = facedetect->mutable_angle(); anglef->set_pitch(detectResult->faces[i].pitch); anglef->set_yaw(detectResult->faces[i].yaw); anglef->set_roll(detectResult->faces[i].roll); facedetect->set_quality(detectResult->faces[i].quality); for (j = 0; j < detectResult->faces[i].pointCount; j++) { PointF *mPoint = facedetect->add_points(); mPoint->set_x(detectResult->faces[i].points[j].x); mPoint->set_y(detectResult->faces[i].points[j].y); //Visibilities *visibilities = facedetect->add_visibilities(); //visibilities->set_visibilities(detectResult->faces[i].visibilities[j]); } facedetect->set_distance(detectResult->faces[i].distance); for (j = 0;j< detectResult->faces[i].attributeCount;j++) { Attr *attributes = facedetect->add_attrs(); //attributes->set_category(detectResult->faces[i].attributes[j].category); attributes->set_name(detectResult->faces[i].attributes[j].category); attributes->set_value(detectResult->faces[i].attributes[j].label); attributes->set_score(detectResult->faces[i].attributes[j].score); } if(detectResult->faces[i].feature != nullptr) { Feature *feature = facedetect->mutable_feature(); feature->set_data((char *)detectResult->faces[i].feature, detectResult->faces[i].featureLen); feature->set_length(detectResult->faces[i].featureLen); } } } if (detectResult->handCount > 0) { HandData *handdetect; for (i = 0; ihandCount; i++) { handdetect = mNNData->add_handdata(); handdetect->set_id(detectResult->hands[i].id); handdetect->set_faceid(detectResult->hands[i].faceId); handdetect->set_quality(1.0); RectF *rectf = handdetect->mutable_rect(); rectf->set_left(detectResult->hands[i].rect.left); rectf->set_top(detectResult->hands[i].rect.top); rectf->set_right(detectResult->hands[i].rect.right); rectf->set_bottom(detectResult->hands[i].rect.bottom); AngleF *anglef = handdetect->mutable_angle(); anglef->set_pitch(0); anglef->set_yaw(0); anglef->set_roll(0); for (j = 0; j < detectResult->hands[i].pointCount; j++) { PointF *hPoint = handdetect->add_points(); hPoint->set_x(detectResult->hands[i].points[j].x); hPoint->set_y(detectResult->hands[i].points[j].y); } Attr *attributes = handdetect->add_attrs(); attributes->set_name("action"); switch(detectResult->hands[i].action) { case 0x00000200: attributes->set_value("0"); break; case 0x00000008: attributes->set_value("1"); break; case 0x00000001: attributes->set_value("2"); break; case 0x00001000: attributes->set_value("3"); break; case 0x00000080: attributes->set_value("4"); break; case 0x00004000: attributes->set_value("5"); break; case 0x00000100: attributes->set_value("6"); break; case 0x00000002: attributes->set_value("7"); break; case 0x00000040: attributes->set_value("8"); break; case 0x00000010: attributes->set_value("9"); break; case 0x00000020: attributes->set_value("10"); break; case 0x00000400: attributes->set_value("11"); break; case 0x00000800: attributes->set_value("12"); break; case 0x00000004: attributes->set_value("13"); break; case 0x00002000: attributes->set_value("14"); break; default: attributes->set_value("999"); break; } //handdetect->set_action(detectResult->hands[i].action); //handdetect->set_event(detectResult->hands[i].event); } } if(detectResult->bodyCount > 0) { BodyData *bodydetect; for (i = 0; ibodyCount; i++) { bodydetect = mNNData->add_bodydata(); bodydetect->set_id(detectResult->bodys[i].id); bodydetect->set_faceid(detectResult->bodys[i].faceId); RectF *rectf = bodydetect->mutable_rect(); rectf->set_left(detectResult->bodys[i].rect.left); rectf->set_top(detectResult->bodys[i].rect.top); rectf->set_right(detectResult->bodys[i].rect.right); rectf->set_bottom(detectResult->bodys[i].rect.bottom); for (j = 0; j < detectResult->bodys[i].pointCount; j++){ PointF *bPoint = bodydetect->add_points(); bPoint->set_x(detectResult->bodys[i].points[j].x); bPoint->set_y(detectResult->bodys[i].points[j].y); //Visibilities *bvisibilities = bodydetect->add_visibilities(); //bvisibilities->set_visibilities(detectResult->bodys[i].visibilities[j]); } bodydetect->set_quality(detectResult->bodys[i].quality); } } } void STTaskHandler::postAIMattingData(void *mattingBuffer, void *imgData) { if (!mattingBuffer || !imgData) return; std::lock_guard lock(mOpMutex); auto mAiMattingResult = (RTKKAIMattingResult *)mattingBuffer; int faceCnt = mAiMattingResult->faceCount; for (int32_t i = 0; i < faceCnt; i++) { RTKKMattingFaceInfo *faceInfo = &mAiMattingResult->faceInfo[i]; MattingFaceHolder *holder = saveMattingFaceInfo(faceInfo, imgData); if (holder == nullptr) continue; if ((!holder->sended && holder->repeatCnt > FACE_FIRST_SEND_DELAY) || (holder->sended && holder->repeatCnt > FACE_SAME_SEND_DELAY)) { LOG_DEBUG("post matting face[%d] repeat[%d]\n", holder->faceID, holder->repeatCnt); holder->sended = 1; holder->repeatCnt = 0; doPostMattingFace(holder); } } } void STTaskHandler::doPostMattingFace(MattingFaceHolder *holder) { RTKKMattingFaceInfo* faceInfo = holder->faceInfo; void* imgData = holder->faceData; KKMessage kkMsg; ImageClip* imgClip = kkMsg.mutable_imgclip(); imgClip->set_data(imgData, faceInfo->dataSize); imgClip->set_height(faceInfo->width); imgClip->set_width(faceInfo->height); imgClip->set_format(faceInfo->format); imgClip->set_angle(faceInfo->angle); imgClip->set_mirror(faceInfo->mirror); imgClip->set_imgid(faceInfo->faceID); kkMsg.set_msg_type(2); kkMsg.set_msg_name("KK_AI_IMAGE_CLIP"); std::string sendbuf; kkMsg.SerializeToString(&sendbuf); mShmNNcontroller->send(sendbuf); } void STTaskHandler::freeMattingFaceInfo(MattingFaceHolder* holder) { if (!holder) return; if (holder->faceInfo != nullptr) { if (holder->faceInfo->feature) { free(holder->faceInfo->feature); holder->faceInfo->feature = nullptr; } free(holder->faceInfo); holder->faceInfo = nullptr; } if (holder->faceData != nullptr) { free(holder->faceData); holder->faceData = nullptr; } free(holder); } MattingFaceHolder* STTaskHandler::saveMattingFaceInfo(RTKKMattingFaceInfo *faceInfo, void *imgData) { MattingFaceHolder* faceHolder = nullptr; std::map::iterator faceIter = mFaceMap.find(faceInfo->faceID); if (faceIter != mFaceMap.end()) { faceHolder = faceIter->second; } if (faceHolder != nullptr && faceInfo->featureLen > 0 && faceInfo->dataSize > 0) { LOG_INFO("remove for update matting face[%d]\n", faceHolder->faceID); mFaceMap.erase(faceHolder->faceID); freeMattingFaceInfo(faceHolder); faceHolder = nullptr; } if (faceHolder == nullptr) { if (faceInfo->featureLen <= 0 || faceInfo->dataSize <= 0) { LOG_DEBUG("empty face[%d] feature\n", faceInfo->faceID); return nullptr; } // remove oldest face if over max size if (mFaceMap.size() >= MAX_KEEP_FACE_COUNT) { int64_t recTime = getNowUs(); MattingFaceHolder *oldHolder = nullptr; std::map::iterator it; for (it = mFaceMap.begin(); it != mFaceMap.end();) { MattingFaceHolder *holder = it->second; if (holder->recTime < recTime) { recTime = holder->recTime; oldHolder = holder; } it++; } if (oldHolder != nullptr) { LOG_DEBUG("remove oldest matting face[%d] repeat[%d]\n", oldHolder->faceID, oldHolder->repeatCnt); mFaceMap.erase(oldHolder->faceID); freeMattingFaceInfo(oldHolder); } else { LOG_ERROR("error record time!!!\n"); } } faceHolder = (MattingFaceHolder *)malloc(sizeof(MattingFaceHolder)); memset(faceHolder, 0, sizeof(MattingFaceHolder)); faceHolder->recTime = getNowUs(); faceHolder->faceID = faceInfo->faceID; faceHolder->dataSize = faceInfo->dataSize; faceHolder->faceData = (unsigned char*)malloc(faceInfo->dataSize); faceHolder->faceInfo = (RTKKMattingFaceInfo *)malloc(sizeof(RTKKMattingFaceInfo)); memcpy(faceHolder->faceData, imgData, faceInfo->dataSize); memcpy(faceHolder->faceInfo, faceInfo, sizeof(RTKKMattingFaceInfo)); faceHolder->faceInfo->feature = (unsigned char*)malloc(faceInfo->featureLen); memcpy(faceHolder->faceInfo->feature, faceInfo->feature, faceInfo->featureLen); mFaceMap[faceInfo->faceID] = faceHolder; LOG_DEBUG("save matting face[%d] size[%d]\n", faceHolder->faceID, mFaceMap.size()); } else { faceHolder->recTime = getNowUs(); faceHolder->repeatCnt += 1; } return faceHolder; } void STTaskHandler::postFeatureData(void* nnResult, const char* uuid) { std::lock_guard lock(mOpMutex); KKMessage kkMsg; KKAIData *mNNData = kkMsg.mutable_kkaidata(); auto st_result = (STDetectResult *)(nnResult); pushDetectInfo(mNNData, st_result, 1); mNNData->set_width(1280); mNNData->set_height(720); mNNData->set_index(0); mNNData->set_function("ANALYSE"); mNNData->set_uuid(uuid); kkMsg.set_msg_type(4); kkMsg.set_msg_name("ANALYSE"); std::string sendBuf; kkMsg.SerializeToString(&sendBuf); mShmNNcontroller->send(sendBuf); } void STTaskHandler::postEmptyFeatureData(const char* uuid) { STDetectResult nnResult; memset(&nnResult, 0, sizeof(STDetectResult)); postFeatureData(&nnResult, uuid); } int32_t STTaskHandler::convertDetectType(int32_t detectType) { detectType = (detectType <= 0) ? 30: detectType; int32_t st_type = 0; if (detectType&(0x2)) { // face detection ( type = 2) st_type = st_type | ST_DETECT_FACE; } if (detectType&(0x4)) { // face attribute ( type = 4), face detection is a precondition st_type = st_type | ST_DETECT_FACE | ST_DETECT_FACE_ATTRIBUTE; } if (detectType&(0x8)) { // face keypoint ( type = 8), equals face detection st_type = st_type | ST_DETECT_FACE; } if (detectType&(0x10)) { // face keypoint ( type = 16), face feature is a precondition st_type = st_type | ST_DETECT_FACE | ST_DETECT_FACE_FEATURE; } LOG_INFO("ai detection type = %x\n", st_type); return st_type; } } // namespace aiserver } // namespace rockchip