/*
|
* cl_post_image_processor.cpp - CL post image processor
|
*
|
* Copyright (c) 2015 Intel Corporation
|
*
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
* you may not use this file except in compliance with the License.
|
* You may obtain a copy of the License at
|
*
|
* http://www.apache.org/licenses/LICENSE-2.0
|
*
|
* Unless required by applicable law or agreed to in writing, software
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* See the License for the specific language governing permissions and
|
* limitations under the License.
|
*
|
* Author: Wind Yuan <feng.yuan@intel.com>
|
* Author: Yinhang Liu <yinhangx.liu@intel.com>
|
*/
|
|
#include "cl_post_image_processor.h"
|
#include "cl_context.h"
|
|
#include "cl_tnr_handler.h"
|
#include "cl_retinex_handler.h"
|
#include "cl_defog_dcp_handler.h"
|
#include "cl_wavelet_denoise_handler.h"
|
#include "cl_newwavelet_denoise_handler.h"
|
#include "cl_3d_denoise_handler.h"
|
#include "cl_image_scaler.h"
|
#include "cl_wire_frame_handler.h"
|
#include "cl_csc_handler.h"
|
#include "cl_image_warp_handler.h"
|
#include "cl_image_360_stitch.h"
|
#include "cl_video_stabilizer.h"
|
|
#define XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE 6
|
#define XCAM_CL_POST_IMAGE_MAX_POOL_SIZE 12
|
|
namespace XCam {
|
|
CLPostImageProcessor::CLPostImageProcessor ()
|
: CLImageProcessor ("CLPostImageProcessor")
|
, _output_fourcc (V4L2_PIX_FMT_NV12)
|
, _out_sample_type (OutSampleYuv)
|
, _scaler_factor (1.0)
|
, _tnr_mode (TnrYuv)
|
, _defog_mode (CLPostImageProcessor::DefogDisabled)
|
, _wavelet_basis (CL_WAVELET_DISABLED)
|
, _wavelet_channel (CL_IMAGE_CHANNEL_UV)
|
, _wavelet_bayes_shrink (false)
|
, _3d_denoise_mode (CLPostImageProcessor::Denoise3DDisabled)
|
, _3d_denoise_ref_count (3)
|
, _enable_scaler (false)
|
, _enable_wireframe (false)
|
, _enable_image_warp (false)
|
, _enable_stitch (false)
|
, _stitch_enable_seam (false)
|
, _stitch_fisheye_map (false)
|
, _stitch_lsc (false)
|
, _stitch_fm_ocl (false)
|
, _stitch_scale_mode (CLBlenderScaleLocal)
|
, _stitch_width (0)
|
, _stitch_height (0)
|
, _stitch_res_mode (0)
|
, _surround_mode (SphereView)
|
{
|
XCAM_LOG_DEBUG ("CLPostImageProcessor constructed");
|
}
|
|
CLPostImageProcessor::~CLPostImageProcessor ()
|
{
|
XCAM_LOG_DEBUG ("CLPostImageProcessor destructed");
|
}
|
|
bool
|
CLPostImageProcessor::set_output_format (uint32_t fourcc)
|
{
|
switch (fourcc) {
|
case XCAM_PIX_FMT_RGBA64:
|
case V4L2_PIX_FMT_XBGR32:
|
case V4L2_PIX_FMT_ABGR32:
|
case V4L2_PIX_FMT_BGR32:
|
case V4L2_PIX_FMT_RGB32:
|
case V4L2_PIX_FMT_ARGB32:
|
case V4L2_PIX_FMT_XRGB32:
|
_out_sample_type = OutSampleRGB;
|
break;
|
case V4L2_PIX_FMT_NV12:
|
_out_sample_type = OutSampleYuv;
|
break;
|
default:
|
XCAM_LOG_WARNING (
|
"cl post processor doesn't support output format: %s",
|
xcam_fourcc_to_string(fourcc));
|
return false;
|
}
|
|
_output_fourcc = fourcc;
|
return true;
|
}
|
|
void
|
CLPostImageProcessor::set_stats_callback (const SmartPtr<StatsCallback> &callback)
|
{
|
XCAM_ASSERT (callback.ptr ());
|
_stats_callback = callback;
|
}
|
|
bool
|
CLPostImageProcessor::set_scaler_factor (const double factor)
|
{
|
_scaler_factor = factor;
|
|
return true;
|
}
|
|
bool
|
CLPostImageProcessor::can_process_result (SmartPtr < X3aResult > & result)
|
{
|
if (!result.ptr ())
|
return false;
|
|
switch (result->get_type ()) {
|
case XCAM_3A_RESULT_TEMPORAL_NOISE_REDUCTION_YUV:
|
case XCAM_3A_RESULT_3D_NOISE_REDUCTION:
|
case XCAM_3A_RESULT_WAVELET_NOISE_REDUCTION:
|
case XCAM_3A_RESULT_FACE_DETECTION:
|
case XCAM_3A_RESULT_DVS:
|
return true;
|
default:
|
return false;
|
}
|
|
return false;
|
}
|
|
XCamReturn
|
CLPostImageProcessor::apply_3a_results (X3aResultList &results)
|
{
|
XCamReturn ret = XCAM_RETURN_NO_ERROR;
|
|
for (X3aResultList::iterator iter = results.begin (); iter != results.end (); ++iter)
|
{
|
SmartPtr<X3aResult> &result = *iter;
|
ret = apply_3a_result (result);
|
if (ret != XCAM_RETURN_NO_ERROR)
|
break;
|
}
|
|
return ret;
|
}
|
|
XCamReturn
|
CLPostImageProcessor::apply_3a_result (SmartPtr<X3aResult> &result)
|
{
|
STREAM_LOCK;
|
|
if (!result.ptr ())
|
return XCAM_RETURN_BYPASS;
|
|
uint32_t res_type = result->get_type ();
|
|
switch (res_type) {
|
case XCAM_3A_RESULT_TEMPORAL_NOISE_REDUCTION_YUV: {
|
SmartPtr<X3aTemporalNoiseReduction> tnr_res = result.dynamic_cast_ptr<X3aTemporalNoiseReduction> ();
|
XCAM_ASSERT (tnr_res.ptr ());
|
if (_tnr.ptr ()) {
|
if (_defog_mode != CLPostImageProcessor::DefogDisabled) {
|
XCam3aResultTemporalNoiseReduction config;
|
xcam_mem_clear (config);
|
// isp processor
|
// config.gain = 0.12;
|
|
// cl processor
|
config.gain = 0.22;
|
|
config.threshold [0] = 0.00081;
|
config.threshold [1] = 0.00072;
|
_tnr->set_yuv_config (config);
|
} else {
|
_tnr->set_yuv_config (tnr_res->get_standard_result ());
|
}
|
}
|
break;
|
}
|
case XCAM_3A_RESULT_3D_NOISE_REDUCTION: {
|
SmartPtr<X3aTemporalNoiseReduction> nr_res = result.dynamic_cast_ptr<X3aTemporalNoiseReduction> ();
|
XCAM_ASSERT (nr_res.ptr ());
|
if (_3d_denoise.ptr ()) {
|
_3d_denoise->set_denoise_config (nr_res->get_standard_result ());
|
}
|
break;
|
}
|
case XCAM_3A_RESULT_WAVELET_NOISE_REDUCTION: {
|
SmartPtr<X3aWaveletNoiseReduction> wavelet_res = result.dynamic_cast_ptr<X3aWaveletNoiseReduction> ();
|
XCAM_ASSERT (wavelet_res.ptr ());
|
if (_wavelet.ptr()) {
|
_wavelet->set_denoise_config (wavelet_res->get_standard_result ());
|
}
|
if (_newwavelet.ptr()) {
|
_newwavelet->set_denoise_config (wavelet_res->get_standard_result ());
|
}
|
break;
|
}
|
case XCAM_3A_RESULT_FACE_DETECTION: {
|
SmartPtr<X3aFaceDetectionResult> fd_res = result.dynamic_cast_ptr<X3aFaceDetectionResult> ();
|
XCAM_ASSERT (fd_res.ptr ());
|
if (_wireframe.ptr ()) {
|
_wireframe->set_wire_frame_config (fd_res->get_standard_result_ptr (), get_scaler_factor ());
|
}
|
break;
|
}
|
case XCAM_3A_RESULT_DVS: {
|
SmartPtr<X3aDVSResult> dvs_res = result.dynamic_cast_ptr<X3aDVSResult> ();
|
XCAM_ASSERT (dvs_res.ptr ());
|
if (_image_warp.ptr ()) {
|
_image_warp->set_warp_config (dvs_res->get_standard_result ());
|
}
|
break;
|
}
|
default:
|
XCAM_LOG_WARNING ("CLPostImageProcessor unknown 3a result: %d", res_type);
|
break;
|
}
|
|
return XCAM_RETURN_NO_ERROR;
|
}
|
|
|
XCamReturn
|
CLPostImageProcessor::create_handlers ()
|
{
|
SmartPtr<CLImageHandler> image_handler;
|
SmartPtr<CLContext> context = get_cl_context ();
|
|
XCAM_ASSERT (context.ptr ());
|
|
/* defog: retinex */
|
image_handler = create_cl_retinex_image_handler (context);
|
_retinex = image_handler.dynamic_cast_ptr<CLRetinexImageHandler> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_retinex.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create retinex handler failed");
|
_retinex->enable_handler (_defog_mode == CLPostImageProcessor::DefogRetinex);
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
|
add_handler (image_handler);
|
|
/* defog: dark channel prior */
|
image_handler = create_cl_defog_dcp_image_handler (context);
|
_defog_dcp = image_handler.dynamic_cast_ptr<CLDefogDcpImageHandler> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_defog_dcp.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create defog handler failed");
|
_defog_dcp->enable_handler (_defog_mode == CLPostImageProcessor::DefogDarkChannelPrior);
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
|
add_handler (image_handler);
|
|
/* Temporal Noise Reduction */
|
if (_defog_mode != CLPostImageProcessor::DefogDisabled) {
|
switch (_tnr_mode) {
|
case TnrYuv: {
|
image_handler = create_cl_tnr_image_handler (context, CL_TNR_TYPE_YUV);
|
_tnr = image_handler.dynamic_cast_ptr<CLTnrImageHandler> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_tnr.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create tnr handler failed");
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
|
add_handler (image_handler);
|
break;
|
}
|
case TnrDisable:
|
XCAM_LOG_DEBUG ("CLPostImageProcessor disable tnr");
|
break;
|
default:
|
XCAM_LOG_WARNING ("CLPostImageProcessor unknown tnr mode (%d)", _tnr_mode);
|
break;
|
}
|
}
|
|
/* wavelet denoise */
|
switch (_wavelet_basis) {
|
case CL_WAVELET_HAT: {
|
image_handler = create_cl_wavelet_denoise_image_handler (context, _wavelet_channel);
|
_wavelet = image_handler.dynamic_cast_ptr<CLWaveletDenoiseImageHandler> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_wavelet.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create wavelet denoise handler failed");
|
_wavelet->enable_handler (true);
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
|
add_handler (image_handler);
|
break;
|
}
|
case CL_WAVELET_HAAR: {
|
image_handler = create_cl_newwavelet_denoise_image_handler (context, _wavelet_channel, _wavelet_bayes_shrink);
|
_newwavelet = image_handler.dynamic_cast_ptr<CLNewWaveletDenoiseImageHandler> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_newwavelet.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create new wavelet denoise handler failed");
|
_newwavelet->enable_handler (true);
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
|
add_handler (image_handler);
|
break;
|
}
|
case CL_WAVELET_DISABLED:
|
default :
|
XCAM_LOG_DEBUG ("unknown or disable wavelet (%d)", _wavelet_basis);
|
break;
|
}
|
|
/* 3D noise reduction */
|
if (_3d_denoise_mode != CLPostImageProcessor::Denoise3DDisabled) {
|
uint32_t denoise_channel = CL_IMAGE_CHANNEL_UV;
|
|
if (_3d_denoise_mode == CLPostImageProcessor::Denoise3DUV) {
|
denoise_channel = CL_IMAGE_CHANNEL_UV;
|
} else if (_3d_denoise_mode == CLPostImageProcessor::Denoise3DYuv) {
|
denoise_channel = CL_IMAGE_CHANNEL_Y | CL_IMAGE_CHANNEL_UV;
|
}
|
|
image_handler = create_cl_3d_denoise_image_handler (context, denoise_channel, _3d_denoise_ref_count);
|
_3d_denoise = image_handler.dynamic_cast_ptr<CL3DDenoiseImageHandler> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_3d_denoise.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CL3aImageProcessor create 3D noise reduction handler failed");
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
|
image_handler->enable_handler (true);
|
add_handler (image_handler);
|
}
|
|
/* image scaler */
|
image_handler = create_cl_image_scaler_handler (context, V4L2_PIX_FMT_NV12);
|
_scaler = image_handler.dynamic_cast_ptr<CLImageScaler> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_scaler.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create scaler handler failed");
|
_scaler->set_scaler_factor (_scaler_factor, _scaler_factor);
|
_scaler->set_buffer_callback (_stats_callback);
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->enable_handler (_enable_scaler);
|
add_handler (image_handler);
|
|
/* wire frame */
|
image_handler = create_cl_wire_frame_image_handler (context);
|
_wireframe = image_handler.dynamic_cast_ptr<CLWireFrameImageHandler> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_wireframe.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create wire frame handler failed");
|
_wireframe->enable_handler (_enable_wireframe);
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
|
add_handler (image_handler);
|
|
/* image warp */
|
image_handler = create_cl_image_warp_handler (context);
|
_image_warp = image_handler.dynamic_cast_ptr<CLImageWarpHandler> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_image_warp.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create image warp handler failed");
|
_image_warp->enable_handler (_enable_image_warp);
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
|
add_handler (image_handler);
|
|
/* video stabilization */
|
image_handler = create_cl_video_stab_handler (context);
|
_video_stab = image_handler.dynamic_cast_ptr<CLVideoStabilizer> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_video_stab.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create video stabilizer failed");
|
_video_stab->enable_handler (false);
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
|
add_handler (image_handler);
|
|
/* image stitch */
|
image_handler =
|
create_image_360_stitch (context, _stitch_enable_seam, _stitch_scale_mode,
|
_stitch_fisheye_map, _stitch_lsc, (SurroundMode) _surround_mode, (StitchResMode) _stitch_res_mode);
|
_stitch = image_handler.dynamic_cast_ptr<CLImage360Stitch> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_stitch.ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create image stitch handler failed");
|
_stitch->set_output_size (_stitch_width, _stitch_height);
|
#if HAVE_OPENCV
|
_stitch->set_feature_match_ocl (_stitch_fm_ocl);
|
#endif
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
|
image_handler->enable_handler (_enable_stitch);
|
add_handler (image_handler);
|
|
/* csc (nv12torgba) */
|
image_handler = create_cl_csc_image_handler (context, CL_CSC_TYPE_NV12TORGBA);
|
_csc = image_handler.dynamic_cast_ptr<CLCscImageHandler> ();
|
XCAM_FAIL_RETURN (
|
WARNING,
|
_csc .ptr (),
|
XCAM_RETURN_ERROR_CL,
|
"CLPostImageProcessor create csc handler failed");
|
_csc->enable_handler (_out_sample_type == OutSampleRGB);
|
_csc->set_output_format (_output_fourcc);
|
image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
|
image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
|
add_handler (image_handler);
|
|
return XCAM_RETURN_NO_ERROR;
|
}
|
|
bool
|
CLPostImageProcessor::set_tnr (CLTnrMode mode)
|
{
|
_tnr_mode = mode;
|
|
STREAM_LOCK;
|
|
return true;
|
}
|
|
bool
|
CLPostImageProcessor::set_defog_mode (CLDefogMode mode)
|
{
|
_defog_mode = mode;
|
|
STREAM_LOCK;
|
|
return true;
|
}
|
|
bool
|
CLPostImageProcessor::set_wavelet (CLWaveletBasis basis, uint32_t channel, bool bayes_shrink)
|
{
|
_wavelet_basis = basis;
|
_wavelet_channel = (CLImageChannel) channel;
|
_wavelet_bayes_shrink = bayes_shrink;
|
|
STREAM_LOCK;
|
|
return true;
|
}
|
|
bool
|
CLPostImageProcessor::set_3ddenoise_mode (CL3DDenoiseMode mode, uint8_t ref_frame_count)
|
{
|
_3d_denoise_mode = mode;
|
_3d_denoise_ref_count = ref_frame_count;
|
|
STREAM_LOCK;
|
|
return true;
|
}
|
|
bool
|
CLPostImageProcessor::set_scaler (bool enable)
|
{
|
_enable_scaler = enable;
|
|
STREAM_LOCK;
|
|
return true;
|
}
|
|
bool
|
CLPostImageProcessor::set_wireframe (bool enable)
|
{
|
_enable_wireframe = enable;
|
|
STREAM_LOCK;
|
|
return true;
|
}
|
|
bool
|
CLPostImageProcessor::set_image_warp (bool enable)
|
{
|
_enable_image_warp = enable;
|
|
STREAM_LOCK;
|
|
return true;
|
}
|
|
bool
|
CLPostImageProcessor::set_image_stitch (
|
bool enable_stitch, bool enable_seam, CLBlenderScaleMode scale_mode, bool enable_fisheye_map,
|
bool lsc, bool fm_ocl, uint32_t stitch_width, uint32_t stitch_height, uint32_t res_mode)
|
{
|
XCAM_ASSERT (scale_mode < CLBlenderScaleMax);
|
|
_enable_stitch = enable_stitch;
|
if (enable_stitch)
|
_stitch_enable_seam = enable_seam;
|
else
|
_stitch_enable_seam = false;
|
|
_stitch_scale_mode = scale_mode;
|
_stitch_fisheye_map = enable_fisheye_map;
|
_stitch_lsc = lsc;
|
_stitch_width = stitch_width;
|
_stitch_height = stitch_height;
|
_stitch_res_mode = res_mode;
|
|
#if HAVE_OPENCV
|
_stitch_fm_ocl = fm_ocl;
|
#else
|
XCAM_UNUSED (fm_ocl);
|
#endif
|
|
STREAM_LOCK;
|
|
return true;
|
}
|
|
};
|
