/*
|
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
*
|
* Use of this source code is governed by a BSD-style license
|
* that can be found in the LICENSE file in the root of the source
|
* tree. An additional intellectual property rights grant can be found
|
* in the file PATENTS. All contributing project authors may
|
* be found in the AUTHORS file in the root of the source tree.
|
*/
|
|
#ifndef WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_
|
#define WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_
|
|
#include "webrtc/common_types.h"
|
#include "webrtc/typedefs.h"
|
|
/******************************************************/
|
/* Quality Modes: Resolution and Robustness settings */
|
/******************************************************/
|
|
namespace webrtc {
|
struct VideoContentMetrics;
|
|
struct VCMResolutionScale {
|
VCMResolutionScale()
|
: codec_width(640),
|
codec_height(480),
|
frame_rate(30.0f),
|
spatial_width_fact(1.0f),
|
spatial_height_fact(1.0f),
|
temporal_fact(1.0f),
|
change_resolution_spatial(false),
|
change_resolution_temporal(false) {}
|
uint16_t codec_width;
|
uint16_t codec_height;
|
float frame_rate;
|
float spatial_width_fact;
|
float spatial_height_fact;
|
float temporal_fact;
|
bool change_resolution_spatial;
|
bool change_resolution_temporal;
|
};
|
|
enum ImageType {
|
kQCIF = 0, // 176x144
|
kHCIF, // 264x216 = half(~3/4x3/4) CIF.
|
kQVGA, // 320x240 = quarter VGA.
|
kCIF, // 352x288
|
kHVGA, // 480x360 = half(~3/4x3/4) VGA.
|
kVGA, // 640x480
|
kQFULLHD, // 960x540 = quarter FULLHD, and half(~3/4x3/4) WHD.
|
kWHD, // 1280x720
|
kFULLHD, // 1920x1080
|
kNumImageTypes
|
};
|
|
const uint32_t kSizeOfImageType[kNumImageTypes] = {
|
25344, 57024, 76800, 101376, 172800, 307200, 518400, 921600, 2073600};
|
|
enum FrameRateLevelClass {
|
kFrameRateLow,
|
kFrameRateMiddle1,
|
kFrameRateMiddle2,
|
kFrameRateHigh
|
};
|
|
enum ContentLevelClass { kLow, kHigh, kDefault };
|
|
struct VCMContFeature {
|
VCMContFeature() : value(0.0f), level(kDefault) {}
|
void Reset() {
|
value = 0.0f;
|
level = kDefault;
|
}
|
float value;
|
ContentLevelClass level;
|
};
|
|
enum UpDownAction { kUpResolution, kDownResolution };
|
|
enum SpatialAction {
|
kNoChangeSpatial,
|
kOneHalfSpatialUniform, // 3/4 x 3/4: 9/6 ~1/2 pixel reduction.
|
kOneQuarterSpatialUniform, // 1/2 x 1/2: 1/4 pixel reduction.
|
kNumModesSpatial
|
};
|
|
enum TemporalAction {
|
kNoChangeTemporal,
|
kTwoThirdsTemporal, // 2/3 frame rate reduction
|
kOneHalfTemporal, // 1/2 frame rate reduction
|
kNumModesTemporal
|
};
|
|
struct ResolutionAction {
|
ResolutionAction() : spatial(kNoChangeSpatial), temporal(kNoChangeTemporal) {}
|
SpatialAction spatial;
|
TemporalAction temporal;
|
};
|
|
// Down-sampling factors for spatial (width and height), and temporal.
|
const float kFactorWidthSpatial[kNumModesSpatial] = {1.0f, 4.0f / 3.0f, 2.0f};
|
|
const float kFactorHeightSpatial[kNumModesSpatial] = {1.0f, 4.0f / 3.0f, 2.0f};
|
|
const float kFactorTemporal[kNumModesTemporal] = {1.0f, 1.5f, 2.0f};
|
|
enum EncoderState {
|
kStableEncoding, // Low rate mis-match, stable buffer levels.
|
kStressedEncoding, // Significant over-shooting of target rate,
|
// Buffer under-flow, etc.
|
kEasyEncoding // Significant under-shooting of target rate.
|
};
|
|
// QmMethod class: main class for resolution and robustness settings
|
|
class VCMQmMethod {
|
public:
|
VCMQmMethod();
|
virtual ~VCMQmMethod();
|
|
// Reset values
|
void ResetQM();
|
virtual void Reset() = 0;
|
|
// Compute content class.
|
uint8_t ComputeContentClass();
|
|
// Update with the content metrics.
|
void UpdateContent(const VideoContentMetrics* content_metrics);
|
|
// Compute spatial texture magnitude and level.
|
// Spatial texture is a spatial prediction error measure.
|
void ComputeSpatial();
|
|
// Compute motion magnitude and level for NFD metric.
|
// NFD is normalized frame difference (normalized by spatial variance).
|
void ComputeMotionNFD();
|
|
// Get the imageType (CIF, VGA, HD, etc) for the system width/height.
|
ImageType GetImageType(uint16_t width, uint16_t height);
|
|
// Return the closest image type.
|
ImageType FindClosestImageType(uint16_t width, uint16_t height);
|
|
// Get the frame rate level.
|
FrameRateLevelClass FrameRateLevel(float frame_rate);
|
|
protected:
|
// Content Data.
|
const VideoContentMetrics* content_metrics_;
|
|
// Encoder frame sizes and native frame sizes.
|
uint16_t width_;
|
uint16_t height_;
|
float user_frame_rate_;
|
uint16_t native_width_;
|
uint16_t native_height_;
|
float native_frame_rate_;
|
float aspect_ratio_;
|
// Image type and frame rate leve, for the current encoder resolution.
|
ImageType image_type_;
|
FrameRateLevelClass framerate_level_;
|
// Content class data.
|
VCMContFeature motion_;
|
VCMContFeature spatial_;
|
uint8_t content_class_;
|
bool init_;
|
};
|
|
// Resolution settings class
|
|
class VCMQmResolution : public VCMQmMethod {
|
public:
|
VCMQmResolution();
|
virtual ~VCMQmResolution();
|
|
// Reset all quantities.
|
virtual void Reset();
|
|
// Reset rate quantities and counters after every SelectResolution() call.
|
void ResetRates();
|
|
// Reset down-sampling state.
|
void ResetDownSamplingState();
|
|
// Get the encoder state.
|
EncoderState GetEncoderState();
|
|
// Initialize after SetEncodingData in media_opt.
|
int Initialize(float bitrate,
|
float user_framerate,
|
uint16_t width,
|
uint16_t height,
|
int num_layers);
|
|
// Update the encoder frame size.
|
void UpdateCodecParameters(float frame_rate, uint16_t width, uint16_t height);
|
|
// Update with actual bit rate (size of the latest encoded frame)
|
// and frame type, after every encoded frame.
|
void UpdateEncodedSize(size_t encoded_size);
|
|
// Update with new target bitrate, actual encoder sent rate, frame_rate,
|
// loss rate: every ~1 sec from SetTargetRates in media_opt.
|
void UpdateRates(float target_bitrate,
|
float encoder_sent_rate,
|
float incoming_framerate,
|
uint8_t packet_loss);
|
|
// Extract ST (spatio-temporal) resolution action.
|
// Inputs: qm: Reference to the quality modes pointer.
|
// Output: the spatial and/or temporal scale change.
|
int SelectResolution(VCMResolutionScale** qm);
|
|
private:
|
// Set the default resolution action.
|
void SetDefaultAction();
|
|
// Compute rates for the selection of down-sampling action.
|
void ComputeRatesForSelection();
|
|
// Compute the encoder state.
|
void ComputeEncoderState();
|
|
// Return true if the action is to go back up in resolution.
|
bool GoingUpResolution();
|
|
// Return true if the action is to go down in resolution.
|
bool GoingDownResolution();
|
|
// Check the condition for going up in resolution by the scale factors:
|
// |facWidth|, |facHeight|, |facTemp|.
|
// |scaleFac| is a scale factor for the transition rate.
|
bool ConditionForGoingUp(float fac_width,
|
float fac_height,
|
float fac_temp,
|
float scale_fac);
|
|
// Get the bitrate threshold for the resolution action.
|
// The case |facWidth|=|facHeight|=|facTemp|==1 is for down-sampling action.
|
// |scaleFac| is a scale factor for the transition rate.
|
float GetTransitionRate(float fac_width,
|
float fac_height,
|
float fac_temp,
|
float scale_fac);
|
|
// Update the down-sampling state.
|
void UpdateDownsamplingState(UpDownAction up_down);
|
|
// Update the codec frame size and frame rate.
|
void UpdateCodecResolution();
|
|
// Return a state based on average target rate relative transition rate.
|
uint8_t RateClass(float transition_rate);
|
|
// Adjust the action selected from the table.
|
void AdjustAction();
|
|
// Covert 2 stages of 3/4 (=9/16) spatial decimation to 1/2.
|
void ConvertSpatialFractionalToWhole();
|
|
// Returns true if the new frame sizes, under the selected spatial action,
|
// are of even size.
|
bool EvenFrameSize();
|
|
// Insert latest down-sampling action into the history list.
|
void InsertLatestDownAction();
|
|
// Remove the last (first element) down-sampling action from the list.
|
void RemoveLastDownAction();
|
|
// Check constraints on the amount of down-sampling allowed.
|
void ConstrainAmountOfDownSampling();
|
|
// For going up in resolution: pick spatial or temporal action,
|
// if both actions were separately selected.
|
void PickSpatialOrTemporal();
|
|
// Select the directional (1x2 or 2x1) spatial down-sampling action.
|
void SelectSpatialDirectionMode(float transition_rate);
|
|
enum { kDownActionHistorySize = 10 };
|
|
VCMResolutionScale* qm_;
|
// Encoder rate control parameters.
|
float target_bitrate_;
|
float incoming_framerate_;
|
float per_frame_bandwidth_;
|
float buffer_level_;
|
|
// Data accumulated every ~1sec from MediaOpt.
|
float sum_target_rate_;
|
float sum_incoming_framerate_;
|
float sum_rate_MM_;
|
float sum_rate_MM_sgn_;
|
float sum_packet_loss_;
|
// Counters.
|
uint32_t frame_cnt_;
|
uint32_t frame_cnt_delta_;
|
uint32_t update_rate_cnt_;
|
uint32_t low_buffer_cnt_;
|
|
// Resolution state parameters.
|
float state_dec_factor_spatial_;
|
float state_dec_factor_temporal_;
|
|
// Quantities used for selection.
|
float avg_target_rate_;
|
float avg_incoming_framerate_;
|
float avg_ratio_buffer_low_;
|
float avg_rate_mismatch_;
|
float avg_rate_mismatch_sgn_;
|
float avg_packet_loss_;
|
EncoderState encoder_state_;
|
ResolutionAction action_;
|
// Short history of the down-sampling actions from the Initialize() state.
|
// This is needed for going up in resolution. Since the total amount of
|
// down-sampling actions are constrained, the length of the list need not be
|
// large: i.e., (4/3) ^{kDownActionHistorySize} <= kMaxDownSample.
|
ResolutionAction down_action_history_[kDownActionHistorySize];
|
int num_layers_;
|
};
|
|
// Robustness settings class.
|
|
class VCMQmRobustness : public VCMQmMethod {
|
public:
|
VCMQmRobustness();
|
~VCMQmRobustness();
|
|
virtual void Reset();
|
|
// Adjust FEC rate based on content: every ~1 sec from SetTargetRates.
|
// Returns an adjustment factor.
|
float AdjustFecFactor(uint8_t code_rate_delta,
|
float total_rate,
|
float framerate,
|
int64_t rtt_time,
|
uint8_t packet_loss);
|
|
// Set the UEP protection on/off.
|
bool SetUepProtection(uint8_t code_rate_delta,
|
float total_rate,
|
uint8_t packet_loss,
|
bool frame_type);
|
|
private:
|
// Previous state of network parameters.
|
float prev_total_rate_;
|
int64_t prev_rtt_time_;
|
uint8_t prev_packet_loss_;
|
uint8_t prev_code_rate_delta_;
|
};
|
} // namespace webrtc
|
#endif // WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_
|
