# Copyright 2016 The Android Open Source Project
|
#
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
# you may not use this file except in compliance with the License.
|
# You may obtain a copy of the License at
|
#
|
# http://www.apache.org/licenses/LICENSE-2.0
|
#
|
# Unless required by applicable law or agreed to in writing, software
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# See the License for the specific language governing permissions and
|
# limitations under the License.
|
|
import os
|
import unittest
|
|
import cv2
|
import its.caps
|
import its.device
|
import its.error
|
import its.image
|
import numpy
|
|
CHART_FILE = os.path.join(os.environ['CAMERA_ITS_TOP'], 'pymodules', 'its',
|
'test_images', 'ISO12233.png')
|
CHART_HEIGHT = 13.5 # cm
|
CHART_DISTANCE_RFOV = 30.0 # cm
|
CHART_DISTANCE_WFOV = 22.0 # cm
|
CHART_SCALE_START = 0.65
|
CHART_SCALE_STOP = 1.35
|
CHART_SCALE_STEP = 0.025
|
|
FOV_THRESH_TELE = 60
|
FOV_THRESH_WFOV = 90
|
|
SCALE_RFOV_IN_WFOV_BOX = 0.67
|
SCALE_TELE_IN_RFOV_BOX = 0.67
|
SCALE_TELE_IN_WFOV_BOX = 0.5
|
|
VGA_HEIGHT = 480
|
VGA_WIDTH = 640
|
|
|
def calc_chart_scaling(chart_distance, camera_fov):
|
chart_scaling = 1.0
|
camera_fov = float(camera_fov)
|
if (FOV_THRESH_TELE < camera_fov < FOV_THRESH_WFOV and
|
numpy.isclose(chart_distance, CHART_DISTANCE_WFOV, rtol=0.1)):
|
chart_scaling = SCALE_RFOV_IN_WFOV_BOX
|
elif (camera_fov <= FOV_THRESH_TELE and
|
numpy.isclose(chart_distance, CHART_DISTANCE_WFOV, rtol=0.1)):
|
chart_scaling = SCALE_TELE_IN_WFOV_BOX
|
elif (camera_fov <= FOV_THRESH_TELE and
|
numpy.isclose(chart_distance, CHART_DISTANCE_RFOV, rtol=0.1)):
|
chart_scaling = SCALE_TELE_IN_RFOV_BOX
|
return chart_scaling
|
|
|
def scale_img(img, scale=1.0):
|
"""Scale and image based on a real number scale factor."""
|
dim = (int(img.shape[1]*scale), int(img.shape[0]*scale))
|
return cv2.resize(img.copy(), dim, interpolation=cv2.INTER_AREA)
|
|
|
def gray_scale_img(img):
|
"""Return gray scale version of image."""
|
if len(img.shape) == 2:
|
img_gray = img.copy()
|
elif len(img.shape) == 3:
|
if img.shape[2] == 1:
|
img_gray = img[:, :, 0].copy()
|
else:
|
img_gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
|
return img_gray
|
|
|
class Chart(object):
|
"""Definition for chart object.
|
|
Defines PNG reference file, chart size and distance, and scaling range.
|
"""
|
|
def __init__(self, chart_file=None, height=None, distance=None,
|
scale_start=None, scale_stop=None, scale_step=None,
|
camera_id=None):
|
"""Initial constructor for class.
|
|
Args:
|
chart_file: str; absolute path to png file of chart
|
height: float; height in cm of displayed chart
|
distance: float; distance in cm from camera of displayed chart
|
scale_start: float; start value for scaling for chart search
|
scale_stop: float; stop value for scaling for chart search
|
scale_step: float; step value for scaling for chart search
|
camera_id: int; camera used for extractor
|
"""
|
self._file = chart_file or CHART_FILE
|
self._height = height or CHART_HEIGHT
|
self._distance = distance or CHART_DISTANCE_RFOV
|
self._scale_start = scale_start or CHART_SCALE_START
|
self._scale_stop = scale_stop or CHART_SCALE_STOP
|
self._scale_step = scale_step or CHART_SCALE_STEP
|
self.xnorm, self.ynorm, self.wnorm, self.hnorm, self.scale = its.image.chart_located_per_argv()
|
if not self.xnorm:
|
with its.device.ItsSession(camera_id) as cam:
|
props = cam.get_camera_properties()
|
if its.caps.read_3a(props):
|
self.locate(cam, props)
|
else:
|
print 'Chart locator skipped.'
|
self._set_scale_factors_to_one()
|
|
def _set_scale_factors_to_one(self):
|
"""Set scale factors to 1.0 for skipped tests."""
|
self.wnorm = 1.0
|
self.hnorm = 1.0
|
self.xnorm = 0.0
|
self.ynorm = 0.0
|
self.scale = 1.0
|
|
def _calc_scale_factors(self, cam, props, fmt, s, e, fd):
|
"""Take an image with s, e, & fd to find the chart location.
|
|
Args:
|
cam: An open device session.
|
props: Properties of cam
|
fmt: Image format for the capture
|
s: Sensitivity for the AF request as defined in
|
android.sensor.sensitivity
|
e: Exposure time for the AF request as defined in
|
android.sensor.exposureTime
|
fd: float; autofocus lens position
|
Returns:
|
template: numpy array; chart template for locator
|
img_3a: numpy array; RGB image for chart location
|
scale_factor: float; scaling factor for chart search
|
"""
|
req = its.objects.manual_capture_request(s, e)
|
req['android.lens.focusDistance'] = fd
|
cap_chart = its.image.stationary_lens_cap(cam, req, fmt)
|
img_3a = its.image.convert_capture_to_rgb_image(cap_chart, props)
|
img_3a = its.image.rotate_img_per_argv(img_3a)
|
its.image.write_image(img_3a, 'af_scene.jpg')
|
template = cv2.imread(self._file, cv2.IMREAD_ANYDEPTH)
|
focal_l = cap_chart['metadata']['android.lens.focalLength']
|
pixel_pitch = (props['android.sensor.info.physicalSize']['height'] /
|
img_3a.shape[0])
|
print ' Chart distance: %.2fcm' % self._distance
|
print ' Chart height: %.2fcm' % self._height
|
print ' Focal length: %.2fmm' % focal_l
|
print ' Pixel pitch: %.2fum' % (pixel_pitch*1E3)
|
print ' Template height: %dpixels' % template.shape[0]
|
chart_pixel_h = self._height * focal_l / (self._distance * pixel_pitch)
|
scale_factor = template.shape[0] / chart_pixel_h
|
print 'Chart/image scale factor = %.2f' % scale_factor
|
return template, img_3a, scale_factor
|
|
def locate(self, cam, props):
|
"""Find the chart in the image, and append location to chart object.
|
|
The values appended are:
|
xnorm: float; [0, 1] left loc of chart in scene
|
ynorm: float; [0, 1] top loc of chart in scene
|
wnorm: float; [0, 1] width of chart in scene
|
hnorm: float; [0, 1] height of chart in scene
|
scale: float; scale factor to extract chart
|
|
Args:
|
cam: An open device session
|
props: Camera properties
|
"""
|
if its.caps.read_3a(props):
|
s, e, _, _, fd = cam.do_3a(get_results=True)
|
fmt = {'format': 'yuv', 'width': VGA_WIDTH, 'height': VGA_HEIGHT}
|
chart, scene, s_factor = self._calc_scale_factors(cam, props, fmt,
|
s, e, fd)
|
else:
|
print 'Chart locator skipped.'
|
self._set_scale_factors_to_one()
|
return
|
scale_start = self._scale_start * s_factor
|
scale_stop = self._scale_stop * s_factor
|
scale_step = self._scale_step * s_factor
|
self.scale = s_factor
|
max_match = []
|
# check for normalized image
|
if numpy.amax(scene) <= 1.0:
|
scene = (scene * 255.0).astype(numpy.uint8)
|
scene_gray = gray_scale_img(scene)
|
print 'Finding chart in scene...'
|
for scale in numpy.arange(scale_start, scale_stop, scale_step):
|
scene_scaled = scale_img(scene_gray, scale)
|
if (scene_scaled.shape[0] < chart.shape[0] or
|
scene_scaled.shape[1] < chart.shape[1]):
|
continue
|
result = cv2.matchTemplate(scene_scaled, chart, cv2.TM_CCOEFF)
|
_, opt_val, _, top_left_scaled = cv2.minMaxLoc(result)
|
# print out scale and match
|
print ' scale factor: %.3f, opt val: %.f' % (scale, opt_val)
|
max_match.append((opt_val, top_left_scaled))
|
|
# determine if optimization results are valid
|
opt_values = [x[0] for x in max_match]
|
if 2.0*min(opt_values) > max(opt_values):
|
estring = ('Warning: unable to find chart in scene!\n'
|
'Check camera distance and self-reported '
|
'pixel pitch, focal length and hyperfocal distance.')
|
print estring
|
self._set_scale_factors_to_one()
|
else:
|
if (max(opt_values) == opt_values[0] or
|
max(opt_values) == opt_values[len(opt_values)-1]):
|
estring = ('Warning: chart is at extreme range of locator '
|
'check.\n')
|
print estring
|
# find max and draw bbox
|
match_index = max_match.index(max(max_match, key=lambda x: x[0]))
|
self.scale = scale_start + scale_step * match_index
|
print 'Optimum scale factor: %.3f' % self.scale
|
top_left_scaled = max_match[match_index][1]
|
h, w = chart.shape
|
bottom_right_scaled = (top_left_scaled[0] + w,
|
top_left_scaled[1] + h)
|
top_left = (int(top_left_scaled[0]/self.scale),
|
int(top_left_scaled[1]/self.scale))
|
bottom_right = (int(bottom_right_scaled[0]/self.scale),
|
int(bottom_right_scaled[1]/self.scale))
|
self.wnorm = float((bottom_right[0]) - top_left[0]) / scene.shape[1]
|
self.hnorm = float((bottom_right[1]) - top_left[1]) / scene.shape[0]
|
self.xnorm = float(top_left[0]) / scene.shape[1]
|
self.ynorm = float(top_left[1]) / scene.shape[0]
|
|
|
def get_angle(input_img):
|
"""Computes anglular inclination of chessboard in input_img.
|
|
Angle estimation algoritm description:
|
Input: 2D grayscale image of chessboard.
|
Output: Angle of rotation of chessboard perpendicular to
|
chessboard. Assumes chessboard and camera are parallel to
|
each other.
|
|
1) Use adaptive threshold to make image binary
|
2) Find countours
|
3) Filter out small contours
|
4) Filter out all non-square contours
|
5) Compute most common square shape.
|
The assumption here is that the most common square instances
|
are the chessboard squares. We've shown that with our current
|
tuning, we can robustly identify the squares on the sensor fusion
|
chessboard.
|
6) Return median angle of most common square shape.
|
|
USAGE NOTE: This function has been tuned to work for the chessboard used in
|
the sensor_fusion tests. See images in test_images/rotated_chessboard/ for
|
sample captures. If this function is used with other chessboards, it may not
|
work as expected.
|
|
TODO: Make algorithm more robust so it works on any type of
|
chessboard.
|
|
Args:
|
input_img (2D numpy.ndarray): Grayscale image stored as a 2D
|
numpy array.
|
|
Returns:
|
Median angle of squares in degrees identified in the image.
|
"""
|
# Tuning parameters
|
min_square_area = (float)(input_img.shape[1] * 0.05)
|
|
# Creates copy of image to avoid modifying original.
|
img = numpy.array(input_img, copy=True)
|
|
# Scale pixel values from 0-1 to 0-255
|
img *= 255
|
img = img.astype(numpy.uint8)
|
|
thresh = cv2.adaptiveThreshold(
|
img, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 201, 2)
|
|
# Find all contours
|
contours = []
|
cv2_version = cv2.__version__
|
if cv2_version.startswith('2.4.'):
|
contours, _ = cv2.findContours(
|
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
|
elif cv2_version.startswith('3.2.'):
|
_, contours, _ = cv2.findContours(
|
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
|
|
# Filter contours to squares only.
|
square_contours = []
|
|
for contour in contours:
|
rect = cv2.minAreaRect(contour)
|
_, (width, height), angle = rect
|
|
# Skip non-squares (with 0.1 tolerance)
|
tolerance = 0.1
|
if width < height * (1 - tolerance) or width > height * (1 + tolerance):
|
continue
|
|
# Remove very small contours.
|
# These are usually just tiny dots due to noise.
|
area = cv2.contourArea(contour)
|
if area < min_square_area:
|
continue
|
|
if cv2_version.startswith('2.4.'):
|
box = numpy.int0(cv2.cv.BoxPoints(rect))
|
elif cv2_version.startswith('3.2.'):
|
box = numpy.int0(cv2.boxPoints(rect))
|
square_contours.append(contour)
|
|
areas = []
|
for contour in square_contours:
|
area = cv2.contourArea(contour)
|
areas.append(area)
|
|
median_area = numpy.median(areas)
|
|
filtered_squares = []
|
filtered_angles = []
|
for square in square_contours:
|
area = cv2.contourArea(square)
|
if area < median_area * 0.90 or area > median_area * 1.10:
|
continue
|
|
filtered_squares.append(square)
|
_, (width, height), angle = cv2.minAreaRect(square)
|
filtered_angles.append(angle)
|
|
if len(filtered_angles) < 10:
|
return None
|
|
return numpy.median(filtered_angles)
|
|
|
class __UnitTest(unittest.TestCase):
|
"""Run a suite of unit tests on this module.
|
"""
|
|
def test_compute_image_sharpness(self):
|
"""Unit test for compute_img_sharpness.
|
|
Test by using PNG of ISO12233 chart and blurring intentionally.
|
'sharpness' should drop off by sqrt(2) for 2x blur of image.
|
|
We do one level of blur as PNG image is not perfect.
|
"""
|
yuv_full_scale = 1023.0
|
chart_file = os.path.join(os.environ['CAMERA_ITS_TOP'], 'pymodules',
|
'its', 'test_images', 'ISO12233.png')
|
chart = cv2.imread(chart_file, cv2.IMREAD_ANYDEPTH)
|
white_level = numpy.amax(chart).astype(float)
|
sharpness = {}
|
for j in [2, 4, 8]:
|
blur = cv2.blur(chart, (j, j))
|
blur = blur[:, :, numpy.newaxis]
|
sharpness[j] = (yuv_full_scale *
|
its.image.compute_image_sharpness(blur /
|
white_level))
|
self.assertTrue(numpy.isclose(sharpness[2]/sharpness[4],
|
numpy.sqrt(2), atol=0.1))
|
self.assertTrue(numpy.isclose(sharpness[4]/sharpness[8],
|
numpy.sqrt(2), atol=0.1))
|
|
def test_get_angle_identify_unrotated_chessboard_angle(self):
|
basedir = os.path.join(
|
os.path.dirname(__file__), 'test_images/rotated_chessboards/')
|
|
normal_img_path = os.path.join(basedir, 'normal.jpg')
|
wide_img_path = os.path.join(basedir, 'wide.jpg')
|
|
normal_img = cv2.cvtColor(
|
cv2.imread(normal_img_path), cv2.COLOR_BGR2GRAY)
|
wide_img = cv2.cvtColor(
|
cv2.imread(wide_img_path), cv2.COLOR_BGR2GRAY)
|
|
assert get_angle(normal_img) == 0
|
assert get_angle(wide_img) == 0
|
|
def test_get_angle_identify_rotated_chessboard_angle(self):
|
basedir = os.path.join(
|
os.path.dirname(__file__), 'test_images/rotated_chessboards/')
|
|
# Array of the image files and angles containing rotated chessboards.
|
test_cases = [
|
('_15_ccw', 15),
|
('_30_ccw', 30),
|
('_45_ccw', 45),
|
('_60_ccw', 60),
|
('_75_ccw', 75),
|
('_90_ccw', 90)
|
]
|
|
# For each rotated image pair (normal, wide). Check if angle is
|
# identified as expected.
|
for suffix, angle in test_cases:
|
# Define image paths
|
normal_img_path = os.path.join(
|
basedir, 'normal{}.jpg'.format(suffix))
|
wide_img_path = os.path.join(
|
basedir, 'wide{}.jpg'.format(suffix))
|
|
# Load and color convert images
|
normal_img = cv2.cvtColor(
|
cv2.imread(normal_img_path), cv2.COLOR_BGR2GRAY)
|
wide_img = cv2.cvtColor(
|
cv2.imread(wide_img_path), cv2.COLOR_BGR2GRAY)
|
|
# Assert angle is as expected up to 2.0 degrees of accuracy.
|
assert numpy.isclose(
|
abs(get_angle(normal_img)), angle, 2.0)
|
assert numpy.isclose(
|
abs(get_angle(wide_img)), angle, 2.0)
|
|
|
if __name__ == '__main__':
|
unittest.main()
|
