From 91913565fb5afffca98e9db0c61dad2e4fe19725 Mon Sep 17 00:00:00 2001
From: Emilian Peev <epeev@google.com>
Date: Fri, 27 Jul 2018 10:34:57 +0100
Subject: [PATCH] Camera: Fix off-by-one error during clamping in distortion
 mapper

The (right, bottom) coordinates of any input rectangles are
exclusive.

Bug: 111885753
Test: Camera CTS,
adb shell /data/nativetest64/cameraservice_test/cameraservice_test
--gtest_filter=DistortionMapperTest.*
Change-Id: Ied956bd154256a10fc4c81d27c93b0ba2e33044d
---
 .../device3/DistortionMapper.cpp              | 16 ++++++-------
 .../tests/DistortionMapperTest.cpp            | 24 +++++++++++++++++++
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/services/camera/libcameraservice/device3/DistortionMapper.cpp b/services/camera/libcameraservice/device3/DistortionMapper.cpp
index 4dafefde55..ae7af8e001 100644
--- a/services/camera/libcameraservice/device3/DistortionMapper.cpp
+++ b/services/camera/libcameraservice/device3/DistortionMapper.cpp
@@ -312,8 +312,8 @@ status_t DistortionMapper::mapRawRectToCorrected(int32_t *rects, int rectCount,
         int32_t coords[4] = {
             rects[i],
             rects[i + 1],
-            rects[i] + rects[i + 2],
-            rects[i + 1] + rects[i + 3]
+            rects[i] + rects[i + 2] - 1,
+            rects[i + 1] + rects[i + 3] - 1
         };
 
         mapRawToCorrected(coords, 2, clamp, simple);
@@ -321,8 +321,8 @@ status_t DistortionMapper::mapRawRectToCorrected(int32_t *rects, int rectCount,
         // Map back to (l, t, width, height)
         rects[i] = coords[0];
         rects[i + 1] = coords[1];
-        rects[i + 2] = coords[2] - coords[0];
-        rects[i + 3] = coords[3] - coords[1];
+        rects[i + 2] = coords[2] - coords[0] + 1;
+        rects[i + 3] = coords[3] - coords[1] + 1;
     }
 
     return OK;
@@ -400,8 +400,8 @@ status_t DistortionMapper::mapCorrectedRectToRaw(int32_t *rects, int rectCount,
         int32_t coords[4] = {
             rects[i],
             rects[i + 1],
-            rects[i] + rects[i + 2],
-            rects[i + 1] + rects[i + 3]
+            rects[i] + rects[i + 2] - 1,
+            rects[i + 1] + rects[i + 3] - 1
         };
 
         mapCorrectedToRaw(coords, 2, clamp, simple);
@@ -409,8 +409,8 @@ status_t DistortionMapper::mapCorrectedRectToRaw(int32_t *rects, int rectCount,
         // Map back to (l, t, width, height)
         rects[i] = coords[0];
         rects[i + 1] = coords[1];
-        rects[i + 2] = coords[2] - coords[0];
-        rects[i + 3] = coords[3] - coords[1];
+        rects[i + 2] = coords[2] - coords[0] + 1;
+        rects[i + 3] = coords[3] - coords[1] + 1;
     }
 
     return OK;
diff --git a/services/camera/libcameraservice/tests/DistortionMapperTest.cpp b/services/camera/libcameraservice/tests/DistortionMapperTest.cpp
index 2a689c6bb3..54935c9485 100644
--- a/services/camera/libcameraservice/tests/DistortionMapperTest.cpp
+++ b/services/camera/libcameraservice/tests/DistortionMapperTest.cpp
@@ -167,6 +167,30 @@ TEST(DistortionMapperTest, IdentityTransform) {
     }
 }
 
+TEST(DistortionMapperTest, ClampConsistency) {
+    status_t res;
+
+    std::array<int32_t, 4> activeArray = {0, 0, 4032, 3024};
+    DistortionMapper m;
+    setupTestMapper(&m, identityDistortion, testICal, /*activeArray*/ activeArray.data(),
+            /*preCorrectionActiveArray*/ activeArray.data());
+
+    auto rectsOrig = activeArray;
+    res = m.mapCorrectedRectToRaw(activeArray.data(), 1, /*clamp*/true, /*simple*/ true);
+    ASSERT_EQ(res, OK);
+
+    for (size_t i = 0; i < activeArray.size(); i++) {
+        EXPECT_EQ(activeArray[i], rectsOrig[i]);
+    }
+
+    res = m.mapRawRectToCorrected(activeArray.data(), 1, /*clamp*/true, /*simple*/ true);
+    ASSERT_EQ(res, OK);
+
+    for (size_t i = 0; i < activeArray.size(); i++) {
+        EXPECT_EQ(activeArray[i], rectsOrig[i]);
+    }
+}
+
 TEST(DistortionMapperTest, SimpleTransform) {
     status_t res;