Check for overflow of crypto size

Bug: 111603051
Test: CTS
Change-Id: Ib5b1802b9b35769a25c16e2b977308cf7a810606

media/ndk/NdkMediaCodec.cpp

@@ -447,7 +447,13 @@ AMediaCodecCryptoInfo *AMediaCodecCryptoInfo_new(
         size_t *encryptedbytes) {
 
     // size needed to store all the crypto data
-    size_t cryptosize = sizeof(AMediaCodecCryptoInfo) + sizeof(size_t) * numsubsamples * 2;
+    size_t cryptosize;
+    // = sizeof(AMediaCodecCryptoInfo) + sizeof(size_t) * numsubsamples * 2;
+    if (__builtin_mul_overflow(sizeof(size_t) * 2, numsubsamples, &cryptosize) ||
+            __builtin_add_overflow(cryptosize, sizeof(AMediaCodecCryptoInfo), &cryptosize)) {
+        ALOGE("crypto size overflow");
+        return NULL;
+    }
     AMediaCodecCryptoInfo *ret = (AMediaCodecCryptoInfo*) malloc(cryptosize);
     if (!ret) {
         ALOGE("couldn't allocate %zu bytes", cryptosize);