av1/encoder/encoder_utils.c & a/av1/common/resize: Stack-buffer-overflow in aom_scaled_2d_ssse3

https://bugs.webkit.org/show_bug.cgi?id=253498
rdar://106063201

Reviewed by Eric Carlson.

Cherry-pick upstream change from https://aomedia.googlesource.com/aom/+/6318378f833b2a0d8e67fb3d12bcdc4e1c26b0e6%5E%21/#F2.

* Source/ThirdParty/libwebrtc/Source/third_party/libaom/source/libaom/av1/common/resize.c:
(av1_realloc_and_scale_if_required):
* Source/ThirdParty/libwebrtc/Source/third_party/libaom/source/libaom/av1/common/resize.h:
* Source/ThirdParty/libwebrtc/Source/third_party/libaom/source/libaom/av1/encoder/encoder_utils.c:
(av1_scale_references):

Originally-landed-as: 259548.381@safari-7615-branch (1de648970cbf). rdar://106063201
Canonical link: https://commits.webkit.org/264327@main

Author: youennf

Source/ThirdParty/libwebrtc/Source/third_party/libaom/source/libaom/av1/common/resize.c

@@ -1384,15 +1384,20 @@ YV12_BUFFER_CONFIG *av1_realloc_and_scale_if_required(
       aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
                          "Failed to allocate scaled buffer");
 
+    const bool has_optimized_scaler = av1_has_optimized_scaler(
+        unscaled->y_crop_width, unscaled->y_crop_height, scaled_width,
+        scaled_height);
+
 #if CONFIG_AV1_HIGHBITDEPTH
-    if (use_optimized_scaler && cm->seq_params->bit_depth == AOM_BITS_8) {
+    if (use_optimized_scaler && has_optimized_scaler &&
+        cm->seq_params->bit_depth == AOM_BITS_8) {
       av1_resize_and_extend_frame(unscaled, scaled, filter, phase, num_planes);
     } else {
       av1_resize_and_extend_frame_nonnormative(
           unscaled, scaled, (int)cm->seq_params->bit_depth, num_planes);
     }
 #else
-    if (use_optimized_scaler) {
+    if (use_optimized_scaler && has_optimized_scaler) {
       av1_resize_and_extend_frame(unscaled, scaled, filter, phase, num_planes);
     } else {
       av1_resize_and_extend_frame_nonnormative(

Source/ThirdParty/libwebrtc/Source/third_party/libaom/source/libaom/av1/common/resize.h

@@ -105,6 +105,24 @@ static INLINE int av1_superres_scaled(const AV1_COMMON *cm) {
   return !(cm->width == cm->superres_upscaled_width);
 }
 
+// There's SIMD optimizations for 1/4, 1/2 and 3/4 downscaling.
+// SSSE3 also has optimizations for 2x upscaling.
+// Use non normative scalers for other scaling ratios.
+static INLINE bool av1_has_optimized_scaler(const int src_width,
+                                            const int src_height,
+                                            const int dst_width,
+                                            const int dst_height) {
+  const bool has_optimized_scaler =
+      (dst_width * 4 == src_width && dst_height * 4 == src_height) ||
+      (dst_width * 2 == src_width && dst_height * 2 == src_height) ||
+      (dst_width * 4 == src_width * 3 && dst_height * 4 == src_height * 3);
+#if HAVE_SSSE3
+  return has_optimized_scaler ||
+         (dst_width == src_width * 2 && dst_height == src_height * 2);
+#endif
+  return has_optimized_scaler;
+}
+
 #define UPSCALE_NORMATIVE_TAPS 8
 extern const int16_t av1_resize_filter_normative[1 << RS_SUBPEL_BITS]
                                                 [UPSCALE_NORMATIVE_TAPS];

Source/ThirdParty/libwebrtc/Source/third_party/libaom/source/libaom/av1/encoder/encoder_utils.c

@@ -733,15 +733,19 @@ void av1_scale_references(AV1_COMP *cpi, const InterpFilter filter,
             aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
                                "Failed to allocate frame buffer");
           }
+          const bool has_optimized_scaler = av1_has_optimized_scaler(
+              cm->width, cm->height, new_fb->buf.y_crop_width,
+              new_fb->buf.y_crop_height);
 #if CONFIG_AV1_HIGHBITDEPTH
-          if (use_optimized_scaler && cm->seq_params->bit_depth == AOM_BITS_8)
+          if (use_optimized_scaler && has_optimized_scaler &&
+              cm->seq_params->bit_depth == AOM_BITS_8)
             av1_resize_and_extend_frame(ref, &new_fb->buf, filter, phase,
                                         num_planes);
           else
             av1_resize_and_extend_frame_nonnormative(
                 ref, &new_fb->buf, (int)cm->seq_params->bit_depth, num_planes);
 #else
-          if (use_optimized_scaler)
+          if (use_optimized_scaler && has_optimized_scaler)
             av1_resize_and_extend_frame(ref, &new_fb->buf, filter, phase,
                                         num_planes);
           else