kc3-lang/libjpeg-turbo/simd/arm/jdmrgext-neon.c

Branch - branch -main

• Show log

  Commit

• Hash : 4e151a4a
  Author :
  Date : 2025-08-26T21:11:07
  

  Remove vestigial filenames from SIMD code headers
  
  These were a relic of libjpeg/SIMD, which attempted to follow the
  conventions of the libjpeg source code, but they are no longer relevant
  (or even accurate in some cases.)
  

Download

• simd/arm/jdmrgext-neon.c

• /*
   * Merged upsampling/color conversion (Arm Neon)
   *
   * Copyright (C) 2020, Arm Limited.  All Rights Reserved.
   * Copyright (C) 2020, D. R. Commander.  All Rights Reserved.
   *
   * This software is provided 'as-is', without any express or implied
   * warranty.  In no event will the authors be held liable for any damages
   * arising from the use of this software.
   *
   * Permission is granted to anyone to use this software for any purpose,
   * including commercial applications, and to alter it and redistribute it
   * freely, subject to the following restrictions:
   *
   * 1. The origin of this software must not be misrepresented; you must not
   *    claim that you wrote the original software. If you use this software
   *    in a product, an acknowledgment in the product documentation would be
   *    appreciated but is not required.
   * 2. Altered source versions must be plainly marked as such, and must not be
   *    misrepresented as being the original software.
   * 3. This notice may not be removed or altered from any source distribution.
   */
  
  /* This file is included by jdmerge-neon.c. */
  
  
  /* These routines combine simple (non-fancy, i.e. non-smooth) h2v1 or h2v2
   * chroma upsampling and YCbCr -> RGB color conversion into a single function.
   *
   * As with the standalone functions, YCbCr -> RGB conversion is defined by the
   * following equations:
   *    R = Y                        + 1.40200 * (Cr - 128)
   *    G = Y - 0.34414 * (Cb - 128) - 0.71414 * (Cr - 128)
   *    B = Y + 1.77200 * (Cb - 128)
   *
   * Scaled integer constants are used to avoid floating-point arithmetic:
   *    0.3441467 = 11277 * 2^-15
   *    0.7141418 = 23401 * 2^-15
   *    1.4020386 = 22971 * 2^-14
   *    1.7720337 = 29033 * 2^-14
   * These constants are defined in jdmerge-neon.c.
   *
   * To ensure correct results, rounding is used when descaling.
   */
  
  /* Notes on safe memory access for merged upsampling/YCbCr -> RGB conversion
   * routines:
   *
   * Input memory buffers can be safely overread up to the next multiple of
   * ALIGN_SIZE bytes, since they are always allocated by alloc_sarray() in
   * jmemmgr.c.
   *
   * The output buffer cannot safely be written beyond output_width, since
   * output_buf points to a possibly unpadded row in the decompressed image
   * buffer allocated by the calling program.
   */
  
  /* Upsample and color convert for the case of 2:1 horizontal and 1:1 vertical.
   */
  
  void jsimd_h2v1_merged_upsample_neon(JDIMENSION output_width,
                                       JSAMPIMAGE input_buf,
                                       JDIMENSION in_row_group_ctr,
                                       JSAMPARRAY output_buf)
  {
    JSAMPROW outptr;
    /* Pointers to Y, Cb, and Cr data */
    JSAMPROW inptr0, inptr1, inptr2;
  
    const int16x4_t consts = vld1_s16(jsimd_ycc_rgb_convert_neon_consts);
    const int16x8_t neg_128 = vdupq_n_s16(-128);
  
    inptr0 = input_buf[0][in_row_group_ctr];
    inptr1 = input_buf[1][in_row_group_ctr];
    inptr2 = input_buf[2][in_row_group_ctr];
    outptr = output_buf[0];
  
    int cols_remaining = output_width;
    for (; cols_remaining >= 16; cols_remaining -= 16) {
      /* De-interleave Y component values into two separate vectors, one
       * containing the component values with even-numbered indices and one
       * containing the component values with odd-numbered indices.
       */
      uint8x8x2_t y = vld2_u8(inptr0);
      uint8x8_t cb = vld1_u8(inptr1);
      uint8x8_t cr = vld1_u8(inptr2);
      /* Subtract 128 from Cb and Cr. */
      int16x8_t cr_128 =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(neg_128), cr));
      int16x8_t cb_128 =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(neg_128), cb));
      /* Compute G-Y: - 0.34414 * (Cb - 128) - 0.71414 * (Cr - 128) */
      int32x4_t g_sub_y_l = vmull_lane_s16(vget_low_s16(cb_128), consts, 0);
      int32x4_t g_sub_y_h = vmull_lane_s16(vget_high_s16(cb_128), consts, 0);
      g_sub_y_l = vmlsl_lane_s16(g_sub_y_l, vget_low_s16(cr_128), consts, 1);
      g_sub_y_h = vmlsl_lane_s16(g_sub_y_h, vget_high_s16(cr_128), consts, 1);
      /* Descale G components: shift right 15, round, and narrow to 16-bit. */
      int16x8_t g_sub_y = vcombine_s16(vrshrn_n_s32(g_sub_y_l, 15),
                                       vrshrn_n_s32(g_sub_y_h, 15));
      /* Compute R-Y: 1.40200 * (Cr - 128) */
      int16x8_t r_sub_y = vqrdmulhq_lane_s16(vshlq_n_s16(cr_128, 1), consts, 2);
      /* Compute B-Y: 1.77200 * (Cb - 128) */
      int16x8_t b_sub_y = vqrdmulhq_lane_s16(vshlq_n_s16(cb_128, 1), consts, 3);
      /* Add the chroma-derived values (G-Y, R-Y, and B-Y) to both the "even" and
       * "odd" Y component values.  This effectively upsamples the chroma
       * components horizontally.
       */
      int16x8_t g_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y.val[0]));
      int16x8_t r_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y.val[0]));
      int16x8_t b_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y.val[0]));
      int16x8_t g_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y.val[1]));
      int16x8_t r_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y.val[1]));
      int16x8_t b_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y.val[1]));
      /* Convert each component to unsigned and narrow, clamping to [0-255].
       * Re-interleave the "even" and "odd" component values.
       */
      uint8x8x2_t r = vzip_u8(vqmovun_s16(r_even), vqmovun_s16(r_odd));
      uint8x8x2_t g = vzip_u8(vqmovun_s16(g_even), vqmovun_s16(g_odd));
      uint8x8x2_t b = vzip_u8(vqmovun_s16(b_even), vqmovun_s16(b_odd));
  
  #ifdef RGB_ALPHA
      uint8x16x4_t rgba;
      rgba.val[RGB_RED] = vcombine_u8(r.val[0], r.val[1]);
      rgba.val[RGB_GREEN] = vcombine_u8(g.val[0], g.val[1]);
      rgba.val[RGB_BLUE] = vcombine_u8(b.val[0], b.val[1]);
      /* Set alpha channel to opaque (0xFF). */
      rgba.val[RGB_ALPHA] = vdupq_n_u8(0xFF);
      /* Store RGBA pixel data to memory. */
      vst4q_u8(outptr, rgba);
  #else
      uint8x16x3_t rgb;
      rgb.val[RGB_RED] = vcombine_u8(r.val[0], r.val[1]);
      rgb.val[RGB_GREEN] = vcombine_u8(g.val[0], g.val[1]);
      rgb.val[RGB_BLUE] = vcombine_u8(b.val[0], b.val[1]);
      /* Store RGB pixel data to memory. */
      vst3q_u8(outptr, rgb);
  #endif
  
      /* Increment pointers. */
      inptr0 += 16;
      inptr1 += 8;
      inptr2 += 8;
      outptr += (RGB_PIXELSIZE * 16);
    }
  
    if (cols_remaining > 0) {
      /* De-interleave Y component values into two separate vectors, one
       * containing the component values with even-numbered indices and one
       * containing the component values with odd-numbered indices.
       */
      uint8x8x2_t y = vld2_u8(inptr0);
      uint8x8_t cb = vld1_u8(inptr1);
      uint8x8_t cr = vld1_u8(inptr2);
      /* Subtract 128 from Cb and Cr. */
      int16x8_t cr_128 =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(neg_128), cr));
      int16x8_t cb_128 =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(neg_128), cb));
      /* Compute G-Y: - 0.34414 * (Cb - 128) - 0.71414 * (Cr - 128) */
      int32x4_t g_sub_y_l = vmull_lane_s16(vget_low_s16(cb_128), consts, 0);
      int32x4_t g_sub_y_h = vmull_lane_s16(vget_high_s16(cb_128), consts, 0);
      g_sub_y_l = vmlsl_lane_s16(g_sub_y_l, vget_low_s16(cr_128), consts, 1);
      g_sub_y_h = vmlsl_lane_s16(g_sub_y_h, vget_high_s16(cr_128), consts, 1);
      /* Descale G components: shift right 15, round, and narrow to 16-bit. */
      int16x8_t g_sub_y = vcombine_s16(vrshrn_n_s32(g_sub_y_l, 15),
                                       vrshrn_n_s32(g_sub_y_h, 15));
      /* Compute R-Y: 1.40200 * (Cr - 128) */
      int16x8_t r_sub_y = vqrdmulhq_lane_s16(vshlq_n_s16(cr_128, 1), consts, 2);
      /* Compute B-Y: 1.77200 * (Cb - 128) */
      int16x8_t b_sub_y = vqrdmulhq_lane_s16(vshlq_n_s16(cb_128, 1), consts, 3);
      /* Add the chroma-derived values (G-Y, R-Y, and B-Y) to both the "even" and
       * "odd" Y component values.  This effectively upsamples the chroma
       * components horizontally.
       */
      int16x8_t g_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y.val[0]));
      int16x8_t r_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y.val[0]));
      int16x8_t b_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y.val[0]));
      int16x8_t g_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y.val[1]));
      int16x8_t r_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y.val[1]));
      int16x8_t b_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y.val[1]));
      /* Convert each component to unsigned and narrow, clamping to [0-255].
       * Re-interleave the "even" and "odd" component values.
       */
      uint8x8x2_t r = vzip_u8(vqmovun_s16(r_even), vqmovun_s16(r_odd));
      uint8x8x2_t g = vzip_u8(vqmovun_s16(g_even), vqmovun_s16(g_odd));
      uint8x8x2_t b = vzip_u8(vqmovun_s16(b_even), vqmovun_s16(b_odd));
  
  #ifdef RGB_ALPHA
      uint8x8x4_t rgba_h;
      rgba_h.val[RGB_RED] = r.val[1];
      rgba_h.val[RGB_GREEN] = g.val[1];
      rgba_h.val[RGB_BLUE] = b.val[1];
      /* Set alpha channel to opaque (0xFF). */
      rgba_h.val[RGB_ALPHA] = vdup_n_u8(0xFF);
      uint8x8x4_t rgba_l;
      rgba_l.val[RGB_RED] = r.val[0];
      rgba_l.val[RGB_GREEN] = g.val[0];
      rgba_l.val[RGB_BLUE] = b.val[0];
      /* Set alpha channel to opaque (0xFF). */
      rgba_l.val[RGB_ALPHA] = vdup_n_u8(0xFF);
      /* Store RGBA pixel data to memory. */
      switch (cols_remaining) {
      case 15:
        vst4_lane_u8(outptr + 14 * RGB_PIXELSIZE, rgba_h, 6);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 14:
        vst4_lane_u8(outptr + 13 * RGB_PIXELSIZE, rgba_h, 5);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 13:
        vst4_lane_u8(outptr + 12 * RGB_PIXELSIZE, rgba_h, 4);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 12:
        vst4_lane_u8(outptr + 11 * RGB_PIXELSIZE, rgba_h, 3);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 11:
        vst4_lane_u8(outptr + 10 * RGB_PIXELSIZE, rgba_h, 2);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 10:
        vst4_lane_u8(outptr + 9 * RGB_PIXELSIZE, rgba_h, 1);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 9:
        vst4_lane_u8(outptr + 8 * RGB_PIXELSIZE, rgba_h, 0);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 8:
        vst4_u8(outptr, rgba_l);
        break;
      case 7:
        vst4_lane_u8(outptr + 6 * RGB_PIXELSIZE, rgba_l, 6);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 6:
        vst4_lane_u8(outptr + 5 * RGB_PIXELSIZE, rgba_l, 5);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 5:
        vst4_lane_u8(outptr + 4 * RGB_PIXELSIZE, rgba_l, 4);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 4:
        vst4_lane_u8(outptr + 3 * RGB_PIXELSIZE, rgba_l, 3);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 3:
        vst4_lane_u8(outptr + 2 * RGB_PIXELSIZE, rgba_l, 2);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 2:
        vst4_lane_u8(outptr + RGB_PIXELSIZE, rgba_l, 1);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 1:
        vst4_lane_u8(outptr, rgba_l, 0);
        FALLTHROUGH               /*FALLTHROUGH*/
      default:
        break;
      }
  #else
      uint8x8x3_t rgb_h;
      rgb_h.val[RGB_RED] = r.val[1];
      rgb_h.val[RGB_GREEN] = g.val[1];
      rgb_h.val[RGB_BLUE] = b.val[1];
      uint8x8x3_t rgb_l;
      rgb_l.val[RGB_RED] = r.val[0];
      rgb_l.val[RGB_GREEN] = g.val[0];
      rgb_l.val[RGB_BLUE] = b.val[0];
      /* Store RGB pixel data to memory. */
      switch (cols_remaining) {
      case 15:
        vst3_lane_u8(outptr + 14 * RGB_PIXELSIZE, rgb_h, 6);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 14:
        vst3_lane_u8(outptr + 13 * RGB_PIXELSIZE, rgb_h, 5);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 13:
        vst3_lane_u8(outptr + 12 * RGB_PIXELSIZE, rgb_h, 4);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 12:
        vst3_lane_u8(outptr + 11 * RGB_PIXELSIZE, rgb_h, 3);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 11:
        vst3_lane_u8(outptr + 10 * RGB_PIXELSIZE, rgb_h, 2);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 10:
        vst3_lane_u8(outptr + 9 * RGB_PIXELSIZE, rgb_h, 1);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 9:
        vst3_lane_u8(outptr + 8 * RGB_PIXELSIZE, rgb_h, 0);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 8:
        vst3_u8(outptr, rgb_l);
        break;
      case 7:
        vst3_lane_u8(outptr + 6 * RGB_PIXELSIZE, rgb_l, 6);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 6:
        vst3_lane_u8(outptr + 5 * RGB_PIXELSIZE, rgb_l, 5);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 5:
        vst3_lane_u8(outptr + 4 * RGB_PIXELSIZE, rgb_l, 4);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 4:
        vst3_lane_u8(outptr + 3 * RGB_PIXELSIZE, rgb_l, 3);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 3:
        vst3_lane_u8(outptr + 2 * RGB_PIXELSIZE, rgb_l, 2);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 2:
        vst3_lane_u8(outptr + RGB_PIXELSIZE, rgb_l, 1);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 1:
        vst3_lane_u8(outptr, rgb_l, 0);
        FALLTHROUGH               /*FALLTHROUGH*/
      default:
        break;
      }
  #endif
    }
  }
  
  
  /* Upsample and color convert for the case of 2:1 horizontal and 2:1 vertical.
   *
   * See comments above for details regarding color conversion and safe memory
   * access.
   */
  
  void jsimd_h2v2_merged_upsample_neon(JDIMENSION output_width,
                                       JSAMPIMAGE input_buf,
                                       JDIMENSION in_row_group_ctr,
                                       JSAMPARRAY output_buf)
  {
    JSAMPROW outptr0, outptr1;
    /* Pointers to Y (both rows), Cb, and Cr data */
    JSAMPROW inptr0_0, inptr0_1, inptr1, inptr2;
  
    const int16x4_t consts = vld1_s16(jsimd_ycc_rgb_convert_neon_consts);
    const int16x8_t neg_128 = vdupq_n_s16(-128);
  
    inptr0_0 = input_buf[0][in_row_group_ctr * 2];
    inptr0_1 = input_buf[0][in_row_group_ctr * 2 + 1];
    inptr1 = input_buf[1][in_row_group_ctr];
    inptr2 = input_buf[2][in_row_group_ctr];
    outptr0 = output_buf[0];
    outptr1 = output_buf[1];
  
    int cols_remaining = output_width;
    for (; cols_remaining >= 16; cols_remaining -= 16) {
      /* For each row, de-interleave Y component values into two separate
       * vectors, one containing the component values with even-numbered indices
       * and one containing the component values with odd-numbered indices.
       */
      uint8x8x2_t y0 = vld2_u8(inptr0_0);
      uint8x8x2_t y1 = vld2_u8(inptr0_1);
      uint8x8_t cb = vld1_u8(inptr1);
      uint8x8_t cr = vld1_u8(inptr2);
      /* Subtract 128 from Cb and Cr. */
      int16x8_t cr_128 =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(neg_128), cr));
      int16x8_t cb_128 =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(neg_128), cb));
      /* Compute G-Y: - 0.34414 * (Cb - 128) - 0.71414 * (Cr - 128) */
      int32x4_t g_sub_y_l = vmull_lane_s16(vget_low_s16(cb_128), consts, 0);
      int32x4_t g_sub_y_h = vmull_lane_s16(vget_high_s16(cb_128), consts, 0);
      g_sub_y_l = vmlsl_lane_s16(g_sub_y_l, vget_low_s16(cr_128), consts, 1);
      g_sub_y_h = vmlsl_lane_s16(g_sub_y_h, vget_high_s16(cr_128), consts, 1);
      /* Descale G components: shift right 15, round, and narrow to 16-bit. */
      int16x8_t g_sub_y = vcombine_s16(vrshrn_n_s32(g_sub_y_l, 15),
                                       vrshrn_n_s32(g_sub_y_h, 15));
      /* Compute R-Y: 1.40200 * (Cr - 128) */
      int16x8_t r_sub_y = vqrdmulhq_lane_s16(vshlq_n_s16(cr_128, 1), consts, 2);
      /* Compute B-Y: 1.77200 * (Cb - 128) */
      int16x8_t b_sub_y = vqrdmulhq_lane_s16(vshlq_n_s16(cb_128, 1), consts, 3);
      /* For each row, add the chroma-derived values (G-Y, R-Y, and B-Y) to both
       * the "even" and "odd" Y component values.  This effectively upsamples the
       * chroma components both horizontally and vertically.
       */
      int16x8_t g0_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y0.val[0]));
      int16x8_t r0_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y0.val[0]));
      int16x8_t b0_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y0.val[0]));
      int16x8_t g0_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y0.val[1]));
      int16x8_t r0_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y0.val[1]));
      int16x8_t b0_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y0.val[1]));
      int16x8_t g1_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y1.val[0]));
      int16x8_t r1_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y1.val[0]));
      int16x8_t b1_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y1.val[0]));
      int16x8_t g1_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y1.val[1]));
      int16x8_t r1_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y1.val[1]));
      int16x8_t b1_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y1.val[1]));
      /* Convert each component to unsigned and narrow, clamping to [0-255].
       * Re-interleave the "even" and "odd" component values.
       */
      uint8x8x2_t r0 = vzip_u8(vqmovun_s16(r0_even), vqmovun_s16(r0_odd));
      uint8x8x2_t r1 = vzip_u8(vqmovun_s16(r1_even), vqmovun_s16(r1_odd));
      uint8x8x2_t g0 = vzip_u8(vqmovun_s16(g0_even), vqmovun_s16(g0_odd));
      uint8x8x2_t g1 = vzip_u8(vqmovun_s16(g1_even), vqmovun_s16(g1_odd));
      uint8x8x2_t b0 = vzip_u8(vqmovun_s16(b0_even), vqmovun_s16(b0_odd));
      uint8x8x2_t b1 = vzip_u8(vqmovun_s16(b1_even), vqmovun_s16(b1_odd));
  
  #ifdef RGB_ALPHA
      uint8x16x4_t rgba0, rgba1;
      rgba0.val[RGB_RED] = vcombine_u8(r0.val[0], r0.val[1]);
      rgba1.val[RGB_RED] = vcombine_u8(r1.val[0], r1.val[1]);
      rgba0.val[RGB_GREEN] = vcombine_u8(g0.val[0], g0.val[1]);
      rgba1.val[RGB_GREEN] = vcombine_u8(g1.val[0], g1.val[1]);
      rgba0.val[RGB_BLUE] = vcombine_u8(b0.val[0], b0.val[1]);
      rgba1.val[RGB_BLUE] = vcombine_u8(b1.val[0], b1.val[1]);
      /* Set alpha channel to opaque (0xFF). */
      rgba0.val[RGB_ALPHA] = vdupq_n_u8(0xFF);
      rgba1.val[RGB_ALPHA] = vdupq_n_u8(0xFF);
      /* Store RGBA pixel data to memory. */
      vst4q_u8(outptr0, rgba0);
      vst4q_u8(outptr1, rgba1);
  #else
      uint8x16x3_t rgb0, rgb1;
      rgb0.val[RGB_RED] = vcombine_u8(r0.val[0], r0.val[1]);
      rgb1.val[RGB_RED] = vcombine_u8(r1.val[0], r1.val[1]);
      rgb0.val[RGB_GREEN] = vcombine_u8(g0.val[0], g0.val[1]);
      rgb1.val[RGB_GREEN] = vcombine_u8(g1.val[0], g1.val[1]);
      rgb0.val[RGB_BLUE] = vcombine_u8(b0.val[0], b0.val[1]);
      rgb1.val[RGB_BLUE] = vcombine_u8(b1.val[0], b1.val[1]);
      /* Store RGB pixel data to memory. */
      vst3q_u8(outptr0, rgb0);
      vst3q_u8(outptr1, rgb1);
  #endif
  
      /* Increment pointers. */
      inptr0_0 += 16;
      inptr0_1 += 16;
      inptr1 += 8;
      inptr2 += 8;
      outptr0 += (RGB_PIXELSIZE * 16);
      outptr1 += (RGB_PIXELSIZE * 16);
    }
  
    if (cols_remaining > 0) {
      /* For each row, de-interleave Y component values into two separate
       * vectors, one containing the component values with even-numbered indices
       * and one containing the component values with odd-numbered indices.
       */
      uint8x8x2_t y0 = vld2_u8(inptr0_0);
      uint8x8x2_t y1 = vld2_u8(inptr0_1);
      uint8x8_t cb = vld1_u8(inptr1);
      uint8x8_t cr = vld1_u8(inptr2);
      /* Subtract 128 from Cb and Cr. */
      int16x8_t cr_128 =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(neg_128), cr));
      int16x8_t cb_128 =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(neg_128), cb));
      /* Compute G-Y: - 0.34414 * (Cb - 128) - 0.71414 * (Cr - 128) */
      int32x4_t g_sub_y_l = vmull_lane_s16(vget_low_s16(cb_128), consts, 0);
      int32x4_t g_sub_y_h = vmull_lane_s16(vget_high_s16(cb_128), consts, 0);
      g_sub_y_l = vmlsl_lane_s16(g_sub_y_l, vget_low_s16(cr_128), consts, 1);
      g_sub_y_h = vmlsl_lane_s16(g_sub_y_h, vget_high_s16(cr_128), consts, 1);
      /* Descale G components: shift right 15, round, and narrow to 16-bit. */
      int16x8_t g_sub_y = vcombine_s16(vrshrn_n_s32(g_sub_y_l, 15),
                                       vrshrn_n_s32(g_sub_y_h, 15));
      /* Compute R-Y: 1.40200 * (Cr - 128) */
      int16x8_t r_sub_y = vqrdmulhq_lane_s16(vshlq_n_s16(cr_128, 1), consts, 2);
      /* Compute B-Y: 1.77200 * (Cb - 128) */
      int16x8_t b_sub_y = vqrdmulhq_lane_s16(vshlq_n_s16(cb_128, 1), consts, 3);
      /* For each row, add the chroma-derived values (G-Y, R-Y, and B-Y) to both
       * the "even" and "odd" Y component values.  This effectively upsamples the
       * chroma components both horizontally and vertically.
       */
      int16x8_t g0_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y0.val[0]));
      int16x8_t r0_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y0.val[0]));
      int16x8_t b0_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y0.val[0]));
      int16x8_t g0_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y0.val[1]));
      int16x8_t r0_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y0.val[1]));
      int16x8_t b0_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y0.val[1]));
      int16x8_t g1_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y1.val[0]));
      int16x8_t r1_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y1.val[0]));
      int16x8_t b1_even =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y1.val[0]));
      int16x8_t g1_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(g_sub_y),
                                       y1.val[1]));
      int16x8_t r1_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(r_sub_y),
                                       y1.val[1]));
      int16x8_t b1_odd =
        vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(b_sub_y),
                                       y1.val[1]));
      /* Convert each component to unsigned and narrow, clamping to [0-255].
       * Re-interleave the "even" and "odd" component values.
       */
      uint8x8x2_t r0 = vzip_u8(vqmovun_s16(r0_even), vqmovun_s16(r0_odd));
      uint8x8x2_t r1 = vzip_u8(vqmovun_s16(r1_even), vqmovun_s16(r1_odd));
      uint8x8x2_t g0 = vzip_u8(vqmovun_s16(g0_even), vqmovun_s16(g0_odd));
      uint8x8x2_t g1 = vzip_u8(vqmovun_s16(g1_even), vqmovun_s16(g1_odd));
      uint8x8x2_t b0 = vzip_u8(vqmovun_s16(b0_even), vqmovun_s16(b0_odd));
      uint8x8x2_t b1 = vzip_u8(vqmovun_s16(b1_even), vqmovun_s16(b1_odd));
  
  #ifdef RGB_ALPHA
      uint8x8x4_t rgba0_h, rgba1_h;
      rgba0_h.val[RGB_RED] = r0.val[1];
      rgba1_h.val[RGB_RED] = r1.val[1];
      rgba0_h.val[RGB_GREEN] = g0.val[1];
      rgba1_h.val[RGB_GREEN] = g1.val[1];
      rgba0_h.val[RGB_BLUE] = b0.val[1];
      rgba1_h.val[RGB_BLUE] = b1.val[1];
      /* Set alpha channel to opaque (0xFF). */
      rgba0_h.val[RGB_ALPHA] = vdup_n_u8(0xFF);
      rgba1_h.val[RGB_ALPHA] = vdup_n_u8(0xFF);
  
      uint8x8x4_t rgba0_l, rgba1_l;
      rgba0_l.val[RGB_RED] = r0.val[0];
      rgba1_l.val[RGB_RED] = r1.val[0];
      rgba0_l.val[RGB_GREEN] = g0.val[0];
      rgba1_l.val[RGB_GREEN] = g1.val[0];
      rgba0_l.val[RGB_BLUE] = b0.val[0];
      rgba1_l.val[RGB_BLUE] = b1.val[0];
      /* Set alpha channel to opaque (0xFF). */
      rgba0_l.val[RGB_ALPHA] = vdup_n_u8(0xFF);
      rgba1_l.val[RGB_ALPHA] = vdup_n_u8(0xFF);
      /* Store RGBA pixel data to memory. */
      switch (cols_remaining) {
      case 15:
        vst4_lane_u8(outptr0 + 14 * RGB_PIXELSIZE, rgba0_h, 6);
        vst4_lane_u8(outptr1 + 14 * RGB_PIXELSIZE, rgba1_h, 6);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 14:
        vst4_lane_u8(outptr0 + 13 * RGB_PIXELSIZE, rgba0_h, 5);
        vst4_lane_u8(outptr1 + 13 * RGB_PIXELSIZE, rgba1_h, 5);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 13:
        vst4_lane_u8(outptr0 + 12 * RGB_PIXELSIZE, rgba0_h, 4);
        vst4_lane_u8(outptr1 + 12 * RGB_PIXELSIZE, rgba1_h, 4);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 12:
        vst4_lane_u8(outptr0 + 11 * RGB_PIXELSIZE, rgba0_h, 3);
        vst4_lane_u8(outptr1 + 11 * RGB_PIXELSIZE, rgba1_h, 3);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 11:
        vst4_lane_u8(outptr0 + 10 * RGB_PIXELSIZE, rgba0_h, 2);
        vst4_lane_u8(outptr1 + 10 * RGB_PIXELSIZE, rgba1_h, 2);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 10:
        vst4_lane_u8(outptr0 + 9 * RGB_PIXELSIZE, rgba0_h, 1);
        vst4_lane_u8(outptr1 + 9 * RGB_PIXELSIZE, rgba1_h, 1);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 9:
        vst4_lane_u8(outptr0 + 8 * RGB_PIXELSIZE, rgba0_h, 0);
        vst4_lane_u8(outptr1 + 8 * RGB_PIXELSIZE, rgba1_h, 0);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 8:
        vst4_u8(outptr0, rgba0_l);
        vst4_u8(outptr1, rgba1_l);
        break;
      case 7:
        vst4_lane_u8(outptr0 + 6 * RGB_PIXELSIZE, rgba0_l, 6);
        vst4_lane_u8(outptr1 + 6 * RGB_PIXELSIZE, rgba1_l, 6);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 6:
        vst4_lane_u8(outptr0 + 5 * RGB_PIXELSIZE, rgba0_l, 5);
        vst4_lane_u8(outptr1 + 5 * RGB_PIXELSIZE, rgba1_l, 5);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 5:
        vst4_lane_u8(outptr0 + 4 * RGB_PIXELSIZE, rgba0_l, 4);
        vst4_lane_u8(outptr1 + 4 * RGB_PIXELSIZE, rgba1_l, 4);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 4:
        vst4_lane_u8(outptr0 + 3 * RGB_PIXELSIZE, rgba0_l, 3);
        vst4_lane_u8(outptr1 + 3 * RGB_PIXELSIZE, rgba1_l, 3);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 3:
        vst4_lane_u8(outptr0 + 2 * RGB_PIXELSIZE, rgba0_l, 2);
        vst4_lane_u8(outptr1 + 2 * RGB_PIXELSIZE, rgba1_l, 2);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 2:
        vst4_lane_u8(outptr0 + 1 * RGB_PIXELSIZE, rgba0_l, 1);
        vst4_lane_u8(outptr1 + 1 * RGB_PIXELSIZE, rgba1_l, 1);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 1:
        vst4_lane_u8(outptr0, rgba0_l, 0);
        vst4_lane_u8(outptr1, rgba1_l, 0);
        FALLTHROUGH               /*FALLTHROUGH*/
      default:
        break;
      }
  #else
      uint8x8x3_t rgb0_h, rgb1_h;
      rgb0_h.val[RGB_RED] = r0.val[1];
      rgb1_h.val[RGB_RED] = r1.val[1];
      rgb0_h.val[RGB_GREEN] = g0.val[1];
      rgb1_h.val[RGB_GREEN] = g1.val[1];
      rgb0_h.val[RGB_BLUE] = b0.val[1];
      rgb1_h.val[RGB_BLUE] = b1.val[1];
  
      uint8x8x3_t rgb0_l, rgb1_l;
      rgb0_l.val[RGB_RED] = r0.val[0];
      rgb1_l.val[RGB_RED] = r1.val[0];
      rgb0_l.val[RGB_GREEN] = g0.val[0];
      rgb1_l.val[RGB_GREEN] = g1.val[0];
      rgb0_l.val[RGB_BLUE] = b0.val[0];
      rgb1_l.val[RGB_BLUE] = b1.val[0];
      /* Store RGB pixel data to memory. */
      switch (cols_remaining) {
      case 15:
        vst3_lane_u8(outptr0 + 14 * RGB_PIXELSIZE, rgb0_h, 6);
        vst3_lane_u8(outptr1 + 14 * RGB_PIXELSIZE, rgb1_h, 6);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 14:
        vst3_lane_u8(outptr0 + 13 * RGB_PIXELSIZE, rgb0_h, 5);
        vst3_lane_u8(outptr1 + 13 * RGB_PIXELSIZE, rgb1_h, 5);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 13:
        vst3_lane_u8(outptr0 + 12 * RGB_PIXELSIZE, rgb0_h, 4);
        vst3_lane_u8(outptr1 + 12 * RGB_PIXELSIZE, rgb1_h, 4);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 12:
        vst3_lane_u8(outptr0 + 11 * RGB_PIXELSIZE, rgb0_h, 3);
        vst3_lane_u8(outptr1 + 11 * RGB_PIXELSIZE, rgb1_h, 3);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 11:
        vst3_lane_u8(outptr0 + 10 * RGB_PIXELSIZE, rgb0_h, 2);
        vst3_lane_u8(outptr1 + 10 * RGB_PIXELSIZE, rgb1_h, 2);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 10:
        vst3_lane_u8(outptr0 + 9 * RGB_PIXELSIZE, rgb0_h, 1);
        vst3_lane_u8(outptr1 + 9 * RGB_PIXELSIZE, rgb1_h, 1);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 9:
        vst3_lane_u8(outptr0 + 8 * RGB_PIXELSIZE, rgb0_h, 0);
        vst3_lane_u8(outptr1 + 8 * RGB_PIXELSIZE, rgb1_h, 0);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 8:
        vst3_u8(outptr0, rgb0_l);
        vst3_u8(outptr1, rgb1_l);
        break;
      case 7:
        vst3_lane_u8(outptr0 + 6 * RGB_PIXELSIZE, rgb0_l, 6);
        vst3_lane_u8(outptr1 + 6 * RGB_PIXELSIZE, rgb1_l, 6);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 6:
        vst3_lane_u8(outptr0 + 5 * RGB_PIXELSIZE, rgb0_l, 5);
        vst3_lane_u8(outptr1 + 5 * RGB_PIXELSIZE, rgb1_l, 5);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 5:
        vst3_lane_u8(outptr0 + 4 * RGB_PIXELSIZE, rgb0_l, 4);
        vst3_lane_u8(outptr1 + 4 * RGB_PIXELSIZE, rgb1_l, 4);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 4:
        vst3_lane_u8(outptr0 + 3 * RGB_PIXELSIZE, rgb0_l, 3);
        vst3_lane_u8(outptr1 + 3 * RGB_PIXELSIZE, rgb1_l, 3);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 3:
        vst3_lane_u8(outptr0 + 2 * RGB_PIXELSIZE, rgb0_l, 2);
        vst3_lane_u8(outptr1 + 2 * RGB_PIXELSIZE, rgb1_l, 2);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 2:
        vst3_lane_u8(outptr0 + 1 * RGB_PIXELSIZE, rgb0_l, 1);
        vst3_lane_u8(outptr1 + 1 * RGB_PIXELSIZE, rgb1_l, 1);
        FALLTHROUGH               /*FALLTHROUGH*/
      case 1:
        vst3_lane_u8(outptr0, rgb0_l, 0);
        vst3_lane_u8(outptr1, rgb1_l, 0);
        FALLTHROUGH               /*FALLTHROUGH*/
      default:
        break;
      }
  #endif
    }
  }
  

Download