https://bugs.chromium.org/p/aomedia/issues/detail?id=3487
https://aomedia.googlesource.com/aom/+/8b65d76f2f95b1bfdc82b8f135246cb367d444e6%5E%21/

From 8b65d76f2f95b1bfdc82b8f135246cb367d444e6 Mon Sep 17 00:00:00 2001
From: Wan-Teh Chang <wtc@google.com>
Date: Tue, 12 Sep 2023 16:38:47 -0700
Subject: [PATCH] Fix byte order issue in write_enc_data_to_out_buf

Fix the byte order issue for big-endian systems introduced in
https://aomedia-review.googlesource.com/c/aom/+/167581.

Use the existing HToBE64 macro defined in aom_util/endian_inl.h and
delete the new get_byteswap64 function from aom_ports/bitops.h.

Based on the patch in bug aomedia:3487.

Bug: aomedia:3487
Change-Id: I56183e062a9f2b18c4c6158fd8e47c5062de85ee
---
 aom_dsp/entenc.h   |  7 ++++---
 aom_ports/bitops.h | 35 +----------------------------------
 2 files changed, 5 insertions(+), 37 deletions(-)

diff --git a/aom_dsp/entenc.h b/aom_dsp/entenc.h
index 467e47bf56..d26f027ed0 100644
--- a/aom_dsp/entenc.h
+++ b/aom_dsp/entenc.h
@@ -13,7 +13,7 @@
 #define AOM_AOM_DSP_ENTENC_H_
 #include <stddef.h>
 #include "aom_dsp/entcode.h"
-#include "aom_ports/bitops.h"
+#include "aom_util/endian_inl.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -87,13 +87,14 @@ static AOM_INLINE void propagate_carry_bwd(unsigned char *buf, uint32_t offs) {
   } while (carry);
 }
 
-// Reverse byte order and write data to buffer adding the carry-bit
+// Convert to big-endian byte order and write data to buffer adding the
+// carry-bit
 static AOM_INLINE void write_enc_data_to_out_buf(unsigned char *out,
                                                  uint32_t offs, uint64_t output,
                                                  uint64_t carry,
                                                  uint32_t *enc_offs,
                                                  uint8_t num_bytes_ready) {
-  const uint64_t reg = get_byteswap64(output) >> ((8 - num_bytes_ready) << 3);
+  const uint64_t reg = HToBE64(output << ((8 - num_bytes_ready) << 3));
   memcpy(&out[offs], &reg, 8);
   // Propagate carry backwards if exists
   if (carry) {
diff --git a/aom_ports/bitops.h b/aom_ports/bitops.h
index 3c5b992bde..7f4c165f5a 100644
--- a/aom_ports/bitops.h
+++ b/aom_ports/bitops.h
@@ -13,7 +13,6 @@
 #define AOM_AOM_PORTS_BITOPS_H_
 
 #include <assert.h>
-#include <stdint.h>
 
 #include "aom_ports/msvc.h"
 #include "config/aom_config.h"
@@ -34,12 +33,8 @@ extern "C" {
 // These versions of get_msb() are only valid when n != 0 because all
 // of the optimized versions are undefined when n == 0:
 
-// get_byteswap64:
-// Returns the number (uint64_t) with byte-positions reversed
-// e.g. input 0x123456789ABCDEF0 returns 0xF0DEBC9A78563412
-
 // GCC compiler: https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
-// MSVC: https://learn.microsoft.com/en-us/cpp/c-runtime-library/
+// MSVC: https://learn.microsoft.com/en-us/cpp/intrinsics/compiler-intrinsics
 
 // use GNU builtins where available.
 #if defined(__GNUC__) && \
@@ -48,10 +43,6 @@ static INLINE int get_msb(unsigned int n) {
   assert(n != 0);
   return 31 ^ __builtin_clz(n);
 }
-
-static INLINE uint64_t get_byteswap64(uint64_t num) {
-  return __builtin_bswap64(num);
-}
 #elif defined(USE_MSC_INTRINSICS)
 #pragma intrinsic(_BitScanReverse)
 
@@ -61,10 +52,6 @@ static INLINE int get_msb(unsigned int n) {
   _BitScanReverse(&first_set_bit, n);
   return first_set_bit;
 }
-
-static INLINE uint64_t get_byteswap64(uint64_t num) {
-  return _byteswap_uint64(num);
-}
 #undef USE_MSC_INTRINSICS
 #else
 static INLINE int get_msb(unsigned int n) {
@@ -82,26 +69,6 @@ static INLINE int get_msb(unsigned int n) {
   }
   return log;
 }
-
-static INLINE uint64_t get_byteswap64(uint64_t num) {
-  uint64_t out = 0x00;
-  uint64_t mask = 0xFF00000000000000;
-  int bit_shift = 56;  // 7 bytes
-  // 4 ms bytes
-  do {
-    out |= (num & mask) >> bit_shift;
-    mask >>= 8;
-    bit_shift -= 16;
-  } while (bit_shift >= 0);
-  // 4 ls bytes
-  bit_shift = 8;  // 1 byte
-  do {
-    out |= (num & mask) << bit_shift;
-    mask >>= 8;
-    bit_shift += 16;
-  } while (bit_shift <= 56);
-  return out;
-}
 #endif
 
 #ifdef __cplusplus
-- 
2.42.0