/* * Copyright (c) Facebook, Inc. and its affiliates. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include #include #include #include #include namespace folly { template struct UnalignedNoASan : public Unaligned {}; // As a general rule, bit operations work on unsigned values only; // right-shift is arithmetic for signed values, and that can lead to // unpleasant bugs. namespace detail { /** * Helper class to make Bits (below) work with both aligned values * (T, where T is an unsigned integral type) or unaligned values * (Unaligned, where T is an unsigned integral type) */ template struct BitsTraits; // Partial specialization for Unaligned, where T is unsigned integral // loadRMW is the same as load, but it indicates that it loads for a // read-modify-write operation (we write back the bits we won't change); // silence the GCC warning in that case. template struct BitsTraits< Unaligned, typename std::enable_if<(std::is_integral::value)>::type> { typedef T UnderlyingType; static T load(const Unaligned& x) { return x.value; } static void store(Unaligned& x, T v) { x.value = v; } static T loadRMW(const Unaligned& x) { FOLLY_PUSH_WARNING FOLLY_GNU_DISABLE_WARNING("-Wuninitialized") FOLLY_GCC_DISABLE_WARNING("-Wmaybe-uninitialized") return x.value; FOLLY_POP_WARNING } }; // Special version that allows one to disable address sanitizer on demand. template struct BitsTraits< UnalignedNoASan, typename std::enable_if<(std::is_integral::value)>::type> { typedef T UnderlyingType; static T FOLLY_DISABLE_ADDRESS_SANITIZER load(const UnalignedNoASan& x) { return x.value; } static void FOLLY_DISABLE_ADDRESS_SANITIZER store(UnalignedNoASan& x, T v) { x.value = v; } static T FOLLY_DISABLE_ADDRESS_SANITIZER loadRMW(const UnalignedNoASan& x) { FOLLY_PUSH_WARNING FOLLY_GNU_DISABLE_WARNING("-Wuninitialized") FOLLY_GCC_DISABLE_WARNING("-Wmaybe-uninitialized") return x.value; FOLLY_POP_WARNING } }; // Partial specialization for T, where T is unsigned integral template struct BitsTraits< T, typename std::enable_if<(std::is_integral::value)>::type> { typedef T UnderlyingType; static T load(const T& x) { return x; } static void store(T& x, T v) { x = v; } static T loadRMW(const T& x) { FOLLY_PUSH_WARNING FOLLY_GNU_DISABLE_WARNING("-Wuninitialized") FOLLY_GCC_DISABLE_WARNING("-Wmaybe-uninitialized") return x; FOLLY_POP_WARNING } }; } // namespace detail /** * Wrapper class with static methods for various bit-level operations, * treating an array of T as an array of bits (in little-endian order). * (T is either an unsigned integral type or Unaligned, where X is * an unsigned integral type) */ template > struct Bits { typedef typename Traits::UnderlyingType UnderlyingType; typedef T type; static_assert(sizeof(T) == sizeof(UnderlyingType), "Size mismatch"); /** * Number of bits in a block. */ static constexpr size_t bitsPerBlock = std::numeric_limits< typename std::make_unsigned::type>::digits; /** * Byte index of the given bit. */ static constexpr size_t blockIndex(size_t bit) { return bit / bitsPerBlock; } /** * Offset in block of the given bit. */ static constexpr size_t bitOffset(size_t bit) { return bit % bitsPerBlock; } /** * Number of blocks used by the given number of bits. */ static constexpr size_t blockCount(size_t nbits) { return nbits / bitsPerBlock + (nbits % bitsPerBlock != 0); } /** * Set the given bit. */ static void set(T* p, size_t bit); /** * Clear the given bit. */ static void clear(T* p, size_t bit); /** * Test the given bit. */ static bool test(const T* p, size_t bit); /** * Set count contiguous bits starting at bitStart to the values * from the least significant count bits of value; little endian. * (value & 1 becomes the bit at bitStart, etc) * Precondition: count <= sizeof(T) * 8 * Precondition: value can fit in 'count' bits */ static void set(T* p, size_t bitStart, size_t count, UnderlyingType value); /** * Get count contiguous bits starting at bitStart. * Precondition: count <= sizeof(T) * 8 */ static UnderlyingType get(const T* p, size_t bitStart, size_t count); /** * Count the number of bits set in a range of blocks. */ static size_t count(const T* begin, const T* end); private: // Same as set, assumes all bits are in the same block. // (bitStart < sizeof(T) * 8, bitStart + count <= sizeof(T) * 8) static void innerSet(T* p, size_t bitStart, size_t count, UnderlyingType value); // Same as get, assumes all bits are in the same block. // (bitStart < sizeof(T) * 8, bitStart + count <= sizeof(T) * 8) static UnderlyingType innerGet(const T* p, size_t bitStart, size_t count); static constexpr UnderlyingType zero = UnderlyingType(0); static constexpr UnderlyingType one = UnderlyingType(1); using UnsignedType = typename std::make_unsigned::type; static constexpr UnderlyingType ones(size_t count) { return (count < bitsPerBlock) ? static_cast((UnsignedType{1} << count) - 1) : ~zero; } }; template inline void Bits::set(T* p, size_t bit) { T& block = p[blockIndex(bit)]; Traits::store(block, Traits::loadRMW(block) | (one << bitOffset(bit))); } template inline void Bits::clear(T* p, size_t bit) { T& block = p[blockIndex(bit)]; Traits::store(block, Traits::loadRMW(block) & ~(one << bitOffset(bit))); } template inline void Bits::set( T* p, size_t bitStart, size_t count, UnderlyingType value) { DCHECK_LE(count, sizeof(UnderlyingType) * 8); size_t cut = bitsPerBlock - count; if (cut != 8 * sizeof(UnderlyingType)) { using U = typename std::make_unsigned::type; DCHECK_EQ(value, UnderlyingType(U(value) << cut) >> cut); } size_t idx = blockIndex(bitStart); size_t offset = bitOffset(bitStart); if (std::is_signed::value) { value &= ones(count); } if (offset + count <= bitsPerBlock) { innerSet(p + idx, offset, count, value); } else { size_t countInThisBlock = bitsPerBlock - offset; size_t countInNextBlock = count - countInThisBlock; UnderlyingType thisBlock = UnderlyingType(value & ones(countInThisBlock)); UnderlyingType nextBlock = UnderlyingType(value >> countInThisBlock); if (std::is_signed::value) { nextBlock &= ones(countInNextBlock); } innerSet(p + idx, offset, countInThisBlock, thisBlock); innerSet(p + idx + 1, 0, countInNextBlock, nextBlock); } } template inline void Bits::innerSet( T* p, size_t offset, size_t count, UnderlyingType value) { // Mask out bits and set new value UnderlyingType v = Traits::loadRMW(*p); v &= ~(ones(count) << offset); v |= (value << offset); Traits::store(*p, v); } template inline bool Bits::test(const T* p, size_t bit) { return Traits::load(p[blockIndex(bit)]) & (one << bitOffset(bit)); } template inline auto Bits::get(const T* p, size_t bitStart, size_t count) -> UnderlyingType { if (count == 0) { return UnderlyingType{}; } DCHECK_LE(count, sizeof(UnderlyingType) * 8); size_t idx = blockIndex(bitStart); size_t offset = bitOffset(bitStart); UnderlyingType ret; if (offset + count <= bitsPerBlock) { ret = innerGet(p + idx, offset, count); } else { size_t countInThisBlock = bitsPerBlock - offset; size_t countInNextBlock = count - countInThisBlock; UnderlyingType thisBlockValue = innerGet(p + idx, offset, countInThisBlock); UnderlyingType nextBlockValue = innerGet(p + idx + 1, 0, countInNextBlock); ret = (nextBlockValue << countInThisBlock) | thisBlockValue; } if (std::is_signed::value) { size_t emptyBits = bitsPerBlock - count; ret <<= emptyBits; ret >>= emptyBits; } return ret; } template inline auto Bits::innerGet(const T* p, size_t offset, size_t count) -> UnderlyingType { return (Traits::load(*p) >> offset) & ones(count); } template inline size_t Bits::count(const T* begin, const T* end) { size_t n = 0; for (; begin != end; ++begin) { n += popcount(Traits::load(*begin)); } return n; } } // namespace folly