Skip to content

Commit

Permalink
Duplicate fix for greg7mdp/parallel-hashmap#254
Browse files Browse the repository at this point in the history
  • Loading branch information
@greg7mdp
greg7mdp committed Oct 28, 2024
1 parent 7301b8e commit 654bc27
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 25 deletions.
73 changes: 49 additions & 24 deletions include/gtl/phmap.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -819,11 +819,15 @@ static_assert(kDeleted == -2,
// A single block of empty control bytes for tables without any slots allocated.
// This enables removing a branch in the hot path of find().
// --------------------------------------------------------------------------
template <class std_alloc_t>
inline ctrl_t* EmptyGroup() {
alignas(16) static constexpr ctrl_t empty_group[] = { kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
kEmpty, kEmpty, kEmpty, kEmpty };
return const_cast<ctrl_t*>(empty_group);
if constexpr (std_alloc_t::value) {
alignas(16) static constexpr ctrl_t empty_group[] = { kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
kEmpty, kEmpty, kEmpty, kEmpty };
return const_cast<ctrl_t*>(empty_group);
}
return nullptr;
}

// --------------------------------------------------------------------------
Expand Down Expand Up @@ -1529,6 +1533,8 @@ class raw_hash_set {
template<class K>
using key_arg = typename KeyArgImpl::template type<K, key_type>;

using std_alloc_t = std::is_same<typename std::decay<Alloc>::type, gtl::priv::Allocator<value_type>>;

private:
// Give an early error when key_type is not hashable/eq.
auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k));
Expand Down Expand Up @@ -1635,6 +1641,11 @@ class raw_hash_set {
, slot_(slot) {}

void skip_empty_or_deleted() {
if constexpr (!std_alloc_t::value) {
// ctrl_ could be nullptr
if (!ctrl_)
return;
}
while (IsEmptyOrDeleted(*ctrl_)) {
// ctrl is not necessarily aligned to Group::kWidth. It is also likely
// to read past the space for ctrl bytes and into slots. This is ok
Expand Down Expand Up @@ -1701,7 +1712,7 @@ class raw_hash_set {
const hasher& hashfn = hasher(),
const key_equal& eq = key_equal(),
const allocator_type& alloc = allocator_type())
: ctrl_(EmptyGroup())
: ctrl_(EmptyGroup<std_alloc_t>())
, settings_(0, hashfn, eq, alloc) {
if (bucket_cnt) {
size_t new_capacity = NormalizeCapacity(bucket_cnt);
Expand Down Expand Up @@ -1824,7 +1835,7 @@ class raw_hash_set {
raw_hash_set(raw_hash_set&& that) noexcept(std::is_nothrow_copy_constructible_v<hasher> &&
std::is_nothrow_copy_constructible_v<key_equal> &&
std::is_nothrow_copy_constructible_v<allocator_type>)
: ctrl_(std::exchange(that.ctrl_, EmptyGroup()))
: ctrl_(std::exchange(that.ctrl_, EmptyGroup<std_alloc_t>()))
, slots_(std::exchange(that.slots_, nullptr))
, size_(std::exchange(that.size_, 0))
, capacity_(std::exchange(that.capacity_, 0))
Expand All @@ -1839,7 +1850,7 @@ class raw_hash_set {
}

raw_hash_set(raw_hash_set&& that, const allocator_type& a)
: ctrl_(EmptyGroup())
: ctrl_(EmptyGroup<std_alloc_t>())
, slots_(nullptr)
, size_(0)
, capacity_(0)
Expand Down Expand Up @@ -2366,22 +2377,13 @@ class raw_hash_set {
// NOTE: This is a very low level operation and should not be used without
// specific benchmarks indicating its importance.
// -----------------------------------------------------------------------
void prefetch_hash(size_t hashval) const {
(void)hashval;
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
auto seq = probe(hashval);
_mm_prefetch((const char*)(ctrl_ + seq.offset()), _MM_HINT_NTA);
_mm_prefetch((const char*)(slots_ + seq.offset()), _MM_HINT_NTA);
#elif defined(__GNUC__)
auto seq = probe(hashval);
__builtin_prefetch(static_cast<const void*>(ctrl_ + seq.offset()));
__builtin_prefetch(static_cast<const void*>(slots_ + seq.offset()));
#endif // __GNUC__
void prefetch_hash(size_t) const {
}

template<class K = key_type>
void prefetch(const key_arg<K>& key) const {
prefetch_hash(this->hash(key));
if constexpr (std_alloc_t::value)
prefetch_hash(this->hash(key));
}

// The API of find() has two extensions.
Expand Down Expand Up @@ -2490,6 +2492,11 @@ class raw_hash_set {

template<class K = key_type>
bool find_impl(const key_arg<K>& key, size_t hashval, size_t& offset) {
if constexpr (!std_alloc_t::value) {
// ctrl_ could be nullptr
if (!ctrl_)
return false;
}
auto seq = probe(hashval);
while (true) {
Group g{ ctrl_ + seq.offset() };
Expand Down Expand Up @@ -2652,7 +2659,7 @@ class raw_hash_set {
// Unpoison before returning the memory to the allocator.
SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_);
Deallocate<Layout::Alignment()>(&alloc_ref(), ctrl_, layout.AllocSize());
ctrl_ = EmptyGroup();
ctrl_ = EmptyGroup<std_alloc_t>();
slots_ = nullptr;
size_ = 0;
capacity_ = 0;
Expand Down Expand Up @@ -2761,6 +2768,11 @@ class raw_hash_set {
}

bool has_element(const value_type& elem, size_t hashval) const {
if constexpr (!std_alloc_t::value) {
// ctrl_ could be nullptr
if (!ctrl_)
return false;
}
auto seq = probe(hashval);
while (true) {
Group g{ ctrl_ + seq.offset() };
Expand Down Expand Up @@ -2823,6 +2835,11 @@ class raw_hash_set {
protected:
template<class K>
size_t _find_key(const K& key, size_t hashval) {
if constexpr (!std_alloc_t::value) {
// ctrl_ could be nullptr
if (!ctrl_)
return (size_t)-1;
}
auto seq = probe(hashval);
while (true) {
Group g{ ctrl_ + seq.offset() };
Expand All @@ -2847,6 +2864,11 @@ class raw_hash_set {
}

size_t prepare_insert(size_t hashval) GTL_ATTRIBUTE_NOINLINE {
if constexpr (!std_alloc_t::value) {
// ctrl_ could be nullptr
if (!ctrl_)
rehash_and_grow_if_necessary();
}
auto target = find_first_non_full(hashval);
if (GTL_PREDICT_FALSE(growth_left() == 0 && !IsDeleted(ctrl_[target.offset]))) {
rehash_and_grow_if_necessary();
Expand Down Expand Up @@ -2966,10 +2988,10 @@ class raw_hash_set {
// - ctrl/slots can be derived from each other
// - size can be moved into the slot array
// -------------------------------------------------------------------------
ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t]
slot_type* slots_ = nullptr; // [capacity * slot_type]
size_t size_ = 0; // number of full slots
size_t capacity_ = 0; // total number of slots
ctrl_t* ctrl_ = EmptyGroup<std_alloc_t>(); // [(capacity + 1) * ctrl_t]
slot_type* slots_ = nullptr; // [capacity * slot_type]
size_t size_ = 0; // number of full slots
size_t capacity_ = 0; // total number of slots
std::tuple<size_t /* growth_left */, hasher, key_equal, allocator_type> settings_{ 0,
hasher{},
key_equal{},
Expand Down Expand Up @@ -4059,6 +4081,7 @@ class parallel_hash_set {
// Do not use erase APIs taking iterators when accessing the map concurrently
// --------------------------------------------------------------------
iterator erase(iterator it) {
assert(it != end());
_erase(it++);
return it;
}
Expand Down Expand Up @@ -4991,6 +5014,8 @@ struct HashtableDebugAccess<Set, std::enable_if_t<has_member_type_raw_hash_set<S
using Slot = typename Traits::slot_type;

static size_t GetNumProbes(const Set& set, const typename Set::key_type& key) {
if (!set.ctrl_)
return 0;
size_t num_probes = 0;
size_t hashval = set.hash(key);
auto seq = set.probe(hashval);
Expand Down
2 changes: 1 addition & 1 deletion tests/phmap/raw_hash_set_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ TEST(BitMask, LeadingTrailing) {

TEST(Group, EmptyGroup) {
for (h2_t h = 0; h != 128; ++h)
EXPECT_FALSE(Group{ EmptyGroup() }.Match(h));
EXPECT_FALSE(Group{ EmptyGroup<std::true_type>() }.Match(h));
}

TEST(Group, Match) {
Expand Down

0 comments on commit 654bc27

Please sign in to comment.