LCOV - code coverage report
Current view: top level - tommyds - check.c (source / functions) Hit Total Coverage
Test: lcov.info Lines: 4053 4053 100.0 %
Date: 2026-11-12 12:42:49 Functions: 122 122 100.0 %

          Line data    Source code
       1             : // SPDX-License-Identifier: BSD-2-Clause
       2             : // Copyright (C) 2010 Andrea Mazzoleni
       3             : 
       4             : /**
       5             :  * Tommy check program.
       6             :  *
       7             :  * Simply run it without any options. If it terminates printing "OK" all the
       8             :  * checks are succesful.
       9             :  */
      10             : 
      11             : #include <math.h>
      12             : #include <stdlib.h>
      13             : #include <stdio.h>
      14             : #include <string.h>
      15             : #include <stddef.h>
      16             : #include <time.h>
      17             : #include <errno.h>
      18             : #include <assert.h>
      19             : #include <limits.h>
      20             : 
      21             : #if defined(_WIN32)
      22             : #include <windows.h>
      23             : #endif
      24             : 
      25             : #if defined(__linux)
      26             : #include <stdint.h>
      27             : #include <unistd.h>
      28             : #include <sys/time.h>
      29             : #endif
      30             : 
      31             : #if defined(__FreeBSD__)
      32             : #include <sys/time.h>
      33             : #endif
      34             : 
      35             : #if defined(__MACH__)
      36             : #include <mach/mach_time.h>
      37             : #endif
      38             : 
      39             : #include "tommyds/tommy.h"
      40             : 
      41             : #define TOMMY_SIZE 100000
      42             : 
      43             : #define PAYLOAD 16 /**< Size of payload data for objects */
      44             : 
      45             : struct object {
      46             :         int value;
      47             :         tommy_node node;
      48             :         char payload[PAYLOAD];
      49             : };
      50             : 
      51             : unsigned compare_counter;
      52             : 
      53    70569407 : int compare(const void* void_a, const void* void_b)
      54             : {
      55    70569407 :         const struct object* a = void_a;
      56    70569407 :         const struct object* b = void_b;
      57             : 
      58    70569407 :         ++compare_counter;
      59             : 
      60    70569407 :         if (a->value < b->value)
      61    36992197 :                 return -1;
      62    33577210 :         if (a->value > b->value)
      63    29305817 :                 return 1;
      64     4271393 :         return 0;
      65             : }
      66             : 
      67             : struct object_vector {
      68             :         int value;
      69             :         char payload[PAYLOAD];
      70             : };
      71             : 
      72    99406352 : int compare_vector(const void* void_a, const void* void_b)
      73             : {
      74    99406352 :         const struct object_vector* a = void_a;
      75    99406352 :         const struct object_vector* b = void_b;
      76             : 
      77    99406352 :         ++compare_counter;
      78             : 
      79    99406352 :         if (a->value < b->value)
      80    49218204 :                 return -1;
      81    50188148 :         if (a->value > b->value)
      82    43580904 :                 return 1;
      83     6607244 :         return 0;
      84             : }
      85             : 
      86             : struct object_hash {
      87             :         int value;
      88             :         tommy_node node;
      89             :         char payload[PAYLOAD];
      90             : };
      91             : 
      92             : struct object_tree {
      93             :         int value;
      94             :         tommy_tree_node node;
      95             :         char payload[PAYLOAD];
      96             : };
      97             : 
      98           8 : int compare_tree_reverse(const void* void_a, const void* void_b)
      99             : {
     100           8 :         const struct object_tree* a = void_a;
     101           8 :         const struct object_tree* b = void_b;
     102             : 
     103           8 :         if (a->value > b->value)
     104           3 :                 return -1;
     105           5 :         if (a->value < b->value)
     106           4 :                 return 1;
     107           1 :         return 0;
     108             : }
     109             : 
     110         225 : int compare_tree_group(const void* void_a, const void* void_b)
     111             : {
     112         225 :         const struct object_tree* a = void_a;
     113         225 :         const struct object_tree* b = void_b;
     114             : 
     115         225 :         if (a->value / 2 < b->value / 2)
     116           8 :                 return -1;
     117         217 :         if (a->value / 2 > b->value / 2)
     118         164 :                 return 1;
     119          53 :         return 0;
     120             : }
     121             : 
     122          31 : int compare_tree_int(const void* void_a, const void* void_b)
     123             : {
     124          31 :         const int* a = void_a;
     125          31 :         const struct object_tree* b = void_b;
     126             : 
     127          31 :         if (*a < b->value)
     128           7 :                 return -1;
     129          24 :         if (*a > b->value)
     130          16 :                 return 1;
     131           8 :         return 0;
     132             : }
     133             : 
     134             : struct object_trie {
     135             :         int value;
     136             :         tommy_trie_node node;
     137             :         char payload[PAYLOAD];
     138             : };
     139             : 
     140             : struct object_trie_inplace {
     141             :         int value;
     142             :         tommy_trie_inplace_node node;
     143             :         char payload[PAYLOAD];
     144             : };
     145             : 
     146             : /******************************************************************************/
     147             : /* time */
     148             : 
     149             : #if defined(_WIN32)
     150             : static LARGE_INTEGER win_frequency;
     151             : #endif
     152             : 
     153           1 : static void nano_init(void)
     154             : {
     155             : #if defined(_WIN32)
     156             :         if (!QueryPerformanceFrequency(&win_frequency)) {
     157             :                 win_frequency.QuadPart = 0;
     158             :         }
     159             : #endif
     160           1 : }
     161             : 
     162        4458 : static tommy_uint64_t nano(void)
     163             : {
     164             : #if defined(_WIN32)
     165             :         LARGE_INTEGER t;
     166             : 
     167             :         if (!QueryPerformanceCounter(&t))
     168             :                 return 0;
     169             : 
     170             :         tommy_uint64_t ret = (t.QuadPart / win_frequency.QuadPart) * 1000000000;
     171             : 
     172             :         ret += (t.QuadPart % win_frequency.QuadPart) * 1000000000 / win_frequency.QuadPart;
     173             :         return ret;
     174             : #elif defined(__MACH__)
     175             :         mach_timebase_info_data_t info;
     176             :         tommy_uint64_t t = mach_absolute_time();
     177             : 
     178             :         kern_return_t r = mach_timebase_info(&info);
     179             :         if (r != 0) {
     180             :                 /* LCOV_EXCL_START */
     181             :                 abort();
     182             :                 /* LCOV_EXCL_STOP */
     183             :         }
     184             : 
     185             :         tommy_uint64_t ret = (t / info.denom) * info.numer;
     186             : 
     187             :         ret += (t % info.denom) * info.numer / info.denom;
     188             :         return ret;
     189             : #elif defined(__linux)
     190             :         struct timespec ts;
     191             : 
     192        4458 :         int r = clock_gettime(CLOCK_MONOTONIC, &ts);
     193        4458 :         if (r != 0) {
     194             :                 /* LCOV_EXCL_START */
     195             :                 abort();
     196             :                 /* LCOV_EXCL_STOP */
     197             :         }
     198             : 
     199        4458 :         return ts.tv_sec * (tommy_uint64_t)1000000000 + ts.tv_nsec;
     200             : #else
     201             :         struct timeval tv;
     202             : 
     203             :         int r = gettimeofday(&tv, 0);
     204             :         if (r != 0) {
     205             :                 /* LCOV_EXCL_START */
     206             :                 abort();
     207             :                 /* LCOV_EXCL_STOP */
     208             :         }
     209             : 
     210             :         return tv.tv_sec * (tommy_uint64_t)1000000000 + tv.tv_usec * 1000;
     211             : #endif
     212             : }
     213             : 
     214             : /******************************************************************************/
     215             : /* random */
     216             : 
     217             : /**
     218             :  * Pseudo random number generator.
     219             :  * Note that using (rand() % max) in Visual C results in totally bogus values,
     220             :  * with *strong* cache effects when accessing elements in a not really random order.
     221             :  * This happen because Visual C uses a simple linear congruential generator with only 32 bits.
     222             :  */
     223             : tommy_uint64_t SEED = 0;
     224             : 
     225      301031 : unsigned rnd(unsigned max)
     226             : {
     227             :         /* linear congruential generator from MMIX by Donald Knuth, http://en.wikipedia.org/wiki/Linear_congruential_generator */
     228             : #ifdef _MSC_VER
     229             :         tommy_uint64_t divider = 0xFFFFFFFFFFFFFFFF / max;
     230             : #else
     231      301031 :         tommy_uint64_t divider = 0xFFFFFFFFFFFFFFFFULL / max;
     232             : #endif
     233             : 
     234           1 :         while (1) {
     235             : #ifdef _MSC_VER
     236             :                 SEED = SEED * 6364136223846793005 + 1442695040888963407;
     237             : #else
     238      301032 :                 SEED = SEED * 6364136223846793005LL + 1442695040888963407LL;
     239             : #endif
     240             : 
     241      301032 :                 unsigned r = (unsigned)(SEED / divider);
     242             : 
     243             :                 /* it may happen as the divider is approximated down */
     244      301032 :                 if (r < max)
     245      301031 :                         return r;
     246             :         }
     247             : }
     248             : 
     249             : /******************************************************************************/
     250             : /* helper */
     251             : 
     252          14 : unsigned isqrt(unsigned n)
     253             : {
     254          14 :         unsigned root = 0;
     255          14 :         unsigned remain = n;
     256          14 :         unsigned place = 0x40000000;
     257             : 
     258         112 :         while (place > remain)
     259          98 :                 place /= 4;
     260             : 
     261         140 :         while (place) {
     262         126 :                 if (remain >= root + place) {
     263          70 :                         remain -= root + place;
     264          70 :                         root += 2 * place;
     265             :                 }
     266             : 
     267         126 :                 root /= 2;
     268         126 :                 place /= 4;
     269             :         }
     270             : 
     271          14 :         return root;
     272             : }
     273             : 
     274             : /**
     275             :  * Cache clearing buffer.
     276             :  */
     277             : static unsigned char the_cache[16 * 1024 * 1024];
     278             : static double the_time;
     279             : static unsigned the_measure;
     280             : static volatile tommy_uint64_t the_hash;
     281             : 
     282          41 : void cache_clear(void)
     283             : {
     284             :         /* read & write */
     285    21495849 :         for (unsigned i = 0; i < sizeof(the_cache); i += 32)
     286    21495808 :                 the_cache[i] += 1;
     287             : 
     288             : #ifdef WIN32
     289             :         Sleep(0);
     290             : #endif
     291          41 : }
     292             : 
     293             : /**
     294             :  * Adaptive measurements, using the portable nano() clock.
     295             :  * Double the repetition count until a sample takes at least 1 ms, then keep
     296             :  * the best sample over 250 ms. Results are normalized to one repetition.
     297             :  * Count only the samples compared to select the final minimum.
     298             :  */
     299             : #define SPEED_START(s) \
     300             :         { \
     301             :                 const char* speed_name_ = s; \
     302             :                 unsigned speed_delta_ = 1; \
     303             :                 tommy_uint64_t elapsed_ = 0; \
     304             :                 tommy_uint64_t best_dt_ = 0xffffffffffffffffULL; \
     305             :                 unsigned best_compare_ = 0; \
     306             :                 the_measure = 0; \
     307             :                 cache_clear(); \
     308             :                 do { \
     309             :                         tommy_uint64_t skipped_ = 0; \
     310             :                         compare_counter = 0; \
     311             :                         tommy_uint64_t start_ = nano(); \
     312             :                         for (unsigned i_ = 0; i_ < speed_delta_; ++i_)
     313             : 
     314             : #define SPEED_STOP \
     315             :         tommy_uint64_t dt_ = nano() - start_ - skipped_; \
     316             :         if (dt_ < 1000000 && speed_delta_ <= UINT_MAX / 2) { \
     317             :                 speed_delta_ *= 2; \
     318             :                 elapsed_ = 0; \
     319             :                 best_dt_ = 0xffffffffffffffffULL; \
     320             :                 the_measure = 0; \
     321             :                 continue; \
     322             :         } \
     323             :         ++the_measure; \
     324             :         elapsed_ += dt_; \
     325             :         if (dt_ < best_dt_) { \
     326             :                 best_dt_ = dt_; \
     327             :                 best_compare_ = compare_counter; \
     328             :         } \
     329             :         } while (elapsed_ < 250000000); \
     330             :         the_time = best_dt_; \
     331             :         the_time /= speed_delta_; \
     332             :         compare_counter = best_compare_ / speed_delta_; \
     333             :         if (speed_name_) \
     334             :         printf("%25s %8.2f [ms], %8u [compare], %8u [measure]\n", speed_name_, the_time / 1000000, compare_counter, the_measure); \
     335             :         }
     336             : 
     337             : /**
     338             :  * Restore mutable inputs between repetitions without timing the restoration.
     339             :  */
     340             : #define SPEED_UNTIMED(...) \
     341             :         do { \
     342             :                 tommy_uint64_t untimed_start_ = nano(); \
     343             :                 __VA_ARGS__; \
     344             :                 skipped_ += nano() - untimed_start_; \
     345             :         } while (0)
     346             : 
     347             : /******************************************************************************/
     348             : /* test */
     349             : 
     350             : static unsigned the_count;
     351             : 
     352     8234460 : static void count_callback(void* data)
     353             : {
     354             :         (void)data;
     355     8234460 :         ++the_count;
     356     8234460 : }
     357             : 
     358      951496 : static void count_arg_callback(void* arg, void* data)
     359             : {
     360      951496 :         unsigned* count = arg;
     361             :         (void)data;
     362      951496 :         ++*count;
     363      951496 : }
     364             : 
     365        4346 : static void array_foreach_check_callback(void* data)
     366             : {
     367        4346 :         if (data != (void*)(tommy_uintptr_t)(the_count + 1)) {
     368             :                 /* LCOV_EXCL_START */
     369             :                 abort();
     370             :                 /* LCOV_EXCL_STOP */
     371             :         }
     372        4346 :         ++the_count;
     373        4346 : }
     374             : 
     375        4346 : static void array_foreach_check_arg_callback(void* arg, void* data)
     376             : {
     377        4346 :         tommy_uintptr_t* expected = arg;
     378        4346 :         if (data != (void*)(*expected + 1)) {
     379             :                 /* LCOV_EXCL_START */
     380             :                 abort();
     381             :                 /* LCOV_EXCL_STOP */
     382             :         }
     383        4346 :         ++*expected;
     384        4346 : }
     385             : 
     386        4346 : static void arrayof_foreach_check_callback(void* data)
     387             : {
     388        4346 :         unsigned* val = data;
     389        4346 :         if (*val != the_count + 1) {
     390             :                 /* LCOV_EXCL_START */
     391             :                 abort();
     392             :                 /* LCOV_EXCL_STOP */
     393             :         }
     394        4346 :         *val = *val * 2;
     395        4346 :         ++the_count;
     396        4346 : }
     397             : 
     398        4346 : static void arrayof_foreach_check_arg_callback(void* arg, void* data)
     399             : {
     400        4346 :         unsigned* expected = arg;
     401        4346 :         unsigned* val = data;
     402        4346 :         if (*val != (*expected + 1) * 2) {
     403             :                 /* LCOV_EXCL_START */
     404             :                 abort();
     405             :                 /* LCOV_EXCL_STOP */
     406             :         }
     407        4346 :         ++*expected;
     408        4346 : }
     409             : 
     410    10437784 : static int search_callback(const void* arg, const void* obj)
     411             : {
     412    10437784 :         return arg != obj;
     413             : }
     414             : 
     415        8500 : static int search_hash_value_callback(const void* arg, const void* obj)
     416             : {
     417        8500 :         const struct object_hash* object = obj;
     418        8500 :         ++compare_counter;
     419        8500 :         return *(const int*)arg != object->value;
     420             : }
     421             : 
     422             : struct hash32_test {
     423             :         char* data;
     424             :         tommy_uint32_t len;
     425             :         tommy_uint32_t hash;
     426             : } HASH32[] = {
     427             :         { "", 0, 0x5e8c3713 },
     428             :         { "a", 1, 0xaf7e9c82 },
     429             :         { "abc", 3, 0xf0b2d6c2 },
     430             :         { "message digest", 14, 0x7dfed9b7 },
     431             :         { "abcdefghijklmnopqrstuvwxyz", 26, 0x0c9ab1e2 },
     432             :         { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 62, 0x2a21fc92 },
     433             :         { "The quick brown fox jumps over the lazy dog", 43, 0x0d73307f },
     434             :         { "\x00", 1, 0xb2a205c2 },
     435             :         { "\x16\x27", 2, 0x9663cb23 },
     436             :         { "\xe2\x56\xb4", 3, 0x981d6b2d },
     437             :         { "\xc9\x4d\x9c\xda", 4, 0xe193aa82 },
     438             :         { "\x79\xf1\x29\x69\x5d", 5, 0x46220359 },
     439             :         { "\x00\x7e\xdf\x1e\x31\x1c", 6, 0xc8b46c21 },
     440             :         { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 7, 0x10b187f4 },
     441             :         { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 8, 0xfc6d3877 },
     442             :         { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 9, 0x09a9ed35 },
     443             :         { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 10, 0x590c5ad7 },
     444             :         { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\x00\xb7\xae", 11, 0x24303014 },
     445             :         { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 12, 0x84d980b7 },
     446             :         { "\x87\xd8\x61\x61\x4c\x89\x17\x4e\xa1\xa4\xef\x13\xa9", 13, 0x25f12a09 },
     447             :         { "\xfe\xa6\x5b\xc2\xda\xe8\x95\xd4\x64\xab\x4c\x39\x58\x29", 14, 0xc36fc936 },
     448             :         { "\x94\x49\xc0\x78\xa0\x80\xda\xc7\x71\x4e\x17\x37\xa9\x7c\x40", 15, 0xe208fe86 },
     449             :         { "\x53\x7e\x36\xb4\x2e\xc9\xb9\xcc\x18\x3e\x9a\x5f\xfc\xb7\xb0\x61", 16, 0x091e6610 },
     450             :         { 0, 0, 0 }
     451             : };
     452             : 
     453             : struct strhash32_test {
     454             :         char* data;
     455             :         tommy_uint32_t hash;
     456             : } STRHASH32[] = {
     457             :         { "", 0x5e8c3713 },
     458             :         { "a", 0xaf7e9c82 },
     459             :         { "abc", 0xf0b2d6c2 },
     460             :         { "message digest", 0x7dfed9b7 },
     461             :         { "abcdefghijklmnopqrstuvwxyz", 0x0c9ab1e2 },
     462             :         { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0x2a21fc92 },
     463             :         { "The quick brown fox jumps over the lazy dog", 0x0d73307f },
     464             :         { "\xff", 0xf8c052c8 },
     465             :         { "\x16\x27", 0x9663cb23 },
     466             :         { "\xe2\x56\xb4", 0x981d6b2d },
     467             :         { "\xc9\x4d\x9c\xda", 0xe193aa82 },
     468             :         { "\x79\xf1\x29\x69\x5d", 0x46220359 },
     469             :         { "\xff\x7e\xdf\x1e\x31\x1c", 0x306a356f },
     470             :         { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 0x10b187f4 },
     471             :         { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 0xfc6d3877 },
     472             :         { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 0x09a9ed35 },
     473             :         { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 0x590c5ad7 },
     474             :         { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\xff\xb7\xae", 0xdbf3b98b },
     475             :         { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 0x84d980b7 },
     476             :         { "\x87\xd8\x61\x61\x4c\x89\x17\x4e\xa1\xa4\xef\x13\xa9", 0x25f12a09 },
     477             :         { "\xfe\xa6\x5b\xc2\xda\xe8\x95\xd4\x64\xab\x4c\x39\x58\x29", 0xc36fc936 },
     478             :         { "\x94\x49\xc0\x78\xa0\x80\xda\xc7\x71\x4e\x17\x37\xa9\x7c\x40", 0xe208fe86 },
     479             :         { "\x53\x7e\x36\xb4\x2e\xc9\xb9\xcc\x18\x3e\x9a\x5f\xfc\xb7\xb0\x61", 0x091e6610 },
     480             :         { 0, 0 }
     481             : };
     482             : 
     483             : struct strhash64_test {
     484             :         char* data;
     485             :         tommy_uint64_t hash;
     486             : } STRHASH64[] = {
     487             :         { "", 0xb9137ac65dfaf9bfULL },
     488             :         { "a", 0x41050925073ab90eULL },
     489             :         { "abc", 0x24bdea1fdbe37bbaULL },
     490             :         { "message digest", 0xde7c0388b3c424f0ULL },
     491             :         { "abcdefghijklmnopqrstuvwxyz", 0xbc31451d15541f84ULL },
     492             :         { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0x10bbb52e5fa644b2ULL },
     493             :         { "The quick brown fox jumps over the lazy dog", 0x3e533e22b5137278ULL },
     494             :         { "\xff", 0x4f1a44c81721ace9ULL },
     495             :         { "\x16\x27", 0x1c32798af082a887ULL },
     496             :         { "\xe2\x56\xb4", 0xcbfa0875fd385560ULL },
     497             :         { "\xc9\x4d\x9c\xda", 0x002109fa17f38af1ULL },
     498             :         { "\x79\xf1\x29\x69\x5d", 0x2b5cf1bd7d4c4fa6ULL },
     499             :         { "\xff\x7e\xdf\x1e\x31\x1c", 0xe780930a5446debfULL },
     500             :         { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 0x7fdfa285f0fd2c48ULL },
     501             :         { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 0x0ebe4815fc3a4ae3ULL },
     502             :         { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 0x7792e5dd45a884d8ULL },
     503             :         { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 0xc6ae75b9f91e3c58ULL },
     504             :         { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\xff\xb7\xae", 0xb324679431d7062aULL },
     505             :         { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 0xd38fb4f296937cc3ULL },
     506             :         { "\x87\xd8\x61\x61\x4c\x89\x17\x4e\xa1\xa4\xef\x13\xa9", 0xbdf09efdf05bda6eULL },
     507             :         { "\xfe\xa6\x5b\xc2\xda\xe8\x95\xd4\x64\xab\x4c\x39\x58\x29", 0x6ef6713df9020be0ULL },
     508             :         { "\x94\x49\xc0\x78\xa0\x80\xda\xc7\x71\x4e\x17\x37\xa9\x7c\x40", 0x3d01826373199d12ULL },
     509             :         { "\x53\x7e\x36\xb4\x2e\xc9\xb9\xcc\x18\x3e\x9a\x5f\xfc\xb7\xb0\x61", 0x245f826dfde52964ULL },
     510             :         { 0, 0 }
     511             : };
     512             : 
     513             : struct hash64_test {
     514             :         char* data;
     515             :         tommy_uint32_t len;
     516             :         tommy_uint64_t hash;
     517             : } HASH64[] = {
     518             :         { "", 0, 0xb9137ac65dfaf9bfULL },
     519             :         { "a", 1, 0x41050925073ab90eULL },
     520             :         { "abc", 3, 0x24bdea1fdbe37bbaULL },
     521             :         { "message digest", 14, 0xde7c0388b3c424f0ULL },
     522             :         { "abcdefghijklmnopqrstuvwxyz", 26, 0xbc31451d15541f84ULL },
     523             :         { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 62, 0x10bbb52e5fa644b2ULL },
     524             :         { "The quick brown fox jumps over the lazy dog", 43, 0x3e533e22b5137278ULL },
     525             :         { "\x00", 1, 0xddf38ca5d36c5e6dULL },
     526             :         { "\x16\x27", 2, 0x1c32798af082a887ULL },
     527             :         { "\xe2\x56\xb4", 3, 0xcbfa0875fd385560ULL },
     528             :         { "\xc9\x4d\x9c\xda", 4, 0x002109fa17f38af1ULL },
     529             :         { "\x79\xf1\x29\x69\x5d", 5, 0x2b5cf1bd7d4c4fa6ULL },
     530             :         { "\x00\x7e\xdf\x1e\x31\x1c", 6, 0x2fd8ea1c888d6a25ULL },
     531             :         { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 7, 0x7fdfa285f0fd2c48ULL },
     532             :         { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 8, 0x0ebe4815fc3a4ae3ULL },
     533             :         { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 9, 0x7792e5dd45a884d8ULL },
     534             :         { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 10, 0xc6ae75b9f91e3c58ULL },
     535             :         { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\x00\xb7\xae", 11, 0x9d7025c65aab3e63ULL },
     536             :         { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 12, 0xd38fb4f296937cc3ULL },
     537             :         { "\x87\xd8\x61\x61\x4c\x89\x17\x4e\xa1\xa4\xef\x13\xa9", 13, 0xbdf09efdf05bda6eULL },
     538             :         { "\xfe\xa6\x5b\xc2\xda\xe8\x95\xd4\x64\xab\x4c\x39\x58\x29", 14, 0x6ef6713df9020be0ULL },
     539             :         { "\x94\x49\xc0\x78\xa0\x80\xda\xc7\x71\x4e\x17\x37\xa9\x7c\x40", 15, 0x3d01826373199d12ULL },
     540             :         { "\x53\x7e\x36\xb4\x2e\xc9\xb9\xcc\x18\x3e\x9a\x5f\xfc\xb7\xb0\x61", 16, 0x245f826dfde52964ULL },
     541             :         { 0, 0, 0 }
     542             : };
     543             : 
     544             : struct inthash32_test {
     545             :         tommy_uint32_t value;
     546             :         tommy_uint32_t hash;
     547             : } INTHASH32[] = {
     548             :         { 0x00000000, 0x00000000 },
     549             :         { 0x00000001, 0xc2b73583 },
     550             :         { 0x00000002, 0xe90f1258 },
     551             :         { 0x00000004, 0x7a10c2d3 },
     552             :         { 0x00000008, 0x200c3457 },
     553             :         { 0x00000010, 0xeb97690a },
     554             :         { 0x00000020, 0x7fb291d3 },
     555             :         { 0x00000040, 0xf50601d8 },
     556             :         { 0x00000080, 0x727dbaed },
     557             :         { 0x00000100, 0x7ef5f77d },
     558             :         { 0x00000200, 0x91a480dc },
     559             :         { 0x00000400, 0x2bad9acc },
     560             :         { 0x00000800, 0xfe4d150e },
     561             :         { 0x00001000, 0xc3add476 },
     562             :         { 0x00002000, 0x23946174 },
     563             :         { 0x00004000, 0x987cfc43 },
     564             :         { 0x00008000, 0x630cdf68 },
     565             :         { 0x00010000, 0x0ac3a767 },
     566             :         { 0x00020000, 0xad086d5b },
     567             :         { 0x00040000, 0x1126ccdf },
     568             :         { 0x00080000, 0x4370dbc4 },
     569             :         { 0x00100000, 0xefd6e5e6 },
     570             :         { 0x00200000, 0x9a93c1b5 },
     571             :         { 0x00400000, 0x10114902 },
     572             :         { 0x00800000, 0x96117e60 },
     573             :         { 0x01000000, 0x5dec9f58 },
     574             :         { 0x02000000, 0xfee234c7 },
     575             :         { 0x04000000, 0x36137e26 },
     576             :         { 0x08000000, 0x6c26fc4c },
     577             :         { 0x10000000, 0xd84df898 },
     578             :         { 0x20000000, 0xb099f131 },
     579             :         { 0x40000000, 0x6131e262 },
     580             :         { 0x80000000, 0xc263c4c4 },
     581             :         { 0x00204a16, 0xd8b97461 },
     582             :         { 0x05542a27, 0x65d0057a },
     583             :         { 0x169c39e2, 0x7c2ff59a },
     584             :         { 0x2eab4956, 0xa8ba89bd },
     585             :         { 0x0bb0b8b4, 0xb790c8de },
     586             :         { 0x0bd068c9, 0x92d30546 },
     587             :         { 0x3e5d224d, 0xd610bf1d },
     588             :         { 0x436c8d9c, 0x27b09019 },
     589             :         { 0x3a2adfda, 0xdfde9385 },
     590             :         { 0x1dd8ca79, 0xc7c10d7b },
     591             :         { 0x6a67c4f1, 0xf92a788d },
     592             :         { 0x7742fa29, 0x892c3519 },
     593             :         { 0x48b62d69, 0x7f642f55 },
     594             :         { 0x472e195d, 0xea2c49e5 },
     595             :         { 0x0681a900, 0x4de5c929 },
     596             :         { 0x622ebb7e, 0x35a7e306 },
     597             :         { 0x026bccdf, 0xa7e4b630 },
     598             :         { 0x204d531e, 0x43ebd664 },
     599             :         { 0x262b5331, 0x7a9a161f },
     600             :         { 0x7020241c, 0xbaed3ef7 },
     601             :         { 0x440a0e2a, 0x0b8c5b29 },
     602             :         { 0x75cb1c4c, 0x19555414 },
     603             :         { 0x41f9a5e1, 0xc6acbc6b },
     604             :         { 0x67bc26ff, 0x54e4411f },
     605             :         { 0x181e279e, 0x979c834f },
     606             :         { 0x7172c06f, 0xe6a179ff },
     607             :         { 0x4909e153, 0x198e5e0f },
     608             :         { 0x09d3bfba, 0x109a6f17 },
     609             :         { 0x685ae502, 0xf9d57a4b },
     610             :         { 0x7e10e8ab, 0x09765bec },
     611             :         { 0x0f262618, 0xe16404cd },
     612             :         { 0x726b8230, 0x55e478b4 },
     613             :         { 0, 0 }
     614             : };
     615             : 
     616             : struct inthash64_test {
     617             :         tommy_uint64_t value;
     618             :         tommy_uint64_t hash;
     619             : } INTHASH64[] = {
     620             :         { 0x0000000000000000ULL, 0x77cfa1eef01bca90ULL },
     621             :         { 0x0000000000000001ULL, 0x5bca7c69b794f8ceULL },
     622             :         { 0x0000000000000002ULL, 0xb795033f6f2a0674ULL },
     623             :         { 0x0000000000000004ULL, 0x6f2a25235e544a31ULL },
     624             :         { 0x0000000000000008ULL, 0xde543f7b3ca87ecbULL },
     625             :         { 0x0000000000000010ULL, 0xbca87eec7950fd82ULL },
     626             :         { 0x0000000000000020ULL, 0x7950fddef2a1fb10ULL },
     627             :         { 0x0000000000000040ULL, 0xf2a1fbbde543f620ULL },
     628             :         { 0x0000000000000080ULL, 0xe543f9324a87efadULL },
     629             :         { 0x0000000000000100ULL, 0xca87f25e950fdf4eULL },
     630             :         { 0x0000000000000200ULL, 0x950fe4bd2a1fbe9cULL },
     631             :         { 0x0000000000000400ULL, 0x2a1fc968543f7d14ULL },
     632             :         { 0x0000000000000800ULL, 0x543f92d0a87efa28ULL },
     633             :         { 0x0000000000001000ULL, 0xa87f259f50fdf44cULL },
     634             :         { 0x0000000000002000ULL, 0x50fe4b3ea1fbe898ULL },
     635             :         { 0x0000000000004000ULL, 0xa1fc967bc3f7d12dULL },
     636             :         { 0x0000000000008000ULL, 0x43f92da207efa3afULL },
     637             :         { 0x0000000000010000ULL, 0x87f25b4e8fdf4773ULL },
     638             :         { 0x0000000000020000ULL, 0x0fe4b6a71fbe8efaULL },
     639             :         { 0x0000000000040000ULL, 0x1fc96d4ebf7d1df5ULL },
     640             :         { 0x0000000000080000ULL, 0x3f92da9dfefa3bebULL },
     641             :         { 0x0000000000100000ULL, 0x7f25b53c7df477d7ULL },
     642             :         { 0x0000000000200000ULL, 0xfe4b6a797be8efafULL },
     643             :         { 0x0000000000400000ULL, 0xfc96d4f377d1df5fULL },
     644             :         { 0x0000000000800000ULL, 0xf92da9e76fa3bebfULL },
     645             :         { 0x0000000001000000ULL, 0xf25b39f0df4749c2ULL },
     646             :         { 0x0000000002000000ULL, 0xe4b673e1be8e9384ULL },
     647             :         { 0x0000000004000000ULL, 0xc96ce7c3fd1d2709ULL },
     648             :         { 0x0000000008000000ULL, 0x92d9cf87fa3a4e12ULL },
     649             :         { 0x0000000010000000ULL, 0x25b39f0ff4749c24ULL },
     650             :         { 0x0000000020000000ULL, 0x4b673e1fe8e93848ULL },
     651             :         { 0x0000000040000000ULL, 0x96ce7c3a51d27085ULL },
     652             :         { 0x0000000080000000ULL, 0x2d9cf88523a4e10bULL },
     653             :         { 0x0000000100000000ULL, 0x5b39f10ac749c217ULL },
     654             :         { 0x0000000200000000ULL, 0xb673e2060e93842fULL },
     655             :         { 0x0000000400000000ULL, 0x6ce7c40c9d27085fULL },
     656             :         { 0x0000000800000000ULL, 0xdc8387ff3f0e10aaULL },
     657             :         { 0x0000001000000000ULL, 0xb9070fee7e1c2154ULL },
     658             :         { 0x0000002000000000ULL, 0x720e1fdcfc3842a8ULL },
     659             :         { 0x0000004000000000ULL, 0xe41c3fa478708545ULL },
     660             :         { 0x0000008000000000ULL, 0xc8387f58f0e10a8aULL },
     661             :         { 0x0000010000000000ULL, 0x9364fec267021515ULL },
     662             :         { 0x0000020000000000ULL, 0x26c9fd954e042a2bULL },
     663             :         { 0x0000040000000000ULL, 0x4d93fb2a9c085456ULL },
     664             :         { 0x0000080000000000ULL, 0x9b27f6553810a8acULL },
     665             :         { 0x0000100000000000ULL, 0xae84159b600d1cc8ULL },
     666             :         { 0x0000200000000000ULL, 0x455932ec903fc254ULL },
     667             :         { 0x0000400000000000ULL, 0xa2d36aa0d044b36fULL },
     668             :         { 0x0000800000000000ULL, 0x7b8ef94d0d2d2844ULL },
     669             :         { 0x0001000000000000ULL, 0xd802248d5ba62df7ULL },
     670             :         { 0x0002000000000000ULL, 0x2c298f47af1d015eULL },
     671             :         { 0x0004000000000000ULL, 0x93a4e2a055abc2f5ULL },
     672             :         { 0x0008000000000000ULL, 0x6f9511373b7145b2ULL },
     673             :         { 0x0010000000000000ULL, 0x6305c0717e1d40d4ULL },
     674             :         { 0x0020000000000000ULL, 0x6237df27b416b76bULL },
     675             :         { 0x0040000000000000ULL, 0x042bc5ffe77f439eULL },
     676             :         { 0x0080000000000000ULL, 0x7241900621a8c72bULL },
     677             :         { 0x0100000000000000ULL, 0x6a298a4b4ecb2ec6ULL },
     678             :         { 0x0200000000000000ULL, 0x789742a5659f92fbULL },
     679             :         { 0x0400000000000000ULL, 0x794ee951db0365e6ULL },
     680             :         { 0x0800000000000000ULL, 0x745424e0b94aec3cULL },
     681             :         { 0x1000000000000000ULL, 0x726e27d005120de7ULL },
     682             :         { 0x2000000000000000ULL, 0x6898adb511c8513eULL },
     683             :         { 0x4000000000000000ULL, 0x59dbb96b3414d7ecULL },
     684             :         { 0x8000000000000000ULL, 0x3be7d0f7780de548ULL },
     685             :         { 0x0cead30e6469f5c5ULL, 0x0f3b67e6407d0ed9ULL },
     686             :         { 0x028a2bec206c7b8aULL, 0xa9be0155bf452972ULL },
     687             :         { 0x56e5747a306eac4eULL, 0x308451cac91706c3ULL },
     688             :         { 0x6058b11e57287c72ULL, 0x6f8b3e2b8bf1abc4ULL },
     689             :         { 0x4fec8f2a00cc2071ULL, 0x11f1e965f20a45a4ULL },
     690             :         { 0x4f0bdf33102febc9ULL, 0x6102a774aa4baacfULL },
     691             :         { 0x17e067e262adc6fdULL, 0x5878e9adb48c4d7cULL },
     692             :         { 0x41374f0f3f94939cULL, 0x55d43d2febf8b7dfULL },
     693             :         { 0x59d163b72870b072ULL, 0x98ee8d1508e8ff20ULL },
     694             :         { 0x702b00ea2f01f008ULL, 0x1abf9db8b3306341ULL },
     695             :         { 0x433b78782f7cc0d0ULL, 0xd33358a085d11f6aULL },
     696             :         { 0x45789bc436c34865ULL, 0xa573c24e116ccf0dULL },
     697             :         { 0x15e7f9b85580528aULL, 0x6d594ec1bb5651dfULL },
     698             :         { 0x6e52ea866a56f880ULL, 0x5dac61fbbe7d64ffULL },
     699             :         { 0x06baec796275d69aULL, 0x193424cca0b43145ULL },
     700             :         { 0x2ac9d0b77bbdcc00ULL, 0x44b2175a2c313151ULL },
     701             :         { 0x2299ab770a8223aeULL, 0x404514aa9d8e5c65ULL },
     702             :         { 0x568e90d709816de9ULL, 0x08907c8b20d9fb59ULL },
     703             :         { 0x3b6b460e67b34680ULL, 0xfc77f1ea859b024eULL },
     704             :         { 0x418fde5c48db0e3fULL, 0x2e6a2d8cbe2d4412ULL },
     705             :         { 0x5e8fa3c84b12c543ULL, 0xa9354017412aba12ULL },
     706             :         { 0x49fad5466f937fc2ULL, 0xe81aeb4429b6264aULL },
     707             :         { 0x2ab116877b7b6839ULL, 0xb189fec8b0994d6eULL },
     708             :         { 0x16a5916132482fd8ULL, 0xa7c48eb9f12b7d37ULL },
     709             :         { 0x6d24a54c0e51b961ULL, 0x701631d776a5b0e2ULL },
     710             :         { 0x7a39bf1767d47a89ULL, 0x6bd08da7d7095b7aULL },
     711             :         { 0x11c7f9a173bf8d4eULL, 0x451390b96dcad08dULL },
     712             :         { 0x10411fef57d4fca4ULL, 0x49bd093dc8d7e040ULL },
     713             :         { 0x001079a900860713ULL, 0x81ab2baf1bf59632ULL },
     714             :         { 0x4b1b9ca618b2a5feULL, 0x152cb5a46ef65f99ULL },
     715             :         { 0x036db8c22ffea95bULL, 0x08a263ec4de57fa9ULL },
     716             :         { 0x64861ee83b7d6ddaULL, 0xd344a694800cea8cULL },
     717             :         { 0, 0 }
     718             : };
     719             : 
     720             : const struct inthash32_test MULHASH32[] = {
     721             :         { 0x00000000U, 0x00000000U },
     722             :         { 0x00000001U, 0x514e28b7U },
     723             :         { 0x00000002U, 0x30f4c306U },
     724             :         { 0x7fffffffU, 0xf9cc0ea8U },
     725             :         { 0x80000000U, 0x6d3c65a0U },
     726             :         { 0xffffffffU, 0x81f16f39U },
     727             :         { 0x55555555U, 0x1d2752bfU },
     728             :         { 0xaaaaaaaaU, 0xabf634c6U },
     729             :         { 0x00010000U, 0x0ea34562U },
     730             :         { 0xffff0000U, 0x4ae38127U },
     731             :         { 0x12345678U, 0xe37cd1bcU },
     732             :         { 0xdeadbeefU, 0x0de5c6a9U },
     733             :         { 0, 0 }
     734             : };
     735             : 
     736             : const struct inthash64_test MULHASH64[] = {
     737             :         { 0x0000000000000000ULL, 0x0000000000000000ULL },
     738             :         { 0x0000000000000001ULL, 0x5692161d100b05e5ULL },
     739             :         { 0x0000000000000002ULL, 0xdbd238973a2b148aULL },
     740             :         { 0x7fffffffffffffffULL, 0x5a682afe7965debdULL },
     741             :         { 0x8000000000000000ULL, 0x25c26ea579cea98aULL },
     742             :         { 0xffffffffffffffffULL, 0xb4d055fcf2cbbd7bULL },
     743             :         { 0x5555555555555555ULL, 0xc4c84c7057e99f63ULL },
     744             :         { 0xaaaaaaaaaaaaaaaaULL, 0x899098e2afd33ec7ULL },
     745             :         { 0x0000000100000000ULL, 0xd820b7e910b0f93fULL },
     746             :         { 0xffffffff00000000ULL, 0xb742092f23ab85b3ULL },
     747             :         { 0x0123456789abcdefULL, 0xb2c058e4ebb5112cULL },
     748             :         { 0xdeadbeefcafebabeULL, 0x7ad6664f09ffe52cULL },
     749             :         { 0, 0 }
     750             : };
     751             : 
     752           1 : void test_strhash_alignment(void)
     753             : {
     754           1 :         const tommy_uint32_t seed[] = { 0, 1, 0xa766795d, 0xffffffff };
     755             :         /* fixed vectors for the TommyDS a5hash32 derivative */
     756             :         const struct {
     757             :                 unsigned len;
     758             :                 tommy_uint32_t seed;
     759             :                 tommy_uint32_t hash;
     760           1 :         } vectors[] = {
     761             :                 { 0, 0xa766795d, 0x5e8c3713 },
     762             :                 { 1, 0xa766795d, 0x77aca6cc },
     763             :                 { 2, 0xa766795d, 0x833558e8 },
     764             :                 { 3, 0xa766795d, 0x9d081a25 },
     765             :                 { 4, 0xa766795d, 0x7144d825 },
     766             :                 { 7, 0xa766795d, 0x1054a067 },
     767             :                 { 8, 0xa766795d, 0xfc859068 },
     768             :                 { 9, 0xa766795d, 0x23d5da05 },
     769             :                 { 15, 0xa766795d, 0xa29b8ee5 },
     770             :                 { 16, 0xa766795d, 0x91d9da06 },
     771             :                 { 17, 0xa766795d, 0x8c8cf4f1 },
     772             :                 { 23, 0xa766795d, 0xc51005a2 },
     773             :                 { 24, 0xa766795d, 0x21312652 },
     774             :                 { 25, 0xa766795d, 0x6d651e24 },
     775             :                 { 31, 0xa766795d, 0x86c71bab },
     776             :                 { 32, 0xa766795d, 0xf2ee8994 },
     777             :                 { 33, 0xa766795d, 0xc58aa8ca },
     778             :                 { 47, 0xa766795d, 0x88e34e82 },
     779             :                 { 48, 0xa766795d, 0xa7c7c3f4 },
     780             :                 { 49, 0xa766795d, 0xc5b7ff8b },
     781             :                 { 63, 0xa766795d, 0x60d02fa3 },
     782             :                 { 64, 0xa766795d, 0x8b08dd79 },
     783             :                 { 65, 0xa766795d, 0xec153e6d },
     784             :                 { 127, 0xa766795d, 0xa9a7475d },
     785             :                 { 128, 0xa766795d, 0x5aed0b3e },
     786             :                 { 129, 0xa766795d, 0x64517586 },
     787             :                 { 255, 0xa766795d, 0xff35a726 },
     788             :                 { 33, 0, 0x471590eb },
     789             :                 { 33, 1, 0xedd9b18d },
     790             :                 { 33, 0xffffffff, 0x8cb3d81a }
     791             :         };
     792             :         unsigned char buffer[4][256 + 8];
     793             :         unsigned char* key[4];
     794             : 
     795           5 :         for (unsigned alignment = 0; alignment < 4; ++alignment) {
     796           4 :                 unsigned char* base = buffer[alignment];
     797           4 :                 key[alignment] = base + ((4 - ((tommy_uintptr_t)base & 3)) & 3) + alignment;
     798             :         }
     799             : 
     800             :         /* verify the string vectors at every alignment */
     801          24 :         for (unsigned i = 0; STRHASH32[i].data; ++i) {
     802         115 :                 for (unsigned alignment = 0; alignment < 4; ++alignment) {
     803          92 :                         memset(buffer[alignment], 0xa5, sizeof(buffer[alignment]));
     804          92 :                         memcpy(key[alignment], STRHASH32[i].data, strlen(STRHASH32[i].data) + 1);
     805          92 :                         if (tommy_strhash_u32(0xa766795d, key[alignment]) != STRHASH32[i].hash) {
     806             :                                 /* LCOV_EXCL_START */
     807             :                                 abort();
     808             :                                 /* LCOV_EXCL_STOP */
     809             :                         }
     810             :                 }
     811             :         }
     812             : 
     813             :         /* check the reference vectors at every alignment, including overlapping bulk tails */
     814          31 :         for (unsigned v = 0; v < sizeof(vectors) / sizeof(vectors[0]); ++v) {
     815          30 :                 unsigned len = vectors[v].len;
     816         150 :                 for (unsigned alignment = 0; alignment < 4; ++alignment) {
     817        5416 :                         for (unsigned i = 0; i < len; ++i)
     818        5296 :                                 key[alignment][i] = (i * 73 + len * 11) % 255 + 1;
     819         120 :                         key[alignment][len] = 0;
     820         120 :                         if (tommy_hash_u32(vectors[v].seed, key[alignment], len) != vectors[v].hash
     821         120 :                                 || tommy_strhash_u32(vectors[v].seed, key[alignment]) != vectors[v].hash) {
     822             :                                 /* LCOV_EXCL_START */
     823             :                                 abort();
     824             :                                 /* LCOV_EXCL_STOP */
     825             :                         }
     826             :                 }
     827             :         }
     828             : 
     829             :         /* cover all tails, complete blocks and non-ASCII bytes with several seeds */
     830         257 :         for (unsigned len = 0; len < 256; ++len) {
     831        1280 :                 for (unsigned alignment = 0; alignment < 4; ++alignment) {
     832        1024 :                         memset(buffer[alignment], 0xa5, sizeof(buffer[alignment]));
     833      131584 :                         for (unsigned i = 0; i < len; ++i)
     834      130560 :                                 key[alignment][i] = (i * 73 + len * 11) % 255 + 1;
     835        1024 :                         key[alignment][len] = 0;
     836             :                 }
     837        1280 :                 for (unsigned s = 0; s < sizeof(seed) / sizeof(seed[0]); ++s) {
     838        1024 :                         tommy_uint32_t expected = tommy_strhash_u32(seed[s], key[0]);
     839        5120 :                         for (unsigned alignment = 0; alignment < 4; ++alignment) {
     840        4096 :                                 if (tommy_strhash_u32(seed[s], key[alignment]) != expected) {
     841             :                                         /* LCOV_EXCL_START */
     842             :                                         abort();
     843             :                                         /* LCOV_EXCL_STOP */
     844             :                                 }
     845        4096 :                                 if (tommy_hash_u32(seed[s], key[alignment], len) != expected) {
     846             :                                         /* LCOV_EXCL_START */
     847             :                                         abort();
     848             :                                         /* LCOV_EXCL_STOP */
     849             :                                 }
     850             :                         }
     851             :                 }
     852             :         }
     853           1 : }
     854             : 
     855           1 : void test_strhash_u64_alignment(void)
     856             : {
     857           1 :         const tommy_uint64_t seed[] = { 0, 1, 0xa766795dULL, 0x2f022773a766795dULL, 0xffffffffffffffffULL };
     858             :         unsigned char buffer[8][256 + 16];
     859             :         unsigned char* key[8];
     860             : 
     861           9 :         for (unsigned alignment = 0; alignment < 8; ++alignment) {
     862           8 :                 unsigned char* base = buffer[alignment];
     863           8 :                 key[alignment] = base + ((8 - ((tommy_uintptr_t)base & 7)) & 7) + alignment;
     864             :         }
     865             : 
     866             :         /* verify the fixed vectors at every alignment */
     867          24 :         for (unsigned i = 0; STRHASH64[i].data; ++i) {
     868         207 :                 for (unsigned alignment = 0; alignment < 8; ++alignment) {
     869         184 :                         memset(buffer[alignment], 0xa5, sizeof(buffer[alignment]));
     870         184 :                         memcpy(key[alignment], STRHASH64[i].data, strlen(STRHASH64[i].data) + 1);
     871         184 :                         if (tommy_strhash_u64(0x2f022773a766795dULL, key[alignment]) != STRHASH64[i].hash) {
     872             :                                 /* LCOV_EXCL_START */
     873             :                                 abort();
     874             :                                 /* LCOV_EXCL_STOP */
     875             :                         }
     876             :                 }
     877             :         }
     878             : 
     879             :         /* cover all tails, complete blocks and non-ASCII bytes with several seeds across all 8 alignments */
     880         257 :         for (unsigned len = 0; len < 256; ++len) {
     881        2304 :                 for (unsigned alignment = 0; alignment < 8; ++alignment) {
     882        2048 :                         memset(buffer[alignment], 0xa5, sizeof(buffer[alignment]));
     883      263168 :                         for (unsigned i = 0; i < len; ++i)
     884      261120 :                                 key[alignment][i] = (i * 73 + len * 11) % 255 + 1;
     885        2048 :                         key[alignment][len] = 0;
     886             :                 }
     887        1536 :                 for (unsigned s = 0; s < sizeof(seed) / sizeof(seed[0]); ++s) {
     888        1280 :                         tommy_uint64_t expected = tommy_strhash_u64(seed[s], key[0]);
     889       11520 :                         for (unsigned alignment = 0; alignment < 8; ++alignment) {
     890       10240 :                                 if (tommy_strhash_u64(seed[s], key[alignment]) != expected) {
     891             :                                         /* LCOV_EXCL_START */
     892             :                                         abort();
     893             :                                         /* LCOV_EXCL_STOP */
     894             :                                 }
     895       10240 :                                 if (tommy_hash_u64(seed[s], key[alignment], len) != expected) {
     896             :                                         /* LCOV_EXCL_START */
     897             :                                         abort();
     898             :                                         /* LCOV_EXCL_STOP */
     899             :                                 }
     900             :                         }
     901             :                 }
     902             :         }
     903           1 : }
     904             : 
     905           1 : void test_hash_u64(void)
     906             : {
     907           1 :         const tommy_uint64_t seed[] = { 0, 1, 0xffffffffULL, 0x100000000ULL, 0x8000000000000000ULL, 0xffffffffffffffffULL, 0x2f022773a766795dULL };
     908             :         const struct {
     909             :                 unsigned len;
     910             :                 tommy_uint64_t hash;
     911           1 :         } vectors[] = {
     912             :                 { 0, 0xb9137ac65dfaf9bfULL },
     913             :                 { 1, 0xbf3b7c7550b99646ULL },
     914             :                 { 2, 0x945a16d5ae03524cULL },
     915             :                 { 3, 0x5b0c3f1ea5b49a50ULL },
     916             :                 { 4, 0x2688f4de88f2c19fULL },
     917             :                 { 5, 0xf5f1b36aa25445c2ULL },
     918             :                 { 6, 0x13f8a0b05914ef0eULL },
     919             :                 { 7, 0xd4e267708faf38f1ULL },
     920             :                 { 8, 0x161458dea00f2480ULL },
     921             :                 { 9, 0x78e2ec94d9d41698ULL },
     922             :                 { 15, 0x8b83f602a1964f48ULL },
     923             :                 { 16, 0xcdcbf07f7d431f92ULL },
     924             :                 { 17, 0xc36da279f3fd3401ULL },
     925             :                 { 23, 0x2a08895959e7162cULL },
     926             :                 { 24, 0xddf5d83f61db21e4ULL },
     927             :                 { 25, 0x4e8a82cf44d8904aULL },
     928             :                 { 31, 0x1d3f4f97935e861aULL },
     929             :                 { 32, 0x35ba36557203e4f3ULL },
     930             :                 { 33, 0x81f60f609f5b3978ULL },
     931             :                 { 63, 0xe97a118e27cae6efULL },
     932             :                 { 64, 0x05e2c49e2f451ef1ULL },
     933             :                 { 65, 0x1cb57d9599c4431eULL },
     934             :                 { 255, 0xea3b82dfcc902757ULL }
     935             :         };
     936             :         unsigned char buffer[8][256 + 16];
     937             :         unsigned char* key[8];
     938             : 
     939           1 :         memset(buffer, 0xa5, sizeof(buffer));
     940           9 :         for (unsigned alignment = 0; alignment < 8; ++alignment) {
     941           8 :                 unsigned char* base = buffer[alignment];
     942           8 :                 key[alignment] = base + ((8 - ((tommy_uintptr_t)base & 7)) & 7) + alignment;
     943             :         }
     944             : 
     945             :         /* fixed vectors for the TommyDS a5hash derivative */
     946          24 :         for (unsigned v = 0; v < sizeof(vectors) / sizeof(vectors[0]); ++v) {
     947          23 :                 unsigned len = vectors[v].len;
     948         731 :                 for (unsigned i = 0; i < len; ++i)
     949         708 :                         key[0][i] = (i * 73 + len * 11) % 255 + 1;
     950          23 :                 key[0][len] = 0;
     951          23 :                 if (tommy_hash_u64(0x2f022773a766795dULL, key[0], len) != vectors[v].hash
     952          23 :                         || tommy_strhash_u64(0x2f022773a766795dULL, key[0]) != vectors[v].hash) {
     953             :                         /* LCOV_EXCL_START */
     954             :                         abort();
     955             :                         /* LCOV_EXCL_STOP */
     956             :                 }
     957             :         }
     958             : 
     959             :         /* a zero byte belongs to a sized buffer but terminates a string */
     960           1 :         if (tommy_hash_u64(0x2f022773a766795dULL, "", 1) != 0xddf38ca5d36c5e6dULL) {
     961             :                 /* LCOV_EXCL_START */
     962             :                 abort();
     963             :                 /* LCOV_EXCL_STOP */
     964             :         }
     965             : 
     966             :         /* cover every tail, seed width, non-ASCII bytes and all eight alignments */
     967         257 :         for (unsigned len = 0; len < 256; ++len) {
     968        2304 :                 for (unsigned alignment = 0; alignment < 8; ++alignment) {
     969      263168 :                         for (unsigned i = 0; i < len; ++i)
     970      261120 :                                 key[alignment][i] = (i * 73 + len * 11) % 255 + 1;
     971        2048 :                         key[alignment][len] = 0;
     972             :                 }
     973        2048 :                 for (unsigned s = 0; s < sizeof(seed) / sizeof(seed[0]); ++s) {
     974        1792 :                         tommy_uint64_t expected = tommy_hash_u64(seed[s], key[0], len);
     975       16128 :                         for (unsigned alignment = 0; alignment < 8; ++alignment) {
     976       14336 :                                 if (tommy_hash_u64(seed[s], key[alignment], len) != expected
     977       14336 :                                         || tommy_strhash_u64(seed[s], key[alignment]) != expected) {
     978             :                                         /* LCOV_EXCL_START */
     979             :                                         abort();
     980             :                                         /* LCOV_EXCL_STOP */
     981             :                                 }
     982             :                         }
     983             :                 }
     984             :         }
     985           1 : }
     986             : 
     987           1 : void test_hash(void)
     988             : {
     989             :         /* test approximation retry in rnd() */
     990           1 :         tommy_uint64_t saved_seed = SEED;
     991           1 :         SEED = 0x1865d6a7bc16fecbULL;
     992           1 :         if (rnd(7) >= 7) {
     993             :                 /* LCOV_EXCL_START */
     994             :                 abort();
     995             :                 /* LCOV_EXCL_STOP */
     996             :         }
     997           1 :         SEED = saved_seed;
     998             : 
     999           1 :         test_strhash_alignment();
    1000           1 :         test_strhash_u64_alignment();
    1001           1 :         test_hash_u64();
    1002             : 
    1003           1 :         if (tommy_hash_u32(0xa766795d, "", 0) != 0x5e8c3713) {
    1004             :                 /* LCOV_EXCL_START */
    1005             :                 abort();
    1006             :                 /* LCOV_EXCL_STOP */
    1007             :         }
    1008             : 
    1009           1 :         if (tommy_hash_u64(0x2f022773a766795dULL, "", 0) != 0xb9137ac65dfaf9bfULL) {
    1010             :                 /* LCOV_EXCL_START */
    1011             :                 abort();
    1012             :                 /* LCOV_EXCL_STOP */
    1013             :         }
    1014             : 
    1015          24 :         for (unsigned i = 0; HASH32[i].data; ++i) {
    1016          23 :                 if (tommy_hash_u32(0xa766795d, HASH32[i].data, HASH32[i].len) != HASH32[i].hash) {
    1017             :                         /* LCOV_EXCL_START */
    1018             :                         abort();
    1019             :                         /* LCOV_EXCL_STOP */
    1020             :                 }
    1021             :         }
    1022             : 
    1023          24 :         for (unsigned i = 0; STRHASH32[i].data; ++i) {
    1024          23 :                 if (tommy_strhash_u32(0xa766795d, STRHASH32[i].data) != STRHASH32[i].hash) {
    1025             :                         /* LCOV_EXCL_START */
    1026             :                         abort();
    1027             :                         /* LCOV_EXCL_STOP */
    1028             :                 }
    1029             :         }
    1030             : 
    1031          24 :         for (unsigned i = 0; HASH64[i].data; ++i) {
    1032          23 :                 if (tommy_hash_u64(0x2f022773a766795dULL, HASH64[i].data, HASH64[i].len) != HASH64[i].hash) {
    1033             :                         /* LCOV_EXCL_START */
    1034             :                         abort();
    1035             :                         /* LCOV_EXCL_STOP */
    1036             :                 }
    1037             :         }
    1038             : 
    1039          24 :         for (unsigned i = 0; STRHASH64[i].data; ++i) {
    1040          23 :                 if (tommy_strhash_u64(0x2f022773a766795dULL, STRHASH64[i].data) != STRHASH64[i].hash) {
    1041             :                         /* LCOV_EXCL_START */
    1042             :                         abort();
    1043             :                         /* LCOV_EXCL_STOP */
    1044             :                 }
    1045             :         }
    1046             : 
    1047          66 :         for (unsigned i = 0; INTHASH32[i].value || !i; ++i) {
    1048          65 :                 if (tommy_inthash_u32(INTHASH32[i].value) != INTHASH32[i].hash) {
    1049             :                         /* LCOV_EXCL_START */
    1050             :                         abort();
    1051             :                         /* LCOV_EXCL_STOP */
    1052             :                 }
    1053             :         }
    1054             : 
    1055          98 :         for (unsigned i = 0; INTHASH64[i].value || !i; ++i) {
    1056          97 :                 if (tommy_inthash_u64(INTHASH64[i].value) != INTHASH64[i].hash) {
    1057             :                         /* LCOV_EXCL_START */
    1058             :                         abort();
    1059             :                         /* LCOV_EXCL_STOP */
    1060             :                 }
    1061             :         }
    1062             : 
    1063          13 :         for (unsigned i = 0; MULHASH32[i].value || !i; ++i) {
    1064          12 :                 if (tommy_mulhash_u32(MULHASH32[i].value) != MULHASH32[i].hash) {
    1065             :                         /* LCOV_EXCL_START */
    1066             :                         abort();
    1067             :                         /* LCOV_EXCL_STOP */
    1068             :                 }
    1069             :         }
    1070             : 
    1071          13 :         for (unsigned i = 0; MULHASH64[i].value || !i; ++i) {
    1072          12 :                 if (tommy_mulhash_u64(MULHASH64[i].value) != MULHASH64[i].hash) {
    1073             :                         /* LCOV_EXCL_START */
    1074             :                         abort();
    1075             :                         /* LCOV_EXCL_STOP */
    1076             :                 }
    1077             :         }
    1078             : 
    1079             :         enum { count = 63 };
    1080             : 
    1081             :         /* prepare aligned strings before timing, using the same inputs for all hashes */
    1082             :         union {
    1083             :                 tommy_uint64_t align;
    1084             :                 char data[264];
    1085             :         } storage[count + 1];
    1086           1 :         tommy_uint32_t hash32 = 0;
    1087           1 :         tommy_uint64_t hash64 = 0;
    1088             : 
    1089           1 :         memset(storage, 0xAA, sizeof(storage));
    1090          65 :         for (unsigned length = 0; length <= count; ++length)
    1091          64 :                 storage[length].data[length] = 0;
    1092             : 
    1093       35215 :         SPEED_START(0) {
    1094     2279615 :                 for (unsigned length = 0; length <= count; ++length) {
    1095     2244544 :                         hash32 = tommy_hash_u32(hash32, storage[length].data, length);
    1096             :                 }
    1097         144 :         } SPEED_STOP
    1098           1 :         printf("%25s %8.2f [ns], %8u [measure]\n", "hash_u32", the_time / (count + 1), the_measure);
    1099             : 
    1100       46780 :         SPEED_START(0) {
    1101     3028415 :                 for (unsigned length = 0; length <= count; ++length) {
    1102     2981824 :                         hash64 = tommy_hash_u64(hash64, storage[length].data, length);
    1103             :                 }
    1104         189 :         } SPEED_STOP
    1105           1 :         printf("%25s %8.2f [ns], %8u [measure]\n", "hash_u64", the_time / (count + 1), the_measure);
    1106             : 
    1107       32131 :         SPEED_START(0) {
    1108     2079935 :                 for (unsigned length = 0; length <= count; ++length) {
    1109     2047936 :                         hash32 = tommy_strhash_u32(hash32, storage[length].data);
    1110             :                 }
    1111         132 :         } SPEED_STOP
    1112           1 :         printf("%25s %8.2f [ns], %8u [measure]\n", "strhash_u32", the_time / (count + 1), the_measure);
    1113             : 
    1114       39971 :         SPEED_START(0) {
    1115     2587455 :                 for (unsigned length = 0; length <= count; ++length) {
    1116     2547648 :                         hash64 = tommy_strhash_u64(hash64, storage[length].data);
    1117             :                 }
    1118         164 :         } SPEED_STOP
    1119           1 :         printf("%25s %8.2f [ns], %8u [measure]\n", "strhash_u64", the_time / (count + 1), the_measure);
    1120             : 
    1121             :         /*
    1122             :          * Measure dependent integer hashes like the seeded byte hashes above.
    1123             :          * The increment prevents adjacent XOR finalizers from cancelling when
    1124             :          * inlined and avoids repeatedly hashing the zero fixed point.
    1125             :          */
    1126           1 :         tommy_uint32_t inthash32 = 1;
    1127      188608 :         SPEED_START(0) {
    1128    12246975 :                 for (unsigned i = 0; i <= count; ++i)
    1129    12058560 :                         inthash32 = tommy_inthash_u32(inthash32 + 1);
    1130         193 :         } SPEED_STOP
    1131           1 :         printf("%25s %8.2f [ns], %8u [measure]\n", "inthash_u32", the_time / (count + 1), the_measure);
    1132             : 
    1133           1 :         tommy_uint32_t mulhash32 = 1;
    1134      251133 :         SPEED_START(0) {
    1135    16307135 :                 for (unsigned i = 0; i <= count; ++i)
    1136    16056256 :                         mulhash32 = tommy_mulhash_u32(mulhash32 + 1);
    1137         254 :         } SPEED_STOP
    1138           1 :         printf("%25s %8.2f [ns], %8u [measure]\n", "mulhash_u32", the_time / (count + 1), the_measure);
    1139             : 
    1140           1 :         tommy_uint64_t inthash64 = 1;
    1141      181433 :         SPEED_START(0) {
    1142    11781055 :                 for (unsigned i = 0; i <= count; ++i)
    1143    11599808 :                         inthash64 = tommy_inthash_u64(inthash64 + 1);
    1144         186 :         } SPEED_STOP
    1145           1 :         printf("%25s %8.2f [ns], %8u [measure]\n", "inthash_u64", the_time / (count + 1), the_measure);
    1146             : 
    1147           1 :         tommy_uint64_t mulhash64 = 1;
    1148      248058 :         SPEED_START(0) {
    1149    16107455 :                 for (unsigned i = 0; i <= count; ++i)
    1150    15859648 :                         mulhash64 = tommy_mulhash_u64(mulhash64 + 1);
    1151         251 :         } SPEED_STOP
    1152           1 :         printf("%25s %8.2f [ns], %8u [measure]\n", "mulhash_u64", the_time / (count + 1), the_measure);
    1153             : 
    1154             :         /* keep all measured hash calls observable */
    1155           1 :         the_hash = hash32;
    1156           1 :         the_hash = hash64;
    1157           1 :         the_hash = inthash32;
    1158           1 :         the_hash = mulhash32;
    1159           1 :         the_hash = inthash64;
    1160           1 :         the_hash = mulhash64;
    1161           1 : }
    1162             : 
    1163           1 : void test_alloc_clear(void)
    1164             : {
    1165           1 :         const tommy_size_t cases[][2] = {
    1166             :                 { sizeof(void*) + 1, sizeof(void*) },
    1167             :                 { 64, 64 },
    1168             :                 { 128000, 64 }
    1169             :         };
    1170             :         void* ptr[128];
    1171             : 
    1172           4 :         for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); ++i) {
    1173             :                 tommy_allocator alloc;
    1174           3 :                 tommy_allocator_init(&alloc, cases[i][0], cases[i][1]);
    1175           3 :                 tommy_size_t block_size = alloc.block_size;
    1176           3 :                 tommy_size_t align_size = alloc.align_size;
    1177             : 
    1178          12 :                 for (unsigned cycle = 0; cycle < 3; ++cycle) {
    1179             :                         /* clear empty and populated allocators, then reuse without initialization */
    1180           9 :                         tommy_allocator_clear(&alloc);
    1181           9 :                         tommy_allocator_clear(&alloc);
    1182           9 :                         if (alloc.free_block != 0 || alloc.used_segment != 0 || alloc.count != 0
    1183           9 :                                 || tommy_allocator_memory_usage(&alloc) != 0
    1184           9 :                                 || alloc.block_size != block_size || alloc.align_size != align_size) {
    1185             :                                 /* LCOV_EXCL_START */
    1186             :                                 abort();
    1187             :                                 /* LCOV_EXCL_STOP */
    1188             :                         }
    1189             : 
    1190        1161 :                         for (unsigned j = 0; j < sizeof(ptr) / sizeof(ptr[0]); ++j) {
    1191        1152 :                                 ptr[j] = tommy_allocator_alloc(&alloc);
    1192        1152 :                                 if ((tommy_uintptr_t)ptr[j] % align_size != 0) {
    1193             :                                         /* LCOV_EXCL_START */
    1194             :                                         abort();
    1195             :                                         /* LCOV_EXCL_STOP */
    1196             :                                 }
    1197        1152 :                                 memset(ptr[j], 0xAA, block_size);
    1198             :                         }
    1199             : 
    1200             :                         /* retain both active and freed blocks across multiple segments */
    1201         585 :                         for (unsigned j = 0; j < sizeof(ptr) / sizeof(ptr[0]) / 2; ++j)
    1202         576 :                                 tommy_allocator_free(&alloc, ptr[j]);
    1203           9 :                         if (tommy_allocator_memory_usage(&alloc) != sizeof(ptr) / sizeof(ptr[0]) / 2 * block_size) {
    1204             :                                 /* LCOV_EXCL_START */
    1205             :                                 abort();
    1206             :                                 /* LCOV_EXCL_STOP */
    1207             :                         }
    1208             :                 }
    1209             : 
    1210           3 :                 tommy_allocator_clear(&alloc);
    1211           3 :                 tommy_allocator_done(&alloc);
    1212             :         }
    1213           1 : }
    1214             : 
    1215           1 : void test_alloc(void)
    1216             : {
    1217           1 :         const unsigned size = 10 * TOMMY_SIZE;
    1218             :         tommy_allocator alloc;
    1219           1 :         void** PTR = malloc(size * sizeof(void*));
    1220             : 
    1221             :         /* ensure at least pointer alignment */
    1222           1 :         tommy_allocator_init(&alloc, sizeof(void*), 1);
    1223           1 :         if (alloc.align_size < sizeof(void*)) {
    1224             :                 /* LCOV_EXCL_START */
    1225             :                 abort();
    1226             :                 /* LCOV_EXCL_STOP */
    1227             :         }
    1228           1 :         tommy_allocator_done(&alloc);
    1229             : 
    1230             :         /* ensure correct alignment */
    1231           1 :         tommy_allocator_init(&alloc, sizeof(void*) - 1, sizeof(void*));
    1232           1 :         if (alloc.block_size != sizeof(void*)) {
    1233             :                 /* LCOV_EXCL_START */
    1234             :                 abort();
    1235             :                 /* LCOV_EXCL_STOP */
    1236             :         }
    1237           1 :         tommy_allocator_done(&alloc);
    1238             : 
    1239             :         /* round a non-multiple block size up to the next alignment boundary */
    1240           1 :         tommy_allocator_init(&alloc, sizeof(void*) + 1, sizeof(void*));
    1241           1 :         if (alloc.block_size != 2 * sizeof(void*)) {
    1242             :                 /* LCOV_EXCL_START */
    1243             :                 abort();
    1244             :                 /* LCOV_EXCL_STOP */
    1245             :         }
    1246           1 :         tommy_allocator_done(&alloc);
    1247             : 
    1248             :         /* check every generated block against the segment bounds */
    1249           1 :         const tommy_size_t cases[][2] = {
    1250             :                 { sizeof(void*), 1 },
    1251             :                 { sizeof(void*) - 1, sizeof(void*) },
    1252             :                 { sizeof(void*) + 1, sizeof(void*) },
    1253             :                 { 32, sizeof(void*) },
    1254             :                 { 64, 64 },
    1255             :                 { 4096 - 64 - sizeof(tommy_allocator_entry), sizeof(void*) },
    1256             :                 { 4096 - 64, sizeof(void*) },
    1257             :                 { 128000, 64 },
    1258             :                 { 64, 8192 }
    1259             :         };
    1260          10 :         for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); ++i) {
    1261           9 :                 tommy_allocator_init(&alloc, cases[i][0], cases[i][1]);
    1262           9 :                 void* ptr = tommy_allocator_alloc(&alloc);
    1263           9 :                 tommy_allocator_free(&alloc, ptr);
    1264             : 
    1265             :                 /* match the segment allocation size in tommy_allocator_alloc() */
    1266           9 :                 tommy_size_t segment_size = 4096 - 64;
    1267           9 :                 if (segment_size < sizeof(tommy_allocator_entry) + alloc.align_size + alloc.block_size)
    1268           4 :                         segment_size = sizeof(tommy_allocator_entry) + alloc.align_size + alloc.block_size;
    1269             : 
    1270        1457 :                 for (tommy_allocator_entry* block = alloc.free_block; block; block = block->next) {
    1271        1448 :                         tommy_uintptr_t address = (tommy_uintptr_t)block;
    1272        1448 :                         tommy_uintptr_t offset = address - (tommy_uintptr_t)alloc.used_segment;
    1273        1448 :                         if (offset < sizeof(tommy_allocator_entry) || offset > segment_size - alloc.block_size
    1274        1448 :                                 || address % alloc.align_size != 0) {
    1275             :                                 /* LCOV_EXCL_START */
    1276             :                                 abort();
    1277             :                                 /* LCOV_EXCL_STOP */
    1278             :                         }
    1279             :                 }
    1280           9 :                 tommy_allocator_done(&alloc);
    1281             :         }
    1282             : 
    1283             :         /* check big blocks */
    1284           1 :         tommy_allocator_init(&alloc, 128000, 64);
    1285           1 :         if (tommy_allocator_alloc(&alloc) == 0) {
    1286             :                 /* LCOV_EXCL_START */
    1287             :                 abort();
    1288             :                 /* LCOV_EXCL_STOP */
    1289             :         }
    1290           1 :         tommy_allocator_done(&alloc);
    1291             : 
    1292           1 :         tommy_allocator_init(&alloc, 64, 64);
    1293             : 
    1294          14 :         SPEED_START("alloc") {
    1295           7 :                 SPEED_UNTIMED(
    1296             :                         tommy_allocator_done(&alloc);
    1297             :                         tommy_allocator_init(&alloc, 64, 64);
    1298             :                 );
    1299     7000007 :                 for (unsigned i = 0; i < size; ++i) {
    1300     7000000 :                         PTR[i] = tommy_allocator_alloc(&alloc);
    1301             :                 }
    1302           7 :         } SPEED_STOP
    1303             : 
    1304          30 :         SPEED_START("free")
    1305             :         {
    1306    15000015 :                 SPEED_UNTIMED(
    1307             :                         tommy_allocator_done(&alloc);
    1308             :                         tommy_allocator_init(&alloc, 64, 64);
    1309             :                         for (unsigned i = 0; i < size; ++i)
    1310             :                                 PTR[i] = tommy_allocator_alloc(&alloc);
    1311             :                 );
    1312    15000015 :                 for (unsigned i = 0; i < size; ++i) {
    1313    15000000 :                         tommy_allocator_free(&alloc, PTR[i]);
    1314             :                 }
    1315             :         }
    1316          15 :         SPEED_STOP
    1317             : 
    1318           1 :         tommy_allocator_done(&alloc);
    1319             : 
    1320           1 :         free(PTR);
    1321           1 : }
    1322             : 
    1323           5 : void test_list_order(tommy_node* list)
    1324             : {
    1325             :         tommy_node* node;
    1326             : 
    1327           5 :         node = list;
    1328      500005 :         while (node) {
    1329      500000 :                 if (node->next) {
    1330      499995 :                         const struct object* a = node->data;
    1331      499995 :                         const struct object* b = node->next->data;
    1332             :                         /* check order */
    1333      499995 :                         if (a->value > b->value) {
    1334             :                                 /* LCOV_EXCL_START */
    1335             :                                 abort();
    1336             :                                 /* LCOV_EXCL_STOP */
    1337             :                         }
    1338             :                         /* check order for stable sort */
    1339      499995 :                         if (a->value == b->value && a > b) {
    1340             :                                 /* LCOV_EXCL_START */
    1341             :                                 abort();
    1342             :                                 /* LCOV_EXCL_STOP */
    1343             :                         }
    1344             :                 }
    1345      500000 :                 node = node->next;
    1346             :         }
    1347           5 : }
    1348             : 
    1349             : /* check exact node identity, both traversal directions and list invariants. */
    1350       31422 : void test_list_sequence(tommy_list* list, struct object* obj, const unsigned* order, unsigned size)
    1351             : {
    1352       31422 :         tommy_node* node = tommy_list_head(list);
    1353       31422 :         tommy_node* prev = tommy_list_tail(list);
    1354             : 
    1355       31422 :         if (tommy_list_empty(list) != (size == 0)
    1356       31422 :                 || tommy_list_tail(list) != (size ? &obj[order[size - 1]].node : 0)) {
    1357             :                 /* LCOV_EXCL_START */
    1358             :                 abort();
    1359             :                 /* LCOV_EXCL_STOP */
    1360             :         }
    1361             : 
    1362      291486 :         for (unsigned i = 0; i < size; ++i) {
    1363      260064 :                 if (node != &obj[order[i]].node || node->prev != prev
    1364      260064 :                         || node->data != &obj[order[i]] || node->index != order[i] + 1) {
    1365             :                         /* LCOV_EXCL_START */
    1366             :                         abort();
    1367             :                         /* LCOV_EXCL_STOP */
    1368             :                 }
    1369      260064 :                 prev = node;
    1370      260064 :                 node = node->next;
    1371             :         }
    1372             : 
    1373       31422 :         if (node != 0 || tommy_list_count(list) != size) {
    1374             :                 /* LCOV_EXCL_START */
    1375             :                 abort();
    1376             :                 /* LCOV_EXCL_STOP */
    1377             :         }
    1378             : 
    1379       31422 :         node = tommy_list_tail(list);
    1380      291486 :         for (unsigned i = size; i > 0; --i) {
    1381      260064 :                 if (node != &obj[order[i - 1]].node) {
    1382             :                         /* LCOV_EXCL_START */
    1383             :                         abort();
    1384             :                         /* LCOV_EXCL_STOP */
    1385             :                 }
    1386      260064 :                 node = tommy_list_prev(list, node);
    1387             :         }
    1388             : 
    1389       31422 :         if (node != 0) {
    1390             :                 /* LCOV_EXCL_START */
    1391             :                 abort();
    1392             :                 /* LCOV_EXCL_STOP */
    1393             :         }
    1394       31422 : }
    1395             : 
    1396       32663 : void test_list_build(tommy_list* list, struct object* obj, unsigned start, unsigned size)
    1397             : {
    1398       32663 :         tommy_list_init(list);
    1399    20092029 :         for (unsigned i = start; i < start + size; ++i) {
    1400    20059366 :                 obj[i].node.index = i + 1;
    1401    20059366 :                 tommy_list_insert_tail(list, &obj[i].node, &obj[i]);
    1402             :         }
    1403       32663 : }
    1404             : 
    1405           1 : void test_list_insert_head(void)
    1406             : {
    1407             :         struct object obj[8];
    1408           1 :         unsigned order[8] = { 0 };
    1409             :         tommy_list list;
    1410             : 
    1411           6 :         for (unsigned n = 0; n <= 4; ++n) {
    1412           5 :                 tommy_list_init(&list);
    1413          15 :                 for (unsigned i = 0; i < n; ++i) {
    1414          10 :                         obj[i].node.index = i + 1;
    1415          10 :                         tommy_list_insert_head(&list, &obj[i].node, &obj[i]);
    1416          10 :                         order[n - 1 - i] = i;
    1417             :                 }
    1418           5 :                 test_list_sequence(&list, obj, order, n);
    1419             :         }
    1420           1 : }
    1421             : 
    1422           1 : void test_list_merge(void)
    1423             : {
    1424             :         struct object obj[8];
    1425             :         unsigned order[8];
    1426           1 :         const unsigned combinations[] = { 1, 3, 9, 27, 81 };
    1427             :         tommy_list first;
    1428             :         tommy_list second;
    1429             : 
    1430             :         /* All sorted sequences of up to four nodes with keys in [0, 2].
    1431             :          * Object identity distinguishes duplicates, including duplicates across lists.
    1432             :          * The oracle groups by key, retaining the original first-then-second order.
    1433             :          */
    1434           6 :         for (unsigned n = 0; n <= 4; ++n) {
    1435          30 :                 for (unsigned m = 0; m <= 4; ++m) {
    1436         630 :                         for (unsigned a = 0; a < combinations[n]; ++a) {
    1437       15246 :                                 for (unsigned b = 0; b < combinations[m]; ++b) {
    1438       14641 :                                         tommy_bool_t sorted = 1;
    1439       14641 :                                         unsigned code = a;
    1440       66187 :                                         for (unsigned i = 0; i < n; ++i) {
    1441       51546 :                                                 obj[i].value = code % 3;
    1442       51546 :                                                 code /= 3;
    1443       51546 :                                                 if (i && obj[i - 1].value > obj[i].value)
    1444       12342 :                                                         sorted = 0;
    1445             :                                         }
    1446       14641 :                                         code = b;
    1447       66187 :                                         for (unsigned i = n; i < n + m; ++i) {
    1448       51546 :                                                 obj[i].value = code % 3;
    1449       51546 :                                                 code /= 3;
    1450       51546 :                                                 if (i != n && obj[i - 1].value > obj[i].value)
    1451       12342 :                                                         sorted = 0;
    1452             :                                         }
    1453       14641 :                                         if (!sorted)
    1454       13416 :                                                 continue;
    1455             : 
    1456        1225 :                                         test_list_build(&first, obj, 0, n);
    1457        1225 :                                         test_list_build(&second, obj, n, m);
    1458        1225 :                                         unsigned size = 0;
    1459        4900 :                                         for (unsigned key = 0; key < 3; ++key)
    1460       25725 :                                                 for (unsigned i = 0; i < n + m; ++i)
    1461       22050 :                                                         if (obj[i].value == (int)key)
    1462        7350 :                                                                 order[size++] = i;
    1463             : 
    1464        1225 :                                         tommy_list_merge(&first, &second, compare);
    1465        1225 :                                         test_list_sequence(&first, obj, order, size);
    1466             :                                 }
    1467             :                         }
    1468             :                 }
    1469             :         }
    1470           1 : }
    1471             : 
    1472           1 : void test_list_sort(void)
    1473             : {
    1474             :         struct object obj[9];
    1475             :         unsigned order[9];
    1476           1 :         const unsigned combinations[] = { 1, 3, 9, 27, 81, 243, 729, 2187, 6561, 19683 };
    1477             :         tommy_list list;
    1478             : 
    1479             :         /* exhaust small inputs, checking stability and links across carry propagation and final merges. */
    1480          11 :         for (unsigned n = 0; n <= 9; ++n) {
    1481       29534 :                 for (unsigned a = 0; a < combinations[n]; ++a) {
    1482       29524 :                         unsigned code = a;
    1483      280483 :                         for (unsigned i = 0; i < n; ++i) {
    1484      250959 :                                 obj[i].value = code % 3;
    1485      250959 :                                 code /= 3;
    1486             :                         }
    1487       29524 :                         test_list_build(&list, obj, 0, n);
    1488       29524 :                         unsigned size = 0;
    1489      118096 :                         for (unsigned key = 0; key < 3; ++key)
    1490      841449 :                                 for (unsigned i = 0; i < n; ++i)
    1491      752877 :                                         if (obj[i].value == (int)key)
    1492      250959 :                                                 order[size++] = i;
    1493             : 
    1494       29524 :                         compare_counter = 0;
    1495       29524 :                         tommy_list_sort(&list, compare);
    1496             :                         /* a singleton pair needs one comparison in either order, including equal keys. */
    1497       29524 :                         if (n <= 2 && compare_counter != (n == 2)) {
    1498             :                                 /* LCOV_EXCL_START */
    1499             :                                 abort();
    1500             :                                 /* LCOV_EXCL_STOP */
    1501             :                         }
    1502       29524 :                         test_list_sequence(&list, obj, order, size);
    1503             :                 }
    1504             :         }
    1505           1 : }
    1506             : 
    1507           1 : void test_list_split(void)
    1508             : {
    1509             :         struct object obj[6];
    1510           1 :         const unsigned order[] = { 0, 1, 2, 3, 4 };
    1511           1 :         const unsigned previous_order[] = { 5 };
    1512             :         tommy_list first;
    1513             :         tommy_list second;
    1514             : 
    1515             :         /* split at every position, including the head, tail and null node. */
    1516           7 :         for (unsigned n = 0; n <= 5; ++n) {
    1517          27 :                 for (unsigned pos = 0; pos <= n; ++pos) {
    1518          21 :                         test_list_build(&first, obj, 0, n);
    1519             :                         /* the previous destination value is overwritten, but its nodes are untouched. */
    1520          21 :                         test_list_build(&second, obj, 5, 1);
    1521          21 :                         tommy_list previous = second;
    1522          21 :                         tommy_list_split(&first, pos == n ? 0 : &obj[pos].node, &second);
    1523          21 :                         test_list_sequence(&first, obj, order, pos);
    1524          21 :                         test_list_sequence(&second, obj, order + pos, n - pos);
    1525          21 :                         test_list_sequence(&previous, obj, previous_order, 1);
    1526             : 
    1527             :                         /* rejoin the lists and check that no node was lost. */
    1528          21 :                         tommy_list_concat(&first, &second);
    1529          21 :                         test_list_sequence(&first, obj, order, n);
    1530             :                 }
    1531             : 
    1532             :                 /* an uninitialized output is valid, including when the input node is 0. */
    1533             :                 {
    1534             :                         tommy_list output;
    1535           6 :                         test_list_build(&first, obj, 0, n);
    1536           6 :                         tommy_list_split(&first, tommy_list_head(&first), &output);
    1537           6 :                         test_list_sequence(&first, obj, 0, 0);
    1538           6 :                         test_list_sequence(&output, obj, order, n);
    1539             :                 }
    1540             :         }
    1541           1 : }
    1542             : 
    1543           1 : void test_list_splice(void)
    1544             : {
    1545             :         struct object obj[8];
    1546             :         unsigned order[8];
    1547             :         tommy_list first;
    1548             :         tommy_list second;
    1549             : 
    1550             :         /* both splice operations, every valid reference and empty/singleton cases. */
    1551           5 :         for (unsigned n = 1; n <= 4; ++n) {
    1552          20 :                 for (unsigned m = 0; m <= 3; ++m) {
    1553          48 :                         for (unsigned operation = 0; operation < 2; ++operation) {
    1554         112 :                                 for (unsigned pos = 0; pos < n; ++pos) {
    1555          80 :                                         test_list_build(&first, obj, 0, n);
    1556          80 :                                         test_list_build(&second, obj, n, m);
    1557          80 :                                         unsigned insert = operation == 0 ? pos : pos + 1;
    1558          80 :                                         unsigned size = 0;
    1559         200 :                                         for (unsigned i = 0; i < insert; ++i)
    1560         120 :                                                 order[size++] = i;
    1561         200 :                                         for (unsigned i = n; i < n + m; ++i)
    1562         120 :                                                 order[size++] = i;
    1563         200 :                                         for (unsigned i = insert; i < n; ++i)
    1564         120 :                                                 order[size++] = i;
    1565             : 
    1566          80 :                                         switch (operation) {
    1567          40 :                                         case 0 :
    1568          40 :                                                 tommy_list_splice_before(&first, &obj[pos].node, &second);
    1569          40 :                                                 break;
    1570          40 :                                         case 1 :
    1571          40 :                                                 tommy_list_splice_after(&first, &obj[pos].node, &second);
    1572          40 :                                                 break;
    1573             :                                         }
    1574          80 :                                         test_list_sequence(&first, obj, order, size);
    1575             : 
    1576             :                                         /* the source can be reused after reinitialization without affecting the destination. */
    1577          80 :                                         unsigned reuse[1] = { n + m };
    1578          80 :                                         test_list_build(&second, obj, reuse[0], 1);
    1579          80 :                                         test_list_sequence(&second, obj, reuse, 1);
    1580          80 :                                         test_list_sequence(&first, obj, order, size);
    1581             :                                 }
    1582             :                         }
    1583             :                 }
    1584             :         }
    1585           1 : }
    1586             : 
    1587           1 : void test_list_concat(void)
    1588             : {
    1589             :         struct object obj[8];
    1590           1 :         unsigned order[8] = { 0 };
    1591             :         tommy_list first;
    1592             :         tommy_list second;
    1593             : 
    1594           6 :         for (unsigned n = 0; n <= 4; ++n) {
    1595          25 :                 for (unsigned m = 0; m <= 3; ++m) {
    1596          20 :                         test_list_build(&first, obj, 0, n);
    1597          20 :                         test_list_build(&second, obj, n, m);
    1598          20 :                         unsigned size = 0;
    1599          90 :                         for (unsigned i = 0; i < n + m; ++i)
    1600          70 :                                 order[size++] = i;
    1601          20 :                         tommy_list_concat(&first, &second);
    1602          20 :                         test_list_sequence(&first, obj, order, size);
    1603             :                 }
    1604             :         }
    1605           1 : }
    1606             : 
    1607           1 : void test_list_prepend(void)
    1608             : {
    1609             :         struct object obj[8];
    1610           1 :         unsigned order[8] = { 0 };
    1611             :         tommy_list first;
    1612             :         tommy_list second;
    1613             : 
    1614           6 :         for (unsigned n = 0; n <= 4; ++n) {
    1615          25 :                 for (unsigned m = 0; m <= 3; ++m) {
    1616          20 :                         test_list_build(&first, obj, 0, n);
    1617          20 :                         test_list_build(&second, obj, n, m);
    1618          20 :                         unsigned size = 0;
    1619          50 :                         for (unsigned i = n; i < n + m; ++i)
    1620          30 :                                 order[size++] = i;
    1621          60 :                         for (unsigned i = 0; i < n; ++i)
    1622          40 :                                 order[size++] = i;
    1623          20 :                         tommy_list_prepend(&first, &second);
    1624          20 :                         test_list_sequence(&first, obj, order, size);
    1625             :                 }
    1626             :         }
    1627           1 : }
    1628             : 
    1629           1 : void test_list_reverse_move(void)
    1630             : {
    1631             :         struct object obj[6];
    1632           1 :         const unsigned forward[] = { 0, 1, 2, 3, 4, 5 };
    1633           1 :         unsigned order[6] = { 0 };
    1634             :         tommy_list list;
    1635             : 
    1636           8 :         for (unsigned n = 0; n <= 6; ++n) {
    1637           7 :                 test_list_build(&list, obj, 0, n);
    1638          28 :                 for (unsigned i = 0; i < n; ++i)
    1639          21 :                         order[i] = n - 1 - i;
    1640           7 :                 tommy_list_reverse(&list);
    1641           7 :                 test_list_sequence(&list, obj, order, n);
    1642           7 :                 tommy_list_reverse(&list);
    1643           7 :                 test_list_sequence(&list, obj, forward, n);
    1644             : 
    1645             :                 /* move every node to either end, including nodes already at that end. */
    1646          28 :                 for (unsigned pos = 0; pos < n; ++pos) {
    1647          63 :                         for (unsigned tail = 0; tail < 2; ++tail) {
    1648          42 :                                 test_list_build(&list, obj, 0, n);
    1649          42 :                                 unsigned size = 0;
    1650          42 :                                 if (!tail)
    1651          21 :                                         order[size++] = pos;
    1652         224 :                                 for (unsigned i = 0; i < n; ++i)
    1653         182 :                                         if (i != pos)
    1654         140 :                                                 order[size++] = i;
    1655          42 :                                 if (tail)
    1656          21 :                                         order[size++] = pos;
    1657             : 
    1658          42 :                                 if (tail)
    1659          21 :                                         tommy_list_move_tail(&list, &obj[pos].node);
    1660             :                                 else
    1661          21 :                                         tommy_list_move_head(&list, &obj[pos].node);
    1662          42 :                                 test_list_sequence(&list, obj, order, n);
    1663             :                         }
    1664             :                 }
    1665             :         }
    1666             : 
    1667             :         /* moving a node with null data preserves it. */
    1668           1 :         test_list_build(&list, obj, 0, 3);
    1669           1 :         obj[1].node.data = 0;
    1670           1 :         tommy_list_move_head(&list, &obj[1].node);
    1671           1 :         tommy_list_move_tail(&list, &obj[1].node);
    1672           1 :         if (tommy_list_tail(&list) != &obj[1].node || obj[1].node.data != 0 || obj[1].node.index != 2) {
    1673             :                 /* LCOV_EXCL_START */
    1674             :                 abort();
    1675             :                 /* LCOV_EXCL_STOP */
    1676             :         }
    1677           1 : }
    1678             : 
    1679          65 : void test_list_removed(tommy_node* node, tommy_node* saved)
    1680             : {
    1681          65 :         if (node->next != saved->next || node->prev != saved->prev
    1682          65 :                 || node->data != saved->data || node->index != saved->index) {
    1683             :                 /* LCOV_EXCL_START */
    1684             :                 abort();
    1685             :                 /* LCOV_EXCL_STOP */
    1686             :         }
    1687          65 : }
    1688             : 
    1689           1 : void test_list_remove_ends(void)
    1690             : {
    1691             :         struct object obj[6];
    1692           1 :         const unsigned order[] = { 0, 1, 2, 3, 4, 5 };
    1693             :         tommy_list list;
    1694             : 
    1695             :         /* drain from the head, from the tail, and alternating between both. */
    1696           8 :         for (unsigned n = 0; n <= 6; ++n) {
    1697          28 :                 for (unsigned mode = 0; mode < 3; ++mode) {
    1698          21 :                         test_list_build(&list, obj, 0, n);
    1699          21 :                         unsigned head = 0;
    1700          21 :                         unsigned tail = n;
    1701          84 :                         while (head != tail) {
    1702          63 :                                 tommy_bool_t remove_tail = mode == 1 || (mode == 2 && (tail - head) % 2 == 0);
    1703          63 :                                 tommy_node* node = &obj[remove_tail ? tail - 1 : head].node;
    1704          63 :                                 tommy_node saved = *node;
    1705             :                                 void* data;
    1706          63 :                                 if (remove_tail) {
    1707          30 :                                         data = tommy_list_remove_tail(&list);
    1708          30 :                                         --tail;
    1709             :                                 } else {
    1710          33 :                                         data = tommy_list_remove_head(&list);
    1711          33 :                                         ++head;
    1712             :                                 }
    1713          63 :                                 if (data != saved.data) {
    1714             :                                         /* LCOV_EXCL_START */
    1715             :                                         abort();
    1716             :                                         /* LCOV_EXCL_STOP */
    1717             :                                 }
    1718          63 :                                 test_list_removed(node, &saved);
    1719          63 :                                 test_list_sequence(&list, obj, order + head, tail - head);
    1720             :                         }
    1721          21 :                         if (tommy_list_remove_head(&list) != 0 || tommy_list_remove_tail(&list) != 0) {
    1722             :                                 /* LCOV_EXCL_START */
    1723             :                                 abort();
    1724             :                                 /* LCOV_EXCL_STOP */
    1725             :                         }
    1726          21 :                         test_list_sequence(&list, obj, 0, 0);
    1727             :                 }
    1728             :         }
    1729             : 
    1730             :         /* null data still removes the node, despite returning 0. */
    1731           3 :         for (unsigned mode = 0; mode < 2; ++mode) {
    1732           2 :                 test_list_build(&list, obj, 0, 1);
    1733           2 :                 obj[0].node.data = 0;
    1734           2 :                 tommy_node saved = obj[0].node;
    1735           2 :                 void* data = mode ? tommy_list_remove_tail(&list) : tommy_list_remove_head(&list);
    1736           2 :                 if (data != 0) {
    1737             :                         /* LCOV_EXCL_START */
    1738             :                         abort();
    1739             :                         /* LCOV_EXCL_STOP */
    1740             :                 }
    1741           2 :                 test_list_removed(&obj[0].node, &saved);
    1742           2 :                 test_list_sequence(&list, obj, 0, 0);
    1743             :         }
    1744           1 : }
    1745             : 
    1746           1 : void test_list_swap(void)
    1747             : {
    1748             :         struct object obj[8];
    1749           1 :         const unsigned order[] = { 0, 1, 2, 3, 4, 5, 6, 7 };
    1750             :         tommy_list first;
    1751             :         tommy_list second;
    1752             : 
    1753           6 :         for (unsigned n = 0; n <= 4; ++n) {
    1754          30 :                 for (unsigned m = 0; m <= 4; ++m) {
    1755          25 :                         test_list_build(&first, obj, 0, n);
    1756          25 :                         test_list_build(&second, obj, n, m);
    1757          25 :                         tommy_list_swap(&first, &second);
    1758          25 :                         test_list_sequence(&first, obj, order + n, m);
    1759          25 :                         test_list_sequence(&second, obj, order, n);
    1760          25 :                         tommy_list_swap(&first, &first);
    1761          25 :                         tommy_list_swap(&second, &second);
    1762          25 :                         test_list_sequence(&first, obj, order + n, m);
    1763          25 :                         test_list_sequence(&second, obj, order, n);
    1764          25 :                         tommy_list_swap(&first, &second);
    1765          25 :                         test_list_sequence(&first, obj, order, n);
    1766          25 :                         test_list_sequence(&second, obj, order + n, m);
    1767             :                 }
    1768             :         }
    1769           1 : }
    1770             : 
    1771           1 : void test_list(void)
    1772             : {
    1773           1 :         const unsigned size = TOMMY_SIZE;
    1774             :         tommy_node* list;
    1775             : 
    1776           1 :         test_list_insert_head();
    1777           1 :         test_list_merge();
    1778           1 :         test_list_sort();
    1779           1 :         test_list_split();
    1780           1 :         test_list_splice();
    1781           1 :         test_list_concat();
    1782           1 :         test_list_prepend();
    1783           1 :         test_list_reverse_move();
    1784           1 :         test_list_remove_ends();
    1785           1 :         test_list_swap();
    1786             : 
    1787           1 :         struct object* LIST = malloc(size * sizeof(struct object));
    1788           1 :         struct object_vector* VECTOR = malloc(size * sizeof(struct object_vector));
    1789             : 
    1790      100001 :         for (unsigned i = 0; i < size; ++i) {
    1791      100000 :                 VECTOR[i].value = LIST[i].value = 0;
    1792             :         }
    1793             : 
    1794           1 :         tommy_list_init(&list);
    1795             : 
    1796             :         /* sort an empty list */
    1797           1 :         tommy_list_sort(&list, compare);
    1798             : 
    1799           1 :         if (!tommy_list_empty(&list)) {
    1800             :                 /* LCOV_EXCL_START */
    1801             :                 abort();
    1802             :                 /* LCOV_EXCL_STOP */
    1803             :         }
    1804             : 
    1805           1 :         if (tommy_list_tail(&list) != 0) {
    1806             :                 /* LCOV_EXCL_START */
    1807             :                 abort();
    1808             :                 /* LCOV_EXCL_STOP */
    1809             :         }
    1810             : 
    1811           1 :         if (tommy_list_head(&list) != 0) {
    1812             :                 /* LCOV_EXCL_START */
    1813             :                 abort();
    1814             :                 /* LCOV_EXCL_STOP */
    1815             :         }
    1816             : 
    1817      100001 :         for (unsigned i = 0; i < size; ++i) {
    1818      100000 :                 VECTOR[i].value = LIST[i].value = rnd(size);
    1819      100000 :                 tommy_list_insert_tail(&list, &LIST[i].node, &LIST[i]);
    1820             :         }
    1821             : 
    1822           1 :         if (tommy_list_tail(&list) == 0) {
    1823             :                 /* LCOV_EXCL_START */
    1824             :                 abort();
    1825             :                 /* LCOV_EXCL_STOP */
    1826             :         }
    1827             : 
    1828           1 :         if (tommy_list_head(&list) == 0) {
    1829             :                 /* LCOV_EXCL_START */
    1830             :                 abort();
    1831             :                 /* LCOV_EXCL_STOP */
    1832             :         }
    1833             : 
    1834          12 :         SPEED_START("sort random") {
    1835           6 :                 SPEED_UNTIMED(test_list_build(&list, LIST, 0, size));
    1836           6 :                 tommy_list_sort(&list, compare);
    1837           6 :         } SPEED_STOP
    1838             : 
    1839          20 :         SPEED_START("C qsort random")
    1840             :         {
    1841     1000010 :                 SPEED_UNTIMED(
    1842             :                         for (unsigned i = 0; i < size; ++i)
    1843             :                                 VECTOR[i].value = LIST[i].value;
    1844             :                 );
    1845          10 :                 qsort(VECTOR, size, sizeof(VECTOR[0]), compare_vector);
    1846             :         }
    1847          10 :         SPEED_STOP
    1848             : 
    1849           1 :         test_list_order(list);
    1850             : 
    1851             :         /* forward order with some (1%) random values */
    1852           1 :         list = 0;
    1853      100001 :         for (unsigned i = 0; i < size; ++i) {
    1854      100000 :                 VECTOR[i].value = LIST[i].value = i;
    1855      100000 :                 if (rnd(100) == 0)
    1856        1030 :                         VECTOR[i].value = LIST[i].value = rnd(size);
    1857      100000 :                 tommy_list_insert_tail(&list, &LIST[i].node, &LIST[i]);
    1858             :         }
    1859             : 
    1860          28 :         SPEED_START("sort partially ordered") {
    1861          14 :                 SPEED_UNTIMED(test_list_build(&list, LIST, 0, size));
    1862          14 :                 tommy_list_sort(&list, compare);
    1863          14 :         } SPEED_STOP
    1864             : 
    1865          40 :         SPEED_START("C qsort partially ordered")
    1866             :         {
    1867     2000020 :                 SPEED_UNTIMED(
    1868             :                         for (unsigned i = 0; i < size; ++i)
    1869             :                                 VECTOR[i].value = LIST[i].value;
    1870             :                 );
    1871          20 :                 qsort(VECTOR, size, sizeof(VECTOR[0]), compare_vector);
    1872             :         }
    1873          20 :         SPEED_STOP
    1874             : 
    1875           1 :         test_list_order(list);
    1876             : 
    1877             :         /* forward order */
    1878           1 :         list = 0;
    1879      100001 :         for (unsigned i = 0; i < size; ++i) {
    1880      100000 :                 VECTOR[i].value = LIST[i].value = i;
    1881      100000 :                 tommy_list_insert_tail(&list, &LIST[i].node, &LIST[i]);
    1882             :         }
    1883             : 
    1884         188 :         SPEED_START("sort forward") {
    1885          94 :                 SPEED_UNTIMED(test_list_build(&list, LIST, 0, size));
    1886          94 :                 tommy_list_sort(&list, compare);
    1887          94 :         } SPEED_STOP
    1888             : 
    1889          56 :         SPEED_START("C qsort forward")
    1890             :         {
    1891     2800028 :                 SPEED_UNTIMED(
    1892             :                         for (unsigned i = 0; i < size; ++i)
    1893             :                                 VECTOR[i].value = LIST[i].value;
    1894             :                 );
    1895          28 :                 qsort(VECTOR, size, sizeof(VECTOR[0]), compare_vector);
    1896             :         }
    1897          28 :         SPEED_STOP
    1898             : 
    1899           1 :         test_list_order(list);
    1900             : 
    1901             :         /* backward order */
    1902           1 :         list = 0;
    1903      100001 :         for (unsigned i = 0; i < size; ++i) {
    1904      100000 :                 VECTOR[i].value = LIST[i].value = size - 1 - i;
    1905      100000 :                 tommy_list_insert_tail(&list, &LIST[i].node, &LIST[i]);
    1906             :         }
    1907             : 
    1908         156 :         SPEED_START("sort backward") {
    1909          78 :                 SPEED_UNTIMED(test_list_build(&list, LIST, 0, size));
    1910          78 :                 tommy_list_sort(&list, compare);
    1911          78 :         } SPEED_STOP
    1912             : 
    1913          42 :         SPEED_START("C qsort backward")
    1914             :         {
    1915     2100021 :                 SPEED_UNTIMED(
    1916             :                         for (unsigned i = 0; i < size; ++i)
    1917             :                                 VECTOR[i].value = LIST[i].value;
    1918             :                 );
    1919          21 :                 qsort(VECTOR, size, sizeof(VECTOR[0]), compare_vector);
    1920             :         }
    1921          21 :         SPEED_STOP
    1922             : 
    1923           1 :         test_list_order(list);
    1924             : 
    1925             :         /* use a small range of random value to insert a lot of duplicates */
    1926           1 :         list = 0;
    1927      100001 :         for (unsigned i = 0; i < size; ++i) {
    1928      100000 :                 VECTOR[i].value = LIST[i].value = rnd(size / 1000 + 2);
    1929      100000 :                 tommy_list_insert_tail(&list, &LIST[i].node, &LIST[i]);
    1930             :         }
    1931             : 
    1932          12 :         SPEED_START("sort random duplicate") {
    1933           6 :                 SPEED_UNTIMED(test_list_build(&list, LIST, 0, size));
    1934           6 :                 tommy_list_sort(&list, compare);
    1935           6 :         } SPEED_STOP
    1936             : 
    1937          24 :         SPEED_START("C qsort random duplicate")
    1938             :         {
    1939     1200012 :                 SPEED_UNTIMED(
    1940             :                         for (unsigned i = 0; i < size; ++i)
    1941             :                                 VECTOR[i].value = LIST[i].value;
    1942             :                 );
    1943          12 :                 qsort(VECTOR, size, sizeof(VECTOR[0]), compare_vector);
    1944             :         }
    1945          12 :         SPEED_STOP
    1946             : 
    1947           1 :         test_list_order(list);
    1948             : 
    1949           1 :         free(LIST);
    1950           1 :         free(VECTOR);
    1951           1 : }
    1952             : 
    1953           1 : void test_list_clear(void)
    1954             : {
    1955             :         tommy_list list;
    1956             :         struct object objects[2];
    1957             : 
    1958           1 :         tommy_list_init(&list);
    1959           1 :         objects[0].value = 1;
    1960           1 :         objects[1].value = 2;
    1961           1 :         tommy_list_insert_tail(&list, &objects[0].node, &objects[0]);
    1962           1 :         tommy_list_insert_tail(&list, &objects[1].node, &objects[1]);
    1963           1 :         tommy_list_clear(&list);
    1964           1 :         if (!tommy_list_empty(&list) || tommy_list_count(&list) != 0) {
    1965             :                 /* LCOV_EXCL_START */
    1966             :                 abort();
    1967             :                 /* LCOV_EXCL_STOP */
    1968             :         }
    1969           1 :         if (tommy_list_head(&list) != 0 || tommy_list_tail(&list) != 0) {
    1970             :                 /* LCOV_EXCL_START */
    1971             :                 abort();
    1972             :                 /* LCOV_EXCL_STOP */
    1973             :         }
    1974             : 
    1975           1 :         tommy_list_insert_tail(&list, &objects[0].node, &objects[0]);
    1976           1 :         if (tommy_list_head(&list) != &objects[0].node) {
    1977             :                 /* LCOV_EXCL_START */
    1978             :                 abort();
    1979             :                 /* LCOV_EXCL_STOP */
    1980             :         }
    1981           1 :         tommy_list_remove_existing(&list, &objects[0].node);
    1982             : 
    1983           1 :         struct object* allocated = malloc(sizeof(*allocated));
    1984           1 :         allocated->value = 3;
    1985           1 :         tommy_list_insert_tail(&list, &allocated->node, allocated);
    1986           1 :         tommy_list_foreach(&list, free);
    1987           1 :         tommy_list_clear(&list);
    1988           1 :         if (!tommy_list_empty(&list)) {
    1989             :                 /* LCOV_EXCL_START */
    1990             :                 abort();
    1991             :                 /* LCOV_EXCL_STOP */
    1992             :         }
    1993           1 : }
    1994             : 
    1995           1 : void test_tree(void)
    1996             : {
    1997           1 :         const unsigned size = TOMMY_SIZE / 4;
    1998           1 :         struct object_tree* OBJ = malloc(size * sizeof(struct object_tree));
    1999             :         tommy_tree tree;
    2000             : 
    2001          12 :         SPEED_START("tree") {
    2002           6 :                 tommy_tree_init(&tree, &compare);
    2003             : 
    2004             :                 /* forward order */
    2005      150006 :                 for (unsigned i = 0; i < size; ++i)
    2006      150000 :                         OBJ[i].value = i;
    2007             : 
    2008             :                 /* insert */
    2009      150006 :                 for (unsigned i = 0; i < size; ++i)
    2010      150000 :                         tommy_tree_insert(&tree, &OBJ[i].node, &OBJ[i]);
    2011             : 
    2012           6 :                 if (tommy_tree_memory_usage(&tree) < size * sizeof(tommy_tree_node)) {
    2013             :                         /* LCOV_EXCL_START */
    2014             :                         abort();
    2015             :                         /* LCOV_EXCL_STOP */
    2016             :                 }
    2017             : 
    2018           6 :                 if (tommy_tree_count(&tree) != size) {
    2019             :                         /* LCOV_EXCL_START */
    2020             :                         abort();
    2021             :                         /* LCOV_EXCL_STOP */
    2022             :                 }
    2023             : 
    2024           6 :                 the_count = 0;
    2025           6 :                 tommy_tree_foreach(&tree, count_callback);
    2026           6 :                 if (the_count != size) {
    2027             :                         /* LCOV_EXCL_START */
    2028             :                         abort();
    2029             :                         /* LCOV_EXCL_STOP */
    2030             :                 }
    2031             : 
    2032           6 :                 the_count = 0;
    2033           6 :                 tommy_tree_foreach_arg(&tree, count_arg_callback, &the_count);
    2034           6 :                 if (the_count != size) {
    2035             :                         /* LCOV_EXCL_START */
    2036             :                         abort();
    2037             :                         /* LCOV_EXCL_STOP */
    2038             :                 }
    2039             : 
    2040             :                 /* search present */
    2041       75006 :                 for (unsigned i = 0; i < size / 2; ++i) {
    2042       75000 :                         if (tommy_tree_search(&tree, &OBJ[i]) == 0) {
    2043             :                                 /* LCOV_EXCL_START */
    2044             :                                 abort();
    2045             :                                 /* LCOV_EXCL_STOP */
    2046             :                         }
    2047       75000 :                         if (tommy_tree_search_compare(&tree, &compare, &OBJ[i]) == 0) {
    2048             :                                 /* LCOV_EXCL_START */
    2049             :                                 abort();
    2050             :                                 /* LCOV_EXCL_STOP */
    2051             :                         }
    2052             :                 }
    2053             : 
    2054             :                 /* insert existing */
    2055      150006 :                 for (unsigned i = 0; i < size; ++i) {
    2056             :                         struct object_tree EXTRA;
    2057      150000 :                         EXTRA.value = i;
    2058      150000 :                         if (tommy_tree_insert_unique(&tree, &EXTRA.node, &EXTRA) == &EXTRA) {
    2059             :                                 /* LCOV_EXCL_START */
    2060             :                                 abort();
    2061             :                                 /* LCOV_EXCL_STOP */
    2062             :                         }
    2063             :                 }
    2064             : 
    2065             :                 /* remove existing */
    2066       75006 :                 for (unsigned i = 0; i < size / 2; ++i)
    2067       75000 :                         tommy_tree_remove_existing(&tree, &OBJ[i].node);
    2068             : 
    2069             :                 /* remove missing */
    2070       75006 :                 for (unsigned i = 0; i < size / 2; ++i)
    2071       75000 :                         if (tommy_tree_remove(&tree, &OBJ[i]) != 0) {
    2072             :                                 /* LCOV_EXCL_START */
    2073             :                                 abort();
    2074             :                                 /* LCOV_EXCL_STOP */
    2075             :                         }
    2076             : 
    2077             :                 /* search missing */
    2078       75006 :                 for (unsigned i = 0; i < size / 2; ++i) {
    2079       75000 :                         if (tommy_tree_search(&tree, &OBJ[i]) != 0) {
    2080             :                                 /* LCOV_EXCL_START */
    2081             :                                 abort();
    2082             :                                 /* LCOV_EXCL_STOP */
    2083             :                         }
    2084       75000 :                         if (tommy_tree_search_compare(&tree, &compare, &OBJ[i]) != 0) {
    2085             :                                 /* LCOV_EXCL_START */
    2086             :                                 abort();
    2087             :                                 /* LCOV_EXCL_STOP */
    2088             :                         }
    2089             :                 }
    2090             : 
    2091             :                 /* remove present */
    2092       75006 :                 for (unsigned i = 0; i < size / 2; ++i)
    2093       75000 :                         if (tommy_tree_remove(&tree, &OBJ[size / 2 + i]) == 0) {
    2094             :                                 /* LCOV_EXCL_START */
    2095             :                                 abort();
    2096             :                                 /* LCOV_EXCL_STOP */
    2097             :                         }
    2098             : 
    2099             :                 /* reverse order */
    2100      150006 :                 for (unsigned i = 0; i < size; ++i)
    2101      150000 :                         OBJ[i].value = size - i;
    2102             : 
    2103             :                 /* insert */
    2104      150006 :                 for (unsigned i = 0; i < size; ++i)
    2105      150000 :                         tommy_tree_insert(&tree, &OBJ[i].node, &OBJ[i]);
    2106             : 
    2107             :                 /* remove all */
    2108      150006 :                 for (unsigned i = 0; i < size; ++i)
    2109      150000 :                         tommy_tree_remove_existing(&tree, &OBJ[i].node);
    2110             : 
    2111             :                 /* random order */
    2112      150006 :                 for (unsigned i = 0; i < size; ++i)
    2113      150000 :                         OBJ[i].value = tommy_inthash_u32(i);
    2114             : 
    2115             :                 /* insert */
    2116      150006 :                 for (unsigned i = 0; i < size; ++i)
    2117      150000 :                         tommy_tree_insert(&tree, &OBJ[i].node, &OBJ[i]);
    2118             : 
    2119             :                 /* remove all */
    2120      150006 :                 for (unsigned i = 0; i < size; ++i)
    2121      150000 :                         tommy_tree_remove_existing(&tree, &OBJ[i].node);
    2122             : 
    2123           6 :         } SPEED_STOP
    2124           1 : }
    2125             : 
    2126           1 : void test_tree_swap(void)
    2127             : {
    2128             :         tommy_tree first;
    2129             :         tommy_tree second;
    2130             :         tommy_tree empty;
    2131             :         struct object_tree objects[5];
    2132             :         tommy_tree_node* first_root;
    2133             :         tommy_tree_node* second_root;
    2134           1 :         const int values[] = { 1, 3, 5, 4, 2 };
    2135             : 
    2136           1 :         tommy_tree_init(&first, &compare);
    2137           1 :         tommy_tree_init(&second, &compare_tree_reverse);
    2138           1 :         tommy_tree_init(&empty, &compare);
    2139           6 :         for (unsigned i = 0; i < 5; ++i)
    2140           5 :                 objects[i].value = values[i];
    2141           4 :         for (unsigned i = 0; i < 3; ++i)
    2142           3 :                 tommy_tree_insert(&first, &objects[i].node, &objects[i]);
    2143           3 :         for (unsigned i = 3; i < 5; ++i)
    2144           2 :                 tommy_tree_insert(&second, &objects[i].node, &objects[i]);
    2145           1 :         first_root = first.root;
    2146           1 :         second_root = second.root;
    2147             : 
    2148           1 :         tommy_tree_swap(&first, &second);
    2149           1 :         if (first.root != second_root || first.cmp != &compare_tree_reverse || tommy_tree_count(&first) != 2
    2150           1 :                 || second.root != first_root || second.cmp != &compare || tommy_tree_count(&second) != 3
    2151           1 :                 || tommy_tree_head(&first) != &objects[3].node
    2152           1 :                 || tommy_tree_next(tommy_tree_head(&first)) != &objects[4].node
    2153           1 :                 || tommy_tree_search(&first, &objects[2]) != 0
    2154           1 :                 || tommy_tree_search_greater(&first, &objects[1]) != &objects[4]
    2155           1 :                 || tommy_tree_search_less(&first, &objects[1]) != &objects[3]
    2156           1 :                 || tommy_tree_head(&second) != &objects[0].node
    2157           1 :                 || tommy_tree_next(tommy_tree_head(&second)) != &objects[1].node
    2158           1 :                 || tommy_tree_tail(&second) != &objects[2].node) {
    2159             :                 /* LCOV_EXCL_START */
    2160             :                 abort();
    2161             :                 /* LCOV_EXCL_STOP */
    2162             :         }
    2163           1 :         tommy_tree_swap(&first, &first);
    2164           1 :         if (first.root != second_root || first.cmp != &compare_tree_reverse || tommy_tree_count(&first) != 2) {
    2165             :                 /* LCOV_EXCL_START */
    2166             :                 abort();
    2167             :                 /* LCOV_EXCL_STOP */
    2168             :         }
    2169           1 :         if (tommy_tree_remove(&first, &objects[4]) != &objects[4]
    2170           1 :                 || tommy_tree_remove(&second, &objects[2]) != &objects[2]) {
    2171             :                 /* LCOV_EXCL_START */
    2172             :                 abort();
    2173             :                 /* LCOV_EXCL_STOP */
    2174             :         }
    2175             : 
    2176           1 :         tommy_tree_swap(&first, &empty);
    2177           1 :         if (!tommy_tree_empty(&first) || first.cmp != &compare || tommy_tree_count(&first) != 0
    2178           1 :                 || empty.root != second_root || empty.cmp != &compare_tree_reverse || tommy_tree_count(&empty) != 1
    2179           1 :                 || tommy_tree_head(&empty) != &objects[3].node) {
    2180             :                 /* LCOV_EXCL_START */
    2181             :                 abort();
    2182             :                 /* LCOV_EXCL_STOP */
    2183             :         }
    2184           1 : }
    2185             : 
    2186           1 : void test_tree_duplicates(void)
    2187             : {
    2188             :         tommy_tree tree;
    2189             :         struct object_tree objects[96];
    2190             : 
    2191           1 :         tommy_tree_init(&tree, &compare);
    2192             : 
    2193          97 :         for (unsigned i = 0; i < 96; ++i) {
    2194          96 :                 objects[i].value = i % 4;
    2195          96 :                 tommy_tree_insert(&tree, &objects[i].node, &objects[i]);
    2196             :         }
    2197             : 
    2198           1 :         if (tommy_tree_count(&tree) != 96) {
    2199             :                 /* LCOV_EXCL_START */
    2200             :                 abort();
    2201             :                 /* LCOV_EXCL_STOP */
    2202             :         }
    2203             : 
    2204           1 :         tommy_tree_node* node = tommy_tree_head(&tree);
    2205           5 :         for (unsigned group = 0; group < 4; ++group) {
    2206         100 :                 for (unsigned i = group; i < 96; i += 4) {
    2207          96 :                         if (node != &objects[i].node) {
    2208             :                                 /* LCOV_EXCL_START */
    2209             :                                 abort();
    2210             :                                 /* LCOV_EXCL_STOP */
    2211             :                         }
    2212          96 :                         node = tommy_tree_next(node);
    2213             :                 }
    2214             :         }
    2215           1 :         if (node) {
    2216             :                 /* LCOV_EXCL_START */
    2217             :                 abort();
    2218             :                 /* LCOV_EXCL_STOP */
    2219             :         }
    2220             : 
    2221           5 :         for (unsigned group = 0; group < 4; ++group) {
    2222           4 :                 const struct object_tree key = { .value = group };
    2223           4 :                 void* greater = group < 3 ? &objects[group + 1] : 0;
    2224           4 :                 void* less = group ? &objects[91 + group] : 0;
    2225           4 :                 if (tommy_tree_search(&tree, &key) != &objects[group]) {
    2226             :                         /* LCOV_EXCL_START */
    2227             :                         abort();
    2228             :                         /* LCOV_EXCL_STOP */
    2229             :                 }
    2230           4 :                 if (tommy_tree_search_compare(&tree, &compare, &key) != &objects[group]) {
    2231             :                         /* LCOV_EXCL_START */
    2232             :                         abort();
    2233             :                         /* LCOV_EXCL_STOP */
    2234             :                 }
    2235           4 :                 if (tommy_tree_search_greater_equal(&tree, &key) != &objects[group]) {
    2236             :                         /* LCOV_EXCL_START */
    2237             :                         abort();
    2238             :                         /* LCOV_EXCL_STOP */
    2239             :                 }
    2240           4 :                 if (tommy_tree_search_greater_equal_compare(&tree, &compare, &key) != &objects[group]) {
    2241             :                         /* LCOV_EXCL_START */
    2242             :                         abort();
    2243             :                         /* LCOV_EXCL_STOP */
    2244             :                 }
    2245           4 :                 if (tommy_tree_search_less_equal(&tree, &key) != &objects[92 + group]) {
    2246             :                         /* LCOV_EXCL_START */
    2247             :                         abort();
    2248             :                         /* LCOV_EXCL_STOP */
    2249             :                 }
    2250           4 :                 if (tommy_tree_search_less_equal_compare(&tree, &compare, &key) != &objects[92 + group]) {
    2251             :                         /* LCOV_EXCL_START */
    2252             :                         abort();
    2253             :                         /* LCOV_EXCL_STOP */
    2254             :                 }
    2255           4 :                 if (tommy_tree_search_greater(&tree, &key) != greater
    2256           4 :                         || tommy_tree_search_greater_compare(&tree, &compare, &key) != greater
    2257           4 :                         || tommy_tree_search_less(&tree, &key) != less
    2258           4 :                         || tommy_tree_search_less_compare(&tree, &compare, &key) != less) {
    2259             :                         /* LCOV_EXCL_START */
    2260             :                         abort();
    2261             :                         /* LCOV_EXCL_STOP */
    2262             :                 }
    2263           4 :                 if (tommy_tree_find(&tree, &key) != &objects[group].node) {
    2264             :                         /* LCOV_EXCL_START */
    2265             :                         abort();
    2266             :                         /* LCOV_EXCL_STOP */
    2267             :                 }
    2268           4 :                 if (tommy_tree_find_compare(&tree, &compare, &key) != &objects[group].node) {
    2269             :                         /* LCOV_EXCL_START */
    2270             :                         abort();
    2271             :                         /* LCOV_EXCL_STOP */
    2272             :                 }
    2273           4 :                 if (tommy_tree_find_greater_equal(&tree, &key) != &objects[group].node) {
    2274             :                         /* LCOV_EXCL_START */
    2275             :                         abort();
    2276             :                         /* LCOV_EXCL_STOP */
    2277             :                 }
    2278           4 :                 if (tommy_tree_find_greater_equal_compare(&tree, &compare, &key) != &objects[group].node) {
    2279             :                         /* LCOV_EXCL_START */
    2280             :                         abort();
    2281             :                         /* LCOV_EXCL_STOP */
    2282             :                 }
    2283           4 :                 if (tommy_tree_find_less_equal(&tree, &key) != &objects[92 + group].node) {
    2284             :                         /* LCOV_EXCL_START */
    2285             :                         abort();
    2286             :                         /* LCOV_EXCL_STOP */
    2287             :                 }
    2288           4 :                 if (tommy_tree_find_less_equal_compare(&tree, &compare, &key) != &objects[92 + group].node) {
    2289             :                         /* LCOV_EXCL_START */
    2290             :                         abort();
    2291             :                         /* LCOV_EXCL_STOP */
    2292             :                 }
    2293           4 :                 tommy_tree_node* greater_node = greater ? &((struct object_tree*)greater)->node : 0;
    2294           4 :                 tommy_tree_node* less_node = less ? &((struct object_tree*)less)->node : 0;
    2295           4 :                 if (tommy_tree_find_greater(&tree, &key) != greater_node
    2296           4 :                         || tommy_tree_find_greater_compare(&tree, &compare, &key) != greater_node
    2297           4 :                         || tommy_tree_find_less(&tree, &key) != less_node
    2298           4 :                         || tommy_tree_find_less_compare(&tree, &compare, &key) != less_node) {
    2299             :                         /* LCOV_EXCL_START */
    2300             :                         abort();
    2301             :                         /* LCOV_EXCL_STOP */
    2302             :                 }
    2303             :         }
    2304             : 
    2305             :         struct object_tree key;
    2306           1 :         key.value = -1;
    2307           1 :         if (tommy_tree_search_greater(&tree, &key) != &objects[0]
    2308           1 :                 || tommy_tree_search_greater_compare(&tree, &compare, &key) != &objects[0]
    2309           1 :                 || tommy_tree_search_less(&tree, &key) != 0
    2310           1 :                 || tommy_tree_search_less_compare(&tree, &compare, &key) != 0) {
    2311             :                 /* LCOV_EXCL_START */
    2312             :                 abort();
    2313             :                 /* LCOV_EXCL_STOP */
    2314             :         }
    2315           1 :         key.value = 4;
    2316           1 :         if (tommy_tree_search_greater(&tree, &key) != 0
    2317           1 :                 || tommy_tree_search_greater_compare(&tree, &compare, &key) != 0
    2318           1 :                 || tommy_tree_search_less(&tree, &key) != &objects[95]
    2319           1 :                 || tommy_tree_search_less_compare(&tree, &compare, &key) != &objects[95]) {
    2320             :                 /* LCOV_EXCL_START */
    2321             :                 abort();
    2322             :                 /* LCOV_EXCL_STOP */
    2323             :         }
    2324           1 :         key.value = 2;
    2325           1 :         if (tommy_tree_search_compare(&tree, &compare_tree_group, &key) != &objects[2]) {
    2326             :                 /* LCOV_EXCL_START */
    2327             :                 abort();
    2328             :                 /* LCOV_EXCL_STOP */
    2329             :         }
    2330           1 :         if (tommy_tree_search_greater_equal_compare(&tree, &compare_tree_group, &key) != &objects[2]) {
    2331             :                 /* LCOV_EXCL_START */
    2332             :                 abort();
    2333             :                 /* LCOV_EXCL_STOP */
    2334             :         }
    2335           1 :         if (tommy_tree_search_less_equal_compare(&tree, &compare_tree_group, &key) != &objects[95]) {
    2336             :                 /* LCOV_EXCL_START */
    2337             :                 abort();
    2338             :                 /* LCOV_EXCL_STOP */
    2339             :         }
    2340           1 :         if (tommy_tree_search_greater_compare(&tree, &compare_tree_group, &key) != 0
    2341           1 :                 || tommy_tree_search_less_compare(&tree, &compare_tree_group, &key) != &objects[93]) {
    2342             :                 /* LCOV_EXCL_START */
    2343             :                 abort();
    2344             :                 /* LCOV_EXCL_STOP */
    2345             :         }
    2346           1 :         key.value = 0;
    2347           1 :         if (tommy_tree_search_greater_compare(&tree, &compare_tree_group, &key) != &objects[2]
    2348           1 :                 || tommy_tree_search_less_compare(&tree, &compare_tree_group, &key) != 0) {
    2349             :                 /* LCOV_EXCL_START */
    2350             :                 abort();
    2351             :                 /* LCOV_EXCL_STOP */
    2352             :         }
    2353           1 :         int value = 2;
    2354           1 :         if (tommy_tree_search_greater_compare(&tree, &compare_tree_int, &value) != &objects[3]
    2355           1 :                 || tommy_tree_search_less_compare(&tree, &compare_tree_int, &value) != &objects[93]) {
    2356             :                 /* LCOV_EXCL_START */
    2357             :                 abort();
    2358             :                 /* LCOV_EXCL_STOP */
    2359             :         }
    2360             : 
    2361             :         struct object_tree candidate;
    2362           1 :         candidate.value = 2;
    2363           1 :         memset(&candidate.node, 0xa5, sizeof(candidate.node));
    2364             :         unsigned char before[sizeof(candidate.node)];
    2365           1 :         memcpy(before, &candidate.node, sizeof(before));
    2366           1 :         if (tommy_tree_insert_unique(&tree, &candidate.node, &candidate) != &objects[2]) {
    2367             :                 /* LCOV_EXCL_START */
    2368             :                 abort();
    2369             :                 /* LCOV_EXCL_STOP */
    2370             :         }
    2371           1 :         if (memcmp(&candidate.node, &before, sizeof(before)) != 0 || tommy_tree_count(&tree) != 96) {
    2372             :                 /* LCOV_EXCL_START */
    2373             :                 abort();
    2374             :                 /* LCOV_EXCL_STOP */
    2375             :         }
    2376             : 
    2377          25 :         for (unsigned i = 2; i < 96; i += 4) {
    2378          24 :                 if (tommy_tree_remove(&tree, &candidate) != &objects[i]) {
    2379             :                         /* LCOV_EXCL_START */
    2380             :                         abort();
    2381             :                         /* LCOV_EXCL_STOP */
    2382             :                 }
    2383             :         }
    2384           1 :         if (tommy_tree_remove(&tree, &candidate) != 0 || tommy_tree_count(&tree) != 72) {
    2385             :                 /* LCOV_EXCL_START */
    2386             :                 abort();
    2387             :                 /* LCOV_EXCL_STOP */
    2388             :         }
    2389           1 :         if (tommy_tree_search_greater_equal(&tree, &candidate) != &objects[3]) {
    2390             :                 /* LCOV_EXCL_START */
    2391             :                 abort();
    2392             :                 /* LCOV_EXCL_STOP */
    2393             :         }
    2394           1 :         if (tommy_tree_search_less_equal(&tree, &candidate) != &objects[93]) {
    2395             :                 /* LCOV_EXCL_START */
    2396             :                 abort();
    2397             :                 /* LCOV_EXCL_STOP */
    2398             :         }
    2399          25 :         for (unsigned i = 3; i < 96; i += 4) {
    2400          24 :                 if (tommy_tree_remove_compare(&tree, &compare_tree_group, &candidate) != &objects[i]) {
    2401             :                         /* LCOV_EXCL_START */
    2402             :                         abort();
    2403             :                         /* LCOV_EXCL_STOP */
    2404             :                 }
    2405             :         }
    2406           1 :         if (tommy_tree_remove_compare(&tree, &compare_tree_group, &candidate) != 0 || tommy_tree_count(&tree) != 48) {
    2407             :                 /* LCOV_EXCL_START */
    2408             :                 abort();
    2409             :                 /* LCOV_EXCL_STOP */
    2410             :         }
    2411           1 :         if (tommy_tree_search_compare(&tree, &compare_tree_group, &candidate) != 0) {
    2412             :                 /* LCOV_EXCL_START */
    2413             :                 abort();
    2414             :                 /* LCOV_EXCL_STOP */
    2415             :         }
    2416             : 
    2417           1 :         candidate.value = 4;
    2418           1 :         if (tommy_tree_insert_unique(&tree, &candidate.node, &candidate) != &candidate) {
    2419             :                 /* LCOV_EXCL_START */
    2420             :                 abort();
    2421             :                 /* LCOV_EXCL_STOP */
    2422             :         }
    2423           1 :         if (tommy_tree_tail(&tree) != &candidate.node || tommy_tree_count(&tree) != 49) {
    2424             :                 /* LCOV_EXCL_START */
    2425             :                 abort();
    2426             :                 /* LCOV_EXCL_STOP */
    2427             :         }
    2428             : 
    2429           1 :         key.value = -2;
    2430           1 :         if (tommy_tree_search_less_equal(&tree, &key) != 0) {
    2431             :                 /* LCOV_EXCL_START */
    2432             :                 abort();
    2433             :                 /* LCOV_EXCL_STOP */
    2434             :         }
    2435           1 :         if (tommy_tree_search_compare(&tree, &compare_tree_group, &key) != 0) {
    2436             :                 /* LCOV_EXCL_START */
    2437             :                 abort();
    2438             :                 /* LCOV_EXCL_STOP */
    2439             :         }
    2440           1 :         key.value = 5;
    2441           1 :         if (tommy_tree_search_greater_equal(&tree, &key) != 0) {
    2442             :                 /* LCOV_EXCL_START */
    2443             :                 abort();
    2444             :                 /* LCOV_EXCL_STOP */
    2445             :         }
    2446             : 
    2447           1 :         value = 4;
    2448           1 :         if (tommy_tree_remove_compare(&tree, &compare_tree_int, &value) != &candidate) {
    2449             :                 /* LCOV_EXCL_START */
    2450             :                 abort();
    2451             :                 /* LCOV_EXCL_STOP */
    2452             :         }
    2453           1 :         if (tommy_tree_remove_compare(&tree, &compare_tree_int, &value) != 0) {
    2454             :                 /* LCOV_EXCL_START */
    2455             :                 abort();
    2456             :                 /* LCOV_EXCL_STOP */
    2457             :         }
    2458           1 :         int val_neg = -1;
    2459           1 :         if (tommy_tree_remove_compare(&tree, &compare_tree_int, &val_neg) != 0) {
    2460             :                 /* LCOV_EXCL_START */
    2461             :                 abort();
    2462             :                 /* LCOV_EXCL_STOP */
    2463             :         }
    2464           5 :         for (unsigned group = 0; group < 4; ++group) {
    2465           4 :                 if (group >= 2)
    2466           2 :                         continue;
    2467          50 :                 for (unsigned i = group; i < 96; i += 4)
    2468          48 :                         tommy_tree_remove_existing(&tree, &objects[i].node);
    2469             :         }
    2470           1 :         if (!tommy_tree_empty(&tree)) {
    2471             :                 /* LCOV_EXCL_START */
    2472             :                 abort();
    2473             :                 /* LCOV_EXCL_STOP */
    2474             :         }
    2475           1 : }
    2476             : 
    2477          64 : static void tree_foreach_free_arg_callback(void* arg, void* data)
    2478             : {
    2479          64 :         int* expected = arg;
    2480          64 :         struct object_tree* object = data;
    2481             : 
    2482          64 :         if (object->value != *expected) {
    2483             :                 /* LCOV_EXCL_START */
    2484             :                 abort();
    2485             :                 /* LCOV_EXCL_STOP */
    2486             :         }
    2487          64 :         ++*expected;
    2488          64 :         free(object);
    2489          64 : }
    2490             : 
    2491           1 : void test_tree_foreach(void)
    2492             : {
    2493             :         tommy_tree tree;
    2494             : 
    2495           3 :         for (unsigned pass = 0; pass < 2; ++pass) {
    2496           2 :                 int expected = 0;
    2497             : 
    2498           2 :                 tommy_tree_init(&tree, &compare);
    2499           2 :                 tommy_tree_foreach(&tree, free);
    2500           2 :                 tommy_tree_foreach_arg(&tree, tree_foreach_free_arg_callback, &expected);
    2501         130 :                 for (unsigned i = 0; i < 64; ++i) {
    2502         128 :                         struct object_tree* object = malloc(sizeof(*object));
    2503             : 
    2504         128 :                         object->value = i * 37 % 64;
    2505         128 :                         tommy_tree_insert(&tree, &object->node, object);
    2506             :                 }
    2507           2 :                 if (pass == 0) {
    2508           1 :                         tommy_tree_foreach(&tree, free);
    2509             :                 } else {
    2510           1 :                         tommy_tree_foreach_arg(&tree, tree_foreach_free_arg_callback, &expected);
    2511           1 :                         if (expected != 64) {
    2512             :                                 /* LCOV_EXCL_START */
    2513             :                                 abort();
    2514             :                                 /* LCOV_EXCL_STOP */
    2515             :                         }
    2516             :                 }
    2517           2 :                 tommy_tree_clear(&tree);
    2518             :         }
    2519           1 : }
    2520             : 
    2521           1 : void test_tree_clear(void)
    2522             : {
    2523             :         tommy_tree tree;
    2524             :         struct object_tree objects[2];
    2525             : 
    2526           1 :         tommy_tree_init(&tree, &compare);
    2527           1 :         objects[0].value = 1;
    2528           1 :         objects[1].value = 2;
    2529           1 :         tommy_tree_insert(&tree, &objects[0].node, &objects[0]);
    2530           1 :         tommy_tree_insert(&tree, &objects[1].node, &objects[1]);
    2531           1 :         tommy_tree_clear(&tree);
    2532           1 :         if (!tommy_tree_empty(&tree) || tommy_tree_count(&tree) != 0) {
    2533             :                 /* LCOV_EXCL_START */
    2534             :                 abort();
    2535             :                 /* LCOV_EXCL_STOP */
    2536             :         }
    2537           1 :         if (tommy_tree_head(&tree) != 0 || tommy_tree_tail(&tree) != 0) {
    2538             :                 /* LCOV_EXCL_START */
    2539             :                 abort();
    2540             :                 /* LCOV_EXCL_STOP */
    2541             :         }
    2542           1 :         if (tommy_tree_search_greater(&tree, &objects[0]) != 0
    2543           1 :                 || tommy_tree_search_greater_compare(&tree, &compare, &objects[0]) != 0
    2544           1 :                 || tommy_tree_search_less(&tree, &objects[0]) != 0
    2545           1 :                 || tommy_tree_search_less_compare(&tree, &compare, &objects[0]) != 0) {
    2546             :                 /* LCOV_EXCL_START */
    2547             :                 abort();
    2548             :                 /* LCOV_EXCL_STOP */
    2549             :         }
    2550             : 
    2551           1 :         tommy_tree_insert(&tree, &objects[0].node, &objects[0]);
    2552           1 :         if (tommy_tree_search(&tree, &objects[0]) != &objects[0]) {
    2553             :                 /* LCOV_EXCL_START */
    2554             :                 abort();
    2555             :                 /* LCOV_EXCL_STOP */
    2556             :         }
    2557           1 :         tommy_tree_remove_existing(&tree, &objects[0].node);
    2558             : 
    2559           1 :         struct object_tree* allocated = malloc(sizeof(*allocated));
    2560           1 :         allocated->value = 3;
    2561           1 :         tommy_tree_insert(&tree, &allocated->node, allocated);
    2562           1 :         tommy_tree_foreach(&tree, free);
    2563           1 :         tommy_tree_clear(&tree);
    2564           1 :         if (!tommy_tree_empty(&tree)) {
    2565             :                 /* LCOV_EXCL_START */
    2566             :                 abort();
    2567             :                 /* LCOV_EXCL_STOP */
    2568             :         }
    2569             : 
    2570           1 :         tommy_tree_insert(&tree, &objects[1].node, &objects[1]);
    2571           1 :         if (tommy_tree_remove(&tree, &objects[1]) != &objects[1]) {
    2572             :                 /* LCOV_EXCL_START */
    2573             :                 abort();
    2574             :                 /* LCOV_EXCL_STOP */
    2575             :         }
    2576           1 : }
    2577             : 
    2578             : /**
    2579             :  * Verifies exact node order and every link, including the circular head prev link.
    2580             :  */
    2581         136 : void test_list_transfer_order(tommy_list* list, tommy_node** expected, unsigned size)
    2582             : {
    2583         136 :         tommy_node* node = tommy_list_head(list);
    2584         136 :         tommy_node* prev = size ? expected[size - 1] : 0;
    2585             : 
    2586        4452 :         for (unsigned i = 0; i < size; ++i) {
    2587        4316 :                 if (node != expected[i] || node->prev != prev) {
    2588             :                         /* LCOV_EXCL_START */
    2589             :                         abort();
    2590             :                         /* LCOV_EXCL_STOP */
    2591             :                 }
    2592        4316 :                 prev = node;
    2593        4316 :                 node = node->next;
    2594             :         }
    2595         136 :         if (node != 0 || tommy_list_tail(list) != prev) {
    2596             :                 /* LCOV_EXCL_START */
    2597             :                 abort();
    2598             :                 /* LCOV_EXCL_STOP */
    2599             :         }
    2600             : 
    2601         136 :         node = tommy_list_tail(list);
    2602        4452 :         for (unsigned i = size; i > 0; --i) {
    2603        4316 :                 if (node != expected[i - 1]) {
    2604             :                         /* LCOV_EXCL_START */
    2605             :                         abort();
    2606             :                         /* LCOV_EXCL_STOP */
    2607             :                 }
    2608        4316 :                 node = tommy_list_prev(list, node);
    2609             :         }
    2610         136 :         if (node != 0) {
    2611             :                 /* LCOV_EXCL_START */
    2612             :                 abort();
    2613             :                 /* LCOV_EXCL_STOP */
    2614             :         }
    2615         136 : }
    2616             : 
    2617           1 : void test_builder(void)
    2618             : {
    2619           1 :         const unsigned counts[] = { 0, 1, 6 };
    2620             :         struct object objects[8];
    2621             :         tommy_node* expected[8];
    2622             :         tommy_builder builder;
    2623             :         tommy_list list;
    2624             : 
    2625           4 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    2626           9 :                 for (unsigned prefix = 0; prefix <= 2; prefix += 2) {
    2627           6 :                         unsigned size = prefix + counts[c];
    2628          18 :                         for (unsigned repeat = 0; repeat < 2; ++repeat) {
    2629             :                                 /* reuse the same builder after its previous contents were transferred */
    2630          12 :                                 tommy_node* builder_tail = tommy_builder_init(&builder);
    2631          12 :                                 tommy_list_init(&list);
    2632          52 :                                 for (unsigned i = 0; i < size; ++i) {
    2633          40 :                                         expected[i] = &objects[i].node;
    2634          40 :                                         objects[i].node.index = i;
    2635          40 :                                         if (i < prefix)
    2636          12 :                                                 tommy_list_insert_tail(&list, &objects[i].node, &objects[i]);
    2637             :                                 }
    2638             : 
    2639          28 :                                 for (unsigned i = prefix; i < size; ) {
    2640          16 :                                         unsigned length = size - i;
    2641          16 :                                         if (length > 2)
    2642           8 :                                                 length = 2;
    2643             :                                         tommy_list source;
    2644          16 :                                         tommy_list_init(&source);
    2645          44 :                                         for (unsigned j = 0; j < length; ++j)
    2646          28 :                                                 tommy_list_insert_tail(&source, &objects[i + j].node, &objects[i + j]);
    2647          16 :                                         tommy_node* head = source;
    2648          16 :                                         tommy_node* tail = head->prev;
    2649             :                                         /* only inner links are required; finalization must repair both boundary links */
    2650          16 :                                         head->prev = 0;
    2651          16 :                                         tail->next = &builder.dummy;
    2652          16 :                                         builder_tail = tommy_builder_concat(builder_tail, head, tail);
    2653          16 :                                         i += length;
    2654             :                                 }
    2655             : 
    2656          12 :                                 tommy_list_concat_builder(&list, &builder, builder_tail);
    2657          12 :                                 test_list_transfer_order(&list, expected, size);
    2658          52 :                                 for (unsigned i = 0; i < size; ++i) {
    2659          40 :                                         if (objects[i].node.data != &objects[i] || objects[i].node.index != i) {
    2660             :                                                 /* LCOV_EXCL_START */
    2661             :                                                 abort();
    2662             :                                                 /* LCOV_EXCL_STOP */
    2663             :                                         }
    2664             :                                 }
    2665             :                         }
    2666             :                 }
    2667             :         }
    2668           1 : }
    2669             : 
    2670           1 : void test_tree_to_list(void)
    2671             : {
    2672           1 :         const unsigned counts[] = { 0, 1, 96 };
    2673             :         struct object_tree objects[98];
    2674             :         tommy_size_t index[98];
    2675             :         tommy_node* expected[98];
    2676             : 
    2677           4 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    2678           9 :                 for (unsigned prefix = 0; prefix <= 2; prefix += 2) {
    2679             :                         tommy_tree tree;
    2680             :                         tommy_list list;
    2681           6 :                         unsigned size = prefix + counts[c];
    2682           6 :                         tommy_tree_init(&tree, &compare);
    2683           6 :                         tommy_list_init(&list);
    2684             : 
    2685         206 :                         for (unsigned i = 0; i < size; ++i) {
    2686         200 :                                 objects[i].value = i % 5;
    2687         200 :                                 if (i < prefix) {
    2688           6 :                                         objects[i].node.index = 0;
    2689           6 :                                         tommy_list_insert_tail(&list, &objects[i].node, &objects[i]);
    2690             :                                 } else {
    2691         194 :                                         tommy_tree_insert(&tree, &objects[i].node, &objects[i]);
    2692             :                                 }
    2693             :                         }
    2694         206 :                         for (unsigned i = 0; i < size; ++i)
    2695         200 :                                 index[i] = objects[i].node.index;
    2696             : 
    2697             :                         /* equal elements must retain insertion order after AVL rotations. */
    2698           6 :                         unsigned pos = 0;
    2699          12 :                         for (unsigned i = 0; i < prefix; ++i) {
    2700           6 :                                 expected[pos] = &objects[i].node;
    2701           6 :                                 ++pos;
    2702             :                         }
    2703          36 :                         for (int group = 0; group < 5; ++group)
    2704        1000 :                                 for (unsigned i = prefix; i < size; ++i)
    2705         970 :                                         if (objects[i].value == group) {
    2706         194 :                                                 expected[pos] = &objects[i].node;
    2707         194 :                                                 ++pos;
    2708             :                                         }
    2709             : 
    2710          18 :                         for (unsigned repeat = 0; repeat < 2; ++repeat) {
    2711          12 :                                 tommy_tree_to_list(&tree, &list);
    2712          12 :                                 test_list_transfer_order(&list, expected, size);
    2713          12 :                                 if (!tommy_tree_empty(&tree) || tommy_tree_count(&tree) != 0 || tree.cmp != &compare) {
    2714             :                                         /* LCOV_EXCL_START */
    2715             :                                         abort();
    2716             :                                         /* LCOV_EXCL_STOP */
    2717             :                                 }
    2718         412 :                                 for (unsigned i = 0; i < size; ++i) {
    2719         400 :                                         if (objects[i].node.data != &objects[i] || objects[i].node.index != index[i]) {
    2720             :                                                 /* LCOV_EXCL_START */
    2721             :                                                 abort();
    2722             :                                                 /* LCOV_EXCL_STOP */
    2723             :                                         }
    2724             :                                 }
    2725             :                         }
    2726             : 
    2727             :                         /* reinsert the transferred nodes, then verify search and removal. */
    2728         206 :                         while (!tommy_list_empty(&list)) {
    2729         200 :                                 struct object_tree* data = tommy_list_remove_head(&list);
    2730         200 :                                 tommy_tree_insert(&tree, &data->node, data);
    2731             :                         }
    2732         206 :                         for (unsigned i = 0; i < size; ++i) {
    2733         200 :                                 if (tommy_tree_search(&tree, &objects[i]) == 0
    2734         200 :                                         || tommy_tree_remove_existing(&tree, &objects[i].node) != &objects[i]
    2735         200 :                                         || tommy_tree_count(&tree) != size - i - 1) {
    2736             :                                         /* LCOV_EXCL_START */
    2737             :                                         abort();
    2738             :                                         /* LCOV_EXCL_STOP */
    2739             :                                 }
    2740             :                         }
    2741             :                 }
    2742             :         }
    2743           1 : }
    2744             : 
    2745           1 : void test_array(void)
    2746             : {
    2747           1 :         const unsigned size = 50 * TOMMY_SIZE;
    2748             :         tommy_array array;
    2749             : 
    2750           1 :         tommy_array_init(&array);
    2751             : 
    2752             :         /* no op */
    2753           1 :         tommy_array_grow(&array, 0);
    2754             : 
    2755           6 :         SPEED_START("array init") {
    2756           3 :                 SPEED_UNTIMED(
    2757             :                         tommy_array_done(&array);
    2758             :                         tommy_array_init(&array);
    2759             :                 );
    2760    15000003 :                 for (tommy_uintptr_t i = 0; i < size; ++i) {
    2761    15000000 :                         tommy_array_grow(&array, i + 1);
    2762    15000000 :                         if (tommy_array_get(&array, i) != 0) {
    2763             :                                 /* LCOV_EXCL_START */
    2764             :                                 abort();
    2765             :                                 /* LCOV_EXCL_STOP */
    2766             :                         }
    2767             :                 }
    2768           3 :         } SPEED_STOP
    2769             : 
    2770           8 :         SPEED_START("array set")
    2771             :         {
    2772    20000004 :                 for (tommy_uintptr_t i = 0; i < size; ++i) {
    2773    20000000 :                         tommy_array_set(&array, i, (void*)i);
    2774             :                 }
    2775             :         }
    2776           4 :         SPEED_STOP
    2777             : 
    2778          10 :         SPEED_START("array get")
    2779             :         {
    2780    25000005 :                 for (tommy_uintptr_t i = 0; i < size; ++i) {
    2781    25000000 :                         if (tommy_array_get(&array, i) != (void*)i) {
    2782             :                                 /* LCOV_EXCL_START */
    2783             :                                 abort();
    2784             :                                 /* LCOV_EXCL_STOP */
    2785             :                         }
    2786             :                 }
    2787             :         }
    2788           5 :         SPEED_STOP
    2789             : 
    2790           1 :         if (tommy_array_memory_usage(&array) < size * sizeof(void*)) {
    2791             :                 /* LCOV_EXCL_START */
    2792             :                 abort();
    2793             :                 /* LCOV_EXCL_STOP */
    2794             :         }
    2795             : 
    2796           1 :         tommy_array_done(&array);
    2797           1 : }
    2798             : 
    2799           1 : void test_arrayof(void)
    2800             : {
    2801           1 :         const unsigned size = 50 * TOMMY_SIZE;
    2802             :         tommy_arrayof arrayof;
    2803             : 
    2804           1 :         tommy_arrayof_init(&arrayof, sizeof(unsigned));
    2805             : 
    2806             :         /* no op */
    2807           1 :         tommy_arrayof_grow(&arrayof, 0);
    2808             : 
    2809           6 :         SPEED_START("arrayof init") {
    2810           3 :                 SPEED_UNTIMED(
    2811             :                         tommy_arrayof_done(&arrayof);
    2812             :                         tommy_arrayof_init(&arrayof, sizeof(unsigned));
    2813             :                 );
    2814    15000003 :                 for (unsigned i = 0; i < size; ++i) {
    2815    15000000 :                         tommy_arrayof_grow(&arrayof, i + 1);
    2816    15000000 :                         unsigned* ref = tommy_arrayof_ref(&arrayof, i);
    2817    15000000 :                         if (*ref != 0) {
    2818             :                                 /* LCOV_EXCL_START */
    2819             :                                 abort();
    2820             :                                 /* LCOV_EXCL_STOP */
    2821             :                         }
    2822             :                 }
    2823           3 :         } SPEED_STOP
    2824             : 
    2825          10 :         SPEED_START("arrayof set")
    2826             :         {
    2827    25000005 :                 for (unsigned i = 0; i < size; ++i) {
    2828    25000000 :                         unsigned* ref = tommy_arrayof_ref(&arrayof, i);
    2829    25000000 :                         *ref = i;
    2830             :                 }
    2831             :         }
    2832           5 :         SPEED_STOP
    2833             : 
    2834          10 :         SPEED_START("arrayof get")
    2835             :         {
    2836    25000005 :                 for (unsigned i = 0; i < size; ++i) {
    2837    25000000 :                         unsigned* ref = tommy_arrayof_ref(&arrayof, i);
    2838    25000000 :                         if (*ref != i) {
    2839             :                                 /* LCOV_EXCL_START */
    2840             :                                 abort();
    2841             :                                 /* LCOV_EXCL_STOP */
    2842             :                         }
    2843             :                 }
    2844             :         }
    2845           5 :         SPEED_STOP
    2846             : 
    2847           1 :         if (tommy_arrayof_memory_usage(&arrayof) < size * sizeof(unsigned)) {
    2848             :                 /* LCOV_EXCL_START */
    2849             :                 abort();
    2850             :                 /* LCOV_EXCL_STOP */
    2851             :         }
    2852             : 
    2853           1 :         tommy_arrayof_done(&arrayof);
    2854           1 : }
    2855             : 
    2856           1 : void test_arrayblk(void)
    2857             : {
    2858           1 :         const unsigned size = 50 * TOMMY_SIZE;
    2859             :         tommy_arrayblk arrayblk;
    2860           1 :         void** first_ref = 0;
    2861             : 
    2862           1 :         tommy_arrayblk_init(&arrayblk);
    2863             : 
    2864             :         /* no op */
    2865           1 :         tommy_arrayblk_grow(&arrayblk, 0);
    2866             : 
    2867           6 :         SPEED_START("arrayblk init") {
    2868           3 :                 SPEED_UNTIMED(
    2869             :                         tommy_arrayblk_done(&arrayblk);
    2870             :                         tommy_arrayblk_init(&arrayblk);
    2871             :                 );
    2872    15000003 :                 for (tommy_uintptr_t i = 0; i < size; ++i) {
    2873    15000000 :                         tommy_arrayblk_grow(&arrayblk, i + 1);
    2874    15000000 :                         if (i == 0)
    2875           3 :                                 first_ref = tommy_arrayblk_ref(&arrayblk, i);
    2876    15000000 :                         if (tommy_arrayblk_get(&arrayblk, i) != 0) {
    2877             :                                 /* LCOV_EXCL_START */
    2878             :                                 abort();
    2879             :                                 /* LCOV_EXCL_STOP */
    2880             :                         }
    2881             :                 }
    2882           3 :         } SPEED_STOP
    2883           1 :         if (first_ref != tommy_arrayblk_ref(&arrayblk, 0)) {
    2884             :                 /* LCOV_EXCL_START */
    2885             :                 abort();
    2886             :                 /* LCOV_EXCL_STOP */
    2887             :         }
    2888             : 
    2889          10 :         SPEED_START("arrayblk set") {
    2890    25000005 :                 for (tommy_uintptr_t i = 0; i < size; ++i) {
    2891    25000000 :                         tommy_arrayblk_set(&arrayblk, i, (void*)i);
    2892             :                 }
    2893           5 :         } SPEED_STOP
    2894             : 
    2895          10 :         SPEED_START("arrayblk get")
    2896             :         {
    2897    25000005 :                 for (tommy_uintptr_t i = 0; i < size; ++i) {
    2898    25000000 :                         if (tommy_arrayblk_get(&arrayblk, i) != (void*)i) {
    2899             :                                 /* LCOV_EXCL_START */
    2900             :                                 abort();
    2901             :                                 /* LCOV_EXCL_STOP */
    2902             :                         }
    2903             :                 }
    2904             :         }
    2905           5 :         SPEED_STOP
    2906             : 
    2907           1 :         if (tommy_arrayblk_memory_usage(&arrayblk) < size * sizeof(void*)) {
    2908             :                 /* LCOV_EXCL_START */
    2909             :                 abort();
    2910             :                 /* LCOV_EXCL_STOP */
    2911             :         }
    2912             : 
    2913           1 :         tommy_arrayblk_done(&arrayblk);
    2914           1 : }
    2915             : 
    2916           1 : void test_arrayblkof(void)
    2917             : {
    2918           1 :         const unsigned size = 50 * TOMMY_SIZE;
    2919             :         tommy_arrayblkof arrayblkof;
    2920           1 :         unsigned* first_ref = 0;
    2921             : 
    2922           1 :         tommy_arrayblkof_init(&arrayblkof, sizeof(unsigned));
    2923             : 
    2924             :         /* no op */
    2925           1 :         tommy_arrayblkof_grow(&arrayblkof, 0);
    2926             : 
    2927           8 :         SPEED_START("arrayblkof init") {
    2928           4 :                 SPEED_UNTIMED(
    2929             :                         tommy_arrayblkof_done(&arrayblkof);
    2930             :                         tommy_arrayblkof_init(&arrayblkof, sizeof(unsigned));
    2931             :                 );
    2932    20000004 :                 for (unsigned i = 0; i < size; ++i) {
    2933    20000000 :                         tommy_arrayblkof_grow(&arrayblkof, i + 1);
    2934    20000000 :                         unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    2935    20000000 :                         if (i == 0)
    2936           4 :                                 first_ref = ref;
    2937    20000000 :                         if (*ref != 0) {
    2938             :                                 /* LCOV_EXCL_START */
    2939             :                                 abort();
    2940             :                                 /* LCOV_EXCL_STOP */
    2941             :                         }
    2942             :                 }
    2943           4 :         } SPEED_STOP
    2944           1 :         if (first_ref != tommy_arrayblkof_ref(&arrayblkof, 0)) {
    2945             :                 /* LCOV_EXCL_START */
    2946             :                 abort();
    2947             :                 /* LCOV_EXCL_STOP */
    2948             :         }
    2949             : 
    2950          14 :         SPEED_START("arrayblkof set") {
    2951    35000007 :                 for (unsigned i = 0; i < size; ++i) {
    2952    35000000 :                         unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    2953    35000000 :                         *ref = i;
    2954             :                 }
    2955           7 :         } SPEED_STOP
    2956             : 
    2957          14 :         SPEED_START("arrayblkof get")
    2958             :         {
    2959    35000007 :                 for (unsigned i = 0; i < size; ++i) {
    2960    35000000 :                         unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    2961    35000000 :                         if (*ref != i) {
    2962             :                                 /* LCOV_EXCL_START */
    2963             :                                 abort();
    2964             :                                 /* LCOV_EXCL_STOP */
    2965             :                         }
    2966             :                 }
    2967             :         }
    2968           7 :         SPEED_STOP
    2969             : 
    2970           1 :         if (tommy_arrayblkof_memory_usage(&arrayblkof) < size * sizeof(unsigned)) {
    2971             :                 /* LCOV_EXCL_START */
    2972             :                 abort();
    2973             :                 /* LCOV_EXCL_STOP */
    2974             :         }
    2975             : 
    2976           1 :         tommy_arrayblkof_done(&arrayblkof);
    2977           1 : }
    2978             : 
    2979           1 : void test_array_tail_ops(void)
    2980             : {
    2981           1 :         unsigned value = 7;
    2982             : 
    2983             :         {
    2984             :                 tommy_array array;
    2985           1 :                 tommy_array_init(&array);
    2986           1 :                 if (!tommy_array_empty(&array)) {
    2987             :                         /* LCOV_EXCL_START */
    2988             :                         abort();
    2989             :                         /* LCOV_EXCL_STOP */
    2990             :                 }
    2991          66 :                 for (tommy_size_t i = 0; i < 65; ++i)
    2992          65 :                         tommy_array_insert_tail(&array, &value);
    2993           1 :                 if (tommy_array_empty(&array) || tommy_array_size(&array) != 65 || tommy_array_tail(&array) != &value) {
    2994             :                         /* LCOV_EXCL_START */
    2995             :                         abort();
    2996             :                         /* LCOV_EXCL_STOP */
    2997             :                 }
    2998           1 :                 if (tommy_array_remove_tail(&array) != &value || tommy_array_size(&array) != 64) {
    2999             :                         /* LCOV_EXCL_START */
    3000             :                         abort();
    3001             :                         /* LCOV_EXCL_STOP */
    3002             :                 }
    3003           1 :                 tommy_array_grow(&array, 65);
    3004           1 :                 if (tommy_array_tail(&array) != 0) {
    3005             :                         /* LCOV_EXCL_START */
    3006             :                         abort();
    3007             :                         /* LCOV_EXCL_STOP */
    3008             :                 }
    3009           1 :                 tommy_array_clear(&array);
    3010           1 :                 if (!tommy_array_empty(&array)) {
    3011             :                         /* LCOV_EXCL_START */
    3012             :                         abort();
    3013             :                         /* LCOV_EXCL_STOP */
    3014             :                 }
    3015           1 :                 tommy_array_done(&array);
    3016             :         }
    3017             : 
    3018             :         {
    3019             :                 tommy_arrayof array;
    3020           1 :                 tommy_arrayof_init(&array, sizeof(unsigned));
    3021           1 :                 if (!tommy_arrayof_empty(&array)) {
    3022             :                         /* LCOV_EXCL_START */
    3023             :                         abort();
    3024             :                         /* LCOV_EXCL_STOP */
    3025             :                 }
    3026          66 :                 for (tommy_size_t i = 0; i < 65; ++i) {
    3027          65 :                         unsigned* ref = tommy_arrayof_insert_tail(&array);
    3028          65 :                         if (ref != tommy_arrayof_tail(&array) || *ref != 0) {
    3029             :                                 /* LCOV_EXCL_START */
    3030             :                                 abort();
    3031             :                                 /* LCOV_EXCL_STOP */
    3032             :                         }
    3033          65 :                         *ref = value;
    3034             :                 }
    3035           1 :                 unsigned* tail = tommy_arrayof_tail(&array);
    3036           1 :                 if (tommy_arrayof_empty(&array) || tommy_arrayof_size(&array) != 65 || *tail != value) {
    3037             :                         /* LCOV_EXCL_START */
    3038             :                         abort();
    3039             :                         /* LCOV_EXCL_STOP */
    3040             :                 }
    3041           1 :                 tommy_arrayof_remove_tail(&array);
    3042           1 :                 tail = tommy_arrayof_insert_tail(&array);
    3043           1 :                 if (tommy_arrayof_size(&array) != 65 || *tail != 0) {
    3044             :                         /* LCOV_EXCL_START */
    3045             :                         abort();
    3046             :                         /* LCOV_EXCL_STOP */
    3047             :                 }
    3048           1 :                 tommy_arrayof_clear(&array);
    3049           1 :                 if (!tommy_arrayof_empty(&array)) {
    3050             :                         /* LCOV_EXCL_START */
    3051             :                         abort();
    3052             :                         /* LCOV_EXCL_STOP */
    3053             :                 }
    3054           1 :                 tommy_arrayof_done(&array);
    3055             :         }
    3056             : 
    3057             :         {
    3058             :                 tommy_arrayblk array;
    3059           1 :                 tommy_arrayblk_init(&array);
    3060           1 :                 if (!tommy_arrayblk_empty(&array)) {
    3061             :                         /* LCOV_EXCL_START */
    3062             :                         abort();
    3063             :                         /* LCOV_EXCL_STOP */
    3064             :                 }
    3065        4098 :                 for (tommy_size_t i = 0; i < TOMMY_ARRAYBLK_SIZE + 1; ++i)
    3066        4097 :                         tommy_arrayblk_insert_tail(&array, &value);
    3067           1 :                 if (tommy_arrayblk_empty(&array) || tommy_arrayblk_size(&array) != TOMMY_ARRAYBLK_SIZE + 1 || tommy_arrayblk_tail(&array) != &value) {
    3068             :                         /* LCOV_EXCL_START */
    3069             :                         abort();
    3070             :                         /* LCOV_EXCL_STOP */
    3071             :                 }
    3072           1 :                 if (tommy_arrayblk_remove_tail(&array) != &value || tommy_arrayblk_size(&array) != TOMMY_ARRAYBLK_SIZE) {
    3073             :                         /* LCOV_EXCL_START */
    3074             :                         abort();
    3075             :                         /* LCOV_EXCL_STOP */
    3076             :                 }
    3077           1 :                 tommy_arrayblk_grow(&array, TOMMY_ARRAYBLK_SIZE + 1);
    3078           1 :                 if (tommy_arrayblk_tail(&array) != 0) {
    3079             :                         /* LCOV_EXCL_START */
    3080             :                         abort();
    3081             :                         /* LCOV_EXCL_STOP */
    3082             :                 }
    3083           1 :                 tommy_arrayblk_clear(&array);
    3084           1 :                 if (!tommy_arrayblk_empty(&array)) {
    3085             :                         /* LCOV_EXCL_START */
    3086             :                         abort();
    3087             :                         /* LCOV_EXCL_STOP */
    3088             :                 }
    3089           1 :                 tommy_arrayblk_done(&array);
    3090             :         }
    3091             : 
    3092             :         {
    3093             :                 tommy_arrayblkof array;
    3094           1 :                 tommy_arrayblkof_init(&array, sizeof(unsigned));
    3095           1 :                 if (!tommy_arrayblkof_empty(&array)) {
    3096             :                         /* LCOV_EXCL_START */
    3097             :                         abort();
    3098             :                         /* LCOV_EXCL_STOP */
    3099             :                 }
    3100        4098 :                 for (tommy_size_t i = 0; i < TOMMY_ARRAYBLKOF_SIZE + 1; ++i) {
    3101        4097 :                         unsigned* ref = tommy_arrayblkof_insert_tail(&array);
    3102        4097 :                         if (ref != tommy_arrayblkof_tail(&array) || *ref != 0) {
    3103             :                                 /* LCOV_EXCL_START */
    3104             :                                 abort();
    3105             :                                 /* LCOV_EXCL_STOP */
    3106             :                         }
    3107        4097 :                         *ref = value;
    3108             :                 }
    3109           1 :                 unsigned* tail = tommy_arrayblkof_tail(&array);
    3110           1 :                 if (tommy_arrayblkof_empty(&array) || tommy_arrayblkof_size(&array) != TOMMY_ARRAYBLKOF_SIZE + 1 || *tail != value) {
    3111             :                         /* LCOV_EXCL_START */
    3112             :                         abort();
    3113             :                         /* LCOV_EXCL_STOP */
    3114             :                 }
    3115           1 :                 tommy_arrayblkof_remove_tail(&array);
    3116           1 :                 tail = tommy_arrayblkof_insert_tail(&array);
    3117           1 :                 if (tommy_arrayblkof_size(&array) != TOMMY_ARRAYBLKOF_SIZE + 1 || *tail != 0) {
    3118             :                         /* LCOV_EXCL_START */
    3119             :                         abort();
    3120             :                         /* LCOV_EXCL_STOP */
    3121             :                 }
    3122           1 :                 tommy_arrayblkof_clear(&array);
    3123           1 :                 if (!tommy_arrayblkof_empty(&array)) {
    3124             :                         /* LCOV_EXCL_START */
    3125             :                         abort();
    3126             :                         /* LCOV_EXCL_STOP */
    3127             :                 }
    3128           1 :                 tommy_arrayblkof_done(&array);
    3129             :         }
    3130           1 : }
    3131             : 
    3132           1 : void test_array_capacity_swap(void)
    3133             : {
    3134           1 :         unsigned value = 7;
    3135             : 
    3136             :         {
    3137             :                 tommy_array first;
    3138             :                 tommy_array second;
    3139           1 :                 tommy_array_init(&first);
    3140           1 :                 tommy_array_init(&second);
    3141           1 :                 if (tommy_array_capacity(&first) != 64 || tommy_array_capacity(&second) != 64) {
    3142             :                         /* LCOV_EXCL_START */
    3143             :                         abort();
    3144             :                         /* LCOV_EXCL_STOP */
    3145             :                 }
    3146           1 :                 tommy_array_reserve(&first, 65);
    3147           1 :                 tommy_array_grow(&first, 65);
    3148           1 :                 tommy_array_set(&first, 0, &value);
    3149           1 :                 tommy_array_set(&first, 64, &value);
    3150           1 :                 void** head = tommy_array_ref(&first, 0);
    3151           1 :                 void** tail = tommy_array_ref(&first, 64);
    3152           1 :                 if (tommy_array_capacity(&first) != 128) {
    3153             :                         /* LCOV_EXCL_START */
    3154             :                         abort();
    3155             :                         /* LCOV_EXCL_STOP */
    3156             :                 }
    3157           1 :                 tommy_array_swap(&first, &second);
    3158           1 :                 tommy_array_swap(&second, &second);
    3159           1 :                 if (tommy_array_size(&first) != 0 || tommy_array_capacity(&first) != 64 || tommy_array_size(&second) != 65 || tommy_array_capacity(&second) != 128 || tommy_array_ref(&second, 0) != head || tommy_array_ref(&second, 64) != tail || tommy_array_get(&second, 64) != &value) {
    3160             :                         /* LCOV_EXCL_START */
    3161             :                         abort();
    3162             :                         /* LCOV_EXCL_STOP */
    3163             :                 }
    3164           1 :                 tommy_array_resize(&second, 1);
    3165           1 :                 tommy_array_shrink(&second);
    3166           1 :                 if (tommy_array_capacity(&second) != 64 || tommy_array_ref(&second, 0) != head || tommy_array_get(&second, 0) != &value) {
    3167             :                         /* LCOV_EXCL_START */
    3168             :                         abort();
    3169             :                         /* LCOV_EXCL_STOP */
    3170             :                 }
    3171           1 :                 tommy_array_done(&first);
    3172           1 :                 tommy_array_done(&second);
    3173             :         }
    3174             : 
    3175             :         {
    3176             :                 tommy_arrayblk first;
    3177             :                 tommy_arrayblk second;
    3178           1 :                 tommy_arrayblk_init(&first);
    3179           1 :                 tommy_arrayblk_init(&second);
    3180           1 :                 if (tommy_arrayblk_capacity(&first) != 0 || tommy_arrayblk_capacity(&second) != 0) {
    3181             :                         /* LCOV_EXCL_START */
    3182             :                         abort();
    3183             :                         /* LCOV_EXCL_STOP */
    3184             :                 }
    3185           1 :                 tommy_arrayblk_reserve(&first, TOMMY_ARRAYBLK_SIZE + 1);
    3186           1 :                 tommy_arrayblk_grow(&first, TOMMY_ARRAYBLK_SIZE + 1);
    3187           1 :                 tommy_arrayblk_set(&first, 0, &value);
    3188           1 :                 tommy_arrayblk_set(&first, TOMMY_ARRAYBLK_SIZE, &value);
    3189           1 :                 void** head = tommy_arrayblk_ref(&first, 0);
    3190           1 :                 void** tail = tommy_arrayblk_ref(&first, TOMMY_ARRAYBLK_SIZE);
    3191           1 :                 if (tommy_arrayblk_capacity(&first) != 2 * TOMMY_ARRAYBLK_SIZE) {
    3192             :                         /* LCOV_EXCL_START */
    3193             :                         abort();
    3194             :                         /* LCOV_EXCL_STOP */
    3195             :                 }
    3196           1 :                 tommy_arrayblk_swap(&first, &second);
    3197           1 :                 tommy_arrayblk_swap(&second, &second);
    3198           1 :                 if (tommy_arrayblk_size(&first) != 0 || tommy_arrayblk_capacity(&first) != 0 || tommy_arrayblk_size(&second) != TOMMY_ARRAYBLK_SIZE + 1 || tommy_arrayblk_capacity(&second) != 2 * TOMMY_ARRAYBLK_SIZE || tommy_arrayblk_ref(&second, 0) != head || tommy_arrayblk_ref(&second, TOMMY_ARRAYBLK_SIZE) != tail || tommy_arrayblk_get(&second, TOMMY_ARRAYBLK_SIZE) != &value) {
    3199             :                         /* LCOV_EXCL_START */
    3200             :                         abort();
    3201             :                         /* LCOV_EXCL_STOP */
    3202             :                 }
    3203           1 :                 tommy_arrayblk_resize(&second, 1);
    3204           1 :                 tommy_arrayblk_shrink(&second);
    3205           1 :                 if (tommy_arrayblk_capacity(&second) != TOMMY_ARRAYBLK_SIZE || tommy_arrayblk_ref(&second, 0) != head || tommy_arrayblk_get(&second, 0) != &value) {
    3206             :                         /* LCOV_EXCL_START */
    3207             :                         abort();
    3208             :                         /* LCOV_EXCL_STOP */
    3209             :                 }
    3210           1 :                 tommy_arrayblk_done(&first);
    3211           1 :                 tommy_arrayblk_done(&second);
    3212             :         }
    3213             : 
    3214             :         {
    3215             :                 tommy_arrayof first;
    3216             :                 tommy_arrayof second;
    3217           1 :                 tommy_arrayof_init(&first, sizeof(unsigned));
    3218           1 :                 tommy_arrayof_init(&second, 2 * sizeof(unsigned));
    3219           1 :                 tommy_arrayof_reserve(&first, 65);
    3220           1 :                 tommy_arrayof_grow(&first, 65);
    3221           1 :                 tommy_arrayof_grow(&second, 1);
    3222           1 :                 unsigned* head = tommy_arrayof_ref(&first, 0);
    3223           1 :                 unsigned* tail = tommy_arrayof_ref(&first, 64);
    3224           1 :                 unsigned* pair = tommy_arrayof_ref(&second, 0);
    3225           1 :                 *head = value;
    3226           1 :                 *tail = value + 1;
    3227           1 :                 pair[0] = value + 2;
    3228           1 :                 pair[1] = value + 3;
    3229           1 :                 if (tommy_arrayof_capacity(&first) != 128 || tommy_arrayof_capacity(&second) != 64) {
    3230             :                         /* LCOV_EXCL_START */
    3231             :                         abort();
    3232             :                         /* LCOV_EXCL_STOP */
    3233             :                 }
    3234           1 :                 tommy_arrayof_swap(&first, &second);
    3235           1 :                 tommy_arrayof_swap(&second, &second);
    3236           1 :                 tommy_arrayof_grow(&first, 2);
    3237           1 :                 if (tommy_arrayof_size(&first) != 2 || tommy_arrayof_size(&second) != 65 || tommy_arrayof_capacity(&first) != 64 || tommy_arrayof_capacity(&second) != 128 || tommy_arrayof_ref(&first, 0) != pair || tommy_arrayof_ref(&first, 1) != pair + 2 || tommy_arrayof_ref(&second, 64) != tail || pair[0] != value + 2 || pair[1] != value + 3 || *tail != value + 1) {
    3238             :                         /* LCOV_EXCL_START */
    3239             :                         abort();
    3240             :                         /* LCOV_EXCL_STOP */
    3241             :                 }
    3242           1 :                 tommy_arrayof_resize(&second, 1);
    3243           1 :                 tommy_arrayof_shrink(&second);
    3244           1 :                 if (tommy_arrayof_capacity(&second) != 64 || tommy_arrayof_ref(&second, 0) != head || *head != value) {
    3245             :                         /* LCOV_EXCL_START */
    3246             :                         abort();
    3247             :                         /* LCOV_EXCL_STOP */
    3248             :                 }
    3249           1 :                 tommy_arrayof_done(&first);
    3250           1 :                 tommy_arrayof_done(&second);
    3251             :         }
    3252             : 
    3253             :         {
    3254             :                 tommy_arrayblkof first;
    3255             :                 tommy_arrayblkof second;
    3256           1 :                 tommy_arrayblkof_init(&first, sizeof(unsigned));
    3257           1 :                 tommy_arrayblkof_init(&second, 2 * sizeof(unsigned));
    3258           1 :                 if (tommy_arrayblkof_capacity(&first) != 0 || tommy_arrayblkof_capacity(&second) != 0) {
    3259             :                         /* LCOV_EXCL_START */
    3260             :                         abort();
    3261             :                         /* LCOV_EXCL_STOP */
    3262             :                 }
    3263           1 :                 tommy_arrayblkof_reserve(&first, TOMMY_ARRAYBLKOF_SIZE + 1);
    3264           1 :                 tommy_arrayblkof_grow(&first, TOMMY_ARRAYBLKOF_SIZE + 1);
    3265           1 :                 tommy_arrayblkof_grow(&second, 1);
    3266           1 :                 unsigned* head = tommy_arrayblkof_ref(&first, 0);
    3267           1 :                 unsigned* tail = tommy_arrayblkof_ref(&first, TOMMY_ARRAYBLKOF_SIZE);
    3268           1 :                 unsigned* pair = tommy_arrayblkof_ref(&second, 0);
    3269           1 :                 *head = value;
    3270           1 :                 *tail = value + 1;
    3271           1 :                 pair[0] = value + 2;
    3272           1 :                 pair[1] = value + 3;
    3273           1 :                 if (tommy_arrayblkof_capacity(&first) != 2 * TOMMY_ARRAYBLKOF_SIZE || tommy_arrayblkof_capacity(&second) != TOMMY_ARRAYBLKOF_SIZE) {
    3274             :                         /* LCOV_EXCL_START */
    3275             :                         abort();
    3276             :                         /* LCOV_EXCL_STOP */
    3277             :                 }
    3278           1 :                 tommy_arrayblkof_swap(&first, &second);
    3279           1 :                 tommy_arrayblkof_swap(&second, &second);
    3280           1 :                 tommy_arrayblkof_grow(&first, 2);
    3281           1 :                 if (tommy_arrayblkof_size(&first) != 2 || tommy_arrayblkof_size(&second) != TOMMY_ARRAYBLKOF_SIZE + 1 || tommy_arrayblkof_capacity(&first) != TOMMY_ARRAYBLKOF_SIZE || tommy_arrayblkof_capacity(&second) != 2 * TOMMY_ARRAYBLKOF_SIZE || tommy_arrayblkof_ref(&first, 0) != pair || tommy_arrayblkof_ref(&first, 1) != pair + 2 || tommy_arrayblkof_ref(&second, TOMMY_ARRAYBLKOF_SIZE) != tail || pair[0] != value + 2 || pair[1] != value + 3 || *tail != value + 1) {
    3282             :                         /* LCOV_EXCL_START */
    3283             :                         abort();
    3284             :                         /* LCOV_EXCL_STOP */
    3285             :                 }
    3286           1 :                 tommy_arrayblkof_resize(&second, 1);
    3287           1 :                 tommy_arrayblkof_shrink(&second);
    3288           1 :                 if (tommy_arrayblkof_capacity(&second) != TOMMY_ARRAYBLKOF_SIZE || tommy_arrayblkof_ref(&second, 0) != head || *head != value) {
    3289             :                         /* LCOV_EXCL_START */
    3290             :                         abort();
    3291             :                         /* LCOV_EXCL_STOP */
    3292             :                 }
    3293           1 :                 tommy_arrayblkof_done(&first);
    3294           1 :                 tommy_arrayblkof_done(&second);
    3295             :         }
    3296           1 : }
    3297             : 
    3298           1 : void test_array_ops(void)
    3299             : {
    3300             :         tommy_array array;
    3301             : 
    3302           1 :         tommy_array_init(&array);
    3303             : 
    3304             :         /* foreach on empty array */
    3305           1 :         the_count = 0;
    3306           1 :         tommy_array_foreach(&array, count_callback);
    3307           1 :         if (the_count != 0) {
    3308             :                 /* LCOV_EXCL_START */
    3309             :                 abort();
    3310             :                 /* LCOV_EXCL_STOP */
    3311             :         }
    3312           1 :         the_count = 0;
    3313           1 :         tommy_array_foreach_arg(&array, count_arg_callback, &the_count);
    3314           1 :         if (the_count != 0) {
    3315             :                 /* LCOV_EXCL_START */
    3316             :                 abort();
    3317             :                 /* LCOV_EXCL_STOP */
    3318             :         }
    3319             : 
    3320             :         /* reserve on empty array */
    3321           1 :         tommy_array_reserve(&array, 0);
    3322           1 :         if (tommy_array_size(&array) != 0) {
    3323             :                 /* LCOV_EXCL_START */
    3324             :                 abort();
    3325             :                 /* LCOV_EXCL_STOP */
    3326             :         }
    3327             : 
    3328           1 :         tommy_array_reserve(&array, 50);
    3329           1 :         if (tommy_array_size(&array) != 0 || tommy_array_memory_usage(&array) < 64 * sizeof(void*)) {
    3330             :                 /* LCOV_EXCL_START */
    3331             :                 abort();
    3332             :                 /* LCOV_EXCL_STOP */
    3333             :         }
    3334             : 
    3335           1 :         tommy_array_reserve(&array, 100);
    3336           1 :         if (tommy_array_size(&array) != 0 || tommy_array_memory_usage(&array) < 128 * sizeof(void*)) {
    3337             :                 /* LCOV_EXCL_START */
    3338             :                 abort();
    3339             :                 /* LCOV_EXCL_STOP */
    3340             :         }
    3341             : 
    3342             :         /* resize growing */
    3343           1 :         tommy_array_resize(&array, 100);
    3344           1 :         if (tommy_array_size(&array) != 100) {
    3345             :                 /* LCOV_EXCL_START */
    3346             :                 abort();
    3347             :                 /* LCOV_EXCL_STOP */
    3348             :         }
    3349         101 :         for (tommy_uintptr_t i = 0; i < 100; ++i) {
    3350         100 :                 if (tommy_array_get(&array, i) != 0) {
    3351             :                         /* LCOV_EXCL_START */
    3352             :                         abort();
    3353             :                         /* LCOV_EXCL_STOP */
    3354             :                 }
    3355         100 :                 tommy_array_set(&array, i, (void*)(i + 1));
    3356             :         }
    3357             : 
    3358             :         /* resize shrinking and invariant check */
    3359           1 :         tommy_array_resize(&array, 50);
    3360           1 :         if (tommy_array_size(&array) != 50) {
    3361             :                 /* LCOV_EXCL_START */
    3362             :                 abort();
    3363             :                 /* LCOV_EXCL_STOP */
    3364             :         }
    3365          51 :         for (tommy_uintptr_t i = 0; i < 50; ++i) {
    3366          50 :                 if (tommy_array_get(&array, i) != (void*)(i + 1)) {
    3367             :                         /* LCOV_EXCL_START */
    3368             :                         abort();
    3369             :                         /* LCOV_EXCL_STOP */
    3370             :                 }
    3371             :         }
    3372           1 :         tommy_array_resize(&array, 100);
    3373          51 :         for (tommy_uintptr_t i = 50; i < 100; ++i) {
    3374          50 :                 if (tommy_array_get(&array, i) != 0) {
    3375             :                         /* LCOV_EXCL_START */
    3376             :                         abort();
    3377             :                         /* LCOV_EXCL_STOP */
    3378             :                 }
    3379             :         }
    3380             : 
    3381             :         /* boundary resizes */
    3382           1 :         tommy_array_resize(&array, 63);
    3383           1 :         tommy_array_resize(&array, 64);
    3384           1 :         tommy_array_resize(&array, 65);
    3385           1 :         tommy_array_resize(&array, 127);
    3386           1 :         tommy_array_resize(&array, 128);
    3387           1 :         tommy_array_resize(&array, 129);
    3388           1 :         tommy_array_resize(&array, 63);
    3389             : 
    3390             :         /* clear */
    3391           1 :         tommy_array_set(&array, 0, (void*)1);
    3392           1 :         tommy_array_clear(&array);
    3393           1 :         if (tommy_array_size(&array) != 0) {
    3394             :                 /* LCOV_EXCL_START */
    3395             :                 abort();
    3396             :                 /* LCOV_EXCL_STOP */
    3397             :         }
    3398           1 :         tommy_array_resize(&array, 10);
    3399          11 :         for (tommy_uintptr_t i = 0; i < 10; ++i) {
    3400          10 :                 if (tommy_array_get(&array, i) != 0) {
    3401             :                         /* LCOV_EXCL_START */
    3402             :                         abort();
    3403             :                         /* LCOV_EXCL_STOP */
    3404             :                 }
    3405             :         }
    3406             : 
    3407             :         /* shrink */
    3408           1 :         tommy_array_resize(&array, 200);
    3409          71 :         for (tommy_uintptr_t i = 0; i < 70; ++i)
    3410          70 :                 tommy_array_set(&array, i, (void*)(i + 1));
    3411           1 :         tommy_array_resize(&array, 70);
    3412           1 :         tommy_array_shrink(&array);
    3413           1 :         if (tommy_array_size(&array) != 70 || tommy_array_memory_usage(&array) != 128 * sizeof(void*)) {
    3414             :                 /* LCOV_EXCL_START */
    3415             :                 abort();
    3416             :                 /* LCOV_EXCL_STOP */
    3417             :         }
    3418          71 :         for (tommy_uintptr_t i = 0; i < 70; ++i) {
    3419          70 :                 if (tommy_array_get(&array, i) != (void*)(i + 1)) {
    3420             :                         /* LCOV_EXCL_START */
    3421             :                         abort();
    3422             :                         /* LCOV_EXCL_STOP */
    3423             :                 }
    3424             :         }
    3425             : 
    3426           1 :         tommy_array_clear(&array);
    3427           1 :         tommy_array_shrink(&array);
    3428           1 :         if (tommy_array_size(&array) != 0 || tommy_array_memory_usage(&array) != 64 * sizeof(void*)) {
    3429             :                 /* LCOV_EXCL_START */
    3430             :                 abort();
    3431             :                 /* LCOV_EXCL_STOP */
    3432             :         }
    3433             : 
    3434           1 :         tommy_array_grow(&array, 1);
    3435           1 :         if (tommy_array_get(&array, 0) != 0) {
    3436             :                 /* LCOV_EXCL_START */
    3437             :                 abort();
    3438             :                 /* LCOV_EXCL_STOP */
    3439             :         }
    3440             : 
    3441             :         /* foreach and foreach_arg on populated array */
    3442           1 :         tommy_array_resize(&array, 200);
    3443         201 :         for (tommy_uintptr_t i = 0; i < 200; ++i)
    3444         200 :                 tommy_array_set(&array, i, (void*)(i + 1));
    3445           1 :         the_count = 0;
    3446           1 :         tommy_array_foreach(&array, array_foreach_check_callback);
    3447           1 :         if (the_count != 200) {
    3448             :                 /* LCOV_EXCL_START */
    3449             :                 abort();
    3450             :                 /* LCOV_EXCL_STOP */
    3451             :         }
    3452             :         {
    3453           1 :                 tommy_uintptr_t expected = 0;
    3454           1 :                 tommy_array_foreach_arg(&array, array_foreach_check_arg_callback, &expected);
    3455           1 :                 if (expected != 200) {
    3456             :                         /* LCOV_EXCL_START */
    3457             :                         abort();
    3458             :                         /* LCOV_EXCL_STOP */
    3459             :                 }
    3460             :         }
    3461             : 
    3462           1 :         tommy_array_done(&array);
    3463           1 : }
    3464             : 
    3465           1 : void test_arrayof_ops(void)
    3466             : {
    3467             :         tommy_arrayof arrayof;
    3468             : 
    3469           1 :         tommy_arrayof_init(&arrayof, sizeof(unsigned));
    3470             : 
    3471             :         /* foreach on empty array */
    3472           1 :         the_count = 0;
    3473           1 :         tommy_arrayof_foreach(&arrayof, count_callback);
    3474           1 :         if (the_count != 0) {
    3475             :                 /* LCOV_EXCL_START */
    3476             :                 abort();
    3477             :                 /* LCOV_EXCL_STOP */
    3478             :         }
    3479           1 :         the_count = 0;
    3480           1 :         tommy_arrayof_foreach_arg(&arrayof, count_arg_callback, &the_count);
    3481           1 :         if (the_count != 0) {
    3482             :                 /* LCOV_EXCL_START */
    3483             :                 abort();
    3484             :                 /* LCOV_EXCL_STOP */
    3485             :         }
    3486             : 
    3487             :         /* reserve on empty array */
    3488           1 :         tommy_arrayof_reserve(&arrayof, 0);
    3489           1 :         if (tommy_arrayof_size(&arrayof) != 0) {
    3490             :                 /* LCOV_EXCL_START */
    3491             :                 abort();
    3492             :                 /* LCOV_EXCL_STOP */
    3493             :         }
    3494             : 
    3495           1 :         tommy_arrayof_reserve(&arrayof, 50);
    3496           1 :         if (tommy_arrayof_size(&arrayof) != 0 || tommy_arrayof_memory_usage(&arrayof) < 64 * sizeof(unsigned)) {
    3497             :                 /* LCOV_EXCL_START */
    3498             :                 abort();
    3499             :                 /* LCOV_EXCL_STOP */
    3500             :         }
    3501             : 
    3502           1 :         tommy_arrayof_reserve(&arrayof, 100);
    3503           1 :         if (tommy_arrayof_size(&arrayof) != 0 || tommy_arrayof_memory_usage(&arrayof) < 128 * sizeof(unsigned)) {
    3504             :                 /* LCOV_EXCL_START */
    3505             :                 abort();
    3506             :                 /* LCOV_EXCL_STOP */
    3507             :         }
    3508             : 
    3509             :         /* resize growing */
    3510           1 :         tommy_arrayof_resize(&arrayof, 100);
    3511           1 :         if (tommy_arrayof_size(&arrayof) != 100) {
    3512             :                 /* LCOV_EXCL_START */
    3513             :                 abort();
    3514             :                 /* LCOV_EXCL_STOP */
    3515             :         }
    3516         101 :         for (unsigned i = 0; i < 100; ++i) {
    3517         100 :                 unsigned* ref = tommy_arrayof_ref(&arrayof, i);
    3518         100 :                 if (*ref != 0) {
    3519             :                         /* LCOV_EXCL_START */
    3520             :                         abort();
    3521             :                         /* LCOV_EXCL_STOP */
    3522             :                 }
    3523         100 :                 *ref = i + 1;
    3524             :         }
    3525             : 
    3526             :         /* resize shrinking and invariant check */
    3527           1 :         tommy_arrayof_resize(&arrayof, 50);
    3528           1 :         if (tommy_arrayof_size(&arrayof) != 50) {
    3529             :                 /* LCOV_EXCL_START */
    3530             :                 abort();
    3531             :                 /* LCOV_EXCL_STOP */
    3532             :         }
    3533          51 :         for (unsigned i = 0; i < 50; ++i) {
    3534          50 :                 unsigned* ref = tommy_arrayof_ref(&arrayof, i);
    3535          50 :                 if (*ref != i + 1) {
    3536             :                         /* LCOV_EXCL_START */
    3537             :                         abort();
    3538             :                         /* LCOV_EXCL_STOP */
    3539             :                 }
    3540             :         }
    3541           1 :         tommy_arrayof_resize(&arrayof, 100);
    3542          51 :         for (unsigned i = 50; i < 100; ++i) {
    3543          50 :                 unsigned* ref = tommy_arrayof_ref(&arrayof, i);
    3544          50 :                 if (*ref != 0) {
    3545             :                         /* LCOV_EXCL_START */
    3546             :                         abort();
    3547             :                         /* LCOV_EXCL_STOP */
    3548             :                 }
    3549             :         }
    3550             : 
    3551             :         /* boundary resizes */
    3552           1 :         tommy_arrayof_resize(&arrayof, 63);
    3553           1 :         tommy_arrayof_resize(&arrayof, 64);
    3554           1 :         tommy_arrayof_resize(&arrayof, 65);
    3555           1 :         tommy_arrayof_resize(&arrayof, 127);
    3556           1 :         tommy_arrayof_resize(&arrayof, 128);
    3557           1 :         tommy_arrayof_resize(&arrayof, 129);
    3558           1 :         tommy_arrayof_resize(&arrayof, 63);
    3559             : 
    3560             :         /* clear */
    3561             :         {
    3562           1 :                 unsigned* ref = tommy_arrayof_ref(&arrayof, 0);
    3563           1 :                 *ref = 1;
    3564             :         }
    3565           1 :         tommy_arrayof_clear(&arrayof);
    3566           1 :         if (tommy_arrayof_size(&arrayof) != 0) {
    3567             :                 /* LCOV_EXCL_START */
    3568             :                 abort();
    3569             :                 /* LCOV_EXCL_STOP */
    3570             :         }
    3571           1 :         tommy_arrayof_resize(&arrayof, 10);
    3572          11 :         for (unsigned i = 0; i < 10; ++i) {
    3573          10 :                 unsigned* ref = tommy_arrayof_ref(&arrayof, i);
    3574          10 :                 if (*ref != 0) {
    3575             :                         /* LCOV_EXCL_START */
    3576             :                         abort();
    3577             :                         /* LCOV_EXCL_STOP */
    3578             :                 }
    3579             :         }
    3580             : 
    3581             :         /* shrink */
    3582           1 :         tommy_arrayof_resize(&arrayof, 200);
    3583          71 :         for (unsigned i = 0; i < 70; ++i) {
    3584          70 :                 unsigned* ref = tommy_arrayof_ref(&arrayof, i);
    3585          70 :                 *ref = i + 1;
    3586             :         }
    3587           1 :         tommy_arrayof_resize(&arrayof, 70);
    3588           1 :         tommy_arrayof_shrink(&arrayof);
    3589           1 :         if (tommy_arrayof_size(&arrayof) != 70 || tommy_arrayof_memory_usage(&arrayof) != 128 * sizeof(unsigned)) {
    3590             :                 /* LCOV_EXCL_START */
    3591             :                 abort();
    3592             :                 /* LCOV_EXCL_STOP */
    3593             :         }
    3594          71 :         for (unsigned i = 0; i < 70; ++i) {
    3595          70 :                 unsigned* ref = tommy_arrayof_ref(&arrayof, i);
    3596          70 :                 if (*ref != i + 1) {
    3597             :                         /* LCOV_EXCL_START */
    3598             :                         abort();
    3599             :                         /* LCOV_EXCL_STOP */
    3600             :                 }
    3601             :         }
    3602             : 
    3603           1 :         tommy_arrayof_clear(&arrayof);
    3604           1 :         tommy_arrayof_shrink(&arrayof);
    3605           1 :         if (tommy_arrayof_size(&arrayof) != 0 || tommy_arrayof_memory_usage(&arrayof) != 64 * sizeof(unsigned)) {
    3606             :                 /* LCOV_EXCL_START */
    3607             :                 abort();
    3608             :                 /* LCOV_EXCL_STOP */
    3609             :         }
    3610             : 
    3611           1 :         tommy_arrayof_grow(&arrayof, 1);
    3612             :         {
    3613           1 :                 unsigned* ref = tommy_arrayof_ref(&arrayof, 0);
    3614           1 :                 if (*ref != 0) {
    3615             :                         /* LCOV_EXCL_START */
    3616             :                         abort();
    3617             :                         /* LCOV_EXCL_STOP */
    3618             :                 }
    3619             :         }
    3620             : 
    3621             :         /* foreach and foreach_arg on populated array */
    3622           1 :         tommy_arrayof_resize(&arrayof, 200);
    3623         201 :         for (unsigned i = 0; i < 200; ++i) {
    3624         200 :                 unsigned* ref = tommy_arrayof_ref(&arrayof, i);
    3625         200 :                 *ref = i + 1;
    3626             :         }
    3627           1 :         the_count = 0;
    3628           1 :         tommy_arrayof_foreach(&arrayof, arrayof_foreach_check_callback);
    3629           1 :         if (the_count != 200) {
    3630             :                 /* LCOV_EXCL_START */
    3631             :                 abort();
    3632             :                 /* LCOV_EXCL_STOP */
    3633             :         }
    3634             :         {
    3635           1 :                 unsigned expected = 0;
    3636           1 :                 tommy_arrayof_foreach_arg(&arrayof, arrayof_foreach_check_arg_callback, &expected);
    3637           1 :                 if (expected != 200) {
    3638             :                         /* LCOV_EXCL_START */
    3639             :                         abort();
    3640             :                         /* LCOV_EXCL_STOP */
    3641             :                 }
    3642             :         }
    3643             : 
    3644           1 :         tommy_arrayof_done(&arrayof);
    3645           1 : }
    3646             : 
    3647           1 : void test_arrayblk_ops(void)
    3648             : {
    3649             :         tommy_arrayblk arrayblk;
    3650             : 
    3651           1 :         tommy_arrayblk_init(&arrayblk);
    3652             : 
    3653             :         /* foreach on empty array */
    3654           1 :         the_count = 0;
    3655           1 :         tommy_arrayblk_foreach(&arrayblk, count_callback);
    3656           1 :         if (the_count != 0) {
    3657             :                 /* LCOV_EXCL_START */
    3658             :                 abort();
    3659             :                 /* LCOV_EXCL_STOP */
    3660             :         }
    3661           1 :         the_count = 0;
    3662           1 :         tommy_arrayblk_foreach_arg(&arrayblk, count_arg_callback, &the_count);
    3663           1 :         if (the_count != 0) {
    3664             :                 /* LCOV_EXCL_START */
    3665             :                 abort();
    3666             :                 /* LCOV_EXCL_STOP */
    3667             :         }
    3668             : 
    3669             :         /* reserve on empty array */
    3670           1 :         tommy_arrayblk_reserve(&arrayblk, 0);
    3671           1 :         if (tommy_arrayblk_size(&arrayblk) != 0) {
    3672             :                 /* LCOV_EXCL_START */
    3673             :                 abort();
    3674             :                 /* LCOV_EXCL_STOP */
    3675             :         }
    3676             : 
    3677           1 :         tommy_arrayblk_reserve(&arrayblk, 100);
    3678           1 :         if (tommy_arrayblk_size(&arrayblk) != 0 || arrayblk.block_count != 1) {
    3679             :                 /* LCOV_EXCL_START */
    3680             :                 abort();
    3681             :                 /* LCOV_EXCL_STOP */
    3682             :         }
    3683             : 
    3684           1 :         tommy_arrayblk_reserve(&arrayblk, TOMMY_ARRAYBLK_SIZE + 1);
    3685           1 :         if (tommy_arrayblk_size(&arrayblk) != 0 || arrayblk.block_count != 2) {
    3686             :                 /* LCOV_EXCL_START */
    3687             :                 abort();
    3688             :                 /* LCOV_EXCL_STOP */
    3689             :         }
    3690             : 
    3691             :         /* test directory expansion beyond initial 64 block entries */
    3692           1 :         tommy_arrayblk_reserve(&arrayblk, 65 * TOMMY_ARRAYBLK_SIZE);
    3693           1 :         if (tommy_arrayblk_size(&arrayblk) != 0 || arrayblk.block_capacity < 65) {
    3694             :                 /* LCOV_EXCL_START */
    3695             :                 abort();
    3696             :                 /* LCOV_EXCL_STOP */
    3697             :         }
    3698             : 
    3699             :         /* resize growing across block boundary */
    3700           1 :         tommy_arrayblk_resize(&arrayblk, TOMMY_ARRAYBLK_SIZE + 50);
    3701           1 :         if (tommy_arrayblk_size(&arrayblk) != TOMMY_ARRAYBLK_SIZE + 50) {
    3702             :                 /* LCOV_EXCL_START */
    3703             :                 abort();
    3704             :                 /* LCOV_EXCL_STOP */
    3705             :         }
    3706        4147 :         for (tommy_uintptr_t i = 0; i < TOMMY_ARRAYBLK_SIZE + 50; ++i) {
    3707        4146 :                 if (tommy_arrayblk_get(&arrayblk, i) != 0) {
    3708             :                         /* LCOV_EXCL_START */
    3709             :                         abort();
    3710             :                         /* LCOV_EXCL_STOP */
    3711             :                 }
    3712        4146 :                 tommy_arrayblk_set(&arrayblk, i, (void*)(i + 1));
    3713             :         }
    3714             : 
    3715             :         /* resize shrinking across block boundary */
    3716           1 :         tommy_arrayblk_resize(&arrayblk, TOMMY_ARRAYBLK_SIZE - 10);
    3717           1 :         if (tommy_arrayblk_size(&arrayblk) != TOMMY_ARRAYBLK_SIZE - 10) {
    3718             :                 /* LCOV_EXCL_START */
    3719             :                 abort();
    3720             :                 /* LCOV_EXCL_STOP */
    3721             :         }
    3722        4087 :         for (tommy_uintptr_t i = 0; i < TOMMY_ARRAYBLK_SIZE - 10; ++i) {
    3723        4086 :                 if (tommy_arrayblk_get(&arrayblk, i) != (void*)(i + 1)) {
    3724             :                         /* LCOV_EXCL_START */
    3725             :                         abort();
    3726             :                         /* LCOV_EXCL_STOP */
    3727             :                 }
    3728             :         }
    3729             : 
    3730             :         /* regrowing and invariant check across block boundary */
    3731           1 :         tommy_arrayblk_resize(&arrayblk, TOMMY_ARRAYBLK_SIZE + 50);
    3732          61 :         for (tommy_uintptr_t i = TOMMY_ARRAYBLK_SIZE - 10; i < TOMMY_ARRAYBLK_SIZE + 50; ++i) {
    3733          60 :                 if (tommy_arrayblk_get(&arrayblk, i) != 0) {
    3734             :                         /* LCOV_EXCL_START */
    3735             :                         abort();
    3736             :                         /* LCOV_EXCL_STOP */
    3737             :                 }
    3738             :         }
    3739             : 
    3740             :         /* boundary resizes */
    3741           1 :         tommy_arrayblk_resize(&arrayblk, 4095);
    3742           1 :         tommy_arrayblk_resize(&arrayblk, 4096);
    3743           1 :         tommy_arrayblk_resize(&arrayblk, 4097);
    3744           1 :         tommy_arrayblk_resize(&arrayblk, 4095);
    3745             : 
    3746             :         /* clear */
    3747           1 :         tommy_arrayblk_set(&arrayblk, 0, (void*)1);
    3748           1 :         tommy_arrayblk_clear(&arrayblk);
    3749           1 :         if (tommy_arrayblk_size(&arrayblk) != 0) {
    3750             :                 /* LCOV_EXCL_START */
    3751             :                 abort();
    3752             :                 /* LCOV_EXCL_STOP */
    3753             :         }
    3754           1 :         tommy_arrayblk_resize(&arrayblk, 10);
    3755          11 :         for (tommy_uintptr_t i = 0; i < 10; ++i) {
    3756          10 :                 if (tommy_arrayblk_get(&arrayblk, i) != 0) {
    3757             :                         /* LCOV_EXCL_START */
    3758             :                         abort();
    3759             :                         /* LCOV_EXCL_STOP */
    3760             :                 }
    3761             :         }
    3762             : 
    3763             :         /* shrink with directory reduction */
    3764           1 :         tommy_arrayblk_reserve(&arrayblk, 65 * TOMMY_ARRAYBLK_SIZE);
    3765           1 :         tommy_arrayblk_resize(&arrayblk, TOMMY_ARRAYBLK_SIZE + 50);
    3766        4107 :         for (tommy_uintptr_t i = 0; i < TOMMY_ARRAYBLK_SIZE + 10; ++i)
    3767        4106 :                 tommy_arrayblk_set(&arrayblk, i, (void*)(i + 1));
    3768           1 :         tommy_arrayblk_resize(&arrayblk, TOMMY_ARRAYBLK_SIZE + 10);
    3769           1 :         tommy_arrayblk_shrink(&arrayblk);
    3770           1 :         if (tommy_arrayblk_size(&arrayblk) != TOMMY_ARRAYBLK_SIZE + 10 || arrayblk.block_count != 2 || arrayblk.block_capacity != 64) {
    3771             :                 /* LCOV_EXCL_START */
    3772             :                 abort();
    3773             :                 /* LCOV_EXCL_STOP */
    3774             :         }
    3775        4107 :         for (tommy_uintptr_t i = 0; i < TOMMY_ARRAYBLK_SIZE + 10; ++i) {
    3776        4106 :                 if (tommy_arrayblk_get(&arrayblk, i) != (void*)(i + 1)) {
    3777             :                         /* LCOV_EXCL_START */
    3778             :                         abort();
    3779             :                         /* LCOV_EXCL_STOP */
    3780             :                 }
    3781             :         }
    3782             : 
    3783             :         /* shrink with directory reduction while capacity > min_capacity */
    3784           1 :         tommy_arrayblk_reserve(&arrayblk, 200 * TOMMY_ARRAYBLK_SIZE);
    3785           1 :         tommy_arrayblk_resize(&arrayblk, 70 * TOMMY_ARRAYBLK_SIZE);
    3786      286721 :         for (tommy_uintptr_t i = 0; i < 70 * TOMMY_ARRAYBLK_SIZE; ++i)
    3787      286720 :                 tommy_arrayblk_set(&arrayblk, i, (void*)(i + 1));
    3788           1 :         tommy_arrayblk_shrink(&arrayblk);
    3789           1 :         if (tommy_arrayblk_size(&arrayblk) != 70 * TOMMY_ARRAYBLK_SIZE || arrayblk.block_count != 70 || arrayblk.block_capacity != 128) {
    3790             :                 /* LCOV_EXCL_START */
    3791             :                 abort();
    3792             :                 /* LCOV_EXCL_STOP */
    3793             :         }
    3794      286721 :         for (tommy_uintptr_t i = 0; i < 70 * TOMMY_ARRAYBLK_SIZE; ++i) {
    3795      286720 :                 if (tommy_arrayblk_get(&arrayblk, i) != (void*)(i + 1)) {
    3796             :                         /* LCOV_EXCL_START */
    3797             :                         abort();
    3798             :                         /* LCOV_EXCL_STOP */
    3799             :                 }
    3800             :         }
    3801             : 
    3802             :         /* shrink to empty */
    3803           1 :         tommy_arrayblk_clear(&arrayblk);
    3804           1 :         tommy_arrayblk_shrink(&arrayblk);
    3805           1 :         if (tommy_arrayblk_size(&arrayblk) != 0 || arrayblk.block_count != 0 || arrayblk.block_capacity != 64) {
    3806             :                 /* LCOV_EXCL_START */
    3807             :                 abort();
    3808             :                 /* LCOV_EXCL_STOP */
    3809             :         }
    3810             : 
    3811           1 :         tommy_arrayblk_grow(&arrayblk, 1);
    3812           1 :         if (tommy_arrayblk_get(&arrayblk, 0) != 0) {
    3813             :                 /* LCOV_EXCL_START */
    3814             :                 abort();
    3815             :                 /* LCOV_EXCL_STOP */
    3816             :         }
    3817             : 
    3818             :         /* foreach and foreach_arg on populated array spanning multiple blocks */
    3819           1 :         tommy_arrayblk_resize(&arrayblk, TOMMY_ARRAYBLK_SIZE + 50);
    3820        4147 :         for (tommy_uintptr_t i = 0; i < TOMMY_ARRAYBLK_SIZE + 50; ++i)
    3821        4146 :                 tommy_arrayblk_set(&arrayblk, i, (void*)(i + 1));
    3822           1 :         the_count = 0;
    3823           1 :         tommy_arrayblk_foreach(&arrayblk, array_foreach_check_callback);
    3824           1 :         if (the_count != TOMMY_ARRAYBLK_SIZE + 50) {
    3825             :                 /* LCOV_EXCL_START */
    3826             :                 abort();
    3827             :                 /* LCOV_EXCL_STOP */
    3828             :         }
    3829             :         {
    3830           1 :                 tommy_uintptr_t expected = 0;
    3831           1 :                 tommy_arrayblk_foreach_arg(&arrayblk, array_foreach_check_arg_callback, &expected);
    3832           1 :                 if (expected != TOMMY_ARRAYBLK_SIZE + 50) {
    3833             :                         /* LCOV_EXCL_START */
    3834             :                         abort();
    3835             :                         /* LCOV_EXCL_STOP */
    3836             :                 }
    3837             :         }
    3838             : 
    3839           1 :         tommy_arrayblk_done(&arrayblk);
    3840           1 : }
    3841             : 
    3842           1 : void test_arrayblkof_ops(void)
    3843             : {
    3844             :         tommy_arrayblkof arrayblkof;
    3845             : 
    3846           1 :         tommy_arrayblkof_init(&arrayblkof, sizeof(unsigned));
    3847             : 
    3848             :         /* foreach on empty array */
    3849           1 :         the_count = 0;
    3850           1 :         tommy_arrayblkof_foreach(&arrayblkof, count_callback);
    3851           1 :         if (the_count != 0) {
    3852             :                 /* LCOV_EXCL_START */
    3853             :                 abort();
    3854             :                 /* LCOV_EXCL_STOP */
    3855             :         }
    3856           1 :         the_count = 0;
    3857           1 :         tommy_arrayblkof_foreach_arg(&arrayblkof, count_arg_callback, &the_count);
    3858           1 :         if (the_count != 0) {
    3859             :                 /* LCOV_EXCL_START */
    3860             :                 abort();
    3861             :                 /* LCOV_EXCL_STOP */
    3862             :         }
    3863             : 
    3864             :         /* reserve on empty array */
    3865           1 :         tommy_arrayblkof_reserve(&arrayblkof, 0);
    3866           1 :         if (tommy_arrayblkof_size(&arrayblkof) != 0) {
    3867             :                 /* LCOV_EXCL_START */
    3868             :                 abort();
    3869             :                 /* LCOV_EXCL_STOP */
    3870             :         }
    3871             : 
    3872           1 :         tommy_arrayblkof_reserve(&arrayblkof, 100);
    3873           1 :         if (tommy_arrayblkof_size(&arrayblkof) != 0 || arrayblkof.block_count != 1) {
    3874             :                 /* LCOV_EXCL_START */
    3875             :                 abort();
    3876             :                 /* LCOV_EXCL_STOP */
    3877             :         }
    3878             : 
    3879           1 :         tommy_arrayblkof_reserve(&arrayblkof, TOMMY_ARRAYBLKOF_SIZE + 1);
    3880           1 :         if (tommy_arrayblkof_size(&arrayblkof) != 0 || arrayblkof.block_count != 2) {
    3881             :                 /* LCOV_EXCL_START */
    3882             :                 abort();
    3883             :                 /* LCOV_EXCL_STOP */
    3884             :         }
    3885             : 
    3886             :         /* test directory expansion beyond initial 64 block entries */
    3887           1 :         tommy_arrayblkof_reserve(&arrayblkof, 65 * TOMMY_ARRAYBLKOF_SIZE);
    3888           1 :         if (tommy_arrayblkof_size(&arrayblkof) != 0 || arrayblkof.block_capacity < 65) {
    3889             :                 /* LCOV_EXCL_START */
    3890             :                 abort();
    3891             :                 /* LCOV_EXCL_STOP */
    3892             :         }
    3893             : 
    3894             :         /* resize growing across block boundary */
    3895           1 :         tommy_arrayblkof_resize(&arrayblkof, TOMMY_ARRAYBLKOF_SIZE + 50);
    3896           1 :         if (tommy_arrayblkof_size(&arrayblkof) != TOMMY_ARRAYBLKOF_SIZE + 50) {
    3897             :                 /* LCOV_EXCL_START */
    3898             :                 abort();
    3899             :                 /* LCOV_EXCL_STOP */
    3900             :         }
    3901        4147 :         for (unsigned i = 0; i < TOMMY_ARRAYBLKOF_SIZE + 50; ++i) {
    3902        4146 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    3903        4146 :                 if (*ref != 0) {
    3904             :                         /* LCOV_EXCL_START */
    3905             :                         abort();
    3906             :                         /* LCOV_EXCL_STOP */
    3907             :                 }
    3908        4146 :                 *ref = i + 1;
    3909             :         }
    3910             : 
    3911             :         /* resize shrinking across block boundary */
    3912           1 :         tommy_arrayblkof_resize(&arrayblkof, TOMMY_ARRAYBLKOF_SIZE - 10);
    3913           1 :         if (tommy_arrayblkof_size(&arrayblkof) != TOMMY_ARRAYBLKOF_SIZE - 10) {
    3914             :                 /* LCOV_EXCL_START */
    3915             :                 abort();
    3916             :                 /* LCOV_EXCL_STOP */
    3917             :         }
    3918        4087 :         for (unsigned i = 0; i < TOMMY_ARRAYBLKOF_SIZE - 10; ++i) {
    3919        4086 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    3920        4086 :                 if (*ref != i + 1) {
    3921             :                         /* LCOV_EXCL_START */
    3922             :                         abort();
    3923             :                         /* LCOV_EXCL_STOP */
    3924             :                 }
    3925             :         }
    3926             : 
    3927             :         /* regrowing and invariant check across block boundary */
    3928           1 :         tommy_arrayblkof_resize(&arrayblkof, TOMMY_ARRAYBLKOF_SIZE + 50);
    3929          61 :         for (unsigned i = TOMMY_ARRAYBLKOF_SIZE - 10; i < TOMMY_ARRAYBLKOF_SIZE + 50; ++i) {
    3930          60 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    3931          60 :                 if (*ref != 0) {
    3932             :                         /* LCOV_EXCL_START */
    3933             :                         abort();
    3934             :                         /* LCOV_EXCL_STOP */
    3935             :                 }
    3936             :         }
    3937             : 
    3938             :         /* boundary resizes */
    3939           1 :         tommy_arrayblkof_resize(&arrayblkof, 4095);
    3940           1 :         tommy_arrayblkof_resize(&arrayblkof, 4096);
    3941           1 :         tommy_arrayblkof_resize(&arrayblkof, 4097);
    3942           1 :         tommy_arrayblkof_resize(&arrayblkof, 4095);
    3943             : 
    3944             :         /* clear */
    3945             :         {
    3946           1 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, 0);
    3947           1 :                 *ref = 1;
    3948             :         }
    3949           1 :         tommy_arrayblkof_clear(&arrayblkof);
    3950           1 :         if (tommy_arrayblkof_size(&arrayblkof) != 0) {
    3951             :                 /* LCOV_EXCL_START */
    3952             :                 abort();
    3953             :                 /* LCOV_EXCL_STOP */
    3954             :         }
    3955           1 :         tommy_arrayblkof_resize(&arrayblkof, 10);
    3956          11 :         for (unsigned i = 0; i < 10; ++i) {
    3957          10 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    3958          10 :                 if (*ref != 0) {
    3959             :                         /* LCOV_EXCL_START */
    3960             :                         abort();
    3961             :                         /* LCOV_EXCL_STOP */
    3962             :                 }
    3963             :         }
    3964             : 
    3965             :         /* shrink with directory reduction */
    3966           1 :         tommy_arrayblkof_reserve(&arrayblkof, 65 * TOMMY_ARRAYBLKOF_SIZE);
    3967           1 :         tommy_arrayblkof_resize(&arrayblkof, TOMMY_ARRAYBLKOF_SIZE + 50);
    3968        4107 :         for (unsigned i = 0; i < TOMMY_ARRAYBLKOF_SIZE + 10; ++i) {
    3969        4106 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    3970        4106 :                 *ref = i + 1;
    3971             :         }
    3972           1 :         tommy_arrayblkof_resize(&arrayblkof, TOMMY_ARRAYBLKOF_SIZE + 10);
    3973           1 :         tommy_arrayblkof_shrink(&arrayblkof);
    3974           1 :         if (tommy_arrayblkof_size(&arrayblkof) != TOMMY_ARRAYBLKOF_SIZE + 10 || arrayblkof.block_count != 2 || arrayblkof.block_capacity != 64) {
    3975             :                 /* LCOV_EXCL_START */
    3976             :                 abort();
    3977             :                 /* LCOV_EXCL_STOP */
    3978             :         }
    3979        4107 :         for (unsigned i = 0; i < TOMMY_ARRAYBLKOF_SIZE + 10; ++i) {
    3980        4106 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    3981        4106 :                 if (*ref != i + 1) {
    3982             :                         /* LCOV_EXCL_START */
    3983             :                         abort();
    3984             :                         /* LCOV_EXCL_STOP */
    3985             :                 }
    3986             :         }
    3987             : 
    3988             :         /* shrink with directory reduction while capacity > min_capacity */
    3989           1 :         tommy_arrayblkof_reserve(&arrayblkof, 200 * TOMMY_ARRAYBLKOF_SIZE);
    3990           1 :         tommy_arrayblkof_resize(&arrayblkof, 70 * TOMMY_ARRAYBLKOF_SIZE);
    3991      286721 :         for (unsigned i = 0; i < 70 * TOMMY_ARRAYBLKOF_SIZE; ++i) {
    3992      286720 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    3993      286720 :                 *ref = i + 1;
    3994             :         }
    3995           1 :         tommy_arrayblkof_shrink(&arrayblkof);
    3996           1 :         if (tommy_arrayblkof_size(&arrayblkof) != 70 * TOMMY_ARRAYBLKOF_SIZE || arrayblkof.block_count != 70 || arrayblkof.block_capacity != 128) {
    3997             :                 /* LCOV_EXCL_START */
    3998             :                 abort();
    3999             :                 /* LCOV_EXCL_STOP */
    4000             :         }
    4001      286721 :         for (unsigned i = 0; i < 70 * TOMMY_ARRAYBLKOF_SIZE; ++i) {
    4002      286720 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    4003      286720 :                 if (*ref != i + 1) {
    4004             :                         /* LCOV_EXCL_START */
    4005             :                         abort();
    4006             :                         /* LCOV_EXCL_STOP */
    4007             :                 }
    4008             :         }
    4009             : 
    4010             :         /* shrink to empty */
    4011           1 :         tommy_arrayblkof_clear(&arrayblkof);
    4012           1 :         tommy_arrayblkof_shrink(&arrayblkof);
    4013           1 :         if (tommy_arrayblkof_size(&arrayblkof) != 0 || arrayblkof.block_count != 0 || arrayblkof.block_capacity != 64) {
    4014             :                 /* LCOV_EXCL_START */
    4015             :                 abort();
    4016             :                 /* LCOV_EXCL_STOP */
    4017             :         }
    4018             : 
    4019           1 :         tommy_arrayblkof_grow(&arrayblkof, 1);
    4020             :         {
    4021           1 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, 0);
    4022           1 :                 if (*ref != 0) {
    4023             :                         /* LCOV_EXCL_START */
    4024             :                         abort();
    4025             :                         /* LCOV_EXCL_STOP */
    4026             :                 }
    4027             :         }
    4028             : 
    4029             :         /* foreach and foreach_arg on populated array spanning multiple blocks */
    4030           1 :         tommy_arrayblkof_resize(&arrayblkof, TOMMY_ARRAYBLKOF_SIZE + 50);
    4031        4147 :         for (unsigned i = 0; i < TOMMY_ARRAYBLKOF_SIZE + 50; ++i) {
    4032        4146 :                 unsigned* ref = tommy_arrayblkof_ref(&arrayblkof, i);
    4033        4146 :                 *ref = i + 1;
    4034             :         }
    4035           1 :         the_count = 0;
    4036           1 :         tommy_arrayblkof_foreach(&arrayblkof, arrayof_foreach_check_callback);
    4037           1 :         if (the_count != TOMMY_ARRAYBLKOF_SIZE + 50) {
    4038             :                 /* LCOV_EXCL_START */
    4039             :                 abort();
    4040             :                 /* LCOV_EXCL_STOP */
    4041             :         }
    4042             :         {
    4043           1 :                 unsigned expected = 0;
    4044           1 :                 tommy_arrayblkof_foreach_arg(&arrayblkof, arrayof_foreach_check_arg_callback, &expected);
    4045           1 :                 if (expected != TOMMY_ARRAYBLKOF_SIZE + 50) {
    4046             :                         /* LCOV_EXCL_START */
    4047             :                         abort();
    4048             :                         /* LCOV_EXCL_STOP */
    4049             :                 }
    4050             :         }
    4051             : 
    4052           1 :         tommy_arrayblkof_done(&arrayblkof);
    4053           1 : }
    4054             : 
    4055             : /* verify node identity, metadata, both list directions and per-bucket order. */
    4056         148 : void test_hash_list(tommy_list* list, struct object_hash* obj, tommy_node* saved, unsigned size, unsigned prefix)
    4057             : {
    4058             :         unsigned order[130], position[130];
    4059         148 :         unsigned seen[130] = { 0 };
    4060         148 :         tommy_node* node = tommy_list_head(list);
    4061         148 :         tommy_node* prev = tommy_list_tail(list);
    4062             : 
    4063        5314 :         for (unsigned i = 0; i < size; ++i) {
    4064             :                 unsigned j;
    4065      221458 :                 for (j = 0; j < size; ++j)
    4066      221458 :                         if (node == &obj[j].node)
    4067        5166 :                                 break;
    4068        5166 :                 if (j == size || seen[j] || (i < prefix && j != i)
    4069        5166 :                         || node->prev != prev || node->data != saved[j].data
    4070        5166 :                         || node->index != saved[j].index) {
    4071             :                         /* LCOV_EXCL_START */
    4072             :                         abort();
    4073             :                         /* LCOV_EXCL_STOP */
    4074             :                 }
    4075        5166 :                 seen[j] = 1;
    4076        5166 :                 order[i] = j;
    4077        5166 :                 position[j] = i;
    4078        5166 :                 prev = node;
    4079        5166 :                 node = node->next;
    4080             :         }
    4081         148 :         if (node != 0 || tommy_list_tail(list) != (size ? prev : 0)) {
    4082             :                 /* LCOV_EXCL_START */
    4083             :                 abort();
    4084             :                 /* LCOV_EXCL_STOP */
    4085             :         }
    4086             : 
    4087             :         /* each original successor must still follow its node, regardless of bucket order. */
    4088        5314 :         for (unsigned i = 0; i < size; ++i) {
    4089        5166 :                 if (saved[i].next == 0)
    4090        1606 :                         continue;
    4091             :                 unsigned j;
    4092      160060 :                 for (j = 0; j < size; ++j)
    4093      160060 :                         if (saved[i].next == &obj[j].node)
    4094        3560 :                                 break;
    4095        3560 :                 if (j == size || position[i] >= position[j]) {
    4096             :                         /* LCOV_EXCL_START */
    4097             :                         abort();
    4098             :                         /* LCOV_EXCL_STOP */
    4099             :                 }
    4100             :         }
    4101             : 
    4102         148 :         node = tommy_list_tail(list);
    4103        5314 :         for (unsigned i = size; i > 0; --i) {
    4104        5166 :                 if (node != &obj[order[i - 1]].node) {
    4105             :                         /* LCOV_EXCL_START */
    4106             :                         abort();
    4107             :                         /* LCOV_EXCL_STOP */
    4108             :                 }
    4109        5166 :                 node = tommy_list_prev(list, node);
    4110             :         }
    4111         148 :         if (node != 0) {
    4112             :                 /* LCOV_EXCL_START */
    4113             :                 abort();
    4114             :                 /* LCOV_EXCL_STOP */
    4115             :         }
    4116         148 : }
    4117             : 
    4118             : /* verify the exact bucket order and circular head/tail links. */
    4119         573 : void test_hash_bucket(tommy_node* node, struct object_hash* obj, const unsigned* order, unsigned size)
    4120             : {
    4121         573 :         tommy_node* prev = size ? &obj[order[size - 1]].node : 0;
    4122             : 
    4123        1575 :         for (unsigned i = 0; i < size; ++i) {
    4124        1002 :                 if (node != &obj[order[i]].node || node->prev != prev || node->data != &obj[order[i]]) {
    4125             :                         /* LCOV_EXCL_START */
    4126             :                         abort();
    4127             :                         /* LCOV_EXCL_STOP */
    4128             :                 }
    4129        1002 :                 prev = node;
    4130        1002 :                 node = node->next;
    4131             :         }
    4132         573 :         if (node != 0) {
    4133             :                 /* LCOV_EXCL_START */
    4134             :                 abort();
    4135             :                 /* LCOV_EXCL_STOP */
    4136             :         }
    4137         573 : }
    4138             : 
    4139           1 : void test_hashtable_swap(void)
    4140             : {
    4141           3 :         for (unsigned first_count = 0; first_count <= 3; first_count += 3) {
    4142           6 :                 for (unsigned second_count = 0; second_count <= 2; second_count += 2) {
    4143             :                         tommy_hashtable first, second;
    4144             :                         struct object_hash obj[7], before[7];
    4145           4 :                         memset(obj, 0, sizeof(obj));
    4146           4 :                         tommy_hashtable_init(&first, 16);
    4147           4 :                         tommy_hashtable_init(&second, 128);
    4148          32 :                         for (unsigned i = 0; i < 7; ++i)
    4149          28 :                                 obj[i].value = i;
    4150          10 :                         for (unsigned i = 0; i < first_count; ++i)
    4151           6 :                                 tommy_hashtable_insert(&first, &obj[i].node, &obj[i], tommy_inthash_u32(i));
    4152           8 :                         for (unsigned i = 0; i < second_count; ++i)
    4153           4 :                                 tommy_hashtable_insert(&second, &obj[3 + i].node, &obj[3 + i], tommy_inthash_u32(3 + i));
    4154           4 :                         tommy_hashtable saved_first = first, saved_second = second;
    4155           4 :                         memcpy(before, obj, sizeof(before));
    4156             : 
    4157           4 :                         tommy_hashtable_swap(&first, &second);
    4158          12 :                         for (unsigned phase = 0; phase < 2; ++phase) {
    4159           8 :                                 if (phase) {
    4160           4 :                                         tommy_hashtable_swap(&first, &first);
    4161           4 :                                         tommy_hashtable_swap(&second, &second);
    4162             :                                 }
    4163           8 :                                 if (first.bucket != saved_second.bucket || first.bucket_max != saved_second.bucket_max
    4164           8 :                                         || first.bucket_mask != saved_second.bucket_mask || first.count != second_count
    4165           8 :                                         || second.bucket != saved_first.bucket || second.bucket_max != saved_first.bucket_max
    4166           8 :                                         || second.bucket_mask != saved_first.bucket_mask || second.count != first_count
    4167           8 :                                         || memcmp(before, obj, sizeof(before)) != 0) {
    4168             :                                         /* LCOV_EXCL_START */
    4169             :                                         abort();
    4170             :                                         /* LCOV_EXCL_STOP */
    4171             :                                 }
    4172          48 :                                 for (unsigned i = 0; i < 5; ++i) {
    4173          40 :                                         if (tommy_hashtable_search(&first, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) != (i >= 3 && i < 3 + second_count ? &obj[i] : 0)
    4174          40 :                                                 || tommy_hashtable_search(&second, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) != (i < first_count ? &obj[i] : 0)) {
    4175             :                                                 /* LCOV_EXCL_START */
    4176             :                                                 abort();
    4177             :                                                 /* LCOV_EXCL_STOP */
    4178             :                                         }
    4179             :                                 }
    4180             :                         }
    4181             : 
    4182             :                         /* both containers accept new nodes after the exchange, including empty ones */
    4183           4 :                         tommy_hashtable_insert(&first, &obj[5].node, &obj[5], tommy_inthash_u32(5));
    4184           4 :                         tommy_hashtable_insert(&second, &obj[6].node, &obj[6], tommy_inthash_u32(6));
    4185           4 :                         if (tommy_hashtable_remove_existing(&first, &obj[5].node) != &obj[5]
    4186           4 :                                 || tommy_hashtable_remove_existing(&second, &obj[6].node) != &obj[6]) {
    4187             :                                 /* LCOV_EXCL_START */
    4188             :                                 abort();
    4189             :                                 /* LCOV_EXCL_STOP */
    4190             :                         }
    4191          24 :                         for (unsigned i = 0; i < 5; ++i) {
    4192          20 :                                 if (i < first_count || (i >= 3 && i < 3 + second_count)) {
    4193          10 :                                         tommy_hashtable* table = i < 3 ? &second : &first;
    4194          10 :                                         void* removed = i % 2 ? tommy_hashtable_remove(table, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) : tommy_hashtable_remove_existing(table, &obj[i].node);
    4195          10 :                                         if (removed != &obj[i] || tommy_hashtable_search(table, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) != 0) {
    4196             :                                                 /* LCOV_EXCL_START */
    4197             :                                                 abort();
    4198             :                                                 /* LCOV_EXCL_STOP */
    4199             :                                         }
    4200             :                                 }
    4201             :                         }
    4202           4 :                         if (!tommy_hashtable_empty(&first) || !tommy_hashtable_empty(&second)) {
    4203             :                                 /* LCOV_EXCL_START */
    4204             :                                 abort();
    4205             :                                 /* LCOV_EXCL_STOP */
    4206             :                         }
    4207           4 :                         tommy_hashtable_done(&first);
    4208           4 :                         tommy_hashtable_done(&second);
    4209             :                 }
    4210             :         }
    4211           1 : }
    4212             : 
    4213           1 : void test_hashtable_insert_unique(void)
    4214             : {
    4215             :         struct object_hash obj[257], candidate;
    4216             :         tommy_hashtable table;
    4217           1 :         const unsigned order[] = { 0, 1, 2, 3, 4, 5 };
    4218             : 
    4219           1 :         tommy_hashtable_init(&table, 16);
    4220           1 :         obj[0].value = 1;
    4221           1 :         compare_counter = 0;
    4222           1 :         if (tommy_hashtable_insert_unique(&table, &obj[0].node, &obj[0], search_hash_value_callback, &obj[0].value, 0) != &obj[0]
    4223           1 :                 || compare_counter != 0 || tommy_hashtable_count(&table) != 1
    4224           1 :                 || tommy_hashtable_search(&table, search_hash_value_callback, &obj[0].value, 0) != &obj[0]) {
    4225             :                 /* LCOV_EXCL_START */
    4226             :                 abort();
    4227             :                 /* LCOV_EXCL_STOP */
    4228             :         }
    4229           1 :         tommy_hashtable_remove_existing(&table, &obj[0].node);
    4230             : 
    4231           1 :         tommy_size_t bucket_max = table.bucket_max;
    4232           1 :         obj[0].value = 10;
    4233           1 :         obj[1].value = 20;
    4234           1 :         obj[2].value = 30;
    4235           1 :         obj[3].value = 20;
    4236           5 :         for (unsigned i = 0; i < 4; ++i)
    4237           4 :                 tommy_hashtable_insert(&table, &obj[i].node, &obj[i], i == 0 ? bucket_max : 0);
    4238             : 
    4239             :         /* the first matching duplicate is returned; the rejected candidate stays intact. */
    4240           1 :         obj[4].value = 20;
    4241           1 :         obj[4].node.next = &obj[4].node;
    4242           1 :         obj[4].node.prev = &obj[4].node;
    4243           1 :         obj[4].node.data = &obj[4];
    4244           1 :         obj[4].node.index = ~(tommy_hash_t)0;
    4245           1 :         tommy_node saved = obj[4].node;
    4246           1 :         compare_counter = 0;
    4247           1 :         if (tommy_hashtable_insert_unique(&table, &obj[4].node, &obj[4], search_hash_value_callback, &obj[4].value, 0) != &obj[1]
    4248           1 :                 || compare_counter != 1 || tommy_hashtable_count(&table) != 4
    4249           1 :                 || obj[4].value != 20 || obj[4].node.next != saved.next || obj[4].node.prev != saved.prev
    4250           1 :                 || obj[4].node.data != saved.data || obj[4].node.index != saved.index) {
    4251             :                 /* LCOV_EXCL_START */
    4252             :                 abort();
    4253             :                 /* LCOV_EXCL_STOP */
    4254             :         }
    4255           1 :         test_hash_bucket(tommy_hashtable_bucket(&table, 0), obj, order, 4);
    4256             : 
    4257             :         /* equal hashes with different keys must allow insertion at the tail. */
    4258           1 :         obj[4].value = 40;
    4259           1 :         compare_counter = 0;
    4260           1 :         if (tommy_hashtable_insert_unique(&table, &obj[4].node, &obj[4], search_hash_value_callback, &obj[4].value, 0) != &obj[4]
    4261           1 :                 || compare_counter != 3 || tommy_hashtable_count(&table) != 5
    4262           1 :                 || obj[4].node.data != &obj[4] || obj[4].node.index != 0) {
    4263             :                 /* LCOV_EXCL_START */
    4264             :                 abort();
    4265             :                 /* LCOV_EXCL_STOP */
    4266             :         }
    4267             : 
    4268             :         /* different full hashes in the same bucket must never invoke the comparator. */
    4269           1 :         obj[5].value = 50;
    4270           1 :         compare_counter = 0;
    4271           1 :         if (tommy_hashtable_insert_unique(&table, &obj[5].node, &obj[5], search_hash_value_callback, &obj[5].value, 2 * bucket_max) != &obj[5]
    4272           1 :                 || compare_counter != 0 || tommy_hashtable_count(&table) != 6) {
    4273             :                 /* LCOV_EXCL_START */
    4274             :                 abort();
    4275             :                 /* LCOV_EXCL_STOP */
    4276             :         }
    4277           1 :         test_hash_bucket(tommy_hashtable_bucket(&table, 0), obj, order, 6);
    4278           1 :         tommy_hashtable_remove_existing(&table, &obj[1].node);
    4279           1 :         obj[6].value = 20;
    4280           1 :         if (tommy_hashtable_insert_unique(&table, &obj[6].node, &obj[6], search_hash_value_callback, &obj[6].value, 0) != &obj[3]
    4281           1 :                 || tommy_hashtable_count(&table) != 5) {
    4282             :                 /* LCOV_EXCL_START */
    4283             :                 abort();
    4284             :                 /* LCOV_EXCL_STOP */
    4285             :         }
    4286           1 :         tommy_hashtable_done(&table);
    4287             : 
    4288             :         /* insert unique keys, reject duplicates, then repeat during contraction. */
    4289           1 :         tommy_hashtable_init(&table, 16);
    4290           1 :         candidate.node.next = &candidate.node;
    4291           1 :         candidate.node.prev = &candidate.node;
    4292           1 :         candidate.node.data = &candidate;
    4293           1 :         candidate.node.index = ~(tommy_hash_t)0;
    4294           3 :         for (unsigned phase = 0; phase < 2; ++phase) {
    4295         514 :                 for (unsigned step = 0; step < 256; ++step) {
    4296         512 :                         unsigned i = phase == 0 ? step : 255 - step;
    4297         512 :                         tommy_hash_t hash = tommy_inthash_u32(i);
    4298         512 :                         if (phase == 0) {
    4299         256 :                                 obj[i].value = i;
    4300         256 :                                 if (tommy_hashtable_insert_unique(&table, &obj[i].node, &obj[i], search_hash_value_callback, &obj[i].value, hash) != &obj[i]
    4301         256 :                                         || tommy_hashtable_count(&table) != i + 1) {
    4302             :                                         /* LCOV_EXCL_START */
    4303             :                                         abort();
    4304             :                                         /* LCOV_EXCL_STOP */
    4305             :                                 }
    4306             :                         }
    4307             : 
    4308         512 :                         candidate.value = i;
    4309         512 :                         saved = obj[i].node;
    4310         512 :                         tommy_size_t count = tommy_hashtable_count(&table);
    4311         512 :                         bucket_max = table.bucket_max;
    4312         512 :                         tommy_size_t bucket_mask = table.bucket_mask;
    4313         512 :                         tommy_hashtable_node** bucket = table.bucket;
    4314         512 :                         compare_counter = 0;
    4315         512 :                         if (tommy_hashtable_insert_unique(&table, &candidate.node, &candidate, search_hash_value_callback, &candidate.value, hash) != &obj[i]
    4316         512 :                                 || compare_counter != 1 || tommy_hashtable_count(&table) != count
    4317         512 :                                 || table.bucket_max != bucket_max || table.bucket_mask != bucket_mask
    4318         512 :                                 || table.bucket != bucket
    4319         512 :                                 || candidate.value != (int)i || candidate.node.next != &candidate.node
    4320         512 :                                 || candidate.node.prev != &candidate.node || candidate.node.data != &candidate
    4321         512 :                                 || candidate.node.index != ~(tommy_hash_t)0 || obj[i].node.next != saved.next
    4322         512 :                                 || obj[i].node.prev != saved.prev || obj[i].node.data != saved.data || obj[i].node.index != saved.index) {
    4323             :                                 /* LCOV_EXCL_START */
    4324             :                                 abort();
    4325             :                                 /* LCOV_EXCL_STOP */
    4326             :                         }
    4327             : 
    4328         512 :                         if (phase != 0) {
    4329         256 :                                 if (tommy_hashtable_remove_existing(&table, &obj[i].node) != &obj[i]
    4330         256 :                                         || tommy_hashtable_count(&table) != count - 1) {
    4331             :                                         /* LCOV_EXCL_START */
    4332             :                                         abort();
    4333             :                                         /* LCOV_EXCL_STOP */
    4334             :                                 }
    4335             :                                 /* also insert a new key while hashlin is shrinking progressively. */
    4336         256 :                                 if (i == 24) {
    4337           1 :                                         obj[256].value = 256;
    4338           1 :                                         if (tommy_hashtable_insert_unique(&table, &obj[256].node, &obj[256], search_hash_value_callback, &obj[256].value, tommy_inthash_u32(256)) != &obj[256]
    4339           1 :                                                 || tommy_hashtable_count(&table) != count
    4340           1 :                                                 || tommy_hashtable_search(&table, search_hash_value_callback, &obj[256].value, tommy_inthash_u32(256)) != &obj[256]
    4341           1 :                                                 || tommy_hashtable_remove_existing(&table, &obj[256].node) != &obj[256]
    4342           1 :                                                 || tommy_hashtable_count(&table) != count - 1) {
    4343             :                                                 /* LCOV_EXCL_START */
    4344             :                                                 abort();
    4345             :                                                 /* LCOV_EXCL_STOP */
    4346             :                                         }
    4347             :                                 }
    4348             :                         }
    4349             :                 }
    4350             :         }
    4351           1 :         tommy_hashtable_done(&table);
    4352           1 : }
    4353             : 
    4354           1 : void test_hashtable_rehash_existing(void)
    4355             : {
    4356           1 :         const unsigned counts[][2] = { { 1, 1 }, { 4, 4 }, { 128, 128 }, { 128, 31 } };
    4357             :         struct object_hash obj[130];
    4358             :         tommy_hash_t hashes[130];
    4359             :         tommy_hashtable table;
    4360             : 
    4361           5 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    4362           4 :                 unsigned n = counts[c][1];
    4363           4 :                 unsigned source_size = n < 3 ? n : 3;
    4364          14 :                 for (unsigned target = 0; target < source_size; ++target) {
    4365          50 :                         for (unsigned mode = 0; mode < 4; ++mode) {
    4366          40 :                                 unsigned source_count = 0, destination_count = 0;
    4367             :                                 unsigned source[3], destination[2];
    4368             : 
    4369          40 :                                 tommy_hashtable_init(&table, 16);
    4370        3164 :                                 for (unsigned i = 0; i < counts[c][0]; ++i) {
    4371        3124 :                                         obj[i].value = i;
    4372        3124 :                                         hashes[i] = i < 3 ? 0 : i == 3 ? 1 : 2;
    4373        3124 :                                         tommy_hashtable_insert(&table, &obj[i].node, &obj[i], hashes[i]);
    4374             :                                 }
    4375        1204 :                                 for (unsigned i = counts[c][0]; i > n; --i)
    4376        1164 :                                         tommy_hashtable_remove_existing(&table, &obj[i - 1].node);
    4377             : 
    4378          40 :                                 tommy_size_t bucket_max = table.bucket_max;
    4379          40 :                                 tommy_size_t bucket_mask = table.bucket_mask;
    4380          40 :                                 tommy_hashtable_node** bucket = table.bucket;
    4381             :                                 /* exercise unchanged hash, same bucket, occupied bucket and empty bucket. */
    4382          40 :                                 tommy_hash_t hash = mode == 0 ? 0 : mode == 1 ? bucket_max : mode == 2 ? 1 : ~(tommy_hash_t)0;
    4383          40 :                                 tommy_node saved = obj[target].node;
    4384          40 :                                 obj[target].value = 1000 + target;
    4385          40 :                                 tommy_hashtable_rehash_existing(&table, &obj[target].node, hash);
    4386          40 :                                 hashes[target] = hash;
    4387             : 
    4388          40 :                                 if (tommy_hashtable_count(&table) != n || table.bucket_max != bucket_max
    4389          40 :                                         || table.bucket_mask != bucket_mask
    4390          40 :                                         || table.bucket != bucket
    4391          40 :                                         || obj[target].node.index != hash || obj[target].node.data != saved.data
    4392          40 :                                         || obj[target].value != (int)(1000 + target)
    4393          40 :                                         || (mode == 0 && (obj[target].node.next != saved.next || obj[target].node.prev != saved.prev))) {
    4394             :                                         /* LCOV_EXCL_START */
    4395             :                                         abort();
    4396             :                                         /* LCOV_EXCL_STOP */
    4397             :                                 }
    4398             : 
    4399         152 :                                 for (unsigned i = 0; i < source_size; ++i)
    4400         112 :                                         if (mode == 0 || i != target)
    4401          82 :                                                 source[source_count++] = i;
    4402          40 :                                 if (mode == 1)
    4403          10 :                                         source[source_count++] = target;
    4404          40 :                                 test_hash_bucket(tommy_hashtable_bucket(&table, 0), obj, source, source_count);
    4405             : 
    4406          40 :                                 if (n > 3)
    4407          36 :                                         destination[destination_count++] = 3;
    4408          40 :                                 if (mode == 2)
    4409          10 :                                         destination[destination_count++] = target;
    4410          40 :                                 test_hash_bucket(tommy_hashtable_bucket(&table, 1), obj, destination, destination_count);
    4411          40 :                                 if (mode == 3)
    4412          10 :                                         test_hash_bucket(tommy_hashtable_bucket(&table, hash), obj, &target, 1);
    4413          40 :                                 if (mode != 0 && tommy_hashtable_search(&table, search_callback, &obj[target], 0) != 0) {
    4414             :                                         /* LCOV_EXCL_START */
    4415             :                                         abort();
    4416             :                                         /* LCOV_EXCL_STOP */
    4417             :                                 }
    4418             :                                 /* also unlink from the new hash, including an unsplit high position. */
    4419          40 :                                 if (mode != 0) {
    4420          30 :                                         tommy_hashtable_rehash_existing(&table, &obj[target].node, 0);
    4421          30 :                                         if (tommy_hashtable_count(&table) != n || obj[target].node.index != 0
    4422          30 :                                                 || tommy_hashtable_search(&table, search_callback, &obj[target], 0) != &obj[target]
    4423          30 :                                                 || tommy_hashtable_search(&table, search_callback, &obj[target], hash) != 0) {
    4424             :                                                 /* LCOV_EXCL_START */
    4425             :                                                 abort();
    4426             :                                                 /* LCOV_EXCL_STOP */
    4427             :                                         }
    4428          30 :                                         tommy_hashtable_rehash_existing(&table, &obj[target].node, hash);
    4429             :                                 }
    4430        2000 :                                 for (unsigned i = 0; i < n; ++i)
    4431        1960 :                                         if (tommy_hashtable_search(&table, search_callback, &obj[i], hashes[i]) != &obj[i]) {
    4432             :                                                 /* LCOV_EXCL_START */
    4433             :                                                 abort();
    4434             :                                                 /* LCOV_EXCL_STOP */
    4435             :                                         }
    4436             : 
    4437             :                                 /* subsequent resizing and removals must use the updated stored hash. */
    4438        3280 :                                 for (unsigned i = n; i < 130; ++i) {
    4439        3240 :                                         obj[i].value = i;
    4440        3240 :                                         hashes[i] = i < 3 ? 0 : i == 3 ? 1 : 2;
    4441        3240 :                                         tommy_hashtable_insert(&table, &obj[i].node, &obj[i], hashes[i]);
    4442             :                                 }
    4443        5240 :                                 for (unsigned i = 0; i < 130; ++i)
    4444        5200 :                                         if (tommy_hashtable_search(&table, search_callback, &obj[i], hashes[i]) != &obj[i]) {
    4445             :                                                 /* LCOV_EXCL_START */
    4446             :                                                 abort();
    4447             :                                                 /* LCOV_EXCL_STOP */
    4448             :                                         }
    4449        5240 :                                 for (unsigned i = 130; i > 0; --i) {
    4450             :                                         void* data;
    4451        5200 :                                         if (i % 2)
    4452        2600 :                                                 data = tommy_hashtable_remove_existing(&table, &obj[i - 1].node);
    4453             :                                         else
    4454        2600 :                                                 data = tommy_hashtable_remove(&table, search_callback, &obj[i - 1], hashes[i - 1]);
    4455        5200 :                                         if (data != &obj[i - 1] || tommy_hashtable_count(&table) != i - 1) {
    4456             :                                                 /* LCOV_EXCL_START */
    4457             :                                                 abort();
    4458             :                                                 /* LCOV_EXCL_STOP */
    4459             :                                         }
    4460             :                                 }
    4461          40 :                                 tommy_hashtable_done(&table);
    4462             :                         }
    4463             :                 }
    4464             :         }
    4465           1 : }
    4466             : 
    4467           1 : void test_hashtable_to_list(void)
    4468             : {
    4469           1 :         const unsigned counts[] = { 0, 1, 16, 128 };
    4470             :         struct object_hash obj[130];
    4471             :         tommy_node saved[130];
    4472             :         tommy_node* expected[130];
    4473             :         tommy_hashtable hashtable;
    4474             :         tommy_list list;
    4475             : 
    4476           5 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    4477          12 :                 for (unsigned prefix = 0; prefix <= 2; prefix += 2) {
    4478           8 :                         unsigned size = prefix + counts[c];
    4479           8 :                         tommy_hashtable_init(&hashtable, counts[c] + 1);
    4480           8 :                         tommy_list_init(&list);
    4481         306 :                         for (unsigned i = 0; i < size; ++i) {
    4482         298 :                                 obj[i].value = i / 2;
    4483         298 :                                 obj[i].node.index = tommy_inthash_u32(obj[i].value);
    4484         298 :                                 if (i < prefix)
    4485           8 :                                         tommy_list_insert_tail(&list, &obj[i].node, &obj[i]);
    4486             :                                 else
    4487         290 :                                         tommy_hashtable_insert(&hashtable, &obj[i].node, &obj[i], obj[i].node.index);
    4488             :                         }
    4489         306 :                         for (unsigned i = 0; i < size; ++i)
    4490         298 :                                 saved[i] = obj[i].node;
    4491           8 :                         tommy_hashtable_node** bucket = hashtable.bucket;
    4492           8 :                         tommy_size_t bucket_max = hashtable.bucket_max;
    4493           8 :                         tommy_size_t bucket_mask = hashtable.bucket_mask;
    4494             : 
    4495             :                         /* capture exact ascending bucket order before any links change. */
    4496           8 :                         unsigned pos = 0;
    4497          16 :                         for (unsigned i = 0; i < prefix; ++i) {
    4498           8 :                                 expected[pos] = &obj[i].node;
    4499           8 :                                 ++pos;
    4500             :                         }
    4501         648 :                         for (tommy_size_t i = 0; i < bucket_max; ++i)
    4502         930 :                                 for (tommy_node* node = bucket[i]; node; node = node->next) {
    4503         290 :                                         expected[pos] = node;
    4504         290 :                                         ++pos;
    4505             :                                 }
    4506           8 :                         if (pos != size) {
    4507             :                                 /* LCOV_EXCL_START */
    4508             :                                 abort();
    4509             :                                 /* LCOV_EXCL_STOP */
    4510             :                         }
    4511             : 
    4512           8 :                         tommy_hashtable_to_list(&hashtable, &list);
    4513           8 :                         test_hash_list(&list, obj, saved, size, prefix);
    4514           8 :                         test_list_transfer_order(&list, expected, size);
    4515           8 :                         if (tommy_hashtable_count(&hashtable) != 0 || hashtable.bucket != bucket
    4516           8 :                                 || hashtable.bucket_max != bucket_max || hashtable.bucket_mask != bucket_mask
    4517           8 :                                 || tommy_hashtable_memory_usage(&hashtable) != bucket_max * sizeof(*bucket)) {
    4518             :                                 /* LCOV_EXCL_START */
    4519             :                                 abort();
    4520             :                                 /* LCOV_EXCL_STOP */
    4521             :                         }
    4522         648 :                         for (unsigned i = 0; i < bucket_max; ++i)
    4523         640 :                                 if (tommy_hashtable_bucket(&hashtable, i) != 0) {
    4524             :                                         /* LCOV_EXCL_START */
    4525             :                                         abort();
    4526             :                                         /* LCOV_EXCL_STOP */
    4527             :                                 }
    4528           8 :                         the_count = 0;
    4529           8 :                         tommy_hashtable_foreach(&hashtable, count_callback);
    4530           8 :                         if (the_count != 0) {
    4531             :                                 /* LCOV_EXCL_START */
    4532             :                                 abort();
    4533             :                                 /* LCOV_EXCL_STOP */
    4534             :                         }
    4535         298 :                         for (unsigned i = prefix; i < size; ++i)
    4536         290 :                                 if (tommy_hashtable_search(&hashtable, search_callback, &obj[i], obj[i].node.index) != 0) {
    4537             :                                         /* LCOV_EXCL_START */
    4538             :                                         abort();
    4539             :                                         /* LCOV_EXCL_STOP */
    4540             :                                 }
    4541             : 
    4542             :                         /* a repeated call on the empty table must preserve the complete list order. */
    4543         306 :                         for (unsigned i = 0; i < size; ++i)
    4544         298 :                                 saved[i] = obj[i].node;
    4545           8 :                         tommy_hashtable_to_list(&hashtable, &list);
    4546           8 :                         test_hash_list(&list, obj, saved, size, prefix);
    4547           8 :                         test_list_transfer_order(&list, expected, size);
    4548             : 
    4549             :                         /* transfer the detached nodes back, then search and remove each one. */
    4550         306 :                         while (!tommy_list_empty(&list)) {
    4551         298 :                                 struct object_hash* data = tommy_list_remove_head(&list);
    4552         298 :                                 tommy_hashtable_insert(&hashtable, &data->node, data, data->node.index);
    4553             :                         }
    4554           8 :                         if (tommy_hashtable_count(&hashtable) != size) {
    4555             :                                 /* LCOV_EXCL_START */
    4556             :                                 abort();
    4557             :                                 /* LCOV_EXCL_STOP */
    4558             :                         }
    4559         306 :                         for (unsigned i = 0; i < size; ++i)
    4560         298 :                                 if (tommy_hashtable_search(&hashtable, search_callback, &obj[i], obj[i].node.index) != &obj[i]
    4561         298 :                                         || tommy_hashtable_remove_existing(&hashtable, &obj[i].node) != &obj[i]
    4562         298 :                                         || tommy_hashtable_count(&hashtable) != size - i - 1) {
    4563             :                                         /* LCOV_EXCL_START */
    4564             :                                         abort();
    4565             :                                         /* LCOV_EXCL_STOP */
    4566             :                                 }
    4567           8 :                         tommy_hashtable_done(&hashtable);
    4568             :                 }
    4569             :         }
    4570           1 : }
    4571             : 
    4572           1 : void test_hashtable_clear(void)
    4573             : {
    4574           1 :         const unsigned counts[] = { 0, 1, 8, 32, 128 };
    4575             :         struct object_hash obj[128];
    4576             :         tommy_hashtable hashtable;
    4577             : 
    4578           6 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    4579           5 :                 unsigned n = counts[c];
    4580           5 :                 tommy_hashtable_init(&hashtable, n + 1);
    4581             : 
    4582             :                 /* include duplicates and different hashes sharing a bucket. */
    4583         174 :                 for (unsigned i = 0; i < n; ++i) {
    4584         169 :                         obj[i].value = i / 2;
    4585         169 :                         tommy_hashtable_insert(&hashtable, &obj[i].node, &obj[i], tommy_inthash_u32(obj[i].value));
    4586             :                 }
    4587             : 
    4588           5 :                 tommy_hashtable_node** bucket = hashtable.bucket;
    4589           5 :                 tommy_size_t bucket_max = hashtable.bucket_max;
    4590           5 :                 tommy_size_t bucket_mask = hashtable.bucket_mask;
    4591           5 :                 tommy_size_t memory_usage = tommy_hashtable_memory_usage(&hashtable) - n * sizeof(tommy_hashtable_node);
    4592             : 
    4593             :                 /* clearing an empty table again must preserve its capacity too. */
    4594          15 :                 for (unsigned j = 0; j < 2; ++j) {
    4595          10 :                         tommy_hashtable_clear(&hashtable);
    4596          10 :                         if (tommy_hashtable_count(&hashtable) != 0
    4597          10 :                                 || hashtable.bucket != bucket || hashtable.bucket_max != bucket_max
    4598          10 :                                 || hashtable.bucket_mask != bucket_mask
    4599          10 :                                 || tommy_hashtable_memory_usage(&hashtable) != memory_usage) {
    4600             :                                 /* LCOV_EXCL_START */
    4601             :                                 abort();
    4602             :                                 /* LCOV_EXCL_STOP */
    4603             :                         }
    4604             : 
    4605         746 :                         for (unsigned i = 0; i < bucket_max; ++i)
    4606         736 :                                 if (tommy_hashtable_bucket(&hashtable, i) != 0) {
    4607             :                                         /* LCOV_EXCL_START */
    4608             :                                         abort();
    4609             :                                         /* LCOV_EXCL_STOP */
    4610             :                                 }
    4611             : 
    4612          10 :                         the_count = 0;
    4613          10 :                         tommy_hashtable_foreach(&hashtable, count_callback);
    4614          10 :                         tommy_hashtable_foreach_arg(&hashtable, count_arg_callback, &the_count);
    4615          10 :                         if (the_count != 0) {
    4616             :                                 /* LCOV_EXCL_START */
    4617             :                                 abort();
    4618             :                                 /* LCOV_EXCL_STOP */
    4619             :                         }
    4620             : 
    4621         348 :                         for (unsigned i = 0; i < n; ++i) {
    4622         338 :                                 tommy_hash_t hash = tommy_inthash_u32(obj[i].value);
    4623         338 :                                 if (obj[i].value != (int)(i / 2)
    4624         338 :                                         || obj[i].node.data != &obj[i] || obj[i].node.index != hash
    4625         338 :                                         || tommy_hashtable_find(&hashtable, search_callback, &obj[i], hash) != 0
    4626         338 :                                         || tommy_hashtable_search(&hashtable, search_callback, &obj[i], hash) != 0
    4627         338 :                                         || tommy_hashtable_remove(&hashtable, search_callback, &obj[i], hash) != 0) {
    4628             :                                         /* LCOV_EXCL_START */
    4629             :                                         abort();
    4630             :                                         /* LCOV_EXCL_STOP */
    4631             :                                 }
    4632             :                         }
    4633             :                 }
    4634             : 
    4635             :                 /* reuse the same nodes without initializing their old links. */
    4636         645 :                 for (unsigned i = 0; i < 128; ++i) {
    4637         640 :                         obj[i].value = i / 2;
    4638         640 :                         tommy_hashtable_insert(&hashtable, &obj[i].node, &obj[i], tommy_inthash_u32(obj[i].value));
    4639             :                 }
    4640           5 :                 if (tommy_hashtable_count(&hashtable) != 128) {
    4641             :                         /* LCOV_EXCL_START */
    4642             :                         abort();
    4643             :                         /* LCOV_EXCL_STOP */
    4644             :                 }
    4645             : 
    4646         645 :                 for (unsigned i = 0; i < 128; ++i)
    4647         640 :                         if (tommy_hashtable_find(&hashtable, search_callback, &obj[i], tommy_inthash_u32(obj[i].value)) != &obj[i].node
    4648         640 :                                 || tommy_hashtable_search(&hashtable, search_callback, &obj[i], tommy_inthash_u32(obj[i].value)) != &obj[i]) {
    4649             :                                 /* LCOV_EXCL_START */
    4650             :                                 abort();
    4651             :                                 /* LCOV_EXCL_STOP */
    4652             :                         }
    4653             : 
    4654         645 :                 for (unsigned i = 128; i > 0; --i) {
    4655             :                         void* data;
    4656         640 :                         if (i % 2)
    4657         320 :                                 data = tommy_hashtable_remove_existing(&hashtable, &obj[i - 1].node);
    4658             :                         else
    4659         320 :                                 data = tommy_hashtable_remove(&hashtable, search_callback, &obj[i - 1], tommy_inthash_u32(obj[i - 1].value));
    4660         640 :                         if (data != &obj[i - 1] || tommy_hashtable_count(&hashtable) != i - 1) {
    4661             :                                 /* LCOV_EXCL_START */
    4662             :                                 abort();
    4663             :                                 /* LCOV_EXCL_STOP */
    4664             :                         }
    4665             :                 }
    4666           5 :                 tommy_hashtable_done(&hashtable);
    4667             :         }
    4668             : 
    4669             :         /* clear must not access objects already freed by foreach. */
    4670           1 :         tommy_hashtable_init(&hashtable, 128);
    4671         129 :         for (unsigned i = 0; i < 128; ++i) {
    4672         128 :                 struct object_hash* allocated = malloc(sizeof(struct object_hash));
    4673         128 :                 allocated->value = i;
    4674         128 :                 tommy_hashtable_insert(&hashtable, &allocated->node, allocated, tommy_inthash_u32(i));
    4675             :         }
    4676           1 :         tommy_hashtable_foreach(&hashtable, free);
    4677           1 :         tommy_hashtable_clear(&hashtable);
    4678           1 :         obj[0].value = 1;
    4679           1 :         tommy_hashtable_insert(&hashtable, &obj[0].node, &obj[0], 1);
    4680           1 :         if (tommy_hashtable_count(&hashtable) != 1
    4681           1 :                 || tommy_hashtable_remove_existing(&hashtable, &obj[0].node) != &obj[0]) {
    4682             :                 /* LCOV_EXCL_START */
    4683             :                 abort();
    4684             :                 /* LCOV_EXCL_STOP */
    4685             :         }
    4686           1 :         tommy_hashtable_done(&hashtable);
    4687           1 : }
    4688             : 
    4689             : static tommy_size_t hashtable_test_mask;
    4690             : 
    4691      104698 : static int hashtable_iteration_compare(const void* first, const void* second)
    4692             : {
    4693      104698 :         const struct object_hash* a = *(struct object_hash* const*)first;
    4694      104698 :         const struct object_hash* b = *(struct object_hash* const*)second;
    4695      104698 :         tommy_size_t pa = a->node.index & hashtable_test_mask;
    4696      104698 :         tommy_size_t pb = b->node.index & hashtable_test_mask;
    4697      104698 :         if (pa != pb)
    4698       48231 :                 return pa < pb ? -1 : 1;
    4699       56467 :         return (a->value > b->value) - (a->value < b->value);
    4700             : }
    4701             : 
    4702       14848 : static tommy_hash_t hashtable_iteration_hash(unsigned i, unsigned mode)
    4703             : {
    4704       14848 :         switch (mode) {
    4705        4096 :         case 0 :
    4706        4096 :                 return tommy_inthash_u32(i / 3);
    4707        3584 :         case 1 :
    4708        3584 :                 return (tommy_hash_t)(i % 19) << 4;
    4709        3584 :         case 2 :
    4710        3584 :                 return ~(tommy_hash_t)0;
    4711        3584 :         default :
    4712        3584 :                 return i % 3 == 0 ? 0 : ((tommy_hash_t)1 << (i % TOMMY_SIZE_BIT));
    4713             :         }
    4714             : }
    4715             : 
    4716          76 : static void hashtable_iteration_check(tommy_hashtable* table, struct object_hash** expected, unsigned count)
    4717             : {
    4718          76 :         tommy_hashtable_node* node = tommy_hashtable_head(table);
    4719        6852 :         for (unsigned i = 0; i < count; ++i) {
    4720        6776 :                 if (!(node == &expected[i]->node)) {
    4721             :                         /* LCOV_EXCL_START */
    4722             :                         abort();
    4723             :                         /* LCOV_EXCL_STOP */
    4724             :                 }
    4725        6776 :                 if (!(tommy_hashtable_prev(table, node) == (i ? &expected[i - 1]->node : 0))) {
    4726             :                         /* LCOV_EXCL_START */
    4727             :                         abort();
    4728             :                         /* LCOV_EXCL_STOP */
    4729             :                 }
    4730        6776 :                 node = tommy_hashtable_next(table, node);
    4731             :         }
    4732          76 :         if (!(node == 0)) {
    4733             :                 /* LCOV_EXCL_START */
    4734             :                 abort();
    4735             :                 /* LCOV_EXCL_STOP */
    4736             :         }
    4737          76 :         node = tommy_hashtable_tail(table);
    4738        6852 :         for (unsigned i = count; i != 0; ) {
    4739        6776 :                 --i;
    4740        6776 :                 if (!(node == &expected[i]->node)) {
    4741             :                         /* LCOV_EXCL_START */
    4742             :                         abort();
    4743             :                         /* LCOV_EXCL_STOP */
    4744             :                 }
    4745        6776 :                 node = tommy_hashtable_prev(table, node);
    4746             :         }
    4747          76 :         if (!(node == 0)) {
    4748             :                 /* LCOV_EXCL_START */
    4749             :                 abort();
    4750             :                 /* LCOV_EXCL_STOP */
    4751             :         }
    4752          76 : }
    4753             : 
    4754           1 : void test_hashtable_iterators(void)
    4755             : {
    4756             :         struct object_hash obj[512];
    4757             :         struct object_hash* expected[512];
    4758           1 :         const unsigned checkpoints[] = { 0, 1, 16, 17, 63, 64, 65, 128, 256, 512 };
    4759             : 
    4760           1 :         hashtable_test_mask = 64 - 1;
    4761             : 
    4762           5 :         for (unsigned mode = 0; mode < 4; ++mode) {
    4763             :                 tommy_hashtable table;
    4764           4 :                 tommy_hashtable_init(&table, 64);
    4765           4 :                 unsigned count = 0;
    4766          44 :                 for (unsigned c = 0; c < sizeof(checkpoints) / sizeof(checkpoints[0]); ++c) {
    4767        2088 :                         while (count < checkpoints[c]) {
    4768        2048 :                                 obj[count].value = count;
    4769        2048 :                                 tommy_hashtable_insert(&table, &obj[count].node, &obj[count], hashtable_iteration_hash(count, mode));
    4770        2048 :                                 ++count;
    4771             :                         }
    4772        4528 :                         for (unsigned i = 0; i < count; ++i)
    4773        4488 :                                 expected[i] = &obj[i];
    4774          40 :                         qsort(expected, count, sizeof(expected[0]), hashtable_iteration_compare);
    4775          40 :                         hashtable_iteration_check(&table, expected, count);
    4776             :                 }
    4777             : 
    4778             :                 /* swapping transfers iterator ownership without changing node order */
    4779             :                 tommy_hashtable other;
    4780           4 :                 tommy_hashtable_init(&other, 64);
    4781           4 :                 tommy_hashtable_swap(&table, &other);
    4782           4 :                 hashtable_iteration_check(&other, expected, count);
    4783           4 :                 hashtable_iteration_check(&table, expected, 0);
    4784           4 :                 tommy_hashtable_clear(&other);
    4785           4 :                 hashtable_iteration_check(&other, expected, 0);
    4786           4 :                 tommy_hashtable_done(&other);
    4787           4 :                 tommy_hashtable_done(&table);
    4788             :         }
    4789             : 
    4790           5 :         for (unsigned mode = 0; mode < 4; ++mode) {
    4791          12 :                 for (unsigned reverse = 0; reverse < 2; ++reverse) {
    4792          32 :                         for (unsigned mutation = 0; mutation < 3; ++mutation) {
    4793             :                                 tommy_hashtable table;
    4794          24 :                                 tommy_hashtable_init(&table, 64);
    4795          24 :                                 unsigned count = 512;
    4796          24 :                                 unsigned char seen[512] = { 0 };
    4797          24 :                                 unsigned char removed[512] = { 0 };
    4798       12312 :                                 for (unsigned i = 0; i < count; ++i) {
    4799       12288 :                                         obj[i].value = i;
    4800       12288 :                                         tommy_hashtable_insert(&table, &obj[i].node, &obj[i], hashtable_iteration_hash(i, mode));
    4801       12288 :                                         expected[i] = &obj[i];
    4802             :                                 }
    4803          24 :                                 qsort(expected, count, sizeof(expected[0]), hashtable_iteration_compare);
    4804          24 :                                 unsigned visited = 0;
    4805          24 :                                 unsigned original_visited = 0;
    4806          24 :                                 unsigned removed_count = 0;
    4807          24 :                                 tommy_hashtable_node* node = reverse ? tommy_hashtable_tail(&table) : tommy_hashtable_head(&table);
    4808       12312 :                                 while (node) {
    4809       12288 :                                         struct object_hash* object = node->data;
    4810       12288 :                                         unsigned id = object->value;
    4811       12288 :                                         if (!(!seen[id])) {
    4812             :                                                 /* LCOV_EXCL_START */
    4813             :                                                 abort();
    4814             :                                                 /* LCOV_EXCL_STOP */
    4815             :                                         }
    4816       12288 :                                         seen[id] = 1;
    4817       12288 :                                         ++visited;
    4818       12288 :                                         if (id < count) {
    4819       12288 :                                                 if (!(original_visited < count)) {
    4820             :                                                         /* LCOV_EXCL_START */
    4821             :                                                         abort();
    4822             :                                                         /* LCOV_EXCL_STOP */
    4823             :                                                 }
    4824       12288 :                                                 if (!(object == expected[reverse ? count - 1 - original_visited : original_visited])) {
    4825             :                                                         /* LCOV_EXCL_START */
    4826             :                                                         abort();
    4827             :                                                         /* LCOV_EXCL_STOP */
    4828             :                                                 }
    4829       12288 :                                                 ++original_visited;
    4830             :                                         }
    4831             : 
    4832             :                                         /* save the neighbor before mutations */
    4833       12288 :                                         tommy_hashtable_node* next = reverse ? tommy_hashtable_prev(&table, node) : tommy_hashtable_next(&table, node);
    4834       12288 :                                         if (mutation == 0 || (mutation == 1 && visited % 17 != 0)) {
    4835        3976 :                                                 void* data = visited % 2 ? tommy_hashtable_remove_existing(&table, node)
    4836        7952 :                                                         : tommy_hashtable_remove(&table, search_callback, object, node->index);
    4837        7952 :                                                 if (!(data == object)) {
    4838             :                                                         /* LCOV_EXCL_START */
    4839             :                                                         abort();
    4840             :                                                         /* LCOV_EXCL_STOP */
    4841             :                                                 }
    4842        7952 :                                                 removed[id] = 1;
    4843        7952 :                                                 ++removed_count;
    4844        4336 :                                         } else if (mutation == 2 && (visited % 16 == 0 || !next)) {
    4845             :                                                 /* remove previously visited nodes as well as the current node */
    4846      131328 :                                                 for (unsigned i = 0; i < count; ++i) {
    4847      131072 :                                                         if (seen[i] && !removed[i]) {
    4848        4096 :                                                                 if (!(tommy_hashtable_remove_existing(&table, &obj[i].node) == &obj[i])) {
    4849             :                                                                         /* LCOV_EXCL_START */
    4850             :                                                                         abort();
    4851             :                                                                         /* LCOV_EXCL_STOP */
    4852             :                                                                 }
    4853        4096 :                                                                 removed[i] = 1;
    4854        4096 :                                                                 ++removed_count;
    4855             :                                                         }
    4856             :                                                 }
    4857             :                                         }
    4858       12288 :                                         node = next;
    4859             :                                 }
    4860          24 :                                 if (!(original_visited == count)) {
    4861             :                                         /* LCOV_EXCL_START */
    4862             :                                         abort();
    4863             :                                         /* LCOV_EXCL_STOP */
    4864             :                                 }
    4865          24 :                                 if (!(tommy_hashtable_count(&table) == count - removed_count)) {
    4866             :                                         /* LCOV_EXCL_START */
    4867             :                                         abort();
    4868             :                                         /* LCOV_EXCL_STOP */
    4869             :                                 }
    4870             : 
    4871             :                                 /* verify survivors in both directions against the independent oracle */
    4872          24 :                                 unsigned survivors = 0;
    4873       12312 :                                 for (unsigned i = 0; i < count; ++i)
    4874       12288 :                                         if (!removed[i])
    4875         240 :                                                 expected[survivors++] = &obj[i];
    4876          24 :                                 qsort(expected, survivors, sizeof(expected[0]), hashtable_iteration_compare);
    4877          24 :                                 hashtable_iteration_check(&table, expected, survivors);
    4878          24 :                                 tommy_hashtable_done(&table);
    4879             :                         }
    4880             :                 }
    4881             :         }
    4882             : 
    4883             :         /* freeing each removed object must not leave iteration dependent on its links */
    4884           3 :         for (unsigned reverse = 0; reverse < 2; ++reverse) {
    4885             :                 tommy_hashtable table;
    4886           2 :                 tommy_hashtable_init(&table, 64);
    4887         514 :                 for (unsigned i = 0; i < 256; ++i) {
    4888         512 :                         struct object_hash* object = malloc(sizeof(struct object_hash));
    4889         512 :                         object->value = i;
    4890         512 :                         tommy_hashtable_insert(&table, &object->node, object, hashtable_iteration_hash(i, 0));
    4891             :                 }
    4892           2 :                 unsigned visited = 0;
    4893           2 :                 tommy_hashtable_node* node = reverse ? tommy_hashtable_tail(&table) : tommy_hashtable_head(&table);
    4894         514 :                 while (node) {
    4895         512 :                         tommy_hashtable_node* next = reverse ? tommy_hashtable_prev(&table, node) : tommy_hashtable_next(&table, node);
    4896         512 :                         free(tommy_hashtable_remove_existing(&table, node));
    4897         512 :                         ++visited;
    4898         512 :                         node = next;
    4899             :                 }
    4900           2 :                 if (visited != 256 || !tommy_hashtable_empty(&table)) {
    4901             :                         /* LCOV_EXCL_START */
    4902             :                         abort();
    4903             :                         /* LCOV_EXCL_STOP */
    4904             :                 }
    4905           2 :                 tommy_hashtable_done(&table);
    4906             :         }
    4907           1 : }
    4908             : 
    4909           1 : void test_hashtable(void)
    4910             : {
    4911             :         tommy_hashtable hashtable;
    4912           1 :         const unsigned size = TOMMY_SIZE;
    4913           1 :         const unsigned module = TOMMY_SIZE / 4;
    4914             : 
    4915           1 :         test_hashtable_clear();
    4916           1 :         test_hashtable_insert_unique();
    4917           1 :         test_hashtable_swap();
    4918           1 :         test_hashtable_rehash_existing();
    4919           1 :         test_hashtable_to_list();
    4920           1 :         test_hashtable_iterators();
    4921             : 
    4922           1 :         struct object_hash* HASH = malloc(size * sizeof(struct object_hash));
    4923             : 
    4924      100001 :         for (unsigned i = 0; i < size; ++i)
    4925      100000 :                 HASH[i].value = i % module;
    4926             : 
    4927             :         /* initialize a very small hashtable */
    4928           1 :         tommy_hashtable_init(&hashtable, 1);
    4929             : 
    4930             :         /* check that we allocated space for more elements */
    4931           1 :         if (hashtable.bucket_max == 1) {
    4932             :                 /* LCOV_EXCL_START */
    4933             :                 abort();
    4934             :                 /* LCOV_EXCL_STOP */
    4935             :         }
    4936             : 
    4937             :         /* destroy it as empty */
    4938           1 :         tommy_hashtable_done(&hashtable);
    4939             : 
    4940           6 :         SPEED_START("hashtable stack") {
    4941           3 :                 unsigned limit = 5 * isqrt(size);
    4942        4746 :                 for (unsigned n = 0; n <= limit; ++n) {
    4943             :                         /* last iteration is full size */
    4944        4743 :                         if (n == limit)
    4945           3 :                                 n = limit = size;
    4946             : 
    4947        4743 :                         tommy_hashtable_init(&hashtable, limit / 2);
    4948             : 
    4949             :                         /* insert */
    4950     4046973 :                         for (unsigned i = 0; i < n; ++i)
    4951     4042230 :                                 tommy_hashtable_insert(&hashtable, &HASH[i].node, &HASH[i], HASH[i].value);
    4952             : 
    4953        4743 :                         if (tommy_hashtable_memory_usage(&hashtable) < n * sizeof(void*)) {
    4954             :                                 /* LCOV_EXCL_START */
    4955             :                                 abort();
    4956             :                                 /* LCOV_EXCL_STOP */
    4957             :                         }
    4958             : 
    4959        4743 :                         if (tommy_hashtable_count(&hashtable) != n) {
    4960             :                                 /* LCOV_EXCL_START */
    4961             :                                 abort();
    4962             :                                 /* LCOV_EXCL_STOP */
    4963             :                         }
    4964             : 
    4965        4743 :                         the_count = 0;
    4966        4743 :                         tommy_hashtable_foreach(&hashtable, count_callback);
    4967        4743 :                         if (the_count != n) {
    4968             :                                 /* LCOV_EXCL_START */
    4969             :                                 abort();
    4970             :                                 /* LCOV_EXCL_STOP */
    4971             :                         }
    4972             : 
    4973             :                         /* remove in backward order */
    4974     2024673 :                         for (unsigned i = 0; i < n / 2; ++i)
    4975     2019930 :                                 tommy_hashtable_remove_existing(&hashtable, &HASH[n - i - 1].node);
    4976             : 
    4977             :                         /* remove missing */
    4978     2024673 :                         for (unsigned i = 0; i < n / 2; ++i)
    4979     2019930 :                                 if (tommy_hashtable_remove(&hashtable, search_callback, &HASH[n - i - 1], HASH[n - i - 1].value) != 0) {
    4980             :                                         /* LCOV_EXCL_START */
    4981             :                                         abort();
    4982             :                                         /* LCOV_EXCL_STOP */
    4983             :                                 }
    4984             : 
    4985             :                         /* remove search */
    4986     2024673 :                         for (unsigned i = 0; i < n / 2; ++i)
    4987     2019930 :                                 if (tommy_hashtable_remove(&hashtable, search_callback, &HASH[n / 2 - i - 1], HASH[n / 2 - i - 1].value) == 0) {
    4988             :                                         /* LCOV_EXCL_START */
    4989             :                                         abort();
    4990             :                                         /* LCOV_EXCL_STOP */
    4991             :                                 }
    4992             : 
    4993        4743 :                         tommy_hashtable_done(&hashtable);
    4994             :                 }
    4995           3 :         } SPEED_STOP
    4996             : 
    4997           6 :         SPEED_START("hashtable queue")
    4998             :         {
    4999           3 :                 unsigned limit = isqrt(size) / 16;
    5000          63 :                 for (unsigned n = 0; n <= limit; ++n) {
    5001             :                         /* last iteration is full size */
    5002          60 :                         if (n == limit)
    5003           3 :                                 n = limit = size;
    5004             : 
    5005          60 :                         tommy_hashtable_init(&hashtable, limit / 2);
    5006             : 
    5007             :                         /* insert first run */
    5008          60 :                         unsigned j = 0, i = 0;
    5009      300573 :                         for (; i < n; ++i)
    5010      300513 :                                 tommy_hashtable_insert(&hashtable, &HASH[i].node, &HASH[i], HASH[i].value);
    5011             : 
    5012          60 :                         the_count = 0;
    5013          60 :                         tommy_hashtable_foreach_arg(&hashtable, count_arg_callback, &the_count);
    5014          60 :                         if (the_count != n) {
    5015             :                                 /* LCOV_EXCL_START */
    5016             :                                 abort();
    5017             :                                 /* LCOV_EXCL_STOP */
    5018             :                         }
    5019             : 
    5020             :                         /* insert all the others */
    5021     5699547 :                         for (; i < size; ++i, ++j) {
    5022             :                                 /* insert one */
    5023     5699487 :                                 tommy_hashtable_insert(&hashtable, &HASH[i].node, &HASH[i], HASH[i].value);
    5024             : 
    5025             :                                 /* remove one */
    5026     5699487 :                                 tommy_hashtable_remove_existing(&hashtable, &HASH[j].node);
    5027             :                         }
    5028             : 
    5029      300573 :                         for (; j < size; ++j)
    5030      300513 :                                 if (tommy_hashtable_remove(&hashtable, search_callback, &HASH[j], HASH[j].value) == 0) {
    5031             :                                         /* LCOV_EXCL_START */
    5032             :                                         abort();
    5033             :                                         /* LCOV_EXCL_STOP */
    5034             :                                 }
    5035             : 
    5036          60 :                         tommy_hashtable_done(&hashtable);
    5037             :                 }
    5038             :         }
    5039           3 :         SPEED_STOP
    5040           1 : }
    5041             : 
    5042           1 : void test_hashdyn_shrink(void)
    5043             : {
    5044           1 :         const unsigned counts[][2] = {
    5045             :                 { 0, 16 }, { 1, 16 }, { 7, 16 }, { 8, 32 },
    5046             :                 { 15, 32 }, { 16, 64 }, { 31, 64 }, { 32, 128 },
    5047             :                 { 63, 128 }, { 64, 256 }, { 65, 256 }, { 128, 512 }
    5048             :         };
    5049             :         struct object_hash obj[130];
    5050             :         tommy_node saved[130];
    5051             : 
    5052          13 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    5053          48 :                 for (unsigned reserved = 0; reserved < 3; ++reserved) {
    5054          36 :                         unsigned n = counts[c][0];
    5055             :                         tommy_hashdyn table;
    5056          36 :                         tommy_hashdyn_init(&table);
    5057          36 :                         tommy_hashdyn_reserve(&table, 0);
    5058             :                         /* reserve far beyond the contents to exercise multi-bit contractions. */
    5059          36 :                         if (reserved == 1)
    5060          12 :                                 tommy_hashdyn_reserve(&table, 1024);
    5061        1326 :                         for (unsigned i = 0; i < n; ++i) {
    5062        1290 :                                 obj[i].value = i / 2;
    5063        1290 :                                 tommy_hashdyn_insert(&table, &obj[i].node, &obj[i], tommy_inthash_u32(obj[i].value));
    5064             :                         }
    5065          36 :                         if (reserved == 2)
    5066          12 :                                 tommy_hashdyn_reserve(&table, 1024);
    5067        1326 :                         for (unsigned i = 0; i < n; ++i)
    5068        1290 :                                 saved[i] = obj[i].node;
    5069          36 :                         tommy_hashdyn_node** bucket = table.bucket;
    5070          36 :                         tommy_hashdyn_shrink(&table);
    5071          36 :                         if (tommy_hashdyn_bucket_count(&table) != counts[c][1]
    5072          36 :                                 || table.bucket_mask != counts[c][1] - 1
    5073          36 :                                 || ((tommy_size_t)1 << table.bucket_bit) != counts[c][1]
    5074          36 :                                 || tommy_hashdyn_count(&table) != n
    5075          36 :                                 || tommy_hashdyn_memory_usage(&table) != counts[c][1] * sizeof(*bucket) + n * sizeof(tommy_hashdyn_node)
    5076          36 :                                 || (!reserved && table.bucket != bucket)) {
    5077             :                                 /* LCOV_EXCL_START */
    5078             :                                 abort();
    5079             :                                 /* LCOV_EXCL_STOP */
    5080             :                         }
    5081             : 
    5082             :                         /* repeated shrinking must not allocate or disturb the contents. */
    5083          36 :                         bucket = table.bucket;
    5084          36 :                         tommy_hashdyn_shrink(&table);
    5085          36 :                         if (table.bucket != bucket || tommy_hashdyn_count(&table) != n) {
    5086             :                                 /* LCOV_EXCL_START */
    5087             :                                 abort();
    5088             :                                 /* LCOV_EXCL_STOP */
    5089             :                         }
    5090        1326 :                         for (unsigned i = 0; i < n; ++i) {
    5091        1290 :                                 if (obj[i].node.index != saved[i].index || obj[i].node.data != saved[i].data
    5092        1290 :                                         || obj[i].value != (int)(i / 2)
    5093        1290 :                                         || tommy_hashdyn_search(&table, search_callback, &obj[i], saved[i].index) != &obj[i]
    5094        1290 :                                         || tommy_hashdyn_search(&table, search_hash_value_callback, &obj[i].value, saved[i].index) != &obj[i - i % 2]) {
    5095             :                                         /* LCOV_EXCL_START */
    5096             :                                         abort();
    5097             :                                         /* LCOV_EXCL_STOP */
    5098             :                                 }
    5099             :                         }
    5100             : 
    5101             :                         /* check all links and original bucket ordering, then shrink the drained table. */
    5102             :                         tommy_list list;
    5103          36 :                         tommy_list_init(&list);
    5104          36 :                         tommy_hashdyn_to_list(&table, &list);
    5105          36 :                         tommy_hashdyn_shrink(&table);
    5106          36 :                         if (!tommy_hashdyn_empty(&table) || tommy_hashdyn_bucket_count(&table) != 16) {
    5107             :                                 /* LCOV_EXCL_START */
    5108             :                                 abort();
    5109             :                                 /* LCOV_EXCL_STOP */
    5110             :                         }
    5111          36 :                         test_hash_list(&list, obj, saved, n, 0);
    5112             : 
    5113             :                         /* reuse the shrunken table, grow it and remove duplicates in insertion order. */
    5114        1326 :                         while (!tommy_list_empty(&list)) {
    5115        1290 :                                 struct object_hash* data = tommy_list_remove_head(&list);
    5116        1290 :                                 tommy_hashdyn_insert(&table, &data->node, data, data->node.index);
    5117             :                         }
    5118        3426 :                         for (unsigned i = n; i < 130; ++i) {
    5119        3390 :                                 obj[i].value = i / 2;
    5120        3390 :                                 tommy_hashdyn_insert(&table, &obj[i].node, &obj[i], tommy_inthash_u32(obj[i].value));
    5121             :                         }
    5122          36 :                         if (tommy_hashdyn_count(&table) != 130 || tommy_hashdyn_bucket_count(&table) != 512) {
    5123             :                                 /* LCOV_EXCL_START */
    5124             :                                 abort();
    5125             :                                 /* LCOV_EXCL_STOP */
    5126             :                         }
    5127        4716 :                         for (unsigned i = 0; i < 130; ++i)
    5128        4680 :                                 if (tommy_hashdyn_remove(&table, search_hash_value_callback, &obj[i].value, obj[i].node.index) != &obj[i]
    5129        4680 :                                         || tommy_hashdyn_count(&table) != 129 - i) {
    5130             :                                         /* LCOV_EXCL_START */
    5131             :                                         abort();
    5132             :                                         /* LCOV_EXCL_STOP */
    5133             :                                 }
    5134          36 :                         tommy_hashdyn_done(&table);
    5135             :                 }
    5136             :         }
    5137           1 : }
    5138             : 
    5139           1 : void test_hashdyn_swap(void)
    5140             : {
    5141           3 :         for (unsigned first_count = 0; first_count <= 3; first_count += 3) {
    5142           6 :                 for (unsigned second_count = 0; second_count <= 2; second_count += 2) {
    5143             :                         tommy_hashdyn first, second;
    5144             :                         struct object_hash obj[7], before[7];
    5145           4 :                         memset(obj, 0, sizeof(obj));
    5146           4 :                         tommy_hashdyn_init(&first);
    5147           4 :                         tommy_hashdyn_init(&second);
    5148           4 :                         tommy_hashdyn_reserve(&first, 128);
    5149          32 :                         for (unsigned i = 0; i < 7; ++i)
    5150          28 :                                 obj[i].value = i;
    5151          10 :                         for (unsigned i = 0; i < first_count; ++i)
    5152           6 :                                 tommy_hashdyn_insert(&first, &obj[i].node, &obj[i], tommy_inthash_u32(i));
    5153           8 :                         for (unsigned i = 0; i < second_count; ++i)
    5154           4 :                                 tommy_hashdyn_insert(&second, &obj[3 + i].node, &obj[3 + i], tommy_inthash_u32(3 + i));
    5155           4 :                         tommy_hashdyn saved_first = first, saved_second = second;
    5156           4 :                         memcpy(before, obj, sizeof(before));
    5157             : 
    5158           4 :                         tommy_hashdyn_swap(&first, &second);
    5159          12 :                         for (unsigned phase = 0; phase < 2; ++phase) {
    5160           8 :                                 if (phase) {
    5161           4 :                                         tommy_hashdyn_swap(&first, &first);
    5162           4 :                                         tommy_hashdyn_swap(&second, &second);
    5163             :                                 }
    5164           8 :                                 if (first.bucket != saved_second.bucket || first.bucket_max != saved_second.bucket_max
    5165           8 :                                         || first.bucket_mask != saved_second.bucket_mask || first.bucket_bit != saved_second.bucket_bit || first.count != second_count
    5166           8 :                                         || second.bucket != saved_first.bucket || second.bucket_max != saved_first.bucket_max
    5167           8 :                                         || second.bucket_mask != saved_first.bucket_mask || second.bucket_bit != saved_first.bucket_bit || second.count != first_count
    5168           8 :                                         || memcmp(before, obj, sizeof(before)) != 0) {
    5169             :                                         /* LCOV_EXCL_START */
    5170             :                                         abort();
    5171             :                                         /* LCOV_EXCL_STOP */
    5172             :                                 }
    5173          48 :                                 for (unsigned i = 0; i < 5; ++i) {
    5174          40 :                                         if (tommy_hashdyn_search(&first, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) != (i >= 3 && i < 3 + second_count ? &obj[i] : 0)
    5175          40 :                                                 || tommy_hashdyn_search(&second, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) != (i < first_count ? &obj[i] : 0)) {
    5176             :                                                 /* LCOV_EXCL_START */
    5177             :                                                 abort();
    5178             :                                                 /* LCOV_EXCL_STOP */
    5179             :                                         }
    5180             :                                 }
    5181             :                         }
    5182             : 
    5183           4 :                         tommy_hashdyn_insert(&first, &obj[5].node, &obj[5], tommy_inthash_u32(5));
    5184           4 :                         tommy_hashdyn_insert(&second, &obj[6].node, &obj[6], tommy_inthash_u32(6));
    5185           4 :                         if (tommy_hashdyn_remove_existing(&first, &obj[5].node) != &obj[5]
    5186           4 :                                 || tommy_hashdyn_remove_existing(&second, &obj[6].node) != &obj[6]) {
    5187             :                                 /* LCOV_EXCL_START */
    5188             :                                 abort();
    5189             :                                 /* LCOV_EXCL_STOP */
    5190             :                         }
    5191          24 :                         for (unsigned i = 0; i < 5; ++i) {
    5192          20 :                                 if (i < first_count || (i >= 3 && i < 3 + second_count)) {
    5193          10 :                                         tommy_hashdyn* table = i < 3 ? &second : &first;
    5194          10 :                                         void* removed = i % 2 ? tommy_hashdyn_remove(table, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) : tommy_hashdyn_remove_existing(table, &obj[i].node);
    5195          10 :                                         if (removed != &obj[i] || tommy_hashdyn_search(table, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) != 0) {
    5196             :                                                 /* LCOV_EXCL_START */
    5197             :                                                 abort();
    5198             :                                                 /* LCOV_EXCL_STOP */
    5199             :                                         }
    5200             :                                 }
    5201             :                         }
    5202           4 :                         if (!tommy_hashdyn_empty(&first) || !tommy_hashdyn_empty(&second)) {
    5203             :                                 /* LCOV_EXCL_START */
    5204             :                                 abort();
    5205             :                                 /* LCOV_EXCL_STOP */
    5206             :                         }
    5207           4 :                         tommy_hashdyn_done(&first);
    5208           4 :                         tommy_hashdyn_done(&second);
    5209             :                 }
    5210             :         }
    5211           1 : }
    5212             : 
    5213           1 : void test_hashdyn_insert_unique(void)
    5214             : {
    5215             :         struct object_hash obj[257];
    5216             :         tommy_hashdyn table;
    5217           1 :         const unsigned order[] = { 0, 1, 2, 3, 4, 5 };
    5218             : 
    5219           1 :         tommy_hashdyn_init(&table);
    5220           1 :         obj[0].value = 1;
    5221           1 :         compare_counter = 0;
    5222           1 :         if (tommy_hashdyn_insert_unique(&table, &obj[0].node, &obj[0], search_hash_value_callback, &obj[0].value, 0) != &obj[0]
    5223           1 :                 || compare_counter != 0 || tommy_hashdyn_count(&table) != 1
    5224           1 :                 || tommy_hashdyn_search(&table, search_hash_value_callback, &obj[0].value, 0) != &obj[0]) {
    5225             :                 /* LCOV_EXCL_START */
    5226             :                 abort();
    5227             :                 /* LCOV_EXCL_STOP */
    5228             :         }
    5229           1 :         tommy_hashdyn_remove_existing(&table, &obj[0].node);
    5230             : 
    5231           1 :         tommy_size_t bucket_max = table.bucket_max;
    5232           1 :         obj[0].value = 10;
    5233           1 :         obj[1].value = 20;
    5234           1 :         obj[2].value = 30;
    5235           1 :         obj[3].value = 20;
    5236           5 :         for (unsigned i = 0; i < 4; ++i)
    5237           4 :                 tommy_hashdyn_insert(&table, &obj[i].node, &obj[i], i == 0 ? bucket_max : 0);
    5238             : 
    5239             :         /* the first matching duplicate is returned; the rejected candidate stays intact. */
    5240           1 :         obj[4].value = 20;
    5241           1 :         obj[4].node.next = &obj[4].node;
    5242           1 :         obj[4].node.prev = &obj[4].node;
    5243           1 :         obj[4].node.data = &obj[4];
    5244           1 :         obj[4].node.index = ~(tommy_hash_t)0;
    5245           1 :         tommy_node saved = obj[4].node;
    5246           1 :         compare_counter = 0;
    5247           1 :         if (tommy_hashdyn_insert_unique(&table, &obj[4].node, &obj[4], search_hash_value_callback, &obj[4].value, 0) != &obj[1]
    5248           1 :                 || compare_counter != 1 || tommy_hashdyn_count(&table) != 4
    5249           1 :                 || obj[4].value != 20 || obj[4].node.next != saved.next || obj[4].node.prev != saved.prev
    5250           1 :                 || obj[4].node.data != saved.data || obj[4].node.index != saved.index) {
    5251             :                 /* LCOV_EXCL_START */
    5252             :                 abort();
    5253             :                 /* LCOV_EXCL_STOP */
    5254             :         }
    5255           1 :         test_hash_bucket(tommy_hashdyn_bucket(&table, 0), obj, order, 4);
    5256             : 
    5257             :         /* equal hashes with different keys must allow insertion at the tail. */
    5258           1 :         obj[4].value = 40;
    5259           1 :         compare_counter = 0;
    5260           1 :         if (tommy_hashdyn_insert_unique(&table, &obj[4].node, &obj[4], search_hash_value_callback, &obj[4].value, 0) != &obj[4]
    5261           1 :                 || compare_counter != 3 || tommy_hashdyn_count(&table) != 5
    5262           1 :                 || obj[4].node.data != &obj[4] || obj[4].node.index != 0) {
    5263             :                 /* LCOV_EXCL_START */
    5264             :                 abort();
    5265             :                 /* LCOV_EXCL_STOP */
    5266             :         }
    5267             : 
    5268             :         /* different full hashes in the same bucket must never invoke the comparator. */
    5269           1 :         obj[5].value = 50;
    5270           1 :         compare_counter = 0;
    5271           1 :         if (tommy_hashdyn_insert_unique(&table, &obj[5].node, &obj[5], search_hash_value_callback, &obj[5].value, 2 * bucket_max) != &obj[5]
    5272           1 :                 || compare_counter != 0 || tommy_hashdyn_count(&table) != 6) {
    5273             :                 /* LCOV_EXCL_START */
    5274             :                 abort();
    5275             :                 /* LCOV_EXCL_STOP */
    5276             :         }
    5277           1 :         test_hash_bucket(tommy_hashdyn_bucket(&table, 0), obj, order, 6);
    5278           1 :         tommy_hashdyn_remove_existing(&table, &obj[1].node);
    5279           1 :         obj[6].value = 20;
    5280           1 :         if (tommy_hashdyn_insert_unique(&table, &obj[6].node, &obj[6], search_hash_value_callback, &obj[6].value, 0) != &obj[3]
    5281           1 :                 || tommy_hashdyn_count(&table) != 5) {
    5282             :                 /* LCOV_EXCL_START */
    5283             :                 abort();
    5284             :                 /* LCOV_EXCL_STOP */
    5285             :         }
    5286           1 :         tommy_hashdyn_done(&table);
    5287             : 
    5288             :         /* insert unique keys, reject duplicates, then repeat during contraction. */
    5289           1 :         tommy_hashdyn_init(&table);
    5290             :         struct object_hash candidate;
    5291           1 :         candidate.node.next = &candidate.node;
    5292           1 :         candidate.node.prev = &candidate.node;
    5293           1 :         candidate.node.data = &candidate;
    5294           1 :         candidate.node.index = ~(tommy_hash_t)0;
    5295           3 :         for (unsigned phase = 0; phase < 2; ++phase) {
    5296         514 :                 for (unsigned step = 0; step < 256; ++step) {
    5297         512 :                         unsigned i = phase == 0 ? step : 255 - step;
    5298         512 :                         tommy_hash_t hash = tommy_inthash_u32(i);
    5299         512 :                         if (phase == 0) {
    5300         256 :                                 obj[i].value = i;
    5301         256 :                                 if (tommy_hashdyn_insert_unique(&table, &obj[i].node, &obj[i], search_hash_value_callback, &obj[i].value, hash) != &obj[i]
    5302         256 :                                         || tommy_hashdyn_count(&table) != i + 1) {
    5303             :                                         /* LCOV_EXCL_START */
    5304             :                                         abort();
    5305             :                                         /* LCOV_EXCL_STOP */
    5306             :                                 }
    5307             :                         }
    5308             : 
    5309         512 :                         candidate.value = i;
    5310         512 :                         saved = obj[i].node;
    5311         512 :                         tommy_size_t count = tommy_hashdyn_count(&table);
    5312         512 :                         bucket_max = table.bucket_max;
    5313         512 :                         tommy_size_t bucket_mask = table.bucket_mask;
    5314         512 :                         tommy_hashdyn_node** bucket = table.bucket;
    5315         512 :                         tommy_uint_t bucket_bit = table.bucket_bit;
    5316         512 :                         compare_counter = 0;
    5317         512 :                         if (tommy_hashdyn_insert_unique(&table, &candidate.node, &candidate, search_hash_value_callback, &candidate.value, hash) != &obj[i]
    5318         512 :                                 || compare_counter != 1 || tommy_hashdyn_count(&table) != count
    5319         512 :                                 || table.bucket_max != bucket_max || table.bucket_mask != bucket_mask
    5320         512 :                                 || table.bucket != bucket || table.bucket_bit != bucket_bit
    5321         512 :                                 || candidate.value != (int)i || candidate.node.next != &candidate.node
    5322         512 :                                 || candidate.node.prev != &candidate.node || candidate.node.data != &candidate
    5323         512 :                                 || candidate.node.index != ~(tommy_hash_t)0 || obj[i].node.next != saved.next
    5324         512 :                                 || obj[i].node.prev != saved.prev || obj[i].node.data != saved.data || obj[i].node.index != saved.index) {
    5325             :                                 /* LCOV_EXCL_START */
    5326             :                                 abort();
    5327             :                                 /* LCOV_EXCL_STOP */
    5328             :                         }
    5329             : 
    5330         512 :                         if (phase != 0) {
    5331         256 :                                 if (tommy_hashdyn_remove_existing(&table, &obj[i].node) != &obj[i]
    5332         256 :                                         || tommy_hashdyn_count(&table) != count - 1) {
    5333             :                                         /* LCOV_EXCL_START */
    5334             :                                         abort();
    5335             :                                         /* LCOV_EXCL_STOP */
    5336             :                                 }
    5337             :                                 /* also insert a new key while hashlin is shrinking progressively. */
    5338         256 :                                 if (i == 24) {
    5339           1 :                                         obj[256].value = 256;
    5340           1 :                                         if (tommy_hashdyn_insert_unique(&table, &obj[256].node, &obj[256], search_hash_value_callback, &obj[256].value, tommy_inthash_u32(256)) != &obj[256]
    5341           1 :                                                 || tommy_hashdyn_count(&table) != count
    5342           1 :                                                 || tommy_hashdyn_search(&table, search_hash_value_callback, &obj[256].value, tommy_inthash_u32(256)) != &obj[256]
    5343           1 :                                                 || tommy_hashdyn_remove_existing(&table, &obj[256].node) != &obj[256]
    5344           1 :                                                 || tommy_hashdyn_count(&table) != count - 1) {
    5345             :                                                 /* LCOV_EXCL_START */
    5346             :                                                 abort();
    5347             :                                                 /* LCOV_EXCL_STOP */
    5348             :                                         }
    5349             :                                 }
    5350             :                         }
    5351             :                 }
    5352             :         }
    5353           1 :         tommy_hashdyn_done(&table);
    5354           1 : }
    5355             : 
    5356           1 : void test_hashdyn_rehash_existing(void)
    5357             : {
    5358           1 :         const unsigned counts[][2] = { { 1, 1 }, { 4, 4 }, { 128, 128 }, { 128, 31 } };
    5359             :         struct object_hash obj[130];
    5360             :         tommy_hash_t hashes[130];
    5361             :         tommy_hashdyn table;
    5362             : 
    5363           5 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    5364           4 :                 unsigned n = counts[c][1];
    5365           4 :                 unsigned source_size = n < 3 ? n : 3;
    5366          14 :                 for (unsigned target = 0; target < source_size; ++target) {
    5367          50 :                         for (unsigned mode = 0; mode < 4; ++mode) {
    5368             :                                 unsigned source[3], destination[2];
    5369          40 :                                 unsigned source_count = 0, destination_count = 0;
    5370             : 
    5371          40 :                                 tommy_hashdyn_init(&table);
    5372          40 :                                 if (n == 4)
    5373          12 :                                         tommy_hashdyn_reserve(&table, 128);
    5374        3164 :                                 for (unsigned i = 0; i < counts[c][0]; ++i) {
    5375        3124 :                                         obj[i].value = i;
    5376        3124 :                                         hashes[i] = i < 3 ? 0 : i == 3 ? 1 : 2;
    5377        3124 :                                         tommy_hashdyn_insert(&table, &obj[i].node, &obj[i], hashes[i]);
    5378             :                                 }
    5379        1204 :                                 for (unsigned i = counts[c][0]; i > n; --i)
    5380        1164 :                                         tommy_hashdyn_remove_existing(&table, &obj[i - 1].node);
    5381             : 
    5382          40 :                                 tommy_size_t bucket_max = table.bucket_max;
    5383          40 :                                 tommy_size_t bucket_mask = table.bucket_mask;
    5384          40 :                                 tommy_hashdyn_node** bucket = table.bucket;
    5385          40 :                                 tommy_uint_t bucket_bit = table.bucket_bit;
    5386             :                                 /* exercise unchanged hash, same bucket, occupied bucket and empty bucket. */
    5387          40 :                                 tommy_hash_t hash = mode == 0 ? 0 : mode == 1 ? bucket_max : mode == 2 ? 1 : ~(tommy_hash_t)0;
    5388          40 :                                 tommy_node saved = obj[target].node;
    5389          40 :                                 obj[target].value = 1000 + target;
    5390          40 :                                 tommy_hashdyn_rehash_existing(&table, &obj[target].node, hash);
    5391          40 :                                 hashes[target] = hash;
    5392             : 
    5393          40 :                                 if (tommy_hashdyn_count(&table) != n || table.bucket_max != bucket_max
    5394          40 :                                         || table.bucket_mask != bucket_mask
    5395          40 :                                         || table.bucket != bucket || table.bucket_bit != bucket_bit
    5396          40 :                                         || obj[target].node.index != hash || obj[target].node.data != saved.data
    5397          40 :                                         || obj[target].value != (int)(1000 + target)
    5398          40 :                                         || (mode == 0 && (obj[target].node.next != saved.next || obj[target].node.prev != saved.prev))) {
    5399             :                                         /* LCOV_EXCL_START */
    5400             :                                         abort();
    5401             :                                         /* LCOV_EXCL_STOP */
    5402             :                                 }
    5403             : 
    5404         152 :                                 for (unsigned i = 0; i < source_size; ++i)
    5405         112 :                                         if (mode == 0 || i != target)
    5406          82 :                                                 source[source_count++] = i;
    5407          40 :                                 if (mode == 1)
    5408          10 :                                         source[source_count++] = target;
    5409          40 :                                 test_hash_bucket(tommy_hashdyn_bucket(&table, 0), obj, source, source_count);
    5410             : 
    5411          40 :                                 if (n > 3)
    5412          36 :                                         destination[destination_count++] = 3;
    5413          40 :                                 if (mode == 2)
    5414          10 :                                         destination[destination_count++] = target;
    5415          40 :                                 test_hash_bucket(tommy_hashdyn_bucket(&table, 1), obj, destination, destination_count);
    5416          40 :                                 if (mode == 3)
    5417          10 :                                         test_hash_bucket(tommy_hashdyn_bucket(&table, hash), obj, &target, 1);
    5418          40 :                                 if (mode != 0 && tommy_hashdyn_search(&table, search_callback, &obj[target], 0) != 0) {
    5419             :                                         /* LCOV_EXCL_START */
    5420             :                                         abort();
    5421             :                                         /* LCOV_EXCL_STOP */
    5422             :                                 }
    5423             :                                 /* also unlink from the new hash, including an unsplit high position. */
    5424          40 :                                 if (mode != 0) {
    5425          30 :                                         tommy_hashdyn_rehash_existing(&table, &obj[target].node, 0);
    5426          30 :                                         if (tommy_hashdyn_count(&table) != n || obj[target].node.index != 0
    5427          30 :                                                 || tommy_hashdyn_search(&table, search_callback, &obj[target], 0) != &obj[target]
    5428          30 :                                                 || tommy_hashdyn_search(&table, search_callback, &obj[target], hash) != 0) {
    5429             :                                                 /* LCOV_EXCL_START */
    5430             :                                                 abort();
    5431             :                                                 /* LCOV_EXCL_STOP */
    5432             :                                         }
    5433          30 :                                         tommy_hashdyn_rehash_existing(&table, &obj[target].node, hash);
    5434             :                                 }
    5435        2000 :                                 for (unsigned i = 0; i < n; ++i)
    5436        1960 :                                         if (tommy_hashdyn_search(&table, search_callback, &obj[i], hashes[i]) != &obj[i]) {
    5437             :                                                 /* LCOV_EXCL_START */
    5438             :                                                 abort();
    5439             :                                                 /* LCOV_EXCL_STOP */
    5440             :                                         }
    5441             : 
    5442             :                                 /* subsequent resizing and removals must use the updated stored hash. */
    5443        3280 :                                 for (unsigned i = n; i < 130; ++i) {
    5444        3240 :                                         obj[i].value = i;
    5445        3240 :                                         hashes[i] = i < 3 ? 0 : i == 3 ? 1 : 2;
    5446        3240 :                                         tommy_hashdyn_insert(&table, &obj[i].node, &obj[i], hashes[i]);
    5447             :                                 }
    5448        5240 :                                 for (unsigned i = 0; i < 130; ++i)
    5449        5200 :                                         if (tommy_hashdyn_search(&table, search_callback, &obj[i], hashes[i]) != &obj[i]) {
    5450             :                                                 /* LCOV_EXCL_START */
    5451             :                                                 abort();
    5452             :                                                 /* LCOV_EXCL_STOP */
    5453             :                                         }
    5454        5240 :                                 for (unsigned i = 130; i > 0; --i) {
    5455             :                                         void* data;
    5456        5200 :                                         if (i % 2)
    5457        2600 :                                                 data = tommy_hashdyn_remove_existing(&table, &obj[i - 1].node);
    5458             :                                         else
    5459        2600 :                                                 data = tommy_hashdyn_remove(&table, search_callback, &obj[i - 1], hashes[i - 1]);
    5460        5200 :                                         if (data != &obj[i - 1] || tommy_hashdyn_count(&table) != i - 1) {
    5461             :                                                 /* LCOV_EXCL_START */
    5462             :                                                 abort();
    5463             :                                                 /* LCOV_EXCL_STOP */
    5464             :                                         }
    5465             :                                 }
    5466          40 :                                 tommy_hashdyn_done(&table);
    5467             :                         }
    5468             :                 }
    5469             :         }
    5470           1 : }
    5471             : 
    5472           1 : void test_hashdyn_to_list(void)
    5473             : {
    5474           1 :         const unsigned counts[] = { 0, 1, 8, 128 };
    5475             :         struct object_hash obj[130];
    5476             :         tommy_node saved[130];
    5477             :         tommy_node* expected[130];
    5478             : 
    5479           5 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    5480          12 :                 for (unsigned reserved = 0; reserved < 2; ++reserved) {
    5481          24 :                         for (unsigned prefix = 0; prefix <= 2; prefix += 2) {
    5482          16 :                                 unsigned size = prefix + counts[c];
    5483             :                                 tommy_hashdyn hashdyn;
    5484          16 :                                 tommy_hashdyn_init(&hashdyn);
    5485             :                                 /* include sparse tables with capacity reserved beyond their contents. */
    5486          16 :                                 if (reserved)
    5487           8 :                                         tommy_hashdyn_reserve(&hashdyn, 512);
    5488             :                                 tommy_list list;
    5489          16 :                                 tommy_list_init(&list);
    5490         580 :                                 for (unsigned i = 0; i < size; ++i) {
    5491         564 :                                         obj[i].value = i / 2;
    5492         564 :                                         obj[i].node.index = tommy_inthash_u32(obj[i].value);
    5493         564 :                                         if (i < prefix)
    5494          16 :                                                 tommy_list_insert_tail(&list, &obj[i].node, &obj[i]);
    5495             :                                         else
    5496         548 :                                                 tommy_hashdyn_insert(&hashdyn, &obj[i].node, &obj[i], obj[i].node.index);
    5497             :                                 }
    5498         580 :                                 for (unsigned i = 0; i < size; ++i)
    5499         564 :                                         saved[i] = obj[i].node;
    5500          16 :                                 tommy_hashdyn_node** bucket = hashdyn.bucket;
    5501          16 :                                 tommy_size_t bucket_max = hashdyn.bucket_max;
    5502          16 :                                 tommy_size_t bucket_mask = hashdyn.bucket_mask;
    5503          16 :                                 tommy_uint_t bucket_bit = hashdyn.bucket_bit;
    5504             : 
    5505             :                                 /* capture exact ascending bucket order before any links change. */
    5506          16 :                                 unsigned pos = 0;
    5507          32 :                                 for (unsigned i = 0; i < prefix; ++i) {
    5508          16 :                                         expected[pos] = &obj[i].node;
    5509          16 :                                         ++pos;
    5510             :                                 }
    5511       17552 :                                 for (tommy_size_t i = 0; i < bucket_max; ++i)
    5512       18084 :                                         for (tommy_node* node = bucket[i]; node; node = node->next) {
    5513         548 :                                                 expected[pos] = node;
    5514         548 :                                                 ++pos;
    5515             :                                         }
    5516          16 :                                 if (pos != size) {
    5517             :                                         /* LCOV_EXCL_START */
    5518             :                                         abort();
    5519             :                                         /* LCOV_EXCL_STOP */
    5520             :                                 }
    5521             : 
    5522          16 :                                 tommy_hashdyn_to_list(&hashdyn, &list);
    5523          16 :                                 test_hash_list(&list, obj, saved, size, prefix);
    5524          16 :                                 test_list_transfer_order(&list, expected, size);
    5525          16 :                                 if (tommy_hashdyn_count(&hashdyn) != 0 || hashdyn.bucket != bucket
    5526          16 :                                         || hashdyn.bucket_max != bucket_max || hashdyn.bucket_mask != bucket_mask
    5527          16 :                                         || hashdyn.bucket_bit != bucket_bit
    5528          16 :                                         || tommy_hashdyn_memory_usage(&hashdyn) != bucket_max * sizeof(*bucket)) {
    5529             :                                         /* LCOV_EXCL_START */
    5530             :                                         abort();
    5531             :                                         /* LCOV_EXCL_STOP */
    5532             :                                 }
    5533       17552 :                                 for (unsigned i = 0; i < bucket_max; ++i)
    5534       17536 :                                         if (tommy_hashdyn_bucket(&hashdyn, i) != 0) {
    5535             :                                                 /* LCOV_EXCL_START */
    5536             :                                                 abort();
    5537             :                                                 /* LCOV_EXCL_STOP */
    5538             :                                         }
    5539          16 :                                 the_count = 0;
    5540          16 :                                 tommy_hashdyn_foreach(&hashdyn, count_callback);
    5541          16 :                                 if (the_count != 0) {
    5542             :                                         /* LCOV_EXCL_START */
    5543             :                                         abort();
    5544             :                                         /* LCOV_EXCL_STOP */
    5545             :                                 }
    5546         564 :                                 for (unsigned i = prefix; i < size; ++i)
    5547         548 :                                         if (tommy_hashdyn_search(&hashdyn, search_callback, &obj[i], obj[i].node.index) != 0) {
    5548             :                                                 /* LCOV_EXCL_START */
    5549             :                                                 abort();
    5550             :                                                 /* LCOV_EXCL_STOP */
    5551             :                                         }
    5552             : 
    5553         580 :                                 for (unsigned i = 0; i < size; ++i)
    5554         564 :                                         saved[i] = obj[i].node;
    5555          16 :                                 tommy_hashdyn_to_list(&hashdyn, &list);
    5556          16 :                                 test_hash_list(&list, obj, saved, size, prefix);
    5557          16 :                                 test_list_transfer_order(&list, expected, size);
    5558             : 
    5559             :                                 /* reuse the nodes and verify removals across shrink thresholds. */
    5560         580 :                                 while (!tommy_list_empty(&list)) {
    5561         564 :                                         struct object_hash* data = tommy_list_remove_head(&list);
    5562         564 :                                         tommy_hashdyn_insert(&hashdyn, &data->node, data, data->node.index);
    5563             :                                 }
    5564          16 :                                 if (tommy_hashdyn_count(&hashdyn) != size) {
    5565             :                                         /* LCOV_EXCL_START */
    5566             :                                         abort();
    5567             :                                         /* LCOV_EXCL_STOP */
    5568             :                                 }
    5569         580 :                                 for (unsigned i = 0; i < size; ++i)
    5570         564 :                                         if (tommy_hashdyn_search(&hashdyn, search_callback, &obj[i], obj[i].node.index) != &obj[i]
    5571         564 :                                                 || tommy_hashdyn_remove_existing(&hashdyn, &obj[i].node) != &obj[i]
    5572         564 :                                                 || tommy_hashdyn_count(&hashdyn) != size - i - 1) {
    5573             :                                                 /* LCOV_EXCL_START */
    5574             :                                                 abort();
    5575             :                                                 /* LCOV_EXCL_STOP */
    5576             :                                         }
    5577          16 :                                 tommy_hashdyn_done(&hashdyn);
    5578             :                         }
    5579             :                 }
    5580             :         }
    5581           1 : }
    5582             : 
    5583           1 : void test_hashdyn_clear(void)
    5584             : {
    5585           1 :         const unsigned counts[] = { 0, 1, 8, 32, 128 };
    5586             :         struct object_hash obj[128];
    5587             : 
    5588           6 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    5589           5 :                 unsigned n = counts[c];
    5590             :                 tommy_hashdyn hashdyn;
    5591           5 :                 tommy_hashdyn_init(&hashdyn);
    5592             :                 /* also exercise an empty table with reserved capacity. */
    5593           5 :                 if (n == 0)
    5594           1 :                         tommy_hashdyn_reserve(&hashdyn, 128);
    5595             : 
    5596         174 :                 for (unsigned i = 0; i < n; ++i) {
    5597         169 :                         obj[i].value = i / 2;
    5598         169 :                         tommy_hashdyn_insert(&hashdyn, &obj[i].node, &obj[i], tommy_inthash_u32(obj[i].value));
    5599             :                 }
    5600             : 
    5601           5 :                 tommy_hashdyn_node** bucket = hashdyn.bucket;
    5602           5 :                 tommy_size_t bucket_max = hashdyn.bucket_max;
    5603           5 :                 tommy_size_t bucket_mask = hashdyn.bucket_mask;
    5604           5 :                 tommy_uint_t bucket_bit = hashdyn.bucket_bit;
    5605           5 :                 tommy_size_t memory_usage = tommy_hashdyn_memory_usage(&hashdyn) - n * sizeof(tommy_hashdyn_node);
    5606             : 
    5607          15 :                 for (unsigned j = 0; j < 2; ++j) {
    5608          10 :                         tommy_hashdyn_clear(&hashdyn);
    5609          10 :                         if (tommy_hashdyn_count(&hashdyn) != 0
    5610          10 :                                 || hashdyn.bucket != bucket || hashdyn.bucket_max != bucket_max
    5611          10 :                                 || hashdyn.bucket_mask != bucket_mask || hashdyn.bucket_bit != bucket_bit
    5612          10 :                                 || tommy_hashdyn_memory_usage(&hashdyn) != memory_usage) {
    5613             :                                 /* LCOV_EXCL_START */
    5614             :                                 abort();
    5615             :                                 /* LCOV_EXCL_STOP */
    5616             :                         }
    5617             : 
    5618        2410 :                         for (unsigned i = 0; i < bucket_max; ++i)
    5619        2400 :                                 if (tommy_hashdyn_bucket(&hashdyn, i) != 0) {
    5620             :                                         /* LCOV_EXCL_START */
    5621             :                                         abort();
    5622             :                                         /* LCOV_EXCL_STOP */
    5623             :                                 }
    5624             : 
    5625          10 :                         the_count = 0;
    5626          10 :                         tommy_hashdyn_foreach(&hashdyn, count_callback);
    5627          10 :                         tommy_hashdyn_foreach_arg(&hashdyn, count_arg_callback, &the_count);
    5628          10 :                         if (the_count != 0) {
    5629             :                                 /* LCOV_EXCL_START */
    5630             :                                 abort();
    5631             :                                 /* LCOV_EXCL_STOP */
    5632             :                         }
    5633             : 
    5634         348 :                         for (unsigned i = 0; i < n; ++i) {
    5635         338 :                                 tommy_hash_t hash = tommy_inthash_u32(obj[i].value);
    5636         338 :                                 if (obj[i].value != (int)(i / 2)
    5637         338 :                                         || obj[i].node.data != &obj[i] || obj[i].node.index != hash
    5638         338 :                                         || tommy_hashdyn_find(&hashdyn, search_callback, &obj[i], hash) != 0
    5639         338 :                                         || tommy_hashdyn_search(&hashdyn, search_callback, &obj[i], hash) != 0
    5640         338 :                                         || tommy_hashdyn_remove(&hashdyn, search_callback, &obj[i], hash) != 0) {
    5641             :                                         /* LCOV_EXCL_START */
    5642             :                                         abort();
    5643             :                                         /* LCOV_EXCL_STOP */
    5644             :                                 }
    5645             :                         }
    5646             :                 }
    5647             : 
    5648             :                 /* reinsert old nodes and exercise growth after clear. */
    5649         645 :                 for (unsigned i = 0; i < 128; ++i) {
    5650         640 :                         obj[i].value = i / 2;
    5651         640 :                         tommy_hashdyn_insert(&hashdyn, &obj[i].node, &obj[i], tommy_inthash_u32(obj[i].value));
    5652             :                 }
    5653           5 :                 if (tommy_hashdyn_count(&hashdyn) != 128) {
    5654             :                         /* LCOV_EXCL_START */
    5655             :                         abort();
    5656             :                         /* LCOV_EXCL_STOP */
    5657             :                 }
    5658             : 
    5659         645 :                 for (unsigned i = 0; i < 128; ++i)
    5660         640 :                         if (tommy_hashdyn_find(&hashdyn, search_callback, &obj[i], tommy_inthash_u32(obj[i].value)) != &obj[i].node
    5661         640 :                                 || tommy_hashdyn_search(&hashdyn, search_callback, &obj[i], tommy_inthash_u32(obj[i].value)) != &obj[i]) {
    5662             :                                 /* LCOV_EXCL_START */
    5663             :                                 abort();
    5664             :                                 /* LCOV_EXCL_STOP */
    5665             :                         }
    5666             : 
    5667             :                 /* both removal paths must still work across shrink thresholds. */
    5668         645 :                 for (unsigned i = 128; i > 0; --i) {
    5669             :                         void* data;
    5670         640 :                         if (i % 2)
    5671         320 :                                 data = tommy_hashdyn_remove_existing(&hashdyn, &obj[i - 1].node);
    5672             :                         else
    5673         320 :                                 data = tommy_hashdyn_remove(&hashdyn, search_callback, &obj[i - 1], tommy_inthash_u32(obj[i - 1].value));
    5674         640 :                         if (data != &obj[i - 1] || tommy_hashdyn_count(&hashdyn) != i - 1) {
    5675             :                                 /* LCOV_EXCL_START */
    5676             :                                 abort();
    5677             :                                 /* LCOV_EXCL_STOP */
    5678             :                         }
    5679             :                 }
    5680           5 :                 tommy_hashdyn_done(&hashdyn);
    5681             :         }
    5682             : 
    5683             :         /* clear must not access objects already freed by foreach. */
    5684             :         tommy_hashdyn hashdyn;
    5685           1 :         tommy_hashdyn_init(&hashdyn);
    5686         129 :         for (unsigned i = 0; i < 128; ++i) {
    5687         128 :                 struct object_hash* allocated = malloc(sizeof(struct object_hash));
    5688         128 :                 allocated->value = i;
    5689         128 :                 tommy_hashdyn_insert(&hashdyn, &allocated->node, allocated, tommy_inthash_u32(i));
    5690             :         }
    5691           1 :         tommy_hashdyn_foreach(&hashdyn, free);
    5692           1 :         tommy_hashdyn_clear(&hashdyn);
    5693           1 :         obj[0].value = 1;
    5694           1 :         tommy_hashdyn_insert(&hashdyn, &obj[0].node, &obj[0], 1);
    5695           1 :         if (tommy_hashdyn_count(&hashdyn) != 1
    5696           1 :                 || tommy_hashdyn_remove_existing(&hashdyn, &obj[0].node) != &obj[0]) {
    5697             :                 /* LCOV_EXCL_START */
    5698             :                 abort();
    5699             :                 /* LCOV_EXCL_STOP */
    5700             :         }
    5701           1 :         tommy_hashdyn_done(&hashdyn);
    5702           1 : }
    5703             : 
    5704      620874 : static int hash_bitreverse_compare(const void* first, const void* second)
    5705             : {
    5706      620874 :         const struct object_hash* a = *(struct object_hash* const*)first;
    5707      620874 :         const struct object_hash* b = *(struct object_hash* const*)second;
    5708      620874 :         tommy_hash_t x = a->node.index;
    5709      620874 :         tommy_hash_t y = b->node.index;
    5710      620874 :         tommy_hash_t rx = 0;
    5711      620874 :         tommy_hash_t ry = 0;
    5712    40356810 :         for (unsigned i = 0; i < TOMMY_SIZE_BIT; ++i) {
    5713    39735936 :                 rx = (rx << 1) | (x & 1);
    5714    39735936 :                 ry = (ry << 1) | (y & 1);
    5715    39735936 :                 x >>= 1;
    5716    39735936 :                 y >>= 1;
    5717             :         }
    5718      620874 :         if (rx != ry)
    5719      387424 :                 return rx < ry ? -1 : 1;
    5720      233450 :         return (a->value > b->value) - (a->value < b->value);
    5721             : }
    5722             : 
    5723       37888 : static tommy_hash_t hashdyn_iteration_hash(unsigned i, unsigned mode)
    5724             : {
    5725       37888 :         switch (mode) {
    5726       10240 :         case 0 :
    5727       10240 :                 return tommy_inthash_u32(i / 3);
    5728        9216 :         case 1 :
    5729        9216 :                 return (tommy_hash_t)(i % 19) << TOMMY_HASHDYN_BIT;
    5730        9216 :         case 2 :
    5731        9216 :                 return ~(tommy_hash_t)0;
    5732        9216 :         default :
    5733        9216 :                 return i % 3 == 0 ? 0 : ((tommy_hash_t)1 << (i % TOMMY_SIZE_BIT));
    5734             :         }
    5735             : }
    5736             : 
    5737         104 : static void hashdyn_iteration_check(tommy_hashdyn* table, struct object_hash** expected, unsigned count)
    5738             : {
    5739         104 :         tommy_hashdyn_node* node = tommy_hashdyn_head(table);
    5740       21460 :         for (unsigned i = 0; i < count; ++i) {
    5741       21356 :                 if (!(node == &expected[i]->node)) {
    5742             :                         /* LCOV_EXCL_START */
    5743             :                         abort();
    5744             :                         /* LCOV_EXCL_STOP */
    5745             :                 }
    5746       21356 :                 if (!(tommy_hashdyn_prev(table, node) == (i ? &expected[i - 1]->node : 0))) {
    5747             :                         /* LCOV_EXCL_START */
    5748             :                         abort();
    5749             :                         /* LCOV_EXCL_STOP */
    5750             :                 }
    5751       21356 :                 node = tommy_hashdyn_next(table, node);
    5752             :         }
    5753         104 :         if (!(node == 0)) {
    5754             :                 /* LCOV_EXCL_START */
    5755             :                 abort();
    5756             :                 /* LCOV_EXCL_STOP */
    5757             :         }
    5758         104 :         node = tommy_hashdyn_tail(table);
    5759       21460 :         for (unsigned i = count; i != 0; ) {
    5760       21356 :                 --i;
    5761       21356 :                 if (!(node == &expected[i]->node)) {
    5762             :                         /* LCOV_EXCL_START */
    5763             :                         abort();
    5764             :                         /* LCOV_EXCL_STOP */
    5765             :                 }
    5766       21356 :                 node = tommy_hashdyn_prev(table, node);
    5767             :         }
    5768         104 :         if (!(node == 0)) {
    5769             :                 /* LCOV_EXCL_START */
    5770             :                 abort();
    5771             :                 /* LCOV_EXCL_STOP */
    5772             :         }
    5773         104 : }
    5774             : 
    5775           1 : void test_hashdyn_iterators(void)
    5776             : {
    5777             :         struct object_hash obj[1024];
    5778             :         struct object_hash* expected[1024];
    5779           1 :         const unsigned checkpoints[] = { 0, 1, 8, 9, 15, 16, 17, 31, 32, 33, 64, 128, 256, 513, 1024 };
    5780             : 
    5781           5 :         for (unsigned mode = 0; mode < 4; ++mode) {
    5782             :                 tommy_hashdyn table;
    5783           4 :                 tommy_hashdyn_init(&table);
    5784           4 :                 unsigned count = 0;
    5785          64 :                 for (unsigned c = 0; c < sizeof(checkpoints) / sizeof(checkpoints[0]); ++c) {
    5786        4156 :                         while (count < checkpoints[c]) {
    5787        4096 :                                 obj[count].value = count;
    5788        4096 :                                 tommy_hashdyn_insert(&table, &obj[count].node, &obj[count], hashdyn_iteration_hash(count, mode));
    5789        4096 :                                 ++count;
    5790             :                         }
    5791        8648 :                         for (unsigned i = 0; i < count; ++i)
    5792        8588 :                                 expected[i] = &obj[i];
    5793          60 :                         qsort(expected, count, sizeof(expected[0]), hash_bitreverse_compare);
    5794          60 :                         hashdyn_iteration_check(&table, expected, count);
    5795             :                 }
    5796             : 
    5797             :                 /* swapping transfers iterator ownership without changing node order */
    5798             :                 tommy_hashdyn other;
    5799           4 :                 tommy_hashdyn_init(&other);
    5800           4 :                 tommy_hashdyn_swap(&table, &other);
    5801           4 :                 hashdyn_iteration_check(&other, expected, count);
    5802           4 :                 hashdyn_iteration_check(&table, expected, 0);
    5803           4 :                 tommy_hashdyn_clear(&other);
    5804           4 :                 hashdyn_iteration_check(&other, expected, 0);
    5805           4 :                 tommy_hashdyn_done(&other);
    5806           4 :                 tommy_hashdyn_done(&table);
    5807             :         }
    5808             : 
    5809           5 :         for (unsigned mode = 0; mode < 4; ++mode) {
    5810          12 :                 for (unsigned reverse = 0; reverse < 2; ++reverse) {
    5811          40 :                         for (unsigned mutation = 0; mutation < 4; ++mutation) {
    5812             :                                 tommy_hashdyn table;
    5813          32 :                                 tommy_hashdyn_init(&table);
    5814          32 :                                 unsigned count = mutation == 3 ? 130 : 1024;
    5815          32 :                                 unsigned char seen[1024] = { 0 };
    5816          32 :                                 unsigned char removed[1024] = { 0 };
    5817       25648 :                                 for (unsigned i = 0; i < count; ++i) {
    5818       25616 :                                         obj[i].value = i;
    5819       25616 :                                         tommy_hashdyn_insert(&table, &obj[i].node, &obj[i], hashdyn_iteration_hash(i, mode));
    5820       25616 :                                         expected[i] = &obj[i];
    5821             :                                 }
    5822          32 :                                 qsort(expected, count, sizeof(expected[0]), hash_bitreverse_compare);
    5823          32 :                                 unsigned visited = 0;
    5824          32 :                                 unsigned original_visited = 0;
    5825          32 :                                 unsigned removed_count = 0;
    5826          32 :                                 tommy_size_t initial_max = table.bucket_max;
    5827          32 :                                 tommy_hashdyn_node* node = reverse ? tommy_hashdyn_tail(&table) : tommy_hashdyn_head(&table);
    5828       31803 :                                 while (node) {
    5829       31771 :                                         struct object_hash* object = node->data;
    5830       31771 :                                         unsigned id = object->value;
    5831       31771 :                                         if (!(!seen[id])) {
    5832             :                                                 /* LCOV_EXCL_START */
    5833             :                                                 abort();
    5834             :                                                 /* LCOV_EXCL_STOP */
    5835             :                                         }
    5836       31771 :                                         seen[id] = 1;
    5837       31771 :                                         ++visited;
    5838       31771 :                                         if (id < count) {
    5839       25616 :                                                 if (!(original_visited < count)) {
    5840             :                                                         /* LCOV_EXCL_START */
    5841             :                                                         abort();
    5842             :                                                         /* LCOV_EXCL_STOP */
    5843             :                                                 }
    5844       25616 :                                                 if (!(object == expected[reverse ? count - 1 - original_visited : original_visited])) {
    5845             :                                                         /* LCOV_EXCL_START */
    5846             :                                                         abort();
    5847             :                                                         /* LCOV_EXCL_STOP */
    5848             :                                                 }
    5849       25616 :                                                 ++original_visited;
    5850             :                                         }
    5851             : 
    5852             :                                         /* save the neighbor before mutations, including resize and batch removal */
    5853       31771 :                                         tommy_hashdyn_node* next = reverse ? tommy_hashdyn_prev(&table, node) : tommy_hashdyn_next(&table, node);
    5854       31771 :                                         if (mutation == 0 || (mutation == 1 && visited % 17 != 0)) {
    5855        7952 :                                                 void* data = visited % 2 ? tommy_hashdyn_remove_existing(&table, node)
    5856       15904 :                                                         : tommy_hashdyn_remove(&table, search_callback, object, node->index);
    5857       15904 :                                                 if (!(data == object)) {
    5858             :                                                         /* LCOV_EXCL_START */
    5859             :                                                         abort();
    5860             :                                                         /* LCOV_EXCL_STOP */
    5861             :                                                 }
    5862       15904 :                                                 removed[id] = 1;
    5863       15904 :                                                 ++removed_count;
    5864       15867 :                                         } else if (mutation == 2 && (visited % 16 == 0 || !next)) {
    5865             :                                                 /* remove previously visited nodes as well as the current node */
    5866      524800 :                                                 for (unsigned i = 0; i < count; ++i) {
    5867      524288 :                                                         if (seen[i] && !removed[i]) {
    5868        8192 :                                                                 if (!(tommy_hashdyn_remove_existing(&table, &obj[i].node) == &obj[i])) {
    5869             :                                                                         /* LCOV_EXCL_START */
    5870             :                                                                         abort();
    5871             :                                                                         /* LCOV_EXCL_STOP */
    5872             :                                                                 }
    5873        8192 :                                                                 removed[i] = 1;
    5874        8192 :                                                                 ++removed_count;
    5875             :                                                         }
    5876             :                                                 }
    5877       15355 :                                         } else if (mutation == 3 && visited == 1) {
    5878             :                                                 /* force growth after saving an iterator, including duplicate hashes */
    5879        7160 :                                                 for (unsigned i = count; i < 1024; ++i) {
    5880        7152 :                                                         obj[i].value = i;
    5881        7152 :                                                         tommy_hashdyn_insert(&table, &obj[i].node, &obj[i], hashdyn_iteration_hash(i, mode));
    5882             :                                                 }
    5883             :                                         }
    5884       31771 :                                         node = next;
    5885             :                                 }
    5886          32 :                                 if (!(original_visited == count)) {
    5887             :                                         /* LCOV_EXCL_START */
    5888             :                                         abort();
    5889             :                                         /* LCOV_EXCL_STOP */
    5890             :                                 }
    5891          32 :                                 if (!(tommy_hashdyn_count(&table) == (mutation == 3 ? 1024 : count) - removed_count)) {
    5892             :                                         /* LCOV_EXCL_START */
    5893             :                                         abort();
    5894             :                                         /* LCOV_EXCL_STOP */
    5895             :                                 }
    5896          32 :                                 if (!(mutation == 3 ? table.bucket_max > initial_max : table.bucket_max < initial_max)) {
    5897             :                                         /* LCOV_EXCL_START */
    5898             :                                         abort();
    5899             :                                         /* LCOV_EXCL_STOP */
    5900             :                                 }
    5901             : 
    5902             :                                 /* verify survivors in both directions against the independent oracle */
    5903          32 :                                 unsigned survivors = 0;
    5904       32800 :                                 for (unsigned i = 0; i < (mutation == 3 ? 1024 : count); ++i)
    5905       32768 :                                         if (!removed[i])
    5906        8672 :                                                 expected[survivors++] = &obj[i];
    5907          32 :                                 qsort(expected, survivors, sizeof(expected[0]), hash_bitreverse_compare);
    5908          32 :                                 hashdyn_iteration_check(&table, expected, survivors);
    5909          32 :                                 tommy_hashdyn_done(&table);
    5910             :                         }
    5911             :                 }
    5912             :         }
    5913             : 
    5914             :         /* freeing each removed object must not leave iteration dependent on its links */
    5915           3 :         for (unsigned reverse = 0; reverse < 2; ++reverse) {
    5916             :                 tommy_hashdyn table;
    5917           2 :                 tommy_hashdyn_init(&table);
    5918        1026 :                 for (unsigned i = 0; i < 512; ++i) {
    5919        1024 :                         struct object_hash* object = malloc(sizeof(struct object_hash));
    5920        1024 :                         object->value = i;
    5921        1024 :                         tommy_hashdyn_insert(&table, &object->node, object, hashdyn_iteration_hash(i, 0));
    5922             :                 }
    5923           2 :                 unsigned visited = 0;
    5924           2 :                 tommy_hashdyn_node* node = reverse ? tommy_hashdyn_tail(&table) : tommy_hashdyn_head(&table);
    5925        1026 :                 while (node) {
    5926        1024 :                         tommy_hashdyn_node* next = reverse ? tommy_hashdyn_prev(&table, node) : tommy_hashdyn_next(&table, node);
    5927        1024 :                         free(tommy_hashdyn_remove_existing(&table, node));
    5928        1024 :                         ++visited;
    5929        1024 :                         node = next;
    5930             :                 }
    5931           2 :                 if (visited != 512 || !tommy_hashdyn_empty(&table)) {
    5932             :                         /* LCOV_EXCL_START */
    5933             :                         abort();
    5934             :                         /* LCOV_EXCL_STOP */
    5935             :                 }
    5936           2 :                 tommy_hashdyn_done(&table);
    5937             :         }
    5938           1 : }
    5939             : 
    5940           1 : void test_hashdyn(void)
    5941             : {
    5942           1 :         const unsigned size = TOMMY_SIZE;
    5943           1 :         const unsigned module = TOMMY_SIZE / 4;
    5944             : 
    5945           1 :         test_hashdyn_clear();
    5946           1 :         test_hashdyn_shrink();
    5947           1 :         test_hashdyn_insert_unique();
    5948           1 :         test_hashdyn_swap();
    5949           1 :         test_hashdyn_rehash_existing();
    5950           1 :         test_hashdyn_to_list();
    5951           1 :         test_hashdyn_iterators();
    5952             : 
    5953           1 :         struct object_hash* HASH = malloc(size * sizeof(struct object_hash));
    5954             : 
    5955      100001 :         for (unsigned i = 0; i < size; ++i)
    5956      100000 :                 HASH[i].value = i % module;
    5957             : 
    5958           4 :         SPEED_START("hashdyn stack") {
    5959           2 :                 unsigned limit = 5 * isqrt(size);
    5960        3164 :                 for (unsigned n = 0; n <= limit; ++n) {
    5961             :                         /* last iteration is full size */
    5962        3162 :                         if (n == limit)
    5963           2 :                                 n = limit = size;
    5964             : 
    5965             :                         tommy_hashdyn hashdyn;
    5966        3162 :                         tommy_hashdyn_init(&hashdyn);
    5967             : 
    5968             :                         /* insert */
    5969     2697982 :                         for (unsigned i = 0; i < n; ++i)
    5970     2694820 :                                 tommy_hashdyn_insert(&hashdyn, &HASH[i].node, &HASH[i], HASH[i].value);
    5971             : 
    5972        3162 :                         if (tommy_hashdyn_memory_usage(&hashdyn) < n * sizeof(void*)) {
    5973             :                                 /* LCOV_EXCL_START */
    5974             :                                 abort();
    5975             :                                 /* LCOV_EXCL_STOP */
    5976             :                         }
    5977             : 
    5978        3162 :                         if (tommy_hashdyn_count(&hashdyn) != n) {
    5979             :                                 /* LCOV_EXCL_START */
    5980             :                                 abort();
    5981             :                                 /* LCOV_EXCL_STOP */
    5982             :                         }
    5983             : 
    5984        3162 :                         the_count = 0;
    5985        3162 :                         tommy_hashdyn_foreach(&hashdyn, count_callback);
    5986        3162 :                         if (the_count != n) {
    5987             :                                 /* LCOV_EXCL_START */
    5988             :                                 abort();
    5989             :                                 /* LCOV_EXCL_STOP */
    5990             :                         }
    5991             : 
    5992             :                         /* remove in backward order */
    5993     1349782 :                         for (unsigned i = 0; i < n / 2; ++i)
    5994     1346620 :                                 tommy_hashdyn_remove_existing(&hashdyn, &HASH[n - i - 1].node);
    5995             : 
    5996             :                         /* remove missing */
    5997     1349782 :                         for (unsigned i = 0; i < n / 2; ++i)
    5998     1346620 :                                 if (tommy_hashdyn_remove(&hashdyn, search_callback, &HASH[n - i - 1], HASH[n - i - 1].value) != 0) {
    5999             :                                         /* LCOV_EXCL_START */
    6000             :                                         abort();
    6001             :                                         /* LCOV_EXCL_STOP */
    6002             :                                 }
    6003             : 
    6004             :                         /* remove search */
    6005     1349782 :                         for (unsigned i = 0; i < n / 2; ++i)
    6006     1346620 :                                 if (tommy_hashdyn_remove(&hashdyn, search_callback, &HASH[n / 2 - i - 1], HASH[n / 2 - i - 1].value) == 0) {
    6007             :                                         /* LCOV_EXCL_START */
    6008             :                                         abort();
    6009             :                                         /* LCOV_EXCL_STOP */
    6010             :                                 }
    6011             : 
    6012        3162 :                         tommy_hashdyn_done(&hashdyn);
    6013             :                 }
    6014           2 :         } SPEED_STOP
    6015             : 
    6016           6 :         SPEED_START("hashdyn queue")
    6017             :         {
    6018           3 :                 unsigned limit = isqrt(size) / 16;
    6019          63 :                 for (unsigned n = 0; n <= limit; ++n) {
    6020             :                         /* last iteration is full size */
    6021          60 :                         if (n == limit)
    6022           3 :                                 n = limit = size;
    6023             : 
    6024             :                         tommy_hashdyn hashdyn;
    6025          60 :                         tommy_hashdyn_init(&hashdyn);
    6026             : 
    6027             :                         /* insert first run */
    6028          60 :                         unsigned j = 0;
    6029          60 :                         unsigned i = 0;
    6030      300573 :                         for (; i < n; ++i)
    6031      300513 :                                 tommy_hashdyn_insert(&hashdyn, &HASH[i].node, &HASH[i], HASH[i].value);
    6032             : 
    6033          60 :                         the_count = 0;
    6034          60 :                         tommy_hashdyn_foreach_arg(&hashdyn, count_arg_callback, &the_count);
    6035          60 :                         if (the_count != n) {
    6036             :                                 /* LCOV_EXCL_START */
    6037             :                                 abort();
    6038             :                                 /* LCOV_EXCL_STOP */
    6039             :                         }
    6040             : 
    6041             :                         /* insert all the others */
    6042     5699547 :                         for (; i < size; ++i, ++j) {
    6043             :                                 /* insert one */
    6044     5699487 :                                 tommy_hashdyn_insert(&hashdyn, &HASH[i].node, &HASH[i], HASH[i].value);
    6045             : 
    6046             :                                 /* remove one */
    6047     5699487 :                                 tommy_hashdyn_remove_existing(&hashdyn, &HASH[j].node);
    6048             :                         }
    6049             : 
    6050      300573 :                         for (; j < size; ++j)
    6051      300513 :                                 if (tommy_hashdyn_remove(&hashdyn, search_callback, &HASH[j], HASH[j].value) == 0) {
    6052             :                                         /* LCOV_EXCL_START */
    6053             :                                         abort();
    6054             :                                         /* LCOV_EXCL_STOP */
    6055             :                                 }
    6056             : 
    6057          60 :                         tommy_hashdyn_done(&hashdyn);
    6058             :                 }
    6059             :         }
    6060           3 :         SPEED_STOP
    6061           1 : }
    6062             : 
    6063           1 : void test_hashlin_swap(void)
    6064             : {
    6065           1 :         const unsigned counts[][2] = { { 0, 0 }, { 33, 33 }, { 48, 48 }, { 128, 24 }, { 128, 17 } };
    6066           1 :         unsigned grow = 0, shrink = 0;
    6067           6 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    6068          15 :                 for (unsigned second_count = 0; second_count <= 3; second_count += 3) {
    6069             :                         tommy_hashlin first, second;
    6070             :                         struct object_hash obj[132], before[132];
    6071          10 :                         memset(obj, 0, sizeof(obj));
    6072          10 :                         tommy_hashlin_init(&first);
    6073          10 :                         tommy_hashlin_init(&second);
    6074        1330 :                         for (unsigned i = 0; i < 132; ++i)
    6075        1320 :                                 obj[i].value = i;
    6076         684 :                         for (unsigned i = 0; i < counts[c][0]; ++i)
    6077         674 :                                 tommy_hashlin_insert(&first, &obj[i].node, &obj[i], tommy_inthash_u32(i));
    6078         440 :                         for (unsigned i = counts[c][0]; i > counts[c][1]; --i)
    6079         430 :                                 tommy_hashlin_remove_existing(&first, &obj[i - 1].node);
    6080          25 :                         for (unsigned i = 0; i < second_count; ++i)
    6081          15 :                                 tommy_hashlin_insert(&second, &obj[128 + i].node, &obj[128 + i], tommy_inthash_u32(128 + i));
    6082             : 
    6083             :                         /* exercise partially split and partially merged bucket segments */
    6084          10 :                         if (c != 0) {
    6085           8 :                                 if (first.state == second.state || first.split == 0 || first.split >= first.low_max) {
    6086             :                                         /* LCOV_EXCL_START */
    6087             :                                         abort();
    6088             :                                         /* LCOV_EXCL_STOP */
    6089             :                                 }
    6090           8 :                                 if (counts[c][0] == counts[c][1])
    6091           4 :                                         ++grow;
    6092             :                                 else
    6093           4 :                                         ++shrink;
    6094             :                         }
    6095          10 :                         tommy_hashlin saved_first = first, saved_second = second;
    6096          10 :                         memcpy(before, obj, sizeof(before));
    6097          10 :                         tommy_hashlin_swap(&first, &second);
    6098          30 :                         for (unsigned phase = 0; phase < 2; ++phase) {
    6099          20 :                                 if (phase) {
    6100          10 :                                         tommy_hashlin_swap(&first, &first);
    6101          10 :                                         tommy_hashlin_swap(&second, &second);
    6102             :                                 }
    6103          60 :                                 for (unsigned side = 0; side < 2; ++side) {
    6104          40 :                                         tommy_hashlin* table = side ? &second : &first;
    6105          40 :                                         const tommy_hashlin* saved = side ? &saved_first : &saved_second;
    6106          40 :                                         if (table->bucket_max != saved->bucket_max || table->bucket_mask != saved->bucket_mask
    6107          40 :                                                 || table->low_max != saved->low_max || table->low_mask != saved->low_mask
    6108          40 :                                                 || table->split != saved->split || table->count != saved->count
    6109          40 :                                                 || table->bucket_bit != saved->bucket_bit || table->state != saved->state) {
    6110             :                                                 /* LCOV_EXCL_START */
    6111             :                                                 abort();
    6112             :                                                 /* LCOV_EXCL_STOP */
    6113             :                                         }
    6114         304 :                                         for (unsigned i = 0; i < saved->bucket_bit; ++i)
    6115         264 :                                                 if (table->bucket[i] != saved->bucket[i]) {
    6116             :                                                         /* LCOV_EXCL_START */
    6117             :                                                         abort();
    6118             :                                                         /* LCOV_EXCL_STOP */
    6119             :                                                 }
    6120             :                                 }
    6121          20 :                                 if (memcmp(before, obj, sizeof(before)) != 0) {
    6122             :                                         /* LCOV_EXCL_START */
    6123             :                                         abort();
    6124             :                                         /* LCOV_EXCL_STOP */
    6125             :                                 }
    6126        2640 :                                 for (unsigned i = 0; i < 131; ++i) {
    6127        2620 :                                         if (tommy_hashlin_search(&first, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) != (i >= 128 && i < 128 + second_count ? &obj[i] : 0)
    6128        2620 :                                                 || tommy_hashlin_search(&second, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) != (i < counts[c][1] ? &obj[i] : 0)) {
    6129             :                                                 /* LCOV_EXCL_START */
    6130             :                                                 abort();
    6131             :                                                 /* LCOV_EXCL_STOP */
    6132             :                                         }
    6133             :                                 }
    6134             :                         }
    6135             : 
    6136             :                         /* subsequent mutations must continue the resize using the moved state */
    6137          30 :                         for (unsigned side = 0; side < 2; ++side) {
    6138          20 :                                 tommy_hashlin* table = side ? &second : &first;
    6139          20 :                                 tommy_hashlin_insert(table, &obj[131].node, &obj[131], tommy_inthash_u32(131));
    6140          20 :                                 if (tommy_hashlin_search(table, search_hash_value_callback, &obj[131].value, tommy_inthash_u32(131)) != &obj[131]
    6141          20 :                                         || tommy_hashlin_remove_existing(table, &obj[131].node) != &obj[131]) {
    6142             :                                         /* LCOV_EXCL_START */
    6143             :                                         abort();
    6144             :                                         /* LCOV_EXCL_STOP */
    6145             :                                 }
    6146             :                         }
    6147        1320 :                         for (unsigned i = 0; i < 131; ++i) {
    6148        1310 :                                 if (i < counts[c][1] || (i >= 128 && i < 128 + second_count)) {
    6149         259 :                                         tommy_hashlin* table = i < 128 ? &second : &first;
    6150         259 :                                         if (tommy_hashlin_search(table, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) != &obj[i]) {
    6151             :                                                 /* LCOV_EXCL_START */
    6152             :                                                 abort();
    6153             :                                                 /* LCOV_EXCL_STOP */
    6154             :                                         }
    6155         259 :                                         void* removed = i % 2 ? tommy_hashlin_remove(table, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) : tommy_hashlin_remove_existing(table, &obj[i].node);
    6156         259 :                                         if (removed != &obj[i] || tommy_hashlin_search(table, search_hash_value_callback, &obj[i].value, tommy_inthash_u32(i)) != 0) {
    6157             :                                                 /* LCOV_EXCL_START */
    6158             :                                                 abort();
    6159             :                                                 /* LCOV_EXCL_STOP */
    6160             :                                         }
    6161             :                                 }
    6162             :                         }
    6163          10 :                         if (!tommy_hashlin_empty(&first) || !tommy_hashlin_empty(&second)) {
    6164             :                                 /* LCOV_EXCL_START */
    6165             :                                 abort();
    6166             :                                 /* LCOV_EXCL_STOP */
    6167             :                         }
    6168          10 :                         tommy_hashlin_done(&first);
    6169          10 :                         tommy_hashlin_done(&second);
    6170             :                 }
    6171             :         }
    6172           1 :         if (!grow || !shrink) {
    6173             :                 /* LCOV_EXCL_START */
    6174             :                 abort();
    6175             :                 /* LCOV_EXCL_STOP */
    6176             :         }
    6177           1 : }
    6178             : 
    6179           1 : void test_hashlin_insert_unique(void)
    6180             : {
    6181             :         struct object_hash obj[257];
    6182             :         tommy_hashlin table;
    6183           1 :         const unsigned order[] = { 0, 1, 2, 3, 4, 5 };
    6184             : 
    6185           1 :         tommy_hashlin_init(&table);
    6186           1 :         obj[0].value = 1;
    6187           1 :         compare_counter = 0;
    6188           1 :         if (tommy_hashlin_insert_unique(&table, &obj[0].node, &obj[0], search_hash_value_callback, &obj[0].value, 0) != &obj[0]
    6189           1 :                 || compare_counter != 0 || tommy_hashlin_count(&table) != 1
    6190           1 :                 || tommy_hashlin_search(&table, search_hash_value_callback, &obj[0].value, 0) != &obj[0]) {
    6191             :                 /* LCOV_EXCL_START */
    6192             :                 abort();
    6193             :                 /* LCOV_EXCL_STOP */
    6194             :         }
    6195           1 :         tommy_hashlin_remove_existing(&table, &obj[0].node);
    6196             : 
    6197           1 :         tommy_size_t bucket_max = table.bucket_max;
    6198           1 :         obj[0].value = 10;
    6199           1 :         obj[1].value = 20;
    6200           1 :         obj[2].value = 30;
    6201           1 :         obj[3].value = 20;
    6202           5 :         for (unsigned i = 0; i < 4; ++i)
    6203           4 :                 tommy_hashlin_insert(&table, &obj[i].node, &obj[i], i == 0 ? bucket_max : 0);
    6204             : 
    6205             :         /* the first matching duplicate is returned; the rejected candidate stays intact. */
    6206           1 :         obj[4].value = 20;
    6207           1 :         obj[4].node.next = &obj[4].node;
    6208           1 :         obj[4].node.prev = &obj[4].node;
    6209           1 :         obj[4].node.data = &obj[4];
    6210           1 :         obj[4].node.index = ~(tommy_hash_t)0;
    6211           1 :         tommy_node saved = obj[4].node;
    6212           1 :         compare_counter = 0;
    6213           1 :         if (tommy_hashlin_insert_unique(&table, &obj[4].node, &obj[4], search_hash_value_callback, &obj[4].value, 0) != &obj[1]
    6214           1 :                 || compare_counter != 1 || tommy_hashlin_count(&table) != 4
    6215           1 :                 || obj[4].value != 20 || obj[4].node.next != saved.next || obj[4].node.prev != saved.prev
    6216           1 :                 || obj[4].node.data != saved.data || obj[4].node.index != saved.index) {
    6217             :                 /* LCOV_EXCL_START */
    6218             :                 abort();
    6219             :                 /* LCOV_EXCL_STOP */
    6220             :         }
    6221           1 :         test_hash_bucket(tommy_hashlin_bucket(&table, 0), obj, order, 4);
    6222             : 
    6223             :         /* equal hashes with different keys must allow insertion at the tail. */
    6224           1 :         obj[4].value = 40;
    6225           1 :         compare_counter = 0;
    6226           1 :         if (tommy_hashlin_insert_unique(&table, &obj[4].node, &obj[4], search_hash_value_callback, &obj[4].value, 0) != &obj[4]
    6227           1 :                 || compare_counter != 3 || tommy_hashlin_count(&table) != 5
    6228           1 :                 || obj[4].node.data != &obj[4] || obj[4].node.index != 0) {
    6229             :                 /* LCOV_EXCL_START */
    6230             :                 abort();
    6231             :                 /* LCOV_EXCL_STOP */
    6232             :         }
    6233             : 
    6234             :         /* different full hashes in the same bucket must never invoke the comparator. */
    6235           1 :         obj[5].value = 50;
    6236           1 :         compare_counter = 0;
    6237           1 :         if (tommy_hashlin_insert_unique(&table, &obj[5].node, &obj[5], search_hash_value_callback, &obj[5].value, 2 * bucket_max) != &obj[5]
    6238           1 :                 || compare_counter != 0 || tommy_hashlin_count(&table) != 6) {
    6239             :                 /* LCOV_EXCL_START */
    6240             :                 abort();
    6241             :                 /* LCOV_EXCL_STOP */
    6242             :         }
    6243           1 :         test_hash_bucket(tommy_hashlin_bucket(&table, 0), obj, order, 6);
    6244           1 :         tommy_hashlin_remove_existing(&table, &obj[1].node);
    6245           1 :         obj[6].value = 20;
    6246           1 :         if (tommy_hashlin_insert_unique(&table, &obj[6].node, &obj[6], search_hash_value_callback, &obj[6].value, 0) != &obj[3]
    6247           1 :                 || tommy_hashlin_count(&table) != 5) {
    6248             :                 /* LCOV_EXCL_START */
    6249             :                 abort();
    6250             :                 /* LCOV_EXCL_STOP */
    6251             :         }
    6252           1 :         tommy_hashlin_done(&table);
    6253             : 
    6254             :         /* insert unique keys, reject duplicates, then repeat during contraction. */
    6255           1 :         tommy_hashlin_init(&table);
    6256             :         struct object_hash candidate;
    6257           1 :         candidate.node.next = &candidate.node;
    6258           1 :         candidate.node.prev = &candidate.node;
    6259           1 :         candidate.node.data = &candidate;
    6260           1 :         candidate.node.index = ~(tommy_hash_t)0;
    6261           3 :         for (unsigned phase = 0; phase < 2; ++phase) {
    6262         514 :                 for (unsigned step = 0; step < 256; ++step) {
    6263         512 :                         unsigned i = phase == 0 ? step : 255 - step;
    6264         512 :                         tommy_hash_t hash = tommy_inthash_u32(i);
    6265         512 :                         if (phase == 0) {
    6266         256 :                                 obj[i].value = i;
    6267         256 :                                 if (tommy_hashlin_insert_unique(&table, &obj[i].node, &obj[i], search_hash_value_callback, &obj[i].value, hash) != &obj[i]
    6268         256 :                                         || tommy_hashlin_count(&table) != i + 1) {
    6269             :                                         /* LCOV_EXCL_START */
    6270             :                                         abort();
    6271             :                                         /* LCOV_EXCL_STOP */
    6272             :                                 }
    6273             :                         }
    6274             : 
    6275         512 :                         candidate.value = i;
    6276         512 :                         saved = obj[i].node;
    6277         512 :                         tommy_size_t count = tommy_hashlin_count(&table);
    6278         512 :                         bucket_max = table.bucket_max;
    6279         512 :                         tommy_size_t bucket_mask = table.bucket_mask;
    6280         512 :                         tommy_hashlin_node** bucket = table.bucket[0];
    6281         512 :                         tommy_uint_t bucket_bit = table.bucket_bit;
    6282         512 :                         tommy_uint_t state = table.state;
    6283         512 :                         tommy_size_t low_max = table.low_max;
    6284         512 :                         tommy_size_t low_mask = table.low_mask;
    6285         512 :                         tommy_size_t split = table.split;
    6286         512 :                         compare_counter = 0;
    6287         512 :                         if (tommy_hashlin_insert_unique(&table, &candidate.node, &candidate, search_hash_value_callback, &candidate.value, hash) != &obj[i]
    6288         512 :                                 || compare_counter != 1 || tommy_hashlin_count(&table) != count
    6289         512 :                                 || table.bucket_max != bucket_max || table.bucket_mask != bucket_mask
    6290         512 :                                 || table.bucket[0] != bucket || table.bucket_bit != bucket_bit || table.state != state
    6291         512 :                                 || table.low_max != low_max || table.low_mask != low_mask || table.split != split
    6292         512 :                                 || candidate.value != (int)i || candidate.node.next != &candidate.node
    6293         512 :                                 || candidate.node.prev != &candidate.node || candidate.node.data != &candidate
    6294         512 :                                 || candidate.node.index != ~(tommy_hash_t)0 || obj[i].node.next != saved.next
    6295         512 :                                 || obj[i].node.prev != saved.prev || obj[i].node.data != saved.data || obj[i].node.index != saved.index) {
    6296             :                                 /* LCOV_EXCL_START */
    6297             :                                 abort();
    6298             :                                 /* LCOV_EXCL_STOP */
    6299             :                         }
    6300             : 
    6301         512 :                         if (phase != 0) {
    6302         256 :                                 if (tommy_hashlin_remove_existing(&table, &obj[i].node) != &obj[i]
    6303         256 :                                         || tommy_hashlin_count(&table) != count - 1) {
    6304             :                                         /* LCOV_EXCL_START */
    6305             :                                         abort();
    6306             :                                         /* LCOV_EXCL_STOP */
    6307             :                                 }
    6308             :                                 /* also insert a new key while hashlin is shrinking progressively. */
    6309         256 :                                 if (i == 24) {
    6310           1 :                                         obj[256].value = 256;
    6311           1 :                                         if (tommy_hashlin_insert_unique(&table, &obj[256].node, &obj[256], search_hash_value_callback, &obj[256].value, tommy_inthash_u32(256)) != &obj[256]
    6312           1 :                                                 || tommy_hashlin_count(&table) != count
    6313           1 :                                                 || tommy_hashlin_search(&table, search_hash_value_callback, &obj[256].value, tommy_inthash_u32(256)) != &obj[256]
    6314           1 :                                                 || tommy_hashlin_remove_existing(&table, &obj[256].node) != &obj[256]
    6315           1 :                                                 || tommy_hashlin_count(&table) != count - 1) {
    6316             :                                                 /* LCOV_EXCL_START */
    6317             :                                                 abort();
    6318             :                                                 /* LCOV_EXCL_STOP */
    6319             :                                         }
    6320             :                                 }
    6321             :                         }
    6322             :                 }
    6323             :         }
    6324           1 :         tommy_hashlin_done(&table);
    6325           1 : }
    6326             : 
    6327           1 : void test_hashlin_rehash_existing(void)
    6328             : {
    6329           1 :         const unsigned counts[][2] = { { 1, 1 }, { 4, 4 }, { 33, 33 }, { 48, 48 }, { 63, 63 }, { 64, 64 },
    6330             :                                        { 65, 65 }, { 128, 128 }, { 128, 31 }, { 128, 24 }, { 128, 17 },
    6331             :                                        { 128, 16 }, { 128, 15 }, { 128, 9 }, { 128, 8 } };
    6332             :         struct object_hash obj[130];
    6333             :         tommy_hash_t hashes[130];
    6334             :         tommy_hashlin table;
    6335             : 
    6336          16 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    6337          15 :                 unsigned n = counts[c][1];
    6338          15 :                 unsigned source_size = n < 3 ? n : 3;
    6339          58 :                 for (unsigned target = 0; target < source_size; ++target) {
    6340         215 :                         for (unsigned mode = 0; mode < 4; ++mode) {
    6341             :                                 unsigned source[3], destination[2];
    6342         172 :                                 unsigned source_count = 0, destination_count = 0;
    6343             : 
    6344         172 :                                 tommy_hashlin_init(&table);
    6345       15788 :                                 for (unsigned i = 0; i < counts[c][0]; ++i) {
    6346       15616 :                                         obj[i].value = i;
    6347       15616 :                                         hashes[i] = i < 3 ? 0 : i == 3 ? 1 : 2;
    6348       15616 :                                         tommy_hashlin_insert(&table, &obj[i].node, &obj[i], hashes[i]);
    6349             :                                 }
    6350        9484 :                                 for (unsigned i = counts[c][0]; i > n; --i)
    6351        9312 :                                         tommy_hashlin_remove_existing(&table, &obj[i - 1].node);
    6352             : 
    6353         172 :                                 tommy_size_t bucket_max = table.bucket_max;
    6354         172 :                                 tommy_size_t bucket_mask = table.bucket_mask;
    6355         172 :                                 tommy_uint_t bucket_bit = table.bucket_bit;
    6356         172 :                                 tommy_uint_t state = table.state;
    6357         172 :                                 tommy_size_t low_max = table.low_max;
    6358         172 :                                 tommy_size_t low_mask = table.low_mask;
    6359         172 :                                 tommy_size_t split = table.split;
    6360             :                                 tommy_hashlin_node** segments[TOMMY_SIZE_BIT];
    6361        1408 :                                 for (unsigned i = 0; i < bucket_bit; ++i)
    6362        1236 :                                         segments[i] = table.bucket[i];
    6363             :                                 /* exercise unchanged hash, same bucket, occupied bucket and empty bucket. */
    6364         172 :                                 tommy_hash_t hash = mode == 0 ? 0 : mode == 1 ? bucket_max : mode == 2 ? 1 : ~(tommy_hash_t)0;
    6365         172 :                                 tommy_node saved = obj[target].node;
    6366         172 :                                 obj[target].value = 1000 + target;
    6367         172 :                                 tommy_hashlin_rehash_existing(&table, &obj[target].node, hash);
    6368         172 :                                 hashes[target] = hash;
    6369             : 
    6370         172 :                                 if (tommy_hashlin_count(&table) != n || table.bucket_max != bucket_max
    6371         172 :                                         || table.bucket_mask != bucket_mask
    6372         172 :                                         || table.bucket_bit != bucket_bit || table.state != state
    6373         172 :                                         || table.low_max != low_max || table.low_mask != low_mask || table.split != split
    6374         172 :                                         || obj[target].node.index != hash || obj[target].node.data != saved.data
    6375         172 :                                         || obj[target].value != (int)(1000 + target)
    6376         172 :                                         || (mode == 0 && (obj[target].node.next != saved.next || obj[target].node.prev != saved.prev))) {
    6377             :                                         /* LCOV_EXCL_START */
    6378             :                                         abort();
    6379             :                                         /* LCOV_EXCL_STOP */
    6380             :                                 }
    6381        1408 :                                 for (unsigned i = 0; i < bucket_bit; ++i)
    6382        1236 :                                         if (table.bucket[i] != segments[i]) {
    6383             :                                                 /* LCOV_EXCL_START */
    6384             :                                                 abort();
    6385             :                                                 /* LCOV_EXCL_STOP */
    6386             :                                         }
    6387             : 
    6388         680 :                                 for (unsigned i = 0; i < source_size; ++i)
    6389         508 :                                         if (mode == 0 || i != target)
    6390         379 :                                                 source[source_count++] = i;
    6391         172 :                                 if (mode == 1)
    6392          43 :                                         source[source_count++] = target;
    6393         172 :                                 test_hash_bucket(tommy_hashlin_bucket(&table, 0), obj, source, source_count);
    6394             : 
    6395         172 :                                 if (n > 3)
    6396         168 :                                         destination[destination_count++] = 3;
    6397         172 :                                 if (mode == 2)
    6398          43 :                                         destination[destination_count++] = target;
    6399         172 :                                 test_hash_bucket(tommy_hashlin_bucket(&table, 1), obj, destination, destination_count);
    6400         172 :                                 if (mode == 3)
    6401          43 :                                         test_hash_bucket(tommy_hashlin_bucket(&table, hash), obj, &target, 1);
    6402         172 :                                 if (mode != 0 && tommy_hashlin_search(&table, search_callback, &obj[target], 0) != 0) {
    6403             :                                         /* LCOV_EXCL_START */
    6404             :                                         abort();
    6405             :                                         /* LCOV_EXCL_STOP */
    6406             :                                 }
    6407             :                                 /* also unlink from the new hash, including an unsplit high position. */
    6408         172 :                                 if (mode != 0) {
    6409         129 :                                         tommy_hashlin_rehash_existing(&table, &obj[target].node, 0);
    6410         129 :                                         if (tommy_hashlin_count(&table) != n || obj[target].node.index != 0
    6411         129 :                                                 || tommy_hashlin_search(&table, search_callback, &obj[target], 0) != &obj[target]
    6412         129 :                                                 || tommy_hashlin_search(&table, search_callback, &obj[target], hash) != 0) {
    6413             :                                                 /* LCOV_EXCL_START */
    6414             :                                                 abort();
    6415             :                                                 /* LCOV_EXCL_STOP */
    6416             :                                         }
    6417         129 :                                         tommy_hashlin_rehash_existing(&table, &obj[target].node, hash);
    6418             :                                 }
    6419        6476 :                                 for (unsigned i = 0; i < n; ++i)
    6420        6304 :                                         if (tommy_hashlin_search(&table, search_callback, &obj[i], hashes[i]) != &obj[i]) {
    6421             :                                                 /* LCOV_EXCL_START */
    6422             :                                                 abort();
    6423             :                                                 /* LCOV_EXCL_STOP */
    6424             :                                         }
    6425             : 
    6426             :                                 /* subsequent resizing and removals must use the updated stored hash. */
    6427       16228 :                                 for (unsigned i = n; i < 130; ++i) {
    6428       16056 :                                         obj[i].value = i;
    6429       16056 :                                         hashes[i] = i < 3 ? 0 : i == 3 ? 1 : 2;
    6430       16056 :                                         tommy_hashlin_insert(&table, &obj[i].node, &obj[i], hashes[i]);
    6431             :                                 }
    6432       22532 :                                 for (unsigned i = 0; i < 130; ++i)
    6433       22360 :                                         if (tommy_hashlin_search(&table, search_callback, &obj[i], hashes[i]) != &obj[i]) {
    6434             :                                                 /* LCOV_EXCL_START */
    6435             :                                                 abort();
    6436             :                                                 /* LCOV_EXCL_STOP */
    6437             :                                         }
    6438       22532 :                                 for (unsigned i = 130; i > 0; --i) {
    6439             :                                         void* data;
    6440       22360 :                                         if (i % 2)
    6441       11180 :                                                 data = tommy_hashlin_remove_existing(&table, &obj[i - 1].node);
    6442             :                                         else
    6443       11180 :                                                 data = tommy_hashlin_remove(&table, search_callback, &obj[i - 1], hashes[i - 1]);
    6444       22360 :                                         if (data != &obj[i - 1] || tommy_hashlin_count(&table) != i - 1) {
    6445             :                                                 /* LCOV_EXCL_START */
    6446             :                                                 abort();
    6447             :                                                 /* LCOV_EXCL_STOP */
    6448             :                                         }
    6449             :                                 }
    6450         172 :                                 tommy_hashlin_done(&table);
    6451             :                         }
    6452             :                 }
    6453             :         }
    6454           1 : }
    6455             : 
    6456             : static struct {
    6457             :         void* expected[128];
    6458             :         unsigned size;
    6459             :         unsigned pos;
    6460             : } hashlin_foreach_check;
    6461             : 
    6462        1108 : static void hashlin_foreach_check_arg_callback(void* arg, void* data)
    6463             : {
    6464        1108 :         unsigned* pos = arg;
    6465        1108 :         if (*pos >= hashlin_foreach_check.size || data != hashlin_foreach_check.expected[*pos]) {
    6466             :                 /* LCOV_EXCL_START */
    6467             :                 abort();
    6468             :                 /* LCOV_EXCL_STOP */
    6469             :         }
    6470        1108 :         ++*pos;
    6471        1108 : }
    6472             : 
    6473         554 : static void hashlin_foreach_check_callback(void* data)
    6474             : {
    6475         554 :         hashlin_foreach_check_arg_callback(&hashlin_foreach_check.pos, data);
    6476         554 : }
    6477             : 
    6478           1 : void test_hashlin_foreach(void)
    6479             : {
    6480           1 :         const unsigned counts[][2] = {
    6481             :                 { 0, 0 }, { 1, 1 }, { 32, 32 }, { 33, 33 }, { 48, 48 }, { 63, 63 },
    6482             :                 { 64, 64 }, { 65, 65 }, { 128, 128 }, { 128, 31 }, { 128, 24 },
    6483             :                 { 128, 17 }, { 128, 16 }, { 128, 15 }, { 128, 9 }, { 128, 8 }, { 128, 0 }
    6484             :         };
    6485             :         struct object_hash obj[128];
    6486             : 
    6487          18 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    6488             :                 tommy_hashlin hashlin;
    6489          17 :                 tommy_hashlin_init(&hashlin);
    6490        1475 :                 for (unsigned i = 0; i < counts[c][0]; ++i) {
    6491        1458 :                         obj[i].value = i / 2;
    6492        1458 :                         tommy_hashlin_insert(&hashlin, &obj[i].node, &obj[i], tommy_inthash_u32(obj[i].value));
    6493             :                 }
    6494         921 :                 for (unsigned i = counts[c][0]; i > counts[c][1]; --i)
    6495         904 :                         tommy_hashlin_remove_existing(&hashlin, &obj[i - 1].node);
    6496             : 
    6497             :                 /* preserve the original bucket and chain order as the reference */
    6498          17 :                 hashlin_foreach_check.size = 0;
    6499          17 :                 tommy_size_t active_max = hashlin.low_max + hashlin.split;
    6500        2035 :                 for (tommy_size_t pos = 0; pos < active_max; ++pos)
    6501        2572 :                         for (tommy_hashlin_node* node = *tommy_hashlin_pos(&hashlin, pos); node; node = node->next)
    6502         554 :                                 hashlin_foreach_check.expected[hashlin_foreach_check.size++] = node->data;
    6503             : 
    6504             :                 /* inactive grow and shrink slots must never reach the callback */
    6505         495 :                 for (tommy_size_t pos = active_max; pos < hashlin.bucket_max; ++pos)
    6506         478 :                         *tommy_hashlin_pos(&hashlin, pos) = &obj[0].node;
    6507             : 
    6508          17 :                 hashlin_foreach_check.pos = 0;
    6509          17 :                 tommy_hashlin_foreach(&hashlin, hashlin_foreach_check_callback);
    6510          17 :                 unsigned pos = 0;
    6511          17 :                 tommy_hashlin_foreach_arg(&hashlin, hashlin_foreach_check_arg_callback, &pos);
    6512          17 :                 if (hashlin_foreach_check.size != counts[c][1] || hashlin_foreach_check.pos != counts[c][1] || pos != counts[c][1]) {
    6513             :                         /* LCOV_EXCL_START */
    6514             :                         abort();
    6515             :                         /* LCOV_EXCL_STOP */
    6516             :                 }
    6517          17 :                 tommy_hashlin_done(&hashlin);
    6518             :         }
    6519           1 : }
    6520             : 
    6521           1 : void test_hashlin_to_list(void)
    6522             : {
    6523           1 :         const tommy_size_t initial_max = (tommy_size_t)1 << TOMMY_HASHLIN_BIT;
    6524           1 :         const unsigned counts[][2] = {
    6525             :                 { 0, 0 }, { 1, 1 }, { 33, 33 }, { 48, 48 }, { 63, 63 }, { 64, 64 },
    6526             :                 { 65, 65 }, { 128, 128 }, { 128, 31 }, { 128, 24 }, { 128, 17 },
    6527             :                 { 128, 16 }, { 128, 15 }, { 128, 9 }, { 128, 8 }, { 128, 0 }
    6528             :         };
    6529             :         struct object_hash obj[130];
    6530             :         tommy_node saved[130];
    6531             :         tommy_node* expected[130];
    6532           1 :         unsigned stable = 0, grow = 0, shrink = 0;
    6533             : 
    6534          17 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    6535          48 :                 for (unsigned prefix = 0; prefix <= 2; prefix += 2) {
    6536          32 :                         unsigned size = prefix + counts[c][1];
    6537             :                         tommy_hashlin hashlin;
    6538          32 :                         tommy_hashlin_init(&hashlin);
    6539          32 :                         tommy_uint_t stable_state = hashlin.state;
    6540             :                         tommy_list list;
    6541          32 :                         tommy_list_init(&list);
    6542        2916 :                         for (unsigned i = 0; i < prefix + counts[c][0]; ++i) {
    6543        2884 :                                 obj[i].value = i / 2;
    6544        2884 :                                 obj[i].node.index = tommy_inthash_u32(obj[i].value);
    6545        2884 :                                 if (i < prefix)
    6546          32 :                                         tommy_list_insert_tail(&list, &obj[i].node, &obj[i]);
    6547             :                                 else
    6548        2852 :                                         tommy_hashlin_insert(&hashlin, &obj[i].node, &obj[i], obj[i].node.index);
    6549             :                         }
    6550        1840 :                         for (unsigned i = prefix + counts[c][0]; i > size; --i)
    6551        1808 :                                 tommy_hashlin_remove_existing(&hashlin, &obj[i - 1].node);
    6552        1108 :                         for (unsigned i = 0; i < size; ++i)
    6553        1076 :                                 saved[i] = obj[i].node;
    6554          32 :                         tommy_size_t bucket_max = hashlin.bucket_max;
    6555          32 :                         tommy_hashlin_node** initial_bucket = hashlin.bucket[0];
    6556          32 :                         if (hashlin.state == stable_state)
    6557          14 :                                 ++stable;
    6558          18 :                         else if (counts[c][0] == counts[c][1])
    6559           8 :                                 ++grow;
    6560             :                         else
    6561          10 :                                 ++shrink;
    6562             : 
    6563             :                         /* preserve the prefix and exact ascending active-bucket order before any links change. */
    6564          32 :                         unsigned expected_size = prefix;
    6565          64 :                         for (unsigned i = 0; i < prefix; ++i)
    6566          32 :                                 expected[i] = &obj[i].node;
    6567        3940 :                         for (tommy_size_t pos = 0; pos < hashlin.low_max + hashlin.split; ++pos)
    6568        4952 :                                 for (tommy_node* node = *tommy_hashlin_pos(&hashlin, pos); node; node = node->next)
    6569        1044 :                                         expected[expected_size++] = node;
    6570          32 :                         if (expected_size != size) {
    6571             :                                 /* LCOV_EXCL_START */
    6572             :                                 abort();
    6573             :                                 /* LCOV_EXCL_STOP */
    6574             :                         }
    6575             : 
    6576             :                         /* inactive slots must never be transferred, including stale shrink pointers. */
    6577         988 :                         for (unsigned i = hashlin.low_max + hashlin.split; i < bucket_max; ++i)
    6578         956 :                                 *tommy_hashlin_pos(&hashlin, i) = &obj[prefix].node;
    6579             : 
    6580          32 :                         tommy_hashlin_to_list(&hashlin, &list);
    6581          32 :                         test_hash_list(&list, obj, saved, size, prefix);
    6582          32 :                         test_list_transfer_order(&list, expected, size);
    6583          32 :                         if (tommy_hashlin_count(&hashlin) != 0 || hashlin.bucket_max != initial_max
    6584          32 :                                 || hashlin.bucket_mask != initial_max - 1 || hashlin.bucket_bit != TOMMY_HASHLIN_BIT
    6585          32 :                                 || hashlin.state != stable_state || hashlin.low_max != initial_max
    6586          32 :                                 || hashlin.low_mask != initial_max - 1 || hashlin.split != 0
    6587          32 :                                 || tommy_hashlin_memory_usage(&hashlin) != initial_max * sizeof(tommy_hashlin_node*)) {
    6588             :                                 /* LCOV_EXCL_START */
    6589             :                                 abort();
    6590             :                                 /* LCOV_EXCL_STOP */
    6591             :                         }
    6592         224 :                         for (unsigned i = 0; i < TOMMY_HASHLIN_BIT; ++i)
    6593         192 :                                 if (hashlin.bucket[i] != initial_bucket) {
    6594             :                                         /* LCOV_EXCL_START */
    6595             :                                         abort();
    6596             :                                         /* LCOV_EXCL_STOP */
    6597             :                                 }
    6598        4896 :                         for (unsigned i = 0; i < bucket_max; ++i)
    6599        4864 :                                 if (tommy_hashlin_bucket(&hashlin, i) != 0) {
    6600             :                                         /* LCOV_EXCL_START */
    6601             :                                         abort();
    6602             :                                         /* LCOV_EXCL_STOP */
    6603             :                                 }
    6604          32 :                         the_count = 0;
    6605          32 :                         tommy_hashlin_foreach(&hashlin, count_callback);
    6606          32 :                         if (the_count != 0) {
    6607             :                                 /* LCOV_EXCL_START */
    6608             :                                 abort();
    6609             :                                 /* LCOV_EXCL_STOP */
    6610             :                         }
    6611        1076 :                         for (unsigned i = prefix; i < size; ++i)
    6612        1044 :                                 if (tommy_hashlin_search(&hashlin, search_callback, &obj[i], obj[i].node.index) != 0) {
    6613             :                                         /* LCOV_EXCL_START */
    6614             :                                         abort();
    6615             :                                         /* LCOV_EXCL_STOP */
    6616             :                                 }
    6617             : 
    6618        1108 :                         for (unsigned i = 0; i < size; ++i)
    6619        1076 :                                 saved[i] = obj[i].node;
    6620          32 :                         tommy_hashlin_to_list(&hashlin, &list);
    6621          32 :                         test_hash_list(&list, obj, saved, size, prefix);
    6622          32 :                         test_list_transfer_order(&list, expected, size);
    6623             : 
    6624             :                         /* reinsert all transferred nodes, then grow again from the minimum capacity. */
    6625        1108 :                         while (!tommy_list_empty(&list)) {
    6626        1076 :                                 struct object_hash* data = tommy_list_remove_head(&list);
    6627        1076 :                                 tommy_hashlin_insert(&hashlin, &data->node, data, data->node.index);
    6628             :                         }
    6629        3116 :                         for (unsigned i = size; i < 130; ++i) {
    6630        3084 :                                 obj[i].value = i / 2;
    6631        3084 :                                 tommy_hashlin_insert(&hashlin, &obj[i].node, &obj[i], tommy_inthash_u32(obj[i].value));
    6632             :                         }
    6633          32 :                         if (tommy_hashlin_count(&hashlin) != 130 || hashlin.bucket_max <= initial_max
    6634          32 :                                 || hashlin.bucket[0] != initial_bucket) {
    6635             :                                 /* LCOV_EXCL_START */
    6636             :                                 abort();
    6637             :                                 /* LCOV_EXCL_STOP */
    6638             :                         }
    6639        4192 :                         for (unsigned i = 0; i < 130; ++i)
    6640        4160 :                                 if (tommy_hashlin_search(&hashlin, search_callback, &obj[i], obj[i].node.index) != &obj[i]
    6641        4160 :                                         || tommy_hashlin_remove_existing(&hashlin, &obj[i].node) != &obj[i]
    6642        4160 :                                         || tommy_hashlin_count(&hashlin) != 130 - i - 1) {
    6643             :                                         /* LCOV_EXCL_START */
    6644             :                                         abort();
    6645             :                                         /* LCOV_EXCL_STOP */
    6646             :                                 }
    6647          32 :                         if (hashlin.bucket_max != (tommy_size_t)1 << TOMMY_HASHLIN_BIT) {
    6648             :                                 /* LCOV_EXCL_START */
    6649             :                                 abort();
    6650             :                                 /* LCOV_EXCL_STOP */
    6651             :                         }
    6652          32 :                         tommy_hashlin_done(&hashlin);
    6653             :                 }
    6654             :         }
    6655           1 :         if (stable == 0 || grow == 0 || shrink == 0) {
    6656             :                 /* LCOV_EXCL_START */
    6657             :                 abort();
    6658             :                 /* LCOV_EXCL_STOP */
    6659             :         }
    6660           1 : }
    6661             : 
    6662           1 : void test_hashlin_clear(void)
    6663             : {
    6664           1 :         const tommy_size_t initial_max = (tommy_size_t)1 << TOMMY_HASHLIN_BIT;
    6665           1 :         const unsigned counts[][2] = {
    6666             :                 { 0, 0 }, { 1, 1 }, { 32, 32 },
    6667             :                 { 33, 33 }, { 48, 48 }, { 63, 63 }, { 64, 64 },
    6668             :                 { 65, 65 }, { 96, 96 }, { 127, 127 }, { 128, 128 },
    6669             :                 { 128, 31 }, { 128, 24 }, { 128, 17 }, { 128, 16 },
    6670             :                 { 128, 15 }, { 128, 12 }, { 128, 9 }, { 128, 8 }, { 128, 0 }
    6671             :         };
    6672             :         struct object_hash obj[256];
    6673             :         tommy_node saved[128];
    6674           1 :         unsigned stable = 0, grow = 0, shrink = 0;
    6675             : 
    6676          21 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    6677          20 :                 unsigned n = counts[c][0];
    6678          20 :                 unsigned retained = counts[c][1];
    6679             :                 tommy_hashlin hashlin;
    6680          20 :                 tommy_hashlin_init(&hashlin);
    6681          20 :                 tommy_uint_t stable_state = hashlin.state;
    6682             : 
    6683        1829 :                 for (unsigned i = 0; i < n; ++i) {
    6684        1809 :                         obj[i].value = i / 2;
    6685        1809 :                         tommy_hashlin_insert(&hashlin, &obj[i].node, &obj[i], tommy_inthash_u32(obj[i].value));
    6686             :                 }
    6687             :                 /* stop at different points of a progressive shrink. */
    6688        1040 :                 for (unsigned i = n; i > retained; --i)
    6689        1020 :                         tommy_hashlin_remove_existing(&hashlin, &obj[i - 1].node);
    6690             : 
    6691          20 :                 if (hashlin.state == stable_state)
    6692           8 :                         ++stable;
    6693          12 :                 else if (n == retained)
    6694           6 :                         ++grow;
    6695             :                 else
    6696           6 :                         ++shrink;
    6697             : 
    6698          20 :                 tommy_size_t bucket_max = hashlin.bucket_max;
    6699          20 :                 tommy_hashlin_node** initial_bucket = hashlin.bucket[0];
    6700         809 :                 for (unsigned i = 0; i < retained; ++i)
    6701         789 :                         memcpy(&saved[i], &obj[i].node, sizeof(saved[i]));
    6702             : 
    6703             :                 /* inactive slots may contain arbitrary values or stale node pointers. */
    6704         596 :                 for (unsigned i = hashlin.low_max + hashlin.split; i < bucket_max; ++i)
    6705         576 :                         *tommy_hashlin_pos(&hashlin, i) = &obj[0].node;
    6706             : 
    6707          60 :                 for (unsigned j = 0; j < 2; ++j) {
    6708          40 :                         tommy_hashlin_clear(&hashlin);
    6709          40 :                         if (tommy_hashlin_count(&hashlin) != 0
    6710          40 :                                 || hashlin.bucket_max != initial_max || hashlin.bucket_mask != initial_max - 1
    6711          40 :                                 || hashlin.bucket_bit != TOMMY_HASHLIN_BIT || hashlin.state != stable_state
    6712          40 :                                 || hashlin.low_max != initial_max || hashlin.low_mask != initial_max - 1
    6713          40 :                                 || hashlin.split != 0 || tommy_hashlin_memory_usage(&hashlin) != initial_max * sizeof(tommy_hashlin_node*)) {
    6714             :                                 /* LCOV_EXCL_START */
    6715             :                                 abort();
    6716             :                                 /* LCOV_EXCL_STOP */
    6717             :                         }
    6718             : 
    6719         280 :                         for (unsigned i = 0; i < TOMMY_HASHLIN_BIT; ++i)
    6720         240 :                                 if (hashlin.bucket[i] != initial_bucket) {
    6721             :                                         /* LCOV_EXCL_START */
    6722             :                                         abort();
    6723             :                                         /* LCOV_EXCL_STOP */
    6724             :                                 }
    6725             : 
    6726        6312 :                         for (unsigned i = 0; i < bucket_max; ++i)
    6727        6272 :                                 if (tommy_hashlin_bucket(&hashlin, i) != 0) {
    6728             :                                         /* LCOV_EXCL_START */
    6729             :                                         abort();
    6730             :                                         /* LCOV_EXCL_STOP */
    6731             :                                 }
    6732             : 
    6733          40 :                         the_count = 0;
    6734          40 :                         tommy_hashlin_foreach(&hashlin, count_callback);
    6735          40 :                         tommy_hashlin_foreach_arg(&hashlin, count_arg_callback, &the_count);
    6736          40 :                         if (the_count != 0) {
    6737             :                                 /* LCOV_EXCL_START */
    6738             :                                 abort();
    6739             :                                 /* LCOV_EXCL_STOP */
    6740             :                         }
    6741             : 
    6742        1618 :                         for (unsigned i = 0; i < retained; ++i) {
    6743        1578 :                                 tommy_hash_t hash = tommy_inthash_u32(obj[i].value);
    6744        1578 :                                 if (obj[i].value != (int)(i / 2)
    6745        1578 :                                         || memcmp(&saved[i], &obj[i].node, sizeof(saved[i])) != 0
    6746        1578 :                                         || obj[i].node.data != &obj[i] || obj[i].node.index != hash
    6747        1578 :                                         || tommy_hashlin_find(&hashlin, search_callback, &obj[i], hash) != 0
    6748        1578 :                                         || tommy_hashlin_search(&hashlin, search_callback, &obj[i], hash) != 0
    6749        1578 :                                         || tommy_hashlin_remove(&hashlin, search_callback, &obj[i], hash) != 0) {
    6750             :                                         /* LCOV_EXCL_START */
    6751             :                                         abort();
    6752             :                                         /* LCOV_EXCL_STOP */
    6753             :                                 }
    6754             :                         }
    6755             :                 }
    6756             : 
    6757             :                 /* reuse old nodes and grow again from the minimum capacity. */
    6758        5140 :                 for (unsigned i = 0; i < 256; ++i) {
    6759        5120 :                         obj[i].value = i / 2;
    6760        5120 :                         tommy_hashlin_insert(&hashlin, &obj[i].node, &obj[i], tommy_inthash_u32(obj[i].value));
    6761        5120 :                         if (tommy_hashlin_count(&hashlin) != i + 1) {
    6762             :                                 /* LCOV_EXCL_START */
    6763             :                                 abort();
    6764             :                                 /* LCOV_EXCL_STOP */
    6765             :                         }
    6766             :                 }
    6767          20 :                 if (hashlin.bucket_max <= initial_max || hashlin.bucket[0] != initial_bucket) {
    6768             :                         /* LCOV_EXCL_START */
    6769             :                         abort();
    6770             :                         /* LCOV_EXCL_STOP */
    6771             :                 }
    6772             : 
    6773        5140 :                 for (unsigned i = 0; i < 256; ++i)
    6774        5120 :                         if (tommy_hashlin_find(&hashlin, search_callback, &obj[i], tommy_inthash_u32(obj[i].value)) != &obj[i].node
    6775        5120 :                                 || tommy_hashlin_search(&hashlin, search_callback, &obj[i], tommy_inthash_u32(obj[i].value)) != &obj[i]) {
    6776             :                                 /* LCOV_EXCL_START */
    6777             :                                 abort();
    6778             :                                 /* LCOV_EXCL_STOP */
    6779             :                         }
    6780             : 
    6781             :                 /* verify both removal paths while the table shrinks again. */
    6782        5140 :                 for (unsigned i = 256; i > 0; --i) {
    6783             :                         void* data;
    6784        5120 :                         if (i % 2)
    6785        2560 :                                 data = tommy_hashlin_remove_existing(&hashlin, &obj[i - 1].node);
    6786             :                         else
    6787        2560 :                                 data = tommy_hashlin_remove(&hashlin, search_callback, &obj[i - 1], tommy_inthash_u32(obj[i - 1].value));
    6788        5120 :                         if (data != &obj[i - 1] || tommy_hashlin_count(&hashlin) != i - 1) {
    6789             :                                 /* LCOV_EXCL_START */
    6790             :                                 abort();
    6791             :                                 /* LCOV_EXCL_STOP */
    6792             :                         }
    6793             :                 }
    6794          20 :                 if (hashlin.bucket_max != (tommy_size_t)1 << TOMMY_HASHLIN_BIT) {
    6795             :                         /* LCOV_EXCL_START */
    6796             :                         abort();
    6797             :                         /* LCOV_EXCL_STOP */
    6798             :                 }
    6799          20 :                 tommy_hashlin_done(&hashlin);
    6800             :         }
    6801             : 
    6802           1 :         if (stable == 0 || grow == 0 || shrink == 0) {
    6803             :                 /* LCOV_EXCL_START */
    6804             :                 abort();
    6805             :                 /* LCOV_EXCL_STOP */
    6806             :         }
    6807             : 
    6808             :         /* free objects during a partial grow or shrink, then clear and reuse. */
    6809           3 :         for (unsigned c = 0; c < 2; ++c) {
    6810             :                 struct object_hash* allocated[128];
    6811           2 :                 unsigned n = c == 0 ? 33 : 128;
    6812           2 :                 unsigned retained = c == 0 ? 33 : 24;
    6813             :                 tommy_hashlin hashlin;
    6814           2 :                 tommy_hashlin_init(&hashlin);
    6815         163 :                 for (unsigned i = 0; i < n; ++i) {
    6816         161 :                         allocated[i] = malloc(sizeof(struct object_hash));
    6817         161 :                         allocated[i]->value = i;
    6818         161 :                         tommy_hashlin_insert(&hashlin, &allocated[i]->node, allocated[i], tommy_inthash_u32(i));
    6819             :                 }
    6820         106 :                 for (unsigned i = n; i > retained; --i)
    6821         104 :                         free(tommy_hashlin_remove_existing(&hashlin, &allocated[i - 1]->node));
    6822           2 :                 tommy_hashlin_foreach(&hashlin, free);
    6823           2 :                 tommy_hashlin_clear(&hashlin);
    6824           2 :                 obj[0].value = 1;
    6825           2 :                 tommy_hashlin_insert(&hashlin, &obj[0].node, &obj[0], 1);
    6826           2 :                 if (tommy_hashlin_count(&hashlin) != 1
    6827           2 :                         || tommy_hashlin_remove_existing(&hashlin, &obj[0].node) != &obj[0]) {
    6828             :                         /* LCOV_EXCL_START */
    6829             :                         abort();
    6830             :                         /* LCOV_EXCL_STOP */
    6831             :                 }
    6832           2 :                 tommy_hashlin_done(&hashlin);
    6833             :         }
    6834           1 : }
    6835             : 
    6836             : /**
    6837             :  * Independent iteration-order oracle: reverse the hash bits, then compare
    6838             :  * insertion sequence numbers for equal hashes.
    6839             :  */
    6840      313504 : static int hashlin_iteration_compare(const void* first, const void* second)
    6841             : {
    6842      313504 :         return hash_bitreverse_compare(first, second);
    6843             : }
    6844             : 
    6845       37888 : static tommy_hash_t hashlin_iteration_hash(unsigned i, unsigned mode)
    6846             : {
    6847       37888 :         switch (mode) {
    6848       10240 :         case 0 :
    6849       10240 :                 return tommy_inthash_u32(i / 3);
    6850        9216 :         case 1 :
    6851        9216 :                 return (tommy_hash_t)(i % 19) << TOMMY_HASHLIN_BIT;
    6852        9216 :         case 2 :
    6853        9216 :                 return ~(tommy_hash_t)0;
    6854        9216 :         default :
    6855        9216 :                 return i % 3 == 0 ? 0 : ((tommy_hash_t)1 << (i % TOMMY_SIZE_BIT));
    6856             :         }
    6857             : }
    6858             : 
    6859          96 : static void hashlin_iteration_check(tommy_hashlin* table, struct object_hash** expected, unsigned count)
    6860             : {
    6861          96 :         tommy_hashlin_node* node = tommy_hashlin_head(table);
    6862       22604 :         for (unsigned i = 0; i < count; ++i) {
    6863       22508 :                 if (!(node == &expected[i]->node)) {
    6864             :                         /* LCOV_EXCL_START */
    6865             :                         abort();
    6866             :                         /* LCOV_EXCL_STOP */
    6867             :                 }
    6868       22508 :                 if (!(tommy_hashlin_prev(table, node) == (i ? &expected[i - 1]->node : 0))) {
    6869             :                         /* LCOV_EXCL_START */
    6870             :                         abort();
    6871             :                         /* LCOV_EXCL_STOP */
    6872             :                 }
    6873       22508 :                 node = tommy_hashlin_next(table, node);
    6874             :         }
    6875          96 :         if (!(node == 0)) {
    6876             :                 /* LCOV_EXCL_START */
    6877             :                 abort();
    6878             :                 /* LCOV_EXCL_STOP */
    6879             :         }
    6880          96 :         node = tommy_hashlin_tail(table);
    6881       22604 :         for (unsigned i = count; i != 0; ) {
    6882       22508 :                 --i;
    6883       22508 :                 if (!(node == &expected[i]->node)) {
    6884             :                         /* LCOV_EXCL_START */
    6885             :                         abort();
    6886             :                         /* LCOV_EXCL_STOP */
    6887             :                 }
    6888       22508 :                 node = tommy_hashlin_prev(table, node);
    6889             :         }
    6890          96 :         if (!(node == 0)) {
    6891             :                 /* LCOV_EXCL_START */
    6892             :                 abort();
    6893             :                 /* LCOV_EXCL_STOP */
    6894             :         }
    6895          96 : }
    6896             : 
    6897           1 : void test_hashlin_iterators(void)
    6898             : {
    6899             :         struct object_hash obj[1024];
    6900             :         struct object_hash* expected[1024];
    6901           1 :         const unsigned checkpoints[] = { 0, 1, 32, 33, 63, 64, 65, 127, 128, 129, 256, 513, 1024 };
    6902             : 
    6903           5 :         for (unsigned mode = 0; mode < 4; ++mode) {
    6904             :                 tommy_hashlin table;
    6905           4 :                 tommy_hashlin_init(&table);
    6906           4 :                 unsigned count = 0;
    6907          56 :                 for (unsigned c = 0; c < sizeof(checkpoints) / sizeof(checkpoints[0]); ++c) {
    6908        4148 :                         while (count < checkpoints[c]) {
    6909        4096 :                                 obj[count].value = count;
    6910        4096 :                                 tommy_hashlin_insert(&table, &obj[count].node, &obj[count], hashlin_iteration_hash(count, mode));
    6911        4096 :                                 ++count;
    6912             :                         }
    6913        9792 :                         for (unsigned i = 0; i < count; ++i)
    6914        9740 :                                 expected[i] = &obj[i];
    6915          52 :                         qsort(expected, count, sizeof(expected[0]), hashlin_iteration_compare);
    6916          52 :                         hashlin_iteration_check(&table, expected, count);
    6917             :                 }
    6918             : 
    6919             :                 /* swapping transfers iterator ownership without changing node order */
    6920             :                 tommy_hashlin other;
    6921           4 :                 tommy_hashlin_init(&other);
    6922           4 :                 tommy_hashlin_swap(&table, &other);
    6923           4 :                 hashlin_iteration_check(&other, expected, count);
    6924           4 :                 hashlin_iteration_check(&table, expected, 0);
    6925           4 :                 tommy_hashlin_clear(&other);
    6926           4 :                 hashlin_iteration_check(&other, expected, 0);
    6927           4 :                 tommy_hashlin_done(&other);
    6928           4 :                 tommy_hashlin_done(&table);
    6929             :         }
    6930             : 
    6931           5 :         for (unsigned mode = 0; mode < 4; ++mode) {
    6932          12 :                 for (unsigned reverse = 0; reverse < 2; ++reverse) {
    6933          40 :                         for (unsigned mutation = 0; mutation < 4; ++mutation) {
    6934             :                                 tommy_hashlin table;
    6935          32 :                                 tommy_hashlin_init(&table);
    6936          32 :                                 unsigned count = mutation == 3 ? 130 : 1024;
    6937          32 :                                 unsigned char seen[1024] = { 0 };
    6938          32 :                                 unsigned char removed[1024] = { 0 };
    6939       25648 :                                 for (unsigned i = 0; i < count; ++i) {
    6940       25616 :                                         obj[i].value = i;
    6941       25616 :                                         tommy_hashlin_insert(&table, &obj[i].node, &obj[i], hashlin_iteration_hash(i, mode));
    6942       25616 :                                         expected[i] = &obj[i];
    6943             :                                 }
    6944          32 :                                 qsort(expected, count, sizeof(expected[0]), hashlin_iteration_compare);
    6945          32 :                                 unsigned visited = 0;
    6946          32 :                                 unsigned original_visited = 0;
    6947          32 :                                 unsigned removed_count = 0;
    6948          32 :                                 tommy_size_t initial_max = table.bucket_max;
    6949          32 :                                 tommy_hashlin_node* node = reverse ? tommy_hashlin_tail(&table) : tommy_hashlin_head(&table);
    6950       31803 :                                 while (node) {
    6951       31771 :                                         struct object_hash* object = node->data;
    6952       31771 :                                         unsigned id = object->value;
    6953       31771 :                                         if (!(!seen[id])) {
    6954             :                                                 /* LCOV_EXCL_START */
    6955             :                                                 abort();
    6956             :                                                 /* LCOV_EXCL_STOP */
    6957             :                                         }
    6958       31771 :                                         seen[id] = 1;
    6959       31771 :                                         ++visited;
    6960       31771 :                                         if (id < count) {
    6961       25616 :                                                 if (!(original_visited < count)) {
    6962             :                                                         /* LCOV_EXCL_START */
    6963             :                                                         abort();
    6964             :                                                         /* LCOV_EXCL_STOP */
    6965             :                                                 }
    6966       25616 :                                                 if (!(object == expected[reverse ? count - 1 - original_visited : original_visited])) {
    6967             :                                                         /* LCOV_EXCL_START */
    6968             :                                                         abort();
    6969             :                                                         /* LCOV_EXCL_STOP */
    6970             :                                                 }
    6971       25616 :                                                 ++original_visited;
    6972             :                                         }
    6973             : 
    6974             :                                         /* save the neighbor before mutations, including resize and batch removal */
    6975       31771 :                                         tommy_hashlin_node* next = reverse ? tommy_hashlin_prev(&table, node) : tommy_hashlin_next(&table, node);
    6976       31771 :                                         if (mutation == 0 || (mutation == 1 && visited % 17 != 0)) {
    6977        7952 :                                                 void* data = visited % 2 ? tommy_hashlin_remove_existing(&table, node)
    6978       15904 :                                                         : tommy_hashlin_remove(&table, search_callback, object, node->index);
    6979       15904 :                                                 if (!(data == object)) {
    6980             :                                                         /* LCOV_EXCL_START */
    6981             :                                                         abort();
    6982             :                                                         /* LCOV_EXCL_STOP */
    6983             :                                                 }
    6984       15904 :                                                 removed[id] = 1;
    6985       15904 :                                                 ++removed_count;
    6986       15867 :                                         } else if (mutation == 2 && (visited % 16 == 0 || !next)) {
    6987             :                                                 /* remove previously visited nodes as well as the current node */
    6988      524800 :                                                 for (unsigned i = 0; i < count; ++i) {
    6989      524288 :                                                         if (seen[i] && !removed[i]) {
    6990        8192 :                                                                 if (!(tommy_hashlin_remove_existing(&table, &obj[i].node) == &obj[i])) {
    6991             :                                                                         /* LCOV_EXCL_START */
    6992             :                                                                         abort();
    6993             :                                                                         /* LCOV_EXCL_STOP */
    6994             :                                                                 }
    6995        8192 :                                                                 removed[i] = 1;
    6996        8192 :                                                                 ++removed_count;
    6997             :                                                         }
    6998             :                                                 }
    6999       15355 :                                         } else if (mutation == 3 && visited == 1) {
    7000             :                                                 /* force growth after saving an iterator, including duplicate hashes */
    7001        7160 :                                                 for (unsigned i = count; i < 1024; ++i) {
    7002        7152 :                                                         obj[i].value = i;
    7003        7152 :                                                         tommy_hashlin_insert(&table, &obj[i].node, &obj[i], hashlin_iteration_hash(i, mode));
    7004             :                                                 }
    7005             :                                         }
    7006       31771 :                                         node = next;
    7007             :                                 }
    7008          32 :                                 if (!(original_visited == count)) {
    7009             :                                         /* LCOV_EXCL_START */
    7010             :                                         abort();
    7011             :                                         /* LCOV_EXCL_STOP */
    7012             :                                 }
    7013          32 :                                 if (!(tommy_hashlin_count(&table) == (mutation == 3 ? 1024 : count) - removed_count)) {
    7014             :                                         /* LCOV_EXCL_START */
    7015             :                                         abort();
    7016             :                                         /* LCOV_EXCL_STOP */
    7017             :                                 }
    7018          32 :                                 if (!(mutation == 3 ? table.bucket_max > initial_max : table.bucket_max < initial_max)) {
    7019             :                                         /* LCOV_EXCL_START */
    7020             :                                         abort();
    7021             :                                         /* LCOV_EXCL_STOP */
    7022             :                                 }
    7023             : 
    7024             :                                 /* verify survivors in both directions against the independent oracle */
    7025          32 :                                 unsigned survivors = 0;
    7026       32800 :                                 for (unsigned i = 0; i < (mutation == 3 ? 1024 : count); ++i)
    7027       32768 :                                         if (!removed[i])
    7028        8672 :                                                 expected[survivors++] = &obj[i];
    7029          32 :                                 qsort(expected, survivors, sizeof(expected[0]), hashlin_iteration_compare);
    7030          32 :                                 hashlin_iteration_check(&table, expected, survivors);
    7031          32 :                                 tommy_hashlin_done(&table);
    7032             :                         }
    7033             :                 }
    7034             :         }
    7035             : 
    7036             :         /* freeing each removed object must not leave iteration dependent on its links */
    7037           3 :         for (unsigned reverse = 0; reverse < 2; ++reverse) {
    7038             :                 tommy_hashlin table;
    7039           2 :                 tommy_hashlin_init(&table);
    7040        1026 :                 for (unsigned i = 0; i < 512; ++i) {
    7041        1024 :                         struct object_hash* object = malloc(sizeof(struct object_hash));
    7042        1024 :                         object->value = i;
    7043        1024 :                         tommy_hashlin_insert(&table, &object->node, object, hashlin_iteration_hash(i, 0));
    7044             :                 }
    7045           2 :                 unsigned visited = 0;
    7046           2 :                 tommy_hashlin_node* node = reverse ? tommy_hashlin_tail(&table) : tommy_hashlin_head(&table);
    7047        1026 :                 while (node) {
    7048        1024 :                         tommy_hashlin_node* next = reverse ? tommy_hashlin_prev(&table, node) : tommy_hashlin_next(&table, node);
    7049        1024 :                         free(tommy_hashlin_remove_existing(&table, node));
    7050        1024 :                         ++visited;
    7051        1024 :                         node = next;
    7052             :                 }
    7053           2 :                 if (visited != 512 || !tommy_hashlin_empty(&table)) {
    7054             :                         /* LCOV_EXCL_START */
    7055             :                         abort();
    7056             :                         /* LCOV_EXCL_STOP */
    7057             :                 }
    7058           2 :                 tommy_hashlin_done(&table);
    7059             :         }
    7060           1 : }
    7061             : 
    7062           1 : void test_hashlin(void)
    7063             : {
    7064           1 :         const unsigned size = TOMMY_SIZE;
    7065           1 :         const unsigned module = TOMMY_SIZE / 4;
    7066             : 
    7067           1 :         test_hashlin_clear();
    7068           1 :         test_hashlin_insert_unique();
    7069           1 :         test_hashlin_swap();
    7070           1 :         test_hashlin_rehash_existing();
    7071           1 :         test_hashlin_foreach();
    7072           1 :         test_hashlin_to_list();
    7073           1 :         test_hashlin_iterators();
    7074             : 
    7075           1 :         struct object_hash* HASH = malloc(size * sizeof(struct object_hash));
    7076             : 
    7077      100001 :         for (unsigned i = 0; i < size; ++i)
    7078      100000 :                 HASH[i].value = i % module;
    7079             : 
    7080             :         tommy_hashlin hashlin;
    7081           1 :         tommy_hashlin_init(&hashlin);
    7082             : 
    7083             :         /* insert */
    7084      100001 :         for (unsigned i = 0; i < size; ++i)
    7085      100000 :                 tommy_hashlin_insert(&hashlin, &HASH[i].node, &HASH[i], HASH[i].value);
    7086             : 
    7087             :         /* get the bucket of the last element */
    7088           1 :         tommy_hashlin_node* bucket = tommy_hashlin_bucket(&hashlin, module - 1);
    7089           1 :         if (bucket == 0) {
    7090             :                 /* LCOV_EXCL_START */
    7091             :                 abort();
    7092             :                 /* LCOV_EXCL_STOP */
    7093             :         }
    7094             : 
    7095             :         /* deinitialize without removing elements to force deallocation */
    7096           1 :         tommy_hashlin_done(&hashlin);
    7097             : 
    7098           2 :         SPEED_START("hashlin stack") {
    7099           1 :                 unsigned limit = 5 * isqrt(size);
    7100        1582 :                 for (unsigned n = 0; n <= limit; ++n) {
    7101             :                         /* last iteration is full size */
    7102        1581 :                         if (n == limit)
    7103           1 :                                 n = limit = size;
    7104             : 
    7105        1581 :                         tommy_hashlin_init(&hashlin);
    7106             : 
    7107             :                         /* insert */
    7108     1348991 :                         for (unsigned i = 0; i < n; ++i)
    7109     1347410 :                                 tommy_hashlin_insert(&hashlin, &HASH[i].node, &HASH[i], HASH[i].value);
    7110             : 
    7111        1581 :                         if (tommy_hashlin_memory_usage(&hashlin) < n * sizeof(void*)) {
    7112             :                                 /* LCOV_EXCL_START */
    7113             :                                 abort();
    7114             :                                 /* LCOV_EXCL_STOP */
    7115             :                         }
    7116             : 
    7117        1581 :                         if (tommy_hashlin_count(&hashlin) != n) {
    7118             :                                 /* LCOV_EXCL_START */
    7119             :                                 abort();
    7120             :                                 /* LCOV_EXCL_STOP */
    7121             :                         }
    7122             : 
    7123        1581 :                         the_count = 0;
    7124        1581 :                         tommy_hashlin_foreach(&hashlin, count_callback);
    7125        1581 :                         if (the_count != n) {
    7126             :                                 /* LCOV_EXCL_START */
    7127             :                                 abort();
    7128             :                                 /* LCOV_EXCL_STOP */
    7129             :                         }
    7130             : 
    7131             :                         /* remove in backward order */
    7132      674891 :                         for (unsigned i = 0; i < n / 2; ++i)
    7133      673310 :                                 tommy_hashlin_remove_existing(&hashlin, &HASH[n - i - 1].node);
    7134             : 
    7135             :                         /* remove missing */
    7136      674891 :                         for (unsigned i = 0; i < n / 2; ++i)
    7137      673310 :                                 if (tommy_hashlin_remove(&hashlin, search_callback, &HASH[n - i - 1], HASH[n - i - 1].value) != 0) {
    7138             :                                         /* LCOV_EXCL_START */
    7139             :                                         abort();
    7140             :                                         /* LCOV_EXCL_STOP */
    7141             :                                 }
    7142             : 
    7143             :                         /* remove search */
    7144      674891 :                         for (unsigned i = 0; i < n / 2; ++i)
    7145      673310 :                                 if (tommy_hashlin_remove(&hashlin, search_callback, &HASH[n / 2 - i - 1], HASH[n / 2 - i - 1].value) == 0) {
    7146             :                                         /* LCOV_EXCL_START */
    7147             :                                         abort();
    7148             :                                         /* LCOV_EXCL_STOP */
    7149             :                                 }
    7150             : 
    7151        1581 :                         tommy_hashlin_done(&hashlin);
    7152             :                 }
    7153           1 :         } SPEED_STOP
    7154             : 
    7155           4 :         SPEED_START("hashlin queue")
    7156             :         {
    7157           2 :                 unsigned limit = isqrt(size) / 16;
    7158          42 :                 for (unsigned n = 0; n <= limit; ++n) {
    7159             :                         /* last iteration is full size */
    7160          40 :                         if (n == limit)
    7161           2 :                                 n = limit = size;
    7162             : 
    7163          40 :                         tommy_hashlin_init(&hashlin);
    7164             : 
    7165             :                         /* insert first run */
    7166          40 :                         unsigned j = 0;
    7167          40 :                         unsigned i = 0;
    7168      200382 :                         for (; i < n; ++i)
    7169      200342 :                                 tommy_hashlin_insert(&hashlin, &HASH[i].node, &HASH[i], HASH[i].value);
    7170             : 
    7171          40 :                         the_count = 0;
    7172          40 :                         tommy_hashlin_foreach_arg(&hashlin, count_arg_callback, &the_count);
    7173          40 :                         if (the_count != n) {
    7174             :                                 /* LCOV_EXCL_START */
    7175             :                                 abort();
    7176             :                                 /* LCOV_EXCL_STOP */
    7177             :                         }
    7178             : 
    7179             :                         /* insert all the others */
    7180     3799698 :                         for (; i < size; ++i, ++j) {
    7181             :                                 /* insert one */
    7182     3799658 :                                 tommy_hashlin_insert(&hashlin, &HASH[i].node, &HASH[i], HASH[i].value);
    7183             : 
    7184             :                                 /* remove one */
    7185     3799658 :                                 tommy_hashlin_remove_existing(&hashlin, &HASH[j].node);
    7186             :                         }
    7187             : 
    7188      200382 :                         for (; j < size; ++j)
    7189      200342 :                                 if (tommy_hashlin_remove(&hashlin, search_callback, &HASH[j], HASH[j].value) == 0) {
    7190             :                                         /* LCOV_EXCL_START */
    7191             :                                         abort();
    7192             :                                         /* LCOV_EXCL_STOP */
    7193             :                                 }
    7194             : 
    7195          40 :                         tommy_hashlin_done(&hashlin);
    7196             :                 }
    7197             :         }
    7198           2 :         SPEED_STOP
    7199           1 : }
    7200             : 
    7201             : static struct {
    7202             :         void* expected[128];
    7203             :         unsigned size;
    7204             :         unsigned pos;
    7205             :         int deallocate;
    7206             : } trie_foreach_check;
    7207             : 
    7208        6892 : static void trie_foreach_check_arg_callback(void* arg, void* data)
    7209             : {
    7210        6892 :         unsigned* pos = arg;
    7211        6892 :         if (*pos >= trie_foreach_check.size || data != trie_foreach_check.expected[*pos]) {
    7212             :                 /* LCOV_EXCL_START */
    7213             :                 abort();
    7214             :                 /* LCOV_EXCL_STOP */
    7215             :         }
    7216        6892 :         ++*pos;
    7217        6892 :         if (trie_foreach_check.deallocate)
    7218        3446 :                 free(data);
    7219        6892 : }
    7220             : 
    7221        3446 : static void trie_foreach_check_callback(void* data)
    7222             : {
    7223        3446 :         trie_foreach_check_arg_callback(&trie_foreach_check.pos, data);
    7224        3446 : }
    7225             : 
    7226           1 : void test_trie_foreach(void)
    7227             : {
    7228           1 :         tommy_key_t high = 1;
    7229           1 :         high <<= TOMMY_TRIE_BIT - 1;
    7230           1 :         tommy_key_t max = high | (high - 1);
    7231           1 :         const tommy_key_t keys[] = { max, 1, 0, max, 1, high, 2, 0 };
    7232           1 :         const unsigned order[] = { 2, 7, 1, 4, 6, 5, 0, 3 };
    7233           1 :         const unsigned sizes[] = { 0, 1, 8 };
    7234             : 
    7235           4 :         for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); ++s) {
    7236          15 :                 for (unsigned mode = 0; mode < 4; ++mode) {
    7237             :                         tommy_allocator alloc;
    7238             :                         tommy_trie trie;
    7239             :                         struct object_trie* obj[8];
    7240             :                         struct object_trie before[8];
    7241          12 :                         tommy_allocator_init(&alloc, TOMMY_TRIE_BLOCK_SIZE, TOMMY_TRIE_BLOCK_SIZE);
    7242          12 :                         tommy_trie_init(&trie, &alloc);
    7243          48 :                         for (unsigned i = 0; i < sizes[s]; ++i) {
    7244          36 :                                 obj[i] = calloc(1, sizeof(*obj[i]));
    7245          36 :                                 obj[i]->value = i;
    7246          36 :                                 tommy_trie_insert(&trie, &obj[i]->node, obj[i], keys[i]);
    7247             :                         }
    7248          48 :                         for (unsigned i = 0; i < sizes[s]; ++i) {
    7249          36 :                                 memcpy(&before[i], obj[i], sizeof(before[i]));
    7250          36 :                                 trie_foreach_check.expected[i] = obj[sizes[s] == 1 ? 0 : order[i]];
    7251             :                         }
    7252             :                         unsigned char saved[sizeof(trie)];
    7253             :                         unsigned char saved_alloc[sizeof(alloc)];
    7254          12 :                         memcpy(saved, &trie, sizeof(saved));
    7255          12 :                         memcpy(saved_alloc, &alloc, sizeof(saved_alloc));
    7256          12 :                         trie_foreach_check.size = sizes[s];
    7257          12 :                         trie_foreach_check.deallocate = mode >= 2;
    7258          12 :                         trie_foreach_check.pos = 0;
    7259          12 :                         unsigned pos = 0;
    7260          12 :                         if (mode % 2)
    7261           6 :                                 tommy_trie_foreach_arg(&trie, trie_foreach_check_arg_callback, &pos);
    7262             :                         else
    7263           6 :                                 tommy_trie_foreach(&trie, trie_foreach_check_callback);
    7264          12 :                         if ((mode % 2 ? pos : trie_foreach_check.pos) != sizes[s]
    7265          12 :                                 || memcmp(saved, &trie, sizeof(trie)) != 0
    7266          12 :                                 || memcmp(saved_alloc, &alloc, sizeof(alloc)) != 0) {
    7267             :                                 /* LCOV_EXCL_START */
    7268             :                                 abort();
    7269             :                                 /* LCOV_EXCL_STOP */
    7270             :                         }
    7271          12 :                         if (!trie_foreach_check.deallocate) {
    7272          24 :                                 for (unsigned i = 0; i < sizes[s]; ++i) {
    7273          18 :                                         if (memcmp(&before[i], obj[i], sizeof(before[i])) != 0) {
    7274             :                                                 /* LCOV_EXCL_START */
    7275             :                                                 abort();
    7276             :                                                 /* LCOV_EXCL_STOP */
    7277             :                                         }
    7278             :                                 }
    7279          24 :                                 for (unsigned i = 0; i < sizes[s]; ++i) {
    7280          18 :                                         if (tommy_trie_remove_existing(&trie, &obj[i]->node) != obj[i]) {
    7281             :                                                 /* LCOV_EXCL_START */
    7282             :                                                 abort();
    7283             :                                                 /* LCOV_EXCL_STOP */
    7284             :                                         }
    7285          18 :                                         free(obj[i]);
    7286             :                                 }
    7287             :                         }
    7288          12 :                         tommy_allocator_done(&alloc);
    7289             :                 }
    7290             :         }
    7291           1 : }
    7292             : 
    7293       29751 : static int trie_foreach_compare(const void* void_a, const void* void_b)
    7294             : {
    7295       29751 :         void* const* ptr_a = void_a;
    7296       29751 :         void* const* ptr_b = void_b;
    7297       29751 :         const struct object_trie* a = *ptr_a;
    7298       29751 :         const struct object_trie* b = *ptr_b;
    7299       29751 :         if (a->node.index < b->node.index)
    7300       10894 :                 return -1;
    7301       18857 :         if (a->node.index > b->node.index)
    7302       12933 :                 return 1;
    7303        5924 :         return (a->value > b->value) - (a->value < b->value);
    7304             : }
    7305             : 
    7306         120 : static void trie_search_bounds_check(tommy_trie* trie, void** expected, unsigned size)
    7307             : {
    7308         120 :         tommy_key_t high = 1;
    7309         120 :         high <<= TOMMY_TRIE_BIT - 1;
    7310         120 :         tommy_key_t max = high | (high - 1);
    7311             : 
    7312             :         /* query existing keys, gaps, arbitrary prefixes, and both key-range endpoints */
    7313       32670 :         for (unsigned i = 0; i < size + 258; ++i) {
    7314             :                 tommy_key_t key;
    7315       32550 :                 if (i < size) {
    7316        1590 :                         struct object_trie* obj = expected[i];
    7317        1590 :                         key = obj->node.index;
    7318       30960 :                 } else if (i < size + 128)
    7319       15360 :                         key = tommy_inthash_u32(i - size) & max;
    7320       15600 :                 else if (i < size + 256)
    7321       15360 :                         key = i - size - 128;
    7322             :                 else
    7323         240 :                         key = i == size + 256 ? 0 : max;
    7324             : 
    7325       32550 :                 const tommy_key_t queries[] = { key ? key - 1 : 0, key, key < max ? key + 1 : max };
    7326      130200 :                 for (unsigned q = 0; q < sizeof(queries) / sizeof(queries[0]); ++q) {
    7327       97650 :                         struct object_trie* less = 0;
    7328       97650 :                         struct object_trie* less_equal = 0;
    7329       97650 :                         struct object_trie* greater_equal = 0;
    7330       97650 :                         struct object_trie* greater = 0;
    7331             :                         /* the oracle uses only independently sorted keys and insertion positions */
    7332     1785384 :                         for (unsigned j = 0; j < size; ++j) {
    7333     1687734 :                                 struct object_trie* obj = expected[j];
    7334     1687734 :                                 if (obj->node.index < queries[q])
    7335      510802 :                                         less = obj;
    7336     1687734 :                                 if (obj->node.index <= queries[q])
    7337      566338 :                                         less_equal = obj;
    7338     1687734 :                                 if (obj->node.index >= queries[q] && !greater_equal)
    7339       36922 :                                         greater_equal = obj;
    7340     1687734 :                                 if (obj->node.index > queries[q] && !greater)
    7341       36180 :                                         greater = obj;
    7342             :                         }
    7343       97650 :                         if (tommy_trie_search_less(trie, queries[q]) != less
    7344       97650 :                                 || tommy_trie_search_less_equal(trie, queries[q]) != less_equal
    7345       97650 :                                 || tommy_trie_search_greater_equal(trie, queries[q]) != greater_equal
    7346       97650 :                                 || tommy_trie_search_greater(trie, queries[q]) != greater) {
    7347             :                                 /* LCOV_EXCL_START */
    7348             :                                 abort();
    7349             :                                 /* LCOV_EXCL_STOP */
    7350             :                         }
    7351       97650 :                         tommy_trie_node* expected_less = less ? &less->node : 0;
    7352       97650 :                         tommy_trie_node* expected_less_equal = less_equal ? &less_equal->node : 0;
    7353       97650 :                         tommy_trie_node* expected_greater_equal = greater_equal ? &greater_equal->node : 0;
    7354       97650 :                         tommy_trie_node* expected_greater = greater ? &greater->node : 0;
    7355       97650 :                         if (tommy_trie_find_less(trie, queries[q]) != expected_less
    7356       97650 :                                 || tommy_trie_find_less_equal(trie, queries[q]) != expected_less_equal
    7357       97650 :                                 || tommy_trie_find_greater_equal(trie, queries[q]) != expected_greater_equal
    7358       97650 :                                 || tommy_trie_find_greater(trie, queries[q]) != expected_greater
    7359       97650 :                                 || tommy_trie_find(trie, queries[q]) != tommy_trie_bucket(trie, queries[q])) {
    7360             :                                 /* LCOV_EXCL_START */
    7361             :                                 abort();
    7362             :                                 /* LCOV_EXCL_STOP */
    7363             :                         }
    7364             :                 }
    7365             :         }
    7366         120 : }
    7367             : 
    7368        2532 : static void trie_iteration_check(tommy_trie* trie, void** expected, unsigned size)
    7369             : {
    7370        2532 :         tommy_trie_node* node = tommy_trie_head(trie);
    7371      150498 :         for (unsigned i = 0; i < size; ++i) {
    7372      147966 :                 struct object_trie* obj = expected[i];
    7373      147966 :                 if (node != &obj->node) {
    7374             :                         /* LCOV_EXCL_START */
    7375             :                         abort();
    7376             :                         /* LCOV_EXCL_STOP */
    7377             :                 }
    7378      147966 :                 node = tommy_trie_next(trie, node);
    7379             :         }
    7380        2532 :         if (node != 0) {
    7381             :                 /* LCOV_EXCL_START */
    7382             :                 abort();
    7383             :                 /* LCOV_EXCL_STOP */
    7384             :         }
    7385             : 
    7386        2532 :         node = tommy_trie_tail(trie);
    7387      150498 :         for (unsigned i = size; i != 0; ) {
    7388      147966 :                 struct object_trie* obj = expected[--i];
    7389      147966 :                 if (node != &obj->node) {
    7390             :                         /* LCOV_EXCL_START */
    7391             :                         abort();
    7392             :                         /* LCOV_EXCL_STOP */
    7393             :                 }
    7394      147966 :                 node = tommy_trie_prev(trie, node);
    7395             :         }
    7396        2532 :         if (node != 0) {
    7397             :                 /* LCOV_EXCL_START */
    7398             :                 abort();
    7399             :                 /* LCOV_EXCL_STOP */
    7400             :         }
    7401        2532 : }
    7402             : 
    7403          30 : static void trie_remove_ends_check(tommy_trie* trie, struct object_trie* obj, unsigned size)
    7404             : {
    7405             :         void* expected[128];
    7406         120 :         for (unsigned mode = 0; mode < 3; ++mode) {
    7407             :                 /* reuse the drained trie with the same keys and original duplicate insertion order */
    7408        2502 :                 for (unsigned i = 0; i < size; ++i) {
    7409        2412 :                         tommy_trie_insert(trie, &obj[i].node, &obj[i], obj[i].node.index);
    7410        2412 :                         expected[i] = &obj[i];
    7411             :                 }
    7412          90 :                 qsort(expected, size, sizeof(expected[0]), trie_foreach_compare);
    7413             : 
    7414          90 :                 unsigned head = 0;
    7415          90 :                 unsigned tail = size;
    7416        2502 :                 while (head != tail) {
    7417             :                         /* drain from either end or alternate, checking the complete remaining order */
    7418        2412 :                         tommy_bool_t remove_tail = mode == 1 || (mode == 2 && (tail - head) % 2 == 0);
    7419             :                         void* data;
    7420             :                         void* removed;
    7421        2412 :                         if (remove_tail) {
    7422        1200 :                                 removed = expected[--tail];
    7423        1200 :                                 data = tommy_trie_remove_tail(trie);
    7424             :                         } else {
    7425        1212 :                                 removed = expected[head];
    7426        1212 :                                 ++head;
    7427        1212 :                                 data = tommy_trie_remove_head(trie);
    7428             :                         }
    7429        2412 :                         if (data != removed || tommy_trie_count(trie) != tail - head) {
    7430             :                                 /* LCOV_EXCL_START */
    7431             :                                 abort();
    7432             :                                 /* LCOV_EXCL_STOP */
    7433             :                         }
    7434        2412 :                         trie_iteration_check(trie, expected + head, tail - head);
    7435             :                 }
    7436          90 :                 if (tommy_trie_remove_head(trie) != 0 || tommy_trie_remove_tail(trie) != 0
    7437          90 :                         || !tommy_trie_empty(trie) || tommy_trie_count(trie) != 0
    7438          90 :                         || trie->node_count != 0 || trie->alloc->count != 0) {
    7439             :                         /* LCOV_EXCL_START */
    7440             :                         abort();
    7441             :                         /* LCOV_EXCL_STOP */
    7442             :                 }
    7443             :         }
    7444             : 
    7445             :         /* a zero data pointer must still remove the node */
    7446             :         tommy_trie_node node;
    7447          90 :         for (unsigned mode = 0; mode < 2; ++mode) {
    7448          60 :                 tommy_trie_insert(trie, &node, 0, 0);
    7449          60 :                 void* data = mode ? tommy_trie_remove_tail(trie) : tommy_trie_remove_head(trie);
    7450          60 :                 if (data != 0 || !tommy_trie_empty(trie) || tommy_trie_count(trie) != 0) {
    7451             :                         /* LCOV_EXCL_START */
    7452             :                         abort();
    7453             :                         /* LCOV_EXCL_STOP */
    7454             :                 }
    7455             :         }
    7456          30 : }
    7457             : 
    7458           1 : void test_trie_iteration(void)
    7459             : {
    7460           1 :         const unsigned sizes[] = { 0, 1, 2, 3, 128 };
    7461           1 :         tommy_key_t high = 1;
    7462           1 :         high <<= TOMMY_TRIE_BIT - 1;
    7463           1 :         tommy_key_t max = high | (high - 1);
    7464             : 
    7465           6 :         for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); ++s) {
    7466          35 :                 for (unsigned shape = 0; shape < 6; ++shape) {
    7467             :                         tommy_allocator alloc;
    7468             :                         tommy_trie trie;
    7469             :                         struct object_trie obj[128];
    7470             :                         void* expected[128];
    7471          30 :                         memset(obj, 0, sizeof(obj));
    7472          30 :                         tommy_allocator_init(&alloc, TOMMY_TRIE_BLOCK_SIZE, TOMMY_TRIE_BLOCK_SIZE);
    7473          30 :                         tommy_trie_init(&trie, &alloc);
    7474         834 :                         for (unsigned i = 0; i < sizes[s]; ++i) {
    7475             :                                 tommy_key_t key;
    7476         804 :                                 if (shape == 0)
    7477         134 :                                         key = i;
    7478         670 :                                 else if (shape == 1)
    7479         134 :                                         key = max - i;
    7480         536 :                                 else if (shape == 2)
    7481         134 :                                         key = i < 2 ? (i ? max : 0) : tommy_inthash_u32(i) & max;
    7482         402 :                                 else if (shape == 3)
    7483         134 :                                         key = i < 6 ? (i % 2 ? max : 0) : tommy_inthash_u32(i / 3) & max;
    7484         268 :                                 else if (shape == 4)
    7485         134 :                                         key = max;
    7486             :                                 else {
    7487             :                                         /* sparse branches at every depth exercise replacement and retention of candidates */
    7488         134 :                                         key = 1;
    7489         134 :                                         key <<= (i / 2) % TOMMY_TRIE_BIT;
    7490         134 :                                         if (i % 2)
    7491          66 :                                                 key ^= max;
    7492             :                                 }
    7493         804 :                                 obj[i].value = i;
    7494         804 :                                 tommy_trie_insert(&trie, &obj[i].node, &obj[i], key);
    7495             :                         }
    7496             : 
    7497         150 :                         for (unsigned round = 0; round < 4; ++round) {
    7498             :                                 /* remove duplicate heads, middle nodes and tails, and collapse internal branches */
    7499         120 :                                 if (round)
    7500         894 :                                         for (unsigned i = round - 1; i < sizes[s]; i += 3)
    7501         804 :                                                 tommy_trie_remove_existing(&trie, &obj[i].node);
    7502         120 :                                 unsigned count = 0;
    7503        3336 :                                 for (unsigned i = 0; i < sizes[s]; ++i)
    7504        3216 :                                         if (i % 3 >= round)
    7505        1590 :                                                 expected[count++] = &obj[i];
    7506             :                                 /* sort independently by numeric key and insertion position */
    7507         120 :                                 qsort(expected, count, sizeof(expected[0]), trie_foreach_compare);
    7508             : 
    7509             :                                 unsigned char saved_trie[sizeof(trie)], saved_alloc[sizeof(alloc)], saved_obj[sizeof(obj)];
    7510         120 :                                 memcpy(saved_trie, &trie, sizeof(trie));
    7511         120 :                                 memcpy(saved_alloc, &alloc, sizeof(alloc));
    7512         120 :                                 memcpy(saved_obj, obj, sizeof(obj));
    7513         120 :                                 trie_iteration_check(&trie, expected, count);
    7514         120 :                                 trie_search_bounds_check(&trie, expected, count);
    7515         120 :                                 if (tommy_trie_count(&trie) != count
    7516         120 :                                         || memcmp(saved_trie, &trie, sizeof(trie)) != 0
    7517         120 :                                         || memcmp(saved_alloc, &alloc, sizeof(alloc)) != 0
    7518         120 :                                         || memcmp(saved_obj, obj, sizeof(obj)) != 0) {
    7519             :                                         /* LCOV_EXCL_START */
    7520             :                                         abort();
    7521             :                                         /* LCOV_EXCL_STOP */
    7522             :                                 }
    7523             :                         }
    7524          30 :                         trie_remove_ends_check(&trie, obj, sizes[s]);
    7525          30 :                         tommy_allocator_done(&alloc);
    7526             :                 }
    7527             :         }
    7528           1 : }
    7529             : 
    7530           1 : void test_trie_clear(void)
    7531             : {
    7532           1 :         const unsigned counts[] = { 0, 1, 2 * TOMMY_TRIE_BIT, 128 };
    7533           1 :         tommy_key_t high = 1;
    7534           1 :         high <<= TOMMY_TRIE_BIT - 1;
    7535           1 :         tommy_key_t max = high | (high - 1);
    7536             : 
    7537           5 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    7538          20 :                 for (unsigned shape = 0; shape < 4; ++shape) {
    7539             :                         struct object_trie obj[128], shared_obj[2];
    7540             :                         tommy_allocator alloc;
    7541             :                         tommy_trie trie, shared;
    7542          16 :                         unsigned size = counts[c];
    7543          16 :                         tommy_allocator_init(&alloc, TOMMY_TRIE_BLOCK_SIZE, TOMMY_TRIE_BLOCK_SIZE);
    7544          16 :                         tommy_trie_init(&trie, &alloc);
    7545          16 :                         tommy_trie_init(&shared, &alloc);
    7546          48 :                         for (unsigned i = 0; i < 2; ++i)
    7547          32 :                                 tommy_trie_insert(&shared, &shared_obj[i].node, &shared_obj[i], i);
    7548             : 
    7549         788 :                         for (unsigned i = 0; i < size; ++i) {
    7550         772 :                                 obj[i].value = i;
    7551             :                                 tommy_key_t key;
    7552         772 :                                 if (shape == 0)
    7553         193 :                                         key = i / 2;
    7554         579 :                                 else if (shape == 1)
    7555         193 :                                         key = max - i / 2;
    7556         386 :                                 else if (shape == 2)
    7557         193 :                                         key = i < 2 ? (i ? max : 0) : tommy_inthash_u32(i / 2) & max;
    7558             :                                 else
    7559         193 :                                         key = max;
    7560         772 :                                 tommy_trie_insert(&trie, &obj[i].node, &obj[i], key);
    7561             :                         }
    7562             : 
    7563          48 :                         for (unsigned repeat = 0; repeat < 2; ++repeat) {
    7564          32 :                                 tommy_trie_clear(&trie);
    7565          32 :                                 if (!tommy_trie_empty(&trie) || tommy_trie_count(&trie) != 0 || trie.node_count != 0
    7566          32 :                                         || trie.alloc != &alloc || tommy_trie_memory_usage(&trie) != 0
    7567          32 :                                         || alloc.count != shared.node_count || tommy_trie_count(&shared) != 2
    7568          32 :                                         || tommy_trie_search(&shared, 0) != &shared_obj[0] || tommy_trie_search(&shared, 1) != &shared_obj[1]) {
    7569             :                                         /* LCOV_EXCL_START */
    7570             :                                         abort();
    7571             :                                         /* LCOV_EXCL_STOP */
    7572             :                                 }
    7573        1056 :                                 for (unsigned i = 0; i < TOMMY_TRIE_BUCKET_MAX; ++i) {
    7574        1024 :                                         if (trie.bucket[i] != 0) {
    7575             :                                                 /* LCOV_EXCL_START */
    7576             :                                                 abort();
    7577             :                                                 /* LCOV_EXCL_STOP */
    7578             :                                         }
    7579             :                                 }
    7580             :                         }
    7581             : 
    7582             :                         /* reinsert the nodes into the cleared trie and exercise both removal paths. */
    7583         788 :                         for (unsigned i = 0; i < size; ++i) {
    7584         772 :                                 tommy_trie_insert(&trie, &obj[i].node, &obj[i], obj[i].node.index);
    7585         772 :                                 if (tommy_trie_search(&trie, obj[i].node.index) == 0) {
    7586             :                                         /* LCOV_EXCL_START */
    7587             :                                         abort();
    7588             :                                         /* LCOV_EXCL_STOP */
    7589             :                                 }
    7590             :                         }
    7591         788 :                         for (unsigned i = 0; i < size; ++i) {
    7592         384 :                                 void* data = i % 2 ? tommy_trie_remove_existing(&trie, &obj[i].node)
    7593         772 :                                         : tommy_trie_remove(&trie, obj[i].node.index);
    7594         772 :                                 if (data != &obj[i]) {
    7595             :                                         /* LCOV_EXCL_START */
    7596             :                                         abort();
    7597             :                                         /* LCOV_EXCL_STOP */
    7598             :                                 }
    7599             :                         }
    7600          16 :                         if (!tommy_trie_empty(&trie) || trie.node_count != 0 || alloc.count != shared.node_count) {
    7601             :                                 /* LCOV_EXCL_START */
    7602             :                                 abort();
    7603             :                                 /* LCOV_EXCL_STOP */
    7604             :                         }
    7605          48 :                         for (unsigned i = 0; i < 2; ++i)
    7606          32 :                                 tommy_trie_remove_existing(&shared, &shared_obj[i].node);
    7607          16 :                         if (alloc.count != 0) {
    7608             :                                 /* LCOV_EXCL_START */
    7609             :                                 abort();
    7610             :                                 /* LCOV_EXCL_STOP */
    7611             :                         }
    7612          16 :                         tommy_allocator_done(&alloc);
    7613             :                 }
    7614             :         }
    7615             : 
    7616             :         /* test foreach deallocation followed by clear */
    7617             :         {
    7618             :                 tommy_allocator alloc;
    7619             :                 tommy_trie trie;
    7620           1 :                 tommy_allocator_init(&alloc, TOMMY_TRIE_BLOCK_SIZE, TOMMY_TRIE_BLOCK_SIZE);
    7621           1 :                 tommy_trie_init(&trie, &alloc);
    7622             : 
    7623          17 :                 for (unsigned i = 0; i < 16; ++i) {
    7624          16 :                         struct object_trie* allocated = malloc(sizeof(*allocated));
    7625          16 :                         allocated->value = i;
    7626          16 :                         tommy_trie_insert(&trie, &allocated->node, allocated, i);
    7627             :                 }
    7628           1 :                 tommy_trie_foreach(&trie, free);
    7629           1 :                 tommy_trie_clear(&trie);
    7630           1 :                 if (!tommy_trie_empty(&trie) || trie.node_count != 0 || alloc.count != 0) {
    7631             :                         /* LCOV_EXCL_START */
    7632             :                         abort();
    7633             :                         /* LCOV_EXCL_STOP */
    7634             :                 }
    7635           1 :                 tommy_allocator_done(&alloc);
    7636             :         }
    7637           1 : }
    7638             : 
    7639           1 : void test_trie_to_list(void)
    7640             : {
    7641           1 :         const unsigned counts[] = { 0, 1, 2 * TOMMY_TRIE_BIT, 128 };
    7642           1 :         tommy_key_t high = 1;
    7643           1 :         high <<= TOMMY_TRIE_BIT - 1;
    7644           1 :         tommy_key_t max = high | (high - 1);
    7645             : 
    7646           5 :         for (unsigned c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) {
    7647          20 :                 for (unsigned shape = 0; shape < 4; ++shape) {
    7648          48 :                         for (unsigned prefix = 0; prefix <= 2; prefix += 2) {
    7649             :                                 struct object_trie obj[130], shared_obj[2];
    7650             :                                 void* expected[130];
    7651             :                                 tommy_key_t saved_key[130];
    7652             :                                 tommy_allocator alloc;
    7653             :                                 tommy_trie trie, shared;
    7654             :                                 tommy_list list;
    7655          32 :                                 unsigned size = prefix + counts[c];
    7656          32 :                                 tommy_allocator_init(&alloc, TOMMY_TRIE_BLOCK_SIZE, TOMMY_TRIE_BLOCK_SIZE);
    7657          32 :                                 tommy_trie_init(&trie, &alloc);
    7658          32 :                                 tommy_trie_init(&shared, &alloc);
    7659          32 :                                 tommy_list_init(&list);
    7660          96 :                                 for (unsigned i = 0; i < 2; ++i)
    7661          64 :                                         tommy_trie_insert(&shared, &shared_obj[i].node, &shared_obj[i], i);
    7662             : 
    7663        1608 :                                 for (unsigned i = 0; i < size; ++i) {
    7664        1576 :                                         obj[i].value = i;
    7665        1576 :                                         expected[i] = &obj[i];
    7666        1576 :                                         if (i < prefix) {
    7667          32 :                                                 obj[i].node.index = max;
    7668          32 :                                                 tommy_list_insert_tail(&list, &obj[i].node, &obj[i]);
    7669             :                                         } else {
    7670        1544 :                                                 unsigned pos = i - prefix;
    7671             :                                                 tommy_key_t key;
    7672        1544 :                                                 if (shape == 0)
    7673         386 :                                                         key = pos / 2;
    7674        1158 :                                                 else if (shape == 1)
    7675         386 :                                                         key = max - pos / 2;
    7676         772 :                                                 else if (shape == 2)
    7677         386 :                                                         key = pos < 2 ? (pos ? max : 0) : tommy_inthash_u32(pos / 2) & max;
    7678             :                                                 else
    7679         386 :                                                         key = max;
    7680        1544 :                                                 tommy_trie_insert(&trie, &obj[i].node, &obj[i], key);
    7681             :                                         }
    7682        1576 :                                         saved_key[i] = obj[i].node.index;
    7683             :                                 }
    7684             :                                 /* the oracle sorts by key and original insertion position, excluding the existing prefix. */
    7685          32 :                                 qsort(expected + prefix, counts[c], sizeof(expected[0]), trie_foreach_compare);
    7686             : 
    7687          96 :                                 for (unsigned repeat = 0; repeat < 2; ++repeat) {
    7688          64 :                                         tommy_trie_to_list(&trie, &list);
    7689          64 :                                         if (!tommy_trie_empty(&trie) || tommy_trie_count(&trie) != 0 || trie.node_count != 0
    7690          64 :                                                 || trie.alloc != &alloc || tommy_trie_memory_usage(&trie) != 0
    7691          64 :                                                 || alloc.count != shared.node_count || tommy_trie_count(&shared) != 2
    7692          64 :                                                 || tommy_trie_search(&shared, 0) != &shared_obj[0] || tommy_trie_search(&shared, 1) != &shared_obj[1]
    7693          64 :                                                 || tommy_list_count(&list) != size) {
    7694             :                                                 /* LCOV_EXCL_START */
    7695             :                                                 abort();
    7696             :                                                 /* LCOV_EXCL_STOP */
    7697             :                                         }
    7698        2112 :                                         for (unsigned i = 0; i < TOMMY_TRIE_BUCKET_MAX; ++i) {
    7699        2048 :                                                 if (trie.bucket[i] != 0) {
    7700             :                                                         /* LCOV_EXCL_START */
    7701             :                                                         abort();
    7702             :                                                         /* LCOV_EXCL_STOP */
    7703             :                                                 }
    7704             :                                         }
    7705          64 :                                         tommy_node* node = tommy_list_head(&list);
    7706          64 :                                         struct object_trie* tail = size ? expected[size - 1] : 0;
    7707          64 :                                         tommy_node* prev = tail ? &tail->node : 0;
    7708        3216 :                                         for (unsigned i = 0; i < size; ++i) {
    7709        3152 :                                                 struct object_trie* object = expected[i];
    7710        3152 :                                                 if (node != &object->node || node->prev != prev
    7711        3152 :                                                         || node->data != object || node->index != saved_key[object->value]) {
    7712             :                                                         /* LCOV_EXCL_START */
    7713             :                                                         abort();
    7714             :                                                         /* LCOV_EXCL_STOP */
    7715             :                                                 }
    7716        3152 :                                                 prev = node;
    7717        3152 :                                                 node = node->next;
    7718             :                                         }
    7719          64 :                                         if (node != 0 || tommy_list_tail(&list) != prev) {
    7720             :                                                 /* LCOV_EXCL_START */
    7721             :                                                 abort();
    7722             :                                                 /* LCOV_EXCL_STOP */
    7723             :                                         }
    7724          64 :                                         node = tommy_list_tail(&list);
    7725        3216 :                                         for (unsigned i = size; i > 0; --i) {
    7726        3152 :                                                 struct object_trie* object = expected[i - 1];
    7727        3152 :                                                 if (node != &object->node) {
    7728             :                                                         /* LCOV_EXCL_START */
    7729             :                                                         abort();
    7730             :                                                         /* LCOV_EXCL_STOP */
    7731             :                                                 }
    7732        3152 :                                                 node = tommy_list_prev(&list, node);
    7733             :                                         }
    7734          64 :                                         if (node != 0) {
    7735             :                                                 /* LCOV_EXCL_START */
    7736             :                                                 abort();
    7737             :                                                 /* LCOV_EXCL_STOP */
    7738             :                                         }
    7739             :                                 }
    7740             : 
    7741             :                                 /* reinsert the transferred nodes into the drained trie and exercise both removal paths. */
    7742        1608 :                                 while (!tommy_list_empty(&list)) {
    7743        1576 :                                         struct object_trie* data = tommy_list_remove_head(&list);
    7744        1576 :                                         tommy_trie_insert(&trie, &data->node, data, data->node.index);
    7745        1576 :                                         if (tommy_trie_search(&trie, data->node.index) == 0) {
    7746             :                                                 /* LCOV_EXCL_START */
    7747             :                                                 abort();
    7748             :                                                 /* LCOV_EXCL_STOP */
    7749             :                                         }
    7750             :                                 }
    7751        1608 :                                 for (unsigned i = 0; i < size; ++i) {
    7752        1576 :                                         struct object_trie* object = expected[i];
    7753         784 :                                         void* data = i % 2 ? tommy_trie_remove_existing(&trie, &object->node)
    7754        1576 :                                                 : tommy_trie_remove(&trie, object->node.index);
    7755        1576 :                                         if (data != expected[i]) {
    7756             :                                                 /* LCOV_EXCL_START */
    7757             :                                                 abort();
    7758             :                                                 /* LCOV_EXCL_STOP */
    7759             :                                         }
    7760             :                                 }
    7761          32 :                                 if (!tommy_trie_empty(&trie) || trie.node_count != 0 || alloc.count != shared.node_count) {
    7762             :                                         /* LCOV_EXCL_START */
    7763             :                                         abort();
    7764             :                                         /* LCOV_EXCL_STOP */
    7765             :                                 }
    7766          96 :                                 for (unsigned i = 0; i < 2; ++i)
    7767          64 :                                         tommy_trie_remove_existing(&shared, &shared_obj[i].node);
    7768          32 :                                 if (alloc.count != 0) {
    7769             :                                         /* LCOV_EXCL_START */
    7770             :                                         abort();
    7771             :                                         /* LCOV_EXCL_STOP */
    7772             :                                 }
    7773          32 :                                 tommy_allocator_done(&alloc);
    7774             :                         }
    7775             :                 }
    7776             :         }
    7777           1 : }
    7778             : 
    7779           1 : void test_trie_foreach_order(void)
    7780             : {
    7781           1 :         const unsigned size = 128;
    7782           1 :         const tommy_key_t example[] = { 5, 10, 2 };
    7783           1 :         tommy_key_t high = 1;
    7784           1 :         high <<= TOMMY_TRIE_BIT - 1;
    7785           1 :         tommy_key_t max = high | (high - 1);
    7786           5 :         for (unsigned shape = 0; shape < 4; ++shape) {
    7787          12 :                 for (unsigned remove = 0; remove < 2; ++remove) {
    7788          24 :                         for (unsigned mode = 0; mode < 2; ++mode) {
    7789             :                                 tommy_allocator alloc;
    7790             :                                 tommy_trie trie;
    7791             :                                 struct object_trie* obj[128];
    7792          16 :                                 tommy_allocator_init(&alloc, TOMMY_TRIE_BLOCK_SIZE, TOMMY_TRIE_BLOCK_SIZE);
    7793          16 :                                 tommy_trie_init(&trie, &alloc);
    7794        2064 :                                 for (unsigned i = 0; i < size; ++i) {
    7795             :                                         tommy_key_t key;
    7796        2048 :                                         if (shape == 0)
    7797         512 :                                                 key = i / 2;
    7798        1536 :                                         else if (shape == 1)
    7799         512 :                                                 key = max - i / 2;
    7800        1024 :                                         else if (shape == 2)
    7801         512 :                                                 key = tommy_inthash_u32(i / 2) & max;
    7802             :                                         else
    7803         512 :                                                 key = i < 3 ? example[i] : tommy_inthash_u32(i) % 16;
    7804        2048 :                                         obj[i] = malloc(sizeof(*obj[i]));
    7805        2048 :                                         obj[i]->value = i;
    7806        2048 :                                         tommy_trie_insert(&trie, &obj[i]->node, obj[i], key);
    7807             :                                 }
    7808             :                                 /* exercise removals from duplicate buckets */
    7809          16 :                                 if (remove) {
    7810         352 :                                         for (unsigned i = 0; i < size; i += 3) {
    7811         344 :                                                 tommy_trie_remove_existing(&trie, &obj[i]->node);
    7812         344 :                                                 free(obj[i]);
    7813         344 :                                                 obj[i] = 0;
    7814             :                                         }
    7815             :                                 }
    7816          16 :                                 trie_foreach_check.size = 0;
    7817        2064 :                                 for (unsigned i = 0; i < size; ++i)
    7818        2048 :                                         if (obj[i])
    7819        1704 :                                                 trie_foreach_check.expected[trie_foreach_check.size++] = obj[i];
    7820             :                                 /* sort independently by key and original insertion position */
    7821          16 :                                 qsort(trie_foreach_check.expected, trie_foreach_check.size, sizeof(trie_foreach_check.expected[0]), trie_foreach_compare);
    7822          48 :                                 for (unsigned deallocate = 0; deallocate < 2; ++deallocate) {
    7823          32 :                                         trie_foreach_check.deallocate = deallocate;
    7824          32 :                                         trie_foreach_check.pos = 0;
    7825          32 :                                         unsigned pos = 0;
    7826          32 :                                         if (mode)
    7827          16 :                                                 tommy_trie_foreach_arg(&trie, trie_foreach_check_arg_callback, &pos);
    7828             :                                         else
    7829          16 :                                                 tommy_trie_foreach(&trie, trie_foreach_check_callback);
    7830          32 :                                         if ((mode ? pos : trie_foreach_check.pos) != trie_foreach_check.size
    7831          32 :                                                 || tommy_trie_count(&trie) != trie_foreach_check.size) {
    7832             :                                                 /* LCOV_EXCL_START */
    7833             :                                                 abort();
    7834             :                                                 /* LCOV_EXCL_STOP */
    7835             :                                         }
    7836             :                                 }
    7837          16 :                                 tommy_allocator_done(&alloc);
    7838             :                         }
    7839             :                 }
    7840             :         }
    7841           1 : }
    7842             : 
    7843           1 : void test_trie_inplace_foreach(void)
    7844             : {
    7845           1 :         tommy_key_t high = 1;
    7846           1 :         high <<= TOMMY_TRIE_INPLACE_BIT - 1;
    7847           1 :         tommy_key_t max = high | (high - 1);
    7848           1 :         tommy_key_t branch = 1;
    7849           1 :         branch <<= TOMMY_TRIE_INPLACE_TREE_SHIFT;
    7850           1 :         const tommy_key_t keys[] = { 0, 3 * branch, branch, 1, 2, 0, branch, max, max };
    7851             :         /* keys increase globally; duplicates retain insertion order */
    7852           1 :         const unsigned order[] = { 0, 5, 3, 4, 2, 6, 1, 7, 8 };
    7853           1 :         const unsigned sizes[] = { 0, 1, 9 };
    7854             : 
    7855           4 :         for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); ++s) {
    7856          15 :                 for (unsigned mode = 0; mode < 4; ++mode) {
    7857             :                         tommy_trie_inplace trie;
    7858             :                         struct object_trie_inplace* obj[9];
    7859             :                         struct object_trie_inplace before[9];
    7860          12 :                         tommy_trie_inplace_init(&trie);
    7861          52 :                         for (unsigned i = 0; i < sizes[s]; ++i) {
    7862          40 :                                 obj[i] = calloc(1, sizeof(*obj[i]));
    7863          40 :                                 obj[i]->value = i;
    7864          40 :                                 tommy_trie_inplace_insert(&trie, &obj[i]->node, obj[i], keys[i]);
    7865             :                         }
    7866          52 :                         for (unsigned i = 0; i < sizes[s]; ++i) {
    7867          40 :                                 memcpy(&before[i], obj[i], sizeof(before[i]));
    7868          40 :                                 trie_foreach_check.expected[i] = obj[sizes[s] == 1 ? 0 : order[i]];
    7869             :                         }
    7870             :                         unsigned char saved[sizeof(trie)];
    7871          12 :                         memcpy(saved, &trie, sizeof(saved));
    7872          12 :                         trie_foreach_check.size = sizes[s];
    7873          12 :                         trie_foreach_check.deallocate = mode >= 2;
    7874          12 :                         trie_foreach_check.pos = 0;
    7875          12 :                         unsigned pos = 0;
    7876          12 :                         if (mode % 2)
    7877           6 :                                 tommy_trie_inplace_foreach_arg(&trie, trie_foreach_check_arg_callback, &pos);
    7878             :                         else
    7879           6 :                                 tommy_trie_inplace_foreach(&trie, trie_foreach_check_callback);
    7880          12 :                         if ((mode % 2 ? pos : trie_foreach_check.pos) != sizes[s]
    7881          12 :                                 || memcmp(saved, &trie, sizeof(trie)) != 0) {
    7882             :                                 /* LCOV_EXCL_START */
    7883             :                                 abort();
    7884             :                                 /* LCOV_EXCL_STOP */
    7885             :                         }
    7886          12 :                         if (!trie_foreach_check.deallocate) {
    7887          26 :                                 for (unsigned i = 0; i < sizes[s]; ++i) {
    7888          20 :                                         if (memcmp(&before[i], obj[i], sizeof(before[i])) != 0) {
    7889             :                                                 /* LCOV_EXCL_START */
    7890             :                                                 abort();
    7891             :                                                 /* LCOV_EXCL_STOP */
    7892             :                                         }
    7893             :                                 }
    7894          26 :                                 for (unsigned i = 0; i < sizes[s]; ++i) {
    7895          20 :                                         if (tommy_trie_inplace_remove_existing(&trie, &obj[i]->node) != obj[i]) {
    7896             :                                                 /* LCOV_EXCL_START */
    7897             :                                                 abort();
    7898             :                                                 /* LCOV_EXCL_STOP */
    7899             :                                         }
    7900          20 :                                         free(obj[i]);
    7901             :                                 }
    7902             :                         }
    7903          12 :                         tommy_trie_inplace_init(&trie);
    7904             :                 }
    7905             :         }
    7906           1 : }
    7907             : 
    7908       30893 : static int trie_inplace_foreach_compare(const void* void_a, const void* void_b)
    7909             : {
    7910       30893 :         void* const* ptr_a = void_a;
    7911       30893 :         void* const* ptr_b = void_b;
    7912       30893 :         const struct object_trie_inplace* a = *ptr_a;
    7913       30893 :         const struct object_trie_inplace* b = *ptr_b;
    7914       30893 :         if (a->node.key < b->node.key)
    7915       10634 :                 return -1;
    7916       20259 :         if (a->node.key > b->node.key)
    7917       14027 :                 return 1;
    7918        6232 :         return (a->value > b->value) - (a->value < b->value);
    7919             : }
    7920             : 
    7921         160 : static void trie_inplace_search_bounds_check(tommy_trie_inplace* trie, void** expected, unsigned size)
    7922             : {
    7923         160 :         tommy_key_t high = 1;
    7924         160 :         high <<= TOMMY_TRIE_INPLACE_BIT - 1;
    7925         160 :         tommy_key_t max = high | (high - 1);
    7926             : 
    7927             :         /* query existing keys, gaps, arbitrary prefixes, and both key-range endpoints */
    7928       43560 :         for (unsigned i = 0; i < size + 258; ++i) {
    7929             :                 tommy_key_t key;
    7930       43400 :                 if (i < size) {
    7931        2120 :                         struct object_trie_inplace* obj = expected[i];
    7932        2120 :                         key = obj->node.key;
    7933       41280 :                 } else if (i < size + 128)
    7934       20480 :                         key = tommy_inthash_u32(i - size) & max;
    7935       20800 :                 else if (i < size + 256)
    7936       20480 :                         key = i - size - 128;
    7937             :                 else
    7938         320 :                         key = i == size + 256 ? 0 : max;
    7939             : 
    7940       43400 :                 const tommy_key_t queries[] = { key ? key - 1 : 0, key, key < max ? key + 1 : max };
    7941      173600 :                 for (unsigned q = 0; q < sizeof(queries) / sizeof(queries[0]); ++q) {
    7942      130200 :                         struct object_trie_inplace* less = 0;
    7943      130200 :                         struct object_trie_inplace* less_equal = 0;
    7944      130200 :                         struct object_trie_inplace* greater_equal = 0;
    7945      130200 :                         struct object_trie_inplace* greater = 0;
    7946             :                         /* the oracle uses only independently sorted keys and insertion positions */
    7947     2380512 :                         for (unsigned j = 0; j < size; ++j) {
    7948     2250312 :                                 struct object_trie_inplace* obj = expected[j];
    7949     2250312 :                                 if (obj->node.key < queries[q])
    7950      960719 :                                         less = obj;
    7951     2250312 :                                 if (obj->node.key <= queries[q])
    7952     1030186 :                                         less_equal = obj;
    7953     2250312 :                                 if (obj->node.key >= queries[q] && !greater_equal)
    7954       39511 :                                         greater_equal = obj;
    7955     2250312 :                                 if (obj->node.key > queries[q] && !greater)
    7956       38564 :                                         greater = obj;
    7957             :                         }
    7958      130200 :                         if (tommy_trie_inplace_search_less(trie, queries[q]) != less
    7959      130200 :                                 || tommy_trie_inplace_search_less_equal(trie, queries[q]) != less_equal
    7960      130200 :                                 || tommy_trie_inplace_search_greater_equal(trie, queries[q]) != greater_equal
    7961      130200 :                                 || tommy_trie_inplace_search_greater(trie, queries[q]) != greater) {
    7962             :                                 /* LCOV_EXCL_START */
    7963             :                                 abort();
    7964             :                                 /* LCOV_EXCL_STOP */
    7965             :                         }
    7966      130200 :                         tommy_trie_inplace_node* expected_inplace_less = less ? &less->node : 0;
    7967      130200 :                         tommy_trie_inplace_node* expected_inplace_less_equal = less_equal ? &less_equal->node : 0;
    7968      130200 :                         tommy_trie_inplace_node* expected_inplace_greater_equal = greater_equal ? &greater_equal->node : 0;
    7969      130200 :                         tommy_trie_inplace_node* expected_inplace_greater = greater ? &greater->node : 0;
    7970      130200 :                         if (tommy_trie_inplace_find_less(trie, queries[q]) != expected_inplace_less
    7971      130200 :                                 || tommy_trie_inplace_find_less_equal(trie, queries[q]) != expected_inplace_less_equal
    7972      130200 :                                 || tommy_trie_inplace_find_greater_equal(trie, queries[q]) != expected_inplace_greater_equal
    7973      130200 :                                 || tommy_trie_inplace_find_greater(trie, queries[q]) != expected_inplace_greater
    7974      130200 :                                 || tommy_trie_inplace_find(trie, queries[q]) != tommy_trie_inplace_bucket(trie, queries[q])) {
    7975             :                                 /* LCOV_EXCL_START */
    7976             :                                 abort();
    7977             :                                 /* LCOV_EXCL_STOP */
    7978             :                         }
    7979             :                 }
    7980             :         }
    7981         160 : }
    7982             : 
    7983        3376 : static void trie_inplace_iteration_check(tommy_trie_inplace* trie, void** expected, unsigned size)
    7984             : {
    7985        3376 :         tommy_trie_inplace_node* node = tommy_trie_inplace_head(trie);
    7986      200664 :         for (unsigned i = 0; i < size; ++i) {
    7987      197288 :                 struct object_trie_inplace* obj = expected[i];
    7988      197288 :                 if (node != &obj->node) {
    7989             :                         /* LCOV_EXCL_START */
    7990             :                         abort();
    7991             :                         /* LCOV_EXCL_STOP */
    7992             :                 }
    7993      197288 :                 node = tommy_trie_inplace_next(trie, node);
    7994             :         }
    7995        3376 :         if (node != 0) {
    7996             :                 /* LCOV_EXCL_START */
    7997             :                 abort();
    7998             :                 /* LCOV_EXCL_STOP */
    7999             :         }
    8000             : 
    8001        3376 :         node = tommy_trie_inplace_tail(trie);
    8002      200664 :         for (unsigned i = size; i != 0; ) {
    8003      197288 :                 struct object_trie_inplace* obj = expected[--i];
    8004      197288 :                 if (node != &obj->node) {
    8005             :                         /* LCOV_EXCL_START */
    8006             :                         abort();
    8007             :                         /* LCOV_EXCL_STOP */
    8008             :                 }
    8009      197288 :                 node = tommy_trie_inplace_prev(trie, node);
    8010             :         }
    8011        3376 :         if (node != 0) {
    8012             :                 /* LCOV_EXCL_START */
    8013             :                 abort();
    8014             :                 /* LCOV_EXCL_STOP */
    8015             :         }
    8016        3376 : }
    8017             : 
    8018          40 : static void trie_inplace_remove_ends_check(tommy_trie_inplace* trie, struct object_trie_inplace* obj, unsigned size)
    8019             : {
    8020             :         void* expected[128];
    8021         160 :         for (unsigned mode = 0; mode < 3; ++mode) {
    8022             :                 /* reuse the drained trie with the same keys and original duplicate insertion order */
    8023        3336 :                 for (unsigned i = 0; i < size; ++i) {
    8024        3216 :                         tommy_trie_inplace_insert(trie, &obj[i].node, &obj[i], obj[i].node.key);
    8025        3216 :                         expected[i] = &obj[i];
    8026             :                 }
    8027         120 :                 qsort(expected, size, sizeof(expected[0]), trie_inplace_foreach_compare);
    8028             : 
    8029         120 :                 unsigned head = 0;
    8030         120 :                 unsigned tail = size;
    8031        3336 :                 while (head != tail) {
    8032             :                         /* drain from either end or alternate, checking the complete remaining order */
    8033        3216 :                         tommy_bool_t remove_tail = mode == 1 || (mode == 2 && (tail - head) % 2 == 0);
    8034             :                         void* data;
    8035             :                         void* removed;
    8036        3216 :                         if (remove_tail) {
    8037        1600 :                                 removed = expected[--tail];
    8038        1600 :                                 data = tommy_trie_inplace_remove_tail(trie);
    8039             :                         } else {
    8040        1616 :                                 removed = expected[head];
    8041        1616 :                                 ++head;
    8042        1616 :                                 data = tommy_trie_inplace_remove_head(trie);
    8043             :                         }
    8044        3216 :                         if (data != removed || tommy_trie_inplace_count(trie) != tail - head) {
    8045             :                                 /* LCOV_EXCL_START */
    8046             :                                 abort();
    8047             :                                 /* LCOV_EXCL_STOP */
    8048             :                         }
    8049        3216 :                         trie_inplace_iteration_check(trie, expected + head, tail - head);
    8050             :                 }
    8051         120 :                 if (tommy_trie_inplace_remove_head(trie) != 0 || tommy_trie_inplace_remove_tail(trie) != 0
    8052         120 :                         || !tommy_trie_inplace_empty(trie) || tommy_trie_inplace_count(trie) != 0) {
    8053             :                         /* LCOV_EXCL_START */
    8054             :                         abort();
    8055             :                         /* LCOV_EXCL_STOP */
    8056             :                 }
    8057             :         }
    8058             : 
    8059             :         /* a zero data pointer must still remove the node */
    8060             :         tommy_trie_inplace_node node;
    8061         120 :         for (unsigned mode = 0; mode < 2; ++mode) {
    8062          80 :                 tommy_trie_inplace_insert(trie, &node, 0, 0);
    8063          80 :                 void* data = mode ? tommy_trie_inplace_remove_tail(trie) : tommy_trie_inplace_remove_head(trie);
    8064          80 :                 if (data != 0 || !tommy_trie_inplace_empty(trie) || tommy_trie_inplace_count(trie) != 0) {
    8065             :                         /* LCOV_EXCL_START */
    8066             :                         abort();
    8067             :                         /* LCOV_EXCL_STOP */
    8068             :                 }
    8069             :         }
    8070          40 : }
    8071             : 
    8072           1 : void test_trie_inplace_iteration(void)
    8073             : {
    8074           1 :         const unsigned sizes[] = { 0, 1, 2, 3, 128 };
    8075           1 :         tommy_key_t high = 1;
    8076           1 :         high <<= TOMMY_TRIE_INPLACE_BIT - 1;
    8077           1 :         tommy_key_t max = high | (high - 1);
    8078             : 
    8079           6 :         for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); ++s) {
    8080          45 :                 for (unsigned shape = 0; shape < 8; ++shape) {
    8081             :                         tommy_trie_inplace trie;
    8082             :                         struct object_trie_inplace obj[128];
    8083             :                         void* expected[128];
    8084          40 :                         memset(obj, 0, sizeof(obj));
    8085          40 :                         tommy_trie_inplace_init(&trie);
    8086        1112 :                         for (unsigned i = 0; i < sizes[s]; ++i) {
    8087             :                                 tommy_key_t key;
    8088        1072 :                                 if (shape == 0)
    8089         134 :                                         key = i;
    8090         938 :                                 else if (shape == 1)
    8091         134 :                                         key = max - i;
    8092         804 :                                 else if (shape == 2)
    8093         134 :                                         key = i < 2 ? (i ? max : 0) : tommy_inthash_u32(i) & max;
    8094         670 :                                 else if (shape == 3)
    8095         134 :                                         key = i < 6 ? (i % 2 ? max : 0) : tommy_inthash_u32(i / 3) & max;
    8096         536 :                                 else if (shape == 4)
    8097         134 :                                         key = max;
    8098         402 :                                 else if (shape == 6) {
    8099         134 :                                         const tommy_key_t example[] = { 5, 10, 2 };
    8100         134 :                                         key = example[i % 3];
    8101         268 :                                 } else if (shape == 7)
    8102         134 :                                         key = tommy_inthash_u32(i) % 64;
    8103             :                                 else {
    8104             :                                         /* sparse branches at every depth exercise replacement and retention of candidates */
    8105         134 :                                         key = 1;
    8106         134 :                                         key <<= (i / 2) % TOMMY_TRIE_INPLACE_BIT;
    8107         134 :                                         if (i % 2)
    8108          66 :                                                 key ^= max;
    8109             :                                 }
    8110        1072 :                                 obj[i].value = i;
    8111        1072 :                                 tommy_trie_inplace_insert(&trie, &obj[i].node, &obj[i], key);
    8112             :                         }
    8113             : 
    8114         200 :                         for (unsigned round = 0; round < 4; ++round) {
    8115             :                                 /* remove duplicate heads, middle nodes and tails, and replace internal nodes */
    8116         160 :                                 if (round)
    8117        1192 :                                         for (unsigned i = round - 1; i < sizes[s]; i += 3)
    8118        1072 :                                                 tommy_trie_inplace_remove_existing(&trie, &obj[i].node);
    8119         160 :                                 unsigned count = 0;
    8120        4448 :                                 for (unsigned i = 0; i < sizes[s]; ++i)
    8121        4288 :                                         if (i % 3 >= round)
    8122        2120 :                                                 expected[count++] = &obj[i];
    8123             :                                 /* sort independently by numeric key and insertion position */
    8124         160 :                                 qsort(expected, count, sizeof(expected[0]), trie_inplace_foreach_compare);
    8125             : 
    8126             :                                 unsigned char saved_trie[sizeof(trie)], saved_obj[sizeof(obj)];
    8127         160 :                                 memcpy(saved_trie, &trie, sizeof(trie));
    8128         160 :                                 memcpy(saved_obj, obj, sizeof(obj));
    8129         160 :                                 trie_inplace_iteration_check(&trie, expected, count);
    8130         160 :                                 trie_inplace_search_bounds_check(&trie, expected, count);
    8131         160 :                                 if (tommy_trie_inplace_count(&trie) != count
    8132         160 :                                         || memcmp(saved_trie, &trie, sizeof(trie)) != 0
    8133         160 :                                         || memcmp(saved_obj, obj, sizeof(obj)) != 0) {
    8134             :                                         /* LCOV_EXCL_START */
    8135             :                                         abort();
    8136             :                                         /* LCOV_EXCL_STOP */
    8137             :                                 }
    8138             :                         }
    8139          40 :                         trie_inplace_remove_ends_check(&trie, obj, sizes[s]);
    8140             :                 }
    8141             :         }
    8142           1 : }
    8143             : 
    8144           1 : void test_trie_inplace_foreach_order(void)
    8145             : {
    8146           1 :         const unsigned size = 128;
    8147           1 :         const tommy_key_t example[] = { 5, 10, 2 };
    8148           1 :         tommy_key_t high = 1;
    8149           1 :         high <<= TOMMY_TRIE_INPLACE_BIT - 1;
    8150           1 :         tommy_key_t max = high | (high - 1);
    8151           5 :         for (unsigned shape = 0; shape < 4; ++shape) {
    8152          12 :                 for (unsigned remove = 0; remove < 2; ++remove) {
    8153          24 :                         for (unsigned mode = 0; mode < 2; ++mode) {
    8154             :                                 tommy_trie_inplace trie;
    8155             :                                 struct object_trie_inplace* obj[128];
    8156          16 :                                 tommy_trie_inplace_init(&trie);
    8157        2064 :                                 for (unsigned i = 0; i < size; ++i) {
    8158             :                                         tommy_key_t key;
    8159        2048 :                                         if (shape == 0)
    8160         512 :                                                 key = i / 2;
    8161        1536 :                                         else if (shape == 1)
    8162         512 :                                                 key = max - i / 2;
    8163        1024 :                                         else if (shape == 2)
    8164         512 :                                                 key = tommy_inthash_u32(i / 2) & max;
    8165             :                                         else
    8166         512 :                                                 key = i < 3 ? example[i] : tommy_inthash_u32(i) % 16;
    8167        2048 :                                         obj[i] = malloc(sizeof(*obj[i]));
    8168        2048 :                                         obj[i]->value = i;
    8169        2048 :                                         tommy_trie_inplace_insert(&trie, &obj[i]->node, obj[i], key);
    8170             :                                 }
    8171             :                                 /* exercise promoted duplicates and replacement of internal nodes */
    8172          16 :                                 if (remove) {
    8173         352 :                                         for (unsigned i = 0; i < size; i += 3) {
    8174         344 :                                                 tommy_trie_inplace_remove_existing(&trie, &obj[i]->node);
    8175         344 :                                                 free(obj[i]);
    8176         344 :                                                 obj[i] = 0;
    8177             :                                         }
    8178             :                                 }
    8179          16 :                                 trie_foreach_check.size = 0;
    8180        2064 :                                 for (unsigned i = 0; i < size; ++i)
    8181        2048 :                                         if (obj[i])
    8182        1704 :                                                 trie_foreach_check.expected[trie_foreach_check.size++] = obj[i];
    8183             :                                 /* sort independently by key and original insertion position */
    8184          16 :                                 qsort(trie_foreach_check.expected, trie_foreach_check.size, sizeof(trie_foreach_check.expected[0]), trie_inplace_foreach_compare);
    8185          48 :                                 for (unsigned deallocate = 0; deallocate < 2; ++deallocate) {
    8186          32 :                                         trie_foreach_check.deallocate = deallocate;
    8187          32 :                                         trie_foreach_check.pos = 0;
    8188          32 :                                         unsigned pos = 0;
    8189          32 :                                         if (mode)
    8190          16 :                                                 tommy_trie_inplace_foreach_arg(&trie, trie_foreach_check_arg_callback, &pos);
    8191             :                                         else
    8192          16 :                                                 tommy_trie_inplace_foreach(&trie, trie_foreach_check_callback);
    8193          32 :                                         if ((mode ? pos : trie_foreach_check.pos) != trie_foreach_check.size
    8194          32 :                                                 || tommy_trie_inplace_count(&trie) != trie_foreach_check.size) {
    8195             :                                                 /* LCOV_EXCL_START */
    8196             :                                                 abort();
    8197             :                                                 /* LCOV_EXCL_STOP */
    8198             :                                         }
    8199             :                                 }
    8200             :                         }
    8201             :                 }
    8202             :         }
    8203           1 : }
    8204             : 
    8205         128 : static void trie_foreach_free_arg_callback(void* arg, void* data)
    8206             : {
    8207         128 :         count_arg_callback(arg, data);
    8208         128 :         free(data);
    8209         128 : }
    8210             : 
    8211           1 : void test_trie_foreach_depth(void)
    8212             : {
    8213             :         /* consecutive small keys force descents through all the remaining key bits */
    8214           1 :         const unsigned size = 2 * TOMMY_TRIE_BIT;
    8215           3 :         for (unsigned mode = 0; mode < 2; ++mode) {
    8216             :                 tommy_allocator alloc;
    8217             :                 tommy_trie trie;
    8218           2 :                 tommy_allocator_init(&alloc, TOMMY_TRIE_BLOCK_SIZE, TOMMY_TRIE_BLOCK_SIZE);
    8219           2 :                 tommy_trie_init(&trie, &alloc);
    8220         130 :                 for (unsigned i = 0; i < size; ++i) {
    8221         128 :                         struct object_trie* obj = malloc(sizeof(*obj));
    8222         128 :                         tommy_trie_insert(&trie, &obj->node, obj, i / 2);
    8223             :                 }
    8224           2 :                 if (mode) {
    8225           1 :                         unsigned count = 0;
    8226           1 :                         tommy_trie_foreach_arg(&trie, trie_foreach_free_arg_callback, &count);
    8227           1 :                         if (count != size) {
    8228             :                                 /* LCOV_EXCL_START */
    8229             :                                 abort();
    8230             :                                 /* LCOV_EXCL_STOP */
    8231             :                         }
    8232             :                 } else {
    8233           1 :                         tommy_trie_foreach(&trie, free);
    8234             :                 }
    8235           2 :                 tommy_allocator_done(&alloc);
    8236             :         }
    8237           1 : }
    8238             : 
    8239           1 : void test_trie_inplace_foreach_depth(void)
    8240             : {
    8241             :         /* consecutive small keys force descents through all the remaining key bits */
    8242           1 :         const unsigned size = 2 * TOMMY_TRIE_INPLACE_BIT;
    8243           3 :         for (unsigned mode = 0; mode < 2; ++mode) {
    8244             :                 tommy_trie_inplace trie_inplace;
    8245           2 :                 tommy_trie_inplace_init(&trie_inplace);
    8246         130 :                 for (unsigned i = 0; i < size; ++i) {
    8247         128 :                         struct object_trie_inplace* obj = malloc(sizeof(*obj));
    8248         128 :                         tommy_trie_inplace_insert(&trie_inplace, &obj->node, obj, i / 2);
    8249             :                 }
    8250           2 :                 if (mode) {
    8251           1 :                         unsigned count = 0;
    8252           1 :                         tommy_trie_inplace_foreach_arg(&trie_inplace, trie_foreach_free_arg_callback, &count);
    8253           1 :                         if (count != size) {
    8254             :                                 /* LCOV_EXCL_START */
    8255             :                                 abort();
    8256             :                                 /* LCOV_EXCL_STOP */
    8257             :                         }
    8258             :                 } else {
    8259           1 :                         tommy_trie_inplace_foreach(&trie_inplace, free);
    8260             :                 }
    8261             :         }
    8262           1 : }
    8263             : 
    8264           1 : void test_trie_swap(void)
    8265             : {
    8266           3 :         for (unsigned first_count = 0; first_count <= 3; first_count += 3) {
    8267           6 :                 for (unsigned second_count = 0; second_count <= 2; second_count += 2) {
    8268             :                         tommy_allocator first_alloc, second_alloc;
    8269             :                         tommy_trie first, second;
    8270             :                         struct object_trie obj[9], before[9];
    8271           4 :                         memset(obj, 0, sizeof(obj));
    8272           4 :                         tommy_allocator_init(&first_alloc, TOMMY_TRIE_BLOCK_SIZE, TOMMY_TRIE_BLOCK_SIZE);
    8273           4 :                         tommy_allocator_init(&second_alloc, TOMMY_TRIE_BLOCK_SIZE, TOMMY_TRIE_BLOCK_SIZE);
    8274           4 :                         tommy_trie_init(&first, &first_alloc);
    8275           4 :                         tommy_trie_init(&second, &second_alloc);
    8276          40 :                         for (unsigned i = 0; i < 9; ++i)
    8277          36 :                                 obj[i].value = i;
    8278          10 :                         for (unsigned i = 0; i < first_count; ++i)
    8279           6 :                                 tommy_trie_insert(&first, &obj[i].node, &obj[i], i);
    8280           8 :                         for (unsigned i = 0; i < second_count; ++i)
    8281           4 :                                 tommy_trie_insert(&second, &obj[3 + i].node, &obj[3 + i], 3 + i);
    8282           4 :                         tommy_trie saved_first = first, saved_second = second;
    8283           4 :                         memcpy(before, obj, sizeof(before));
    8284             : 
    8285           4 :                         tommy_trie_swap(&first, &second);
    8286          12 :                         for (unsigned phase = 0; phase < 2; ++phase) {
    8287           8 :                                 if (phase) {
    8288           4 :                                         tommy_trie_swap(&first, &first);
    8289           4 :                                         tommy_trie_swap(&second, &second);
    8290             :                                 }
    8291           8 :                                 if (first.alloc != &second_alloc || second.alloc != &first_alloc
    8292           8 :                                         || first.count != second_count || second.count != first_count
    8293           8 :                                         || first.node_count != saved_second.node_count || second.node_count != saved_first.node_count
    8294           8 :                                         || first_alloc.count != saved_first.node_count || second_alloc.count != saved_second.node_count
    8295           8 :                                         || memcmp(before, obj, sizeof(before)) != 0) {
    8296             :                                         /* LCOV_EXCL_START */
    8297             :                                         abort();
    8298             :                                         /* LCOV_EXCL_STOP */
    8299             :                                 }
    8300         264 :                                 for (unsigned i = 0; i < TOMMY_TRIE_BUCKET_MAX; ++i)
    8301         256 :                                         if (first.bucket[i] != saved_second.bucket[i] || second.bucket[i] != saved_first.bucket[i]) {
    8302             :                                                 /* LCOV_EXCL_START */
    8303             :                                                 abort();
    8304             :                                                 /* LCOV_EXCL_STOP */
    8305             :                                         }
    8306          48 :                                 for (unsigned i = 0; i < 5; ++i) {
    8307          40 :                                         if (tommy_trie_search(&first, i) != (i >= 3 && i < 3 + second_count ? &obj[i] : 0)
    8308          40 :                                                 || tommy_trie_search(&second, i) != (i < first_count ? &obj[i] : 0)) {
    8309             :                                                 /* LCOV_EXCL_START */
    8310             :                                                 abort();
    8311             :                                                 /* LCOV_EXCL_STOP */
    8312             :                                         }
    8313             :                                 }
    8314             :                         }
    8315             : 
    8316             :                         /* expansion and reduction must use the allocator that moved with the roots */
    8317           4 :                         tommy_key_t key = 4 * TOMMY_TRIE_TREE_MAX;
    8318           4 :                         tommy_trie_insert(&first, &obj[5].node, &obj[5], key);
    8319           4 :                         tommy_trie_insert(&first, &obj[6].node, &obj[6], key + 1);
    8320           4 :                         tommy_trie_insert(&second, &obj[7].node, &obj[7], key);
    8321           4 :                         tommy_trie_insert(&second, &obj[8].node, &obj[8], key + 1);
    8322           4 :                         if (first.node_count != second_alloc.count || second.node_count != first_alloc.count
    8323           4 :                                 || first.node_count <= saved_second.node_count || second.node_count <= saved_first.node_count
    8324           4 :                                 || tommy_trie_search(&first, key) != &obj[5] || tommy_trie_search(&first, key + 1) != &obj[6]
    8325           4 :                                 || tommy_trie_search(&second, key) != &obj[7] || tommy_trie_search(&second, key + 1) != &obj[8]
    8326           4 :                                 || tommy_trie_remove_existing(&first, &obj[5].node) != &obj[5]
    8327           4 :                                 || tommy_trie_remove_existing(&first, &obj[6].node) != &obj[6]
    8328           4 :                                 || tommy_trie_remove_existing(&second, &obj[7].node) != &obj[7]
    8329           4 :                                 || tommy_trie_remove_existing(&second, &obj[8].node) != &obj[8]) {
    8330             :                                 /* LCOV_EXCL_START */
    8331             :                                 abort();
    8332             :                                 /* LCOV_EXCL_STOP */
    8333             :                         }
    8334          24 :                         for (unsigned i = 0; i < 5; ++i) {
    8335          20 :                                 if (i < first_count || (i >= 3 && i < 3 + second_count)) {
    8336          10 :                                         tommy_trie* trie = i < 3 ? &second : &first;
    8337          10 :                                         void* removed = i % 2 ? tommy_trie_remove(trie, i) : tommy_trie_remove_existing(trie, &obj[i].node);
    8338          10 :                                         if (removed != &obj[i] || tommy_trie_search(trie, i) != 0) {
    8339             :                                                 /* LCOV_EXCL_START */
    8340             :                                                 abort();
    8341             :                                                 /* LCOV_EXCL_STOP */
    8342             :                                         }
    8343             :                                 }
    8344             :                         }
    8345           4 :                         if (!tommy_trie_empty(&first) || !tommy_trie_empty(&second)
    8346           4 :                                 || first.node_count != 0 || second.node_count != 0 || first_alloc.count != 0 || second_alloc.count != 0) {
    8347             :                                 /* LCOV_EXCL_START */
    8348             :                                 abort();
    8349             :                                 /* LCOV_EXCL_STOP */
    8350             :                         }
    8351           4 :                         tommy_allocator_done(&first_alloc);
    8352           4 :                         tommy_allocator_done(&second_alloc);
    8353             :                 }
    8354             :         }
    8355           1 : }
    8356             : 
    8357           1 : void test_trie_inplace_swap(void)
    8358             : {
    8359           3 :         for (unsigned first_count = 0; first_count <= 3; first_count += 3) {
    8360           6 :                 for (unsigned second_count = 0; second_count <= 2; second_count += 2) {
    8361             :                         tommy_trie_inplace first, second;
    8362             :                         struct object_trie_inplace obj[7], before[7];
    8363           4 :                         memset(obj, 0, sizeof(obj));
    8364           4 :                         tommy_trie_inplace_init(&first);
    8365           4 :                         tommy_trie_inplace_init(&second);
    8366          32 :                         for (unsigned i = 0; i < 7; ++i)
    8367          28 :                                 obj[i].value = i;
    8368          10 :                         for (unsigned i = 0; i < first_count; ++i)
    8369           6 :                                 tommy_trie_inplace_insert(&first, &obj[i].node, &obj[i], i);
    8370           8 :                         for (unsigned i = 0; i < second_count; ++i)
    8371           4 :                                 tommy_trie_inplace_insert(&second, &obj[3 + i].node, &obj[3 + i], 3 + i);
    8372           4 :                         tommy_trie_inplace saved_first = first, saved_second = second;
    8373           4 :                         memcpy(before, obj, sizeof(before));
    8374             : 
    8375           4 :                         tommy_trie_inplace_swap(&first, &second);
    8376          12 :                         for (unsigned phase = 0; phase < 2; ++phase) {
    8377           8 :                                 if (phase) {
    8378           4 :                                         tommy_trie_inplace_swap(&first, &first);
    8379           4 :                                         tommy_trie_inplace_swap(&second, &second);
    8380             :                                 }
    8381           8 :                                 if (first.count != second_count || second.count != first_count || memcmp(before, obj, sizeof(before)) != 0) {
    8382             :                                         /* LCOV_EXCL_START */
    8383             :                                         abort();
    8384             :                                         /* LCOV_EXCL_STOP */
    8385             :                                 }
    8386         520 :                                 for (unsigned i = 0; i < TOMMY_TRIE_INPLACE_BUCKET_MAX; ++i)
    8387         512 :                                         if (first.bucket[i] != saved_second.bucket[i] || second.bucket[i] != saved_first.bucket[i]) {
    8388             :                                                 /* LCOV_EXCL_START */
    8389             :                                                 abort();
    8390             :                                                 /* LCOV_EXCL_STOP */
    8391             :                                         }
    8392          48 :                                 for (unsigned i = 0; i < 5; ++i) {
    8393          40 :                                         if (tommy_trie_inplace_search(&first, i) != (i >= 3 && i < 3 + second_count ? &obj[i] : 0)
    8394          40 :                                                 || tommy_trie_inplace_search(&second, i) != (i < first_count ? &obj[i] : 0)) {
    8395             :                                                 /* LCOV_EXCL_START */
    8396             :                                                 abort();
    8397             :                                                 /* LCOV_EXCL_STOP */
    8398             :                                         }
    8399             :                                 }
    8400             :                         }
    8401             : 
    8402           4 :                         tommy_trie_inplace_insert(&first, &obj[5].node, &obj[5], 5);
    8403           4 :                         tommy_trie_inplace_insert(&second, &obj[6].node, &obj[6], 6);
    8404           4 :                         if (tommy_trie_inplace_search(&first, 5) != &obj[5] || tommy_trie_inplace_search(&second, 6) != &obj[6]
    8405           4 :                                 || tommy_trie_inplace_remove_existing(&first, &obj[5].node) != &obj[5]
    8406           4 :                                 || tommy_trie_inplace_remove_existing(&second, &obj[6].node) != &obj[6]) {
    8407             :                                 /* LCOV_EXCL_START */
    8408             :                                 abort();
    8409             :                                 /* LCOV_EXCL_STOP */
    8410             :                         }
    8411          24 :                         for (unsigned i = 0; i < 5; ++i) {
    8412          20 :                                 if (i < first_count || (i >= 3 && i < 3 + second_count)) {
    8413          10 :                                         tommy_trie_inplace* trie = i < 3 ? &second : &first;
    8414          10 :                                         void* removed = i % 2 ? tommy_trie_inplace_remove(trie, i) : tommy_trie_inplace_remove_existing(trie, &obj[i].node);
    8415          10 :                                         if (removed != &obj[i] || tommy_trie_inplace_search(trie, i) != 0) {
    8416             :                                                 /* LCOV_EXCL_START */
    8417             :                                                 abort();
    8418             :                                                 /* LCOV_EXCL_STOP */
    8419             :                                         }
    8420             :                                 }
    8421             :                         }
    8422           4 :                         if (!tommy_trie_inplace_empty(&first) || !tommy_trie_inplace_empty(&second)) {
    8423             :                                 /* LCOV_EXCL_START */
    8424             :                                 abort();
    8425             :                                 /* LCOV_EXCL_STOP */
    8426             :                         }
    8427             :                 }
    8428             :         }
    8429           1 : }
    8430             : 
    8431           1 : void test_trie(void)
    8432             : {
    8433           1 :         const unsigned size = TOMMY_SIZE * 4;
    8434             : 
    8435           1 :         struct object_trie* OBJ = malloc(size * sizeof(struct object_trie));
    8436             : 
    8437      400001 :         for (unsigned i = 0; i < size; ++i)
    8438      400000 :                 OBJ[i].value = i;
    8439             : 
    8440           6 :         SPEED_START("trie") {
    8441             :                 tommy_allocator alloc;
    8442           3 :                 tommy_allocator_init(&alloc, TOMMY_TRIE_BLOCK_SIZE, TOMMY_TRIE_BLOCK_SIZE);
    8443             :                 tommy_trie trie;
    8444           3 :                 tommy_trie_init(&trie, &alloc);
    8445             : 
    8446           3 :                 if (!tommy_trie_empty(&trie)) {
    8447             :                         /* LCOV_EXCL_START */
    8448             :                         abort();
    8449             :                         /* LCOV_EXCL_STOP */
    8450             :                 }
    8451             : 
    8452             :                 /* unique insertion uses the numeric key and preserves normal duplicate order */
    8453           3 :                 if (tommy_trie_insert_unique(&trie, &OBJ[0].node, &OBJ[0], 0) != &OBJ[0]
    8454           3 :                         || tommy_trie_insert_unique(&trie, &OBJ[1].node, &OBJ[1], 1) != &OBJ[1]
    8455           3 :                         || tommy_trie_count(&trie) != 2 || tommy_trie_empty(&trie)
    8456           3 :                         || tommy_trie_search(&trie, 0) != &OBJ[0] || tommy_trie_search(&trie, 1) != &OBJ[1]) {
    8457             :                         /* LCOV_EXCL_START */
    8458             :                         abort();
    8459             :                         /* LCOV_EXCL_STOP */
    8460             :                 }
    8461             :                 struct object_trie duplicate;
    8462           3 :                 tommy_trie_insert(&trie, &duplicate.node, &duplicate, 0);
    8463             : 
    8464             :                 /* rejection must not modify any candidate bytes, trie state, or allocator state */
    8465             :                 struct object_trie candidate;
    8466           3 :                 memset(&candidate.node, 0xa5, sizeof(candidate.node));
    8467             :                 unsigned char node_before[sizeof(candidate.node)];
    8468             :                 unsigned char trie_before[sizeof(trie)];
    8469             :                 unsigned char alloc_before[sizeof(alloc)];
    8470           3 :                 memcpy(node_before, &candidate.node, sizeof(node_before));
    8471           3 :                 memcpy(trie_before, &trie, sizeof(trie_before));
    8472           3 :                 memcpy(alloc_before, &alloc, sizeof(alloc_before));
    8473           9 :                 for (unsigned i = 0; i < 2; ++i) {
    8474           6 :                         if (tommy_trie_insert_unique(&trie, &candidate.node, &candidate, i) != &OBJ[i]
    8475           6 :                                 || tommy_trie_count(&trie) != 3 || tommy_trie_empty(&trie)
    8476           6 :                                 || memcmp(node_before, &candidate.node, sizeof(node_before)) != 0
    8477           6 :                                 || memcmp(trie_before, &trie, sizeof(trie_before)) != 0
    8478           6 :                                 || memcmp(alloc_before, &alloc, sizeof(alloc_before)) != 0
    8479           6 :                                 || tommy_trie_search(&trie, i) != &OBJ[i]) {
    8480             :                                 /* LCOV_EXCL_START */
    8481             :                                 abort();
    8482             :                                 /* LCOV_EXCL_STOP */
    8483             :                         }
    8484             :                 }
    8485             : 
    8486             :                 /* continue unique insertion from empty child pointers and colliding leaves */
    8487           3 :                 const unsigned keys[] = { 2, 2 * TOMMY_TRIE_TREE_MAX, 2 * TOMMY_TRIE_TREE_MAX + 1 };
    8488          12 :                 for (unsigned i = 0; i < sizeof(keys) / sizeof(keys[0]); ++i) {
    8489           9 :                         unsigned key = keys[i];
    8490           9 :                         if (tommy_trie_insert_unique(&trie, &OBJ[key].node, &OBJ[key], key) != &OBJ[key]
    8491           9 :                                 || tommy_trie_count(&trie) != i + 4 || tommy_trie_search(&trie, key) != &OBJ[key]) {
    8492             :                                 /* LCOV_EXCL_START */
    8493             :                                 abort();
    8494             :                                 /* LCOV_EXCL_STOP */
    8495             :                         }
    8496             :                 }
    8497             : 
    8498             :                 /* drain through both removal paths, preserving the newly inserted branches */
    8499           3 :                 if (tommy_trie_remove(&trie, 0) != &OBJ[0] || tommy_trie_search(&trie, 0) != &duplicate
    8500           3 :                         || tommy_trie_remove_existing(&trie, &duplicate.node) != &duplicate || tommy_trie_search(&trie, 0) != 0
    8501           3 :                         || tommy_trie_empty(&trie) || tommy_trie_search(&trie, 1) != &OBJ[1]
    8502           3 :                         || tommy_trie_remove_existing(&trie, &OBJ[1].node) != &OBJ[1] || tommy_trie_search(&trie, 1) != 0) {
    8503             :                         /* LCOV_EXCL_START */
    8504             :                         abort();
    8505             :                         /* LCOV_EXCL_STOP */
    8506             :                 }
    8507          12 :                 for (unsigned i = 0; i < sizeof(keys) / sizeof(keys[0]); ++i) {
    8508           9 :                         unsigned key = keys[i];
    8509           9 :                         if (tommy_trie_search(&trie, key) != &OBJ[key] || tommy_trie_remove(&trie, key) != &OBJ[key]
    8510           9 :                                 || tommy_trie_search(&trie, key) != 0) {
    8511             :                                 /* LCOV_EXCL_START */
    8512             :                                 abort();
    8513             :                                 /* LCOV_EXCL_STOP */
    8514             :                         }
    8515             :                 }
    8516           3 :                 if (!tommy_trie_empty(&trie) || tommy_trie_count(&trie) != 0 || trie.node_count != 0 || alloc.count != 0) {
    8517             :                         /* LCOV_EXCL_START */
    8518             :                         abort();
    8519             :                         /* LCOV_EXCL_STOP */
    8520             :                 }
    8521             : 
    8522             :                 /* insert */
    8523     1200003 :                 for (unsigned i = 0; i < size; ++i)
    8524     1200000 :                         tommy_trie_insert(&trie, &OBJ[i].node, &OBJ[i], OBJ[i].value);
    8525             : 
    8526           3 :                 if (tommy_trie_memory_usage(&trie) < size * sizeof(tommy_trie_node)) {
    8527             :                         /* LCOV_EXCL_START */
    8528             :                         abort();
    8529             :                         /* LCOV_EXCL_STOP */
    8530             :                 }
    8531             : 
    8532           3 :                 if (tommy_allocator_memory_usage(&alloc) < trie.node_count * TOMMY_TRIE_BLOCK_SIZE) {
    8533             :                         /* LCOV_EXCL_START */
    8534             :                         abort();
    8535             :                         /* LCOV_EXCL_STOP */
    8536             :                 }
    8537             : 
    8538           3 :                 if (tommy_trie_count(&trie) != size) {
    8539             :                         /* LCOV_EXCL_START */
    8540             :                         abort();
    8541             :                         /* LCOV_EXCL_STOP */
    8542             :                 }
    8543             : 
    8544             :                 /* insert duplicate */
    8545             :                 struct object_trie DUP[2];
    8546           9 :                 for (unsigned i = 0; i < 2; ++i) {
    8547           6 :                         DUP[i].value = 0;
    8548           6 :                         tommy_trie_insert(&trie, &DUP[i].node, &DUP[i], DUP[i].value);
    8549             :                 }
    8550             : 
    8551             :                 /* search present */
    8552      600003 :                 for (unsigned i = 0; i < size / 2; ++i)
    8553      600000 :                         if (tommy_trie_search(&trie, OBJ[i].value) == 0) {
    8554             :                                 /* LCOV_EXCL_START */
    8555             :                                 abort();
    8556             :                                 /* LCOV_EXCL_STOP */
    8557             :                         }
    8558             : 
    8559             :                 /* remove first duplicate */
    8560           3 :                 tommy_trie_remove_existing(&trie, &DUP[0].node);
    8561             : 
    8562             :                 /* remove existing */
    8563      600003 :                 for (unsigned i = 0; i < size / 2; ++i)
    8564      600000 :                         tommy_trie_remove_existing(&trie, &OBJ[i].node);
    8565             : 
    8566             :                 /* remove missing using the same bucket of the duplicate */
    8567           3 :                 if (tommy_trie_remove(&trie, 1) != 0) {
    8568             :                         /* LCOV_EXCL_START */
    8569             :                         abort();
    8570             :                         /* LCOV_EXCL_STOP */
    8571             :                 }
    8572             : 
    8573             :                 /* search missing using the same bucket of the duplicate */
    8574           3 :                 if (tommy_trie_search(&trie, 1) != 0) {
    8575             :                         /* LCOV_EXCL_START */
    8576             :                         abort();
    8577             :                         /* LCOV_EXCL_STOP */
    8578             :                 }
    8579             : 
    8580             :                 /* remove second duplicate */
    8581           3 :                 tommy_trie_remove_existing(&trie, &DUP[1].node);
    8582             : 
    8583             :                 /* remove missing */
    8584      600003 :                 for (unsigned i = 0; i < size / 2; ++i)
    8585      600000 :                         if (tommy_trie_remove(&trie, OBJ[i].value) != 0) {
    8586             :                                 /* LCOV_EXCL_START */
    8587             :                                 abort();
    8588             :                                 /* LCOV_EXCL_STOP */
    8589             :                         }
    8590             : 
    8591             :                 /* search missing */
    8592      600003 :                 for (unsigned i = 0; i < size / 2; ++i)
    8593      600000 :                         if (tommy_trie_search(&trie, OBJ[i].value) != 0) {
    8594             :                                 /* LCOV_EXCL_START */
    8595             :                                 abort();
    8596             :                                 /* LCOV_EXCL_STOP */
    8597             :                         }
    8598             : 
    8599             :                 /* remove present */
    8600      600003 :                 for (unsigned i = 0; i < size / 2; ++i)
    8601      600000 :                         if (tommy_trie_remove(&trie, OBJ[size / 2 + i].value) == 0) {
    8602             :                                 /* LCOV_EXCL_START */
    8603             :                                 abort();
    8604             :                                 /* LCOV_EXCL_STOP */
    8605             :                         }
    8606             : 
    8607           3 :                 tommy_allocator_done(&alloc);
    8608           3 :         } SPEED_STOP
    8609           1 : }
    8610             : 
    8611           1 : void test_trie_inplace(void)
    8612             : {
    8613           1 :         const unsigned size = TOMMY_SIZE * 4;
    8614             : 
    8615           1 :         struct object_trie_inplace* OBJ = malloc(size * sizeof(struct object_trie_inplace));
    8616             : 
    8617      400001 :         for (unsigned i = 0; i < size; ++i)
    8618      400000 :                 OBJ[i].value = i;
    8619             : 
    8620           4 :         SPEED_START("trie_inplace") {
    8621             :                 tommy_trie_inplace trie_inplace;
    8622           2 :                 tommy_trie_inplace_init(&trie_inplace);
    8623             : 
    8624           2 :                 if (!tommy_trie_inplace_empty(&trie_inplace)) {
    8625             :                         /* LCOV_EXCL_START */
    8626             :                         abort();
    8627             :                         /* LCOV_EXCL_STOP */
    8628             :                 }
    8629             : 
    8630             :                 /* unique insertion uses the numeric key and preserves normal duplicate order */
    8631           2 :                 if (tommy_trie_inplace_insert_unique(&trie_inplace, &OBJ[0].node, &OBJ[0], 0) != &OBJ[0]
    8632           2 :                         || tommy_trie_inplace_insert_unique(&trie_inplace, &OBJ[1].node, &OBJ[1], 1) != &OBJ[1]
    8633           2 :                         || tommy_trie_inplace_count(&trie_inplace) != 2 || tommy_trie_inplace_empty(&trie_inplace)
    8634           2 :                         || tommy_trie_inplace_search(&trie_inplace, 0) != &OBJ[0] || tommy_trie_inplace_search(&trie_inplace, 1) != &OBJ[1]) {
    8635             :                         /* LCOV_EXCL_START */
    8636             :                         abort();
    8637             :                         /* LCOV_EXCL_STOP */
    8638             :                 }
    8639             :                 struct object_trie_inplace duplicate;
    8640           2 :                 tommy_trie_inplace_insert(&trie_inplace, &duplicate.node, &duplicate, 0);
    8641             : 
    8642             :                 /* rejection must preserve every candidate field, including its child pointers */
    8643             :                 struct object_trie_inplace candidate;
    8644           2 :                 memset(&candidate.node, 0xa5, sizeof(candidate.node));
    8645             :                 unsigned char node_before[sizeof(candidate.node)];
    8646             :                 unsigned char trie_before[sizeof(trie_inplace)];
    8647           2 :                 memcpy(node_before, &candidate.node, sizeof(node_before));
    8648           2 :                 memcpy(trie_before, &trie_inplace, sizeof(trie_before));
    8649           6 :                 for (unsigned i = 0; i < 2; ++i) {
    8650           4 :                         if (tommy_trie_inplace_insert_unique(&trie_inplace, &candidate.node, &candidate, i) != &OBJ[i]
    8651           4 :                                 || tommy_trie_inplace_count(&trie_inplace) != 3 || tommy_trie_inplace_empty(&trie_inplace)
    8652           4 :                                 || memcmp(node_before, &candidate.node, sizeof(node_before)) != 0
    8653           4 :                                 || memcmp(trie_before, &trie_inplace, sizeof(trie_before)) != 0
    8654           4 :                                 || tommy_trie_inplace_search(&trie_inplace, i) != &OBJ[i]) {
    8655             :                                 /* LCOV_EXCL_START */
    8656             :                                 abort();
    8657             :                                 /* LCOV_EXCL_STOP */
    8658             :                         }
    8659             :                 }
    8660             : 
    8661             :                 /* continue unique insertion from child pointers reached during the search */
    8662           2 :                 const unsigned keys[] = { 2, 8, 9 };
    8663           8 :                 for (unsigned i = 0; i < sizeof(keys) / sizeof(keys[0]); ++i) {
    8664           6 :                         unsigned key = keys[i];
    8665           6 :                         if (tommy_trie_inplace_insert_unique(&trie_inplace, &OBJ[key].node, &OBJ[key], key) != &OBJ[key]
    8666           6 :                                 || tommy_trie_inplace_count(&trie_inplace) != i + 4 || tommy_trie_inplace_search(&trie_inplace, key) != &OBJ[key]) {
    8667             :                                 /* LCOV_EXCL_START */
    8668             :                                 abort();
    8669             :                                 /* LCOV_EXCL_STOP */
    8670             :                         }
    8671             :                 }
    8672             : 
    8673             :                 /* drain through both removal paths, including duplicate and child replacement */
    8674           2 :                 if (tommy_trie_inplace_remove(&trie_inplace, 0) != &OBJ[0] || tommy_trie_inplace_search(&trie_inplace, 0) != &duplicate
    8675           2 :                         || tommy_trie_inplace_remove_existing(&trie_inplace, &duplicate.node) != &duplicate || tommy_trie_inplace_search(&trie_inplace, 0) != 0
    8676           2 :                         || tommy_trie_inplace_empty(&trie_inplace) || tommy_trie_inplace_search(&trie_inplace, 1) != &OBJ[1]
    8677           2 :                         || tommy_trie_inplace_remove_existing(&trie_inplace, &OBJ[1].node) != &OBJ[1] || tommy_trie_inplace_search(&trie_inplace, 1) != 0) {
    8678             :                         /* LCOV_EXCL_START */
    8679             :                         abort();
    8680             :                         /* LCOV_EXCL_STOP */
    8681             :                 }
    8682           8 :                 for (unsigned i = 0; i < sizeof(keys) / sizeof(keys[0]); ++i) {
    8683           6 :                         unsigned key = keys[i];
    8684           6 :                         if (tommy_trie_inplace_search(&trie_inplace, key) != &OBJ[key] || tommy_trie_inplace_remove(&trie_inplace, key) != &OBJ[key]
    8685           6 :                                 || tommy_trie_inplace_search(&trie_inplace, key) != 0) {
    8686             :                                 /* LCOV_EXCL_START */
    8687             :                                 abort();
    8688             :                                 /* LCOV_EXCL_STOP */
    8689             :                         }
    8690             :                 }
    8691           2 :                 if (!tommy_trie_inplace_empty(&trie_inplace) || tommy_trie_inplace_count(&trie_inplace) != 0) {
    8692             :                         /* LCOV_EXCL_START */
    8693             :                         abort();
    8694             :                         /* LCOV_EXCL_STOP */
    8695             :                 }
    8696             : 
    8697             :                 /* insert */
    8698      800002 :                 for (unsigned i = 0; i < size; ++i)
    8699      800000 :                         tommy_trie_inplace_insert(&trie_inplace, &OBJ[i].node, &OBJ[i], OBJ[i].value);
    8700             : 
    8701           2 :                 if (tommy_trie_inplace_memory_usage(&trie_inplace) < size * sizeof(tommy_trie_inplace_node)) {
    8702             :                         /* LCOV_EXCL_START */
    8703             :                         abort();
    8704             :                         /* LCOV_EXCL_STOP */
    8705             :                 }
    8706             : 
    8707           2 :                 if (tommy_trie_inplace_count(&trie_inplace) != size) {
    8708             :                         /* LCOV_EXCL_START */
    8709             :                         abort();
    8710             :                         /* LCOV_EXCL_STOP */
    8711             :                 }
    8712             : 
    8713             :                 /* insert duplicates */
    8714             :                 struct object_trie_inplace DUP[2];
    8715           6 :                 for (unsigned i = 0; i < 2; ++i) {
    8716           4 :                         DUP[i].value = 0;
    8717           4 :                         tommy_trie_inplace_insert(&trie_inplace, &DUP[i].node, &DUP[i], DUP[i].value);
    8718             :                 }
    8719             : 
    8720             :                 /* search present */
    8721      400002 :                 for (unsigned i = 0; i < size / 2; ++i)
    8722      400000 :                         if (tommy_trie_inplace_search(&trie_inplace, OBJ[i].value) == 0) {
    8723             :                                 /* LCOV_EXCL_START */
    8724             :                                 abort();
    8725             :                                 /* LCOV_EXCL_STOP */
    8726             :                         }
    8727             : 
    8728             :                 /* remove first duplicate */
    8729           2 :                 tommy_trie_inplace_remove_existing(&trie_inplace, &DUP[0].node);
    8730             : 
    8731             :                 /* remove existing */
    8732      400002 :                 for (unsigned i = 0; i < size / 2; ++i)
    8733      400000 :                         tommy_trie_inplace_remove_existing(&trie_inplace, &OBJ[i].node);
    8734             : 
    8735             :                 /* remove second duplicate */
    8736           2 :                 tommy_trie_inplace_remove_existing(&trie_inplace, &DUP[1].node);
    8737             : 
    8738             :                 /* remove missing */
    8739      400002 :                 for (unsigned i = 0; i < size / 2; ++i)
    8740      400000 :                         if (tommy_trie_inplace_remove(&trie_inplace, OBJ[i].value) != 0) {
    8741             :                                 /* LCOV_EXCL_START */
    8742             :                                 abort();
    8743             :                                 /* LCOV_EXCL_STOP */
    8744             :                         }
    8745             : 
    8746             :                 /* search missing */
    8747      400002 :                 for (unsigned i = 0; i < size / 2; ++i)
    8748      400000 :                         if (tommy_trie_inplace_search(&trie_inplace, OBJ[i].value) != 0) {
    8749             :                                 /* LCOV_EXCL_START */
    8750             :                                 abort();
    8751             :                                 /* LCOV_EXCL_STOP */
    8752             :                         }
    8753             : 
    8754             :                 /* remove present */
    8755      400002 :                 for (unsigned i = 0; i < size / 2; ++i)
    8756      400000 :                         if (tommy_trie_inplace_remove(&trie_inplace, OBJ[size / 2 + i].value) == 0) {
    8757             :                                 /* LCOV_EXCL_START */
    8758             :                                 abort();
    8759             :                                 /* LCOV_EXCL_STOP */
    8760             :                         }
    8761             : 
    8762           2 :         } SPEED_STOP
    8763           1 : }
    8764             : 
    8765           1 : void test_trie_inplace_clear(void)
    8766             : {
    8767             :         tommy_trie_inplace trie;
    8768             :         struct object_trie_inplace objects[2];
    8769             : 
    8770           1 :         tommy_trie_inplace_init(&trie);
    8771           1 :         objects[0].value = 1;
    8772           1 :         objects[1].value = 2;
    8773           1 :         tommy_trie_inplace_insert(&trie, &objects[0].node, &objects[0], 1);
    8774           1 :         tommy_trie_inplace_insert(&trie, &objects[1].node, &objects[1], 2);
    8775           1 :         tommy_trie_inplace_clear(&trie);
    8776           1 :         if (!tommy_trie_inplace_empty(&trie) || tommy_trie_inplace_count(&trie) != 0) {
    8777             :                 /* LCOV_EXCL_START */
    8778             :                 abort();
    8779             :                 /* LCOV_EXCL_STOP */
    8780             :         }
    8781          65 :         for (unsigned i = 0; i < TOMMY_TRIE_INPLACE_BUCKET_MAX; ++i) {
    8782          64 :                 if (trie.bucket[i] != 0) {
    8783             :                         /* LCOV_EXCL_START */
    8784             :                         abort();
    8785             :                         /* LCOV_EXCL_STOP */
    8786             :                 }
    8787             :         }
    8788           1 :         if (tommy_trie_inplace_search(&trie, 1) != 0 || tommy_trie_inplace_search(&trie, 2) != 0) {
    8789             :                 /* LCOV_EXCL_START */
    8790             :                 abort();
    8791             :                 /* LCOV_EXCL_STOP */
    8792             :         }
    8793             : 
    8794           1 :         tommy_trie_inplace_insert(&trie, &objects[0].node, &objects[0], 1);
    8795           1 :         if (tommy_trie_inplace_search(&trie, 1) != &objects[0]) {
    8796             :                 /* LCOV_EXCL_START */
    8797             :                 abort();
    8798             :                 /* LCOV_EXCL_STOP */
    8799             :         }
    8800           1 :         tommy_trie_inplace_remove_existing(&trie, &objects[0].node);
    8801             : 
    8802           1 :         struct object_trie_inplace* allocated = malloc(sizeof(*allocated));
    8803           1 :         allocated->value = 3;
    8804           1 :         tommy_trie_inplace_insert(&trie, &allocated->node, allocated, 3);
    8805           1 :         tommy_trie_inplace_foreach(&trie, free);
    8806           1 :         tommy_trie_inplace_clear(&trie);
    8807           1 :         if (!tommy_trie_inplace_empty(&trie)) {
    8808             :                 /* LCOV_EXCL_START */
    8809             :                 abort();
    8810             :                 /* LCOV_EXCL_STOP */
    8811             :         }
    8812           1 : }
    8813             : 
    8814           1 : int main(void)
    8815             : {
    8816           1 :         nano_init();
    8817             : 
    8818           1 :         printf("Tommy check program. Tests with %d elements\n", TOMMY_SIZE);
    8819             : 
    8820           1 :         test_hash();
    8821           1 :         test_alloc();
    8822           1 :         test_alloc_clear();
    8823           1 :         test_list();
    8824           1 :         test_list_clear();
    8825           1 :         test_builder();
    8826           1 :         test_tree();
    8827           1 :         test_tree_swap();
    8828           1 :         test_tree_duplicates();
    8829           1 :         test_tree_foreach();
    8830           1 :         test_tree_clear();
    8831           1 :         test_tree_to_list();
    8832           1 :         test_array();
    8833           1 :         test_array_ops();
    8834           1 :         test_arrayof();
    8835           1 :         test_arrayof_ops();
    8836           1 :         test_arrayblk();
    8837           1 :         test_arrayblk_ops();
    8838           1 :         test_arrayblkof();
    8839           1 :         test_arrayblkof_ops();
    8840           1 :         test_array_tail_ops();
    8841           1 :         test_array_capacity_swap();
    8842           1 :         test_hashtable();
    8843           1 :         test_hashdyn();
    8844           1 :         test_hashlin();
    8845           1 :         test_trie();
    8846           1 :         test_trie_iteration();
    8847           1 :         test_trie_clear();
    8848           1 :         test_trie_to_list();
    8849           1 :         test_trie_foreach();
    8850           1 :         test_trie_foreach_order();
    8851           1 :         test_trie_foreach_depth();
    8852           1 :         test_trie_swap();
    8853           1 :         test_trie_inplace();
    8854           1 :         test_trie_inplace_iteration();
    8855           1 :         test_trie_inplace_clear();
    8856           1 :         test_trie_inplace_foreach();
    8857           1 :         test_trie_inplace_foreach_order();
    8858           1 :         test_trie_inplace_foreach_depth();
    8859           1 :         test_trie_inplace_swap();
    8860             : 
    8861           1 :         printf("OK\n");
    8862             : 
    8863           1 :         return EXIT_SUCCESS;
    8864             : }
    8865             : 

Generated by: LCOV version 1.0