diff --git a/.vscode/launch.json b/.vscode/launch.json index 4a1e72a5..6377fc6e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,8 +5,15 @@ "name": "Debug libCacheSim", "type": "cppdbg", "request": "launch", - "program": "${workspaceFolder}/_build_dbg/bin/cachesim", - "args": ["data/cloudPhysicsIO.vscsi", "vscsi", "lru", "100m,1gb"], + "program": "${workspaceFolder}/build/bin/cachesim", + "args": [ + "${workspaceFolder}/data/cloudPhysicsIO.vscsi", + "vscsi", + "fifo,lru,arc,qdlp", + "0.01,0.05,0.1,0.2", + "--ignore-obj-size", + "1" + ], "stopAtEntry": false, "cwd": "${workspaceFolder}", "environment": [], @@ -18,9 +25,7 @@ "text": "-enable-pretty-printing", "ignoreFailures": true } - ], - "preLaunchTask": "build-debug" + ] } ] -} - +} \ No newline at end of file diff --git a/doc/datastructures.md b/doc/datastructures.md new file mode 100644 index 00000000..ebd9b5be --- /dev/null +++ b/doc/datastructures.md @@ -0,0 +1,275 @@ +# Reference for common data structures used throughout the project + +In May 2025 we began to remove GLib dependencies from this project. Thus, we need to find replacements for various GLib objects used in the project, especially `GHashTable` and `GThreadPool`. This document is a brief introduction on how to write new code using replacement datastructure libraries. + +## Hash Table + +- Type: Header-only C library +- Use: Generic (non-cache related) +- Path: *include/libCacheSim/hashmap.h* + +This is a generic hash table imported from [sheredom/hashmap.h](https://github.com/sheredom/hashmap.h). It is used everywhere except with cache objects, where there exists an optimised hashtable implementation. + +#### Structs + +```c +struct hashmap_element_s { + const void *key; + void *data; +} +``` + +This structure is a representation of the hash table element. + + + +```c +struct hashmap_create_options_s { + hashmap_hasher_t hasher; + hashmap_comparer_t comparer; + hashmap_uint32_t initial_capacity; +} +``` + +This structure needs to be populated if custom hasher/comparer is desired. If it's not fully populated, the default will be used. + + + + +```c +hashmap_uint32_t hasher(const hashmap_uint32_t seed, const void *const s, + const hashmap_uint32_t len) +``` + +- Signature of custom hasher. + + - `seed`: random seed, if **reproducible** randomness is desired. + + - `s`: key of the element to be stored in the hash table. + + - `len`: length of `s`. By design `s` is a C string. If it is not, then the use of `len` can be user-defined (e.g. ignored.) + +```c +int obj_id_comparer(const void *const a, const hashmap_uint32_t a_len, + const void *const b, const hashmap_uint32_t b_len) +``` + +- Signature of custom comparer. + - Return **0** on equal, **1** on non-equal (regardless of `a` being greater of lesser.) + +#### Methods + +```c +int hashmap_create(const hashmap_uint32_t initial_capacity, + struct hashmap_s *const out_hashmap) +``` + +- Create a hash table using all defaults. + - `out_hashmap`: hash table structure. It does not need to be on the heap, but it MUST be initialized before this call. + + + +```c +int hashmap_create_ex(struct hashmap_create_options_s options, + struct hashmap_s *const out_hashmap) +``` + +- Create a hashmap using customised options. + + + +```c +int hashmap_put(struct hashmap_s *const hashmap, const void *const key, + const hashmap_uint32_t len, void *const value) +``` + +- Put a key-value pair into the hash table. If the key exists, the value will be updated. + + - `key`: by design it's a C string, but it can be something else as long as a custom hasher is provided. + + **MUST** exist during the entire lifecycle of the hash table. It will **NOT** automatically be copied. + + - `len`: the length of the key if it's a C string. If the key is something else, it's user-defined. + + - `value`: it can be anything, but like the key, it **MUST** exist during the entire lifecycle of the hash table. + + + +```c +void *hashmap_get(const struct hashmap_s *const hashmap, + const void *const key, const hashmap_uint32_t len) +``` + +- Get a **value** from the hash table. + + + +```c +int hashmap_remove(struct hashmap_s *const hashmap, const void *const key, + const hashmap_uint32_t len) +``` + +- Remove a key-value pair from the hash table. + + + +```c +int hashmap_iterate(const struct hashmap_s *const hashmap, int (*iterator) + (void *const context, void *const value), void *const context) +``` + +- Iterate through all **values** from the hash table. + + - `iterator`: function pointer to pass to the iterator. Can pass in an additional context as the first parameter. + + To exit from iteration at any time, return **non-zero**. To continue, return **0**. + + + +```c +int hashmap_iterate_pairs(struct hashmap_s *const hashmap, int (*iterator) + (void *const, struct hashmap_element_s *const), + void *const context) +``` + +- Iterate through all key-value pairs from the hash table. + + - `iterator`: function pointer to pass to the iterator. Can pass in an additional context as the first parameter. + + To remove the current element (**only removes record, not memory**) and continue iteration, return **-1**. + + To end iteration, return a **positive integer**. To continue, return **0**. + + + +```c +void hashmap_destroy(struct hashmap_s *const hashmap) +``` + +- Destroys the hash table. + + - If the hash table is created on heap, it is still needed to call `free` on `hashmap` to free its memory. + + + +#### Example + +```c +struct hashmap_create_options_s options = { .initial_capacity = 16 }; +struct hashmap_s hashmap; +hashmap_create(options, &hashmap); + +char* value = (char *)malloc(sizeof(char) * 5); +strcpy(value, "value"); +if (hashmap_put(&hashmap, "key", 3, value) != 0) + printf("encountered unknown error in hashmap_put()\n"); + +char* content = (char *)hashmap_get(&hashmap, "key", 3); +printf("content is %s\n", content); + +hashmap_destroy(&hashmap); free(value); +free(content); // double free +``` + + + +## Thread Pool + +- Type: C library +- Use: Generic +- Path: *utils/include/threadpool.h* +- Link: Add *utils* to `target_link_libraries` in CMakeLists + +This is a simple fixed-size thread pool adapted from https://nachtimwald.com/2019/04/12/thread-pool-in-c, which uses semantics similar to that of GLib. It currently does not support adjusting priority of jobs, etc. + +#### Methods + +```c +bool threadpool_create(threadpool_t *tm, size_t num_threads) +``` + +- Create a thread pool + + - `tm`: structure of thread pool. **MUST** be pre-allocated on the heap. + + + +```c +void threadpool_destroy(threadpool_t *tm) +``` + +- Destroys the thread pool + - The memory associated with `tm` is also freed, so calling `free()` on `tm` again will cause double free! + + + +```c +bool threadpool_push(threadpool_t *tm, void (*job)(void *arg1, void *arg2), + void *arg, void *arg2) +``` + +- Pushes a job to the execution queue of the thread pool. + - `job`: a function that accepts 2 arguments. It is easy to modify this library to make `job` accept more arguments. + + + +#### Example + +```c +// Adapted from https://nachtimwald.com/2019/04/12/thread-pool-in-c/ +void worker(void *arg1, void* arg2) +{ + int *old = arg1; + int *new = arg2; + + *new += 1000; + printf("tid=%p, old=%d, new=%d\n", pthread_self(), *old, *new); + + if (*old % 2) + usleep(100000); +} + +int main(int argc, char **argv) +{ + threadpool_t *tm = (threadpool_t *)malloc(sizeof(threadpool_t)); + int *vals; + size_t i; + + tm = threadpool_create(num_threads); + vals = calloc(num_items, sizeof(*vals)); + + for (i=0; idistance_tree, params->lookup_hash, (int64_t)timestamp, &last_access); + get_stack_dist_add_req(req, ¶ms->distance_tree, params->lookup_hash, + (int64_t)timestamp, &last_access); return distance; } // Compute reuse distance for each request in fixed-size mode. -int64_t compute_distance_fixed_size(struct PARAM *params, request_t *req, uint64_t timestamp) { +int64_t compute_distance_fixed_size(struct PARAM *params, request_t *req, + uint64_t timestamp) { int64_t last_access = -1; int64_t distance = - get_stack_dist_add_req(req, ¶ms->distance_tree, params->lookup_hash, (int64_t)timestamp, &last_access); + get_stack_dist_add_req(req, ¶ms->distance_tree, params->lookup_hash, + (int64_t)timestamp, &last_access); - // If the object has not been accessed before, insert it into the priority tree. + // If the object has not been accessed before, insert it into the priority + // tree. if (distance == -1) { struct key *new_tuple = malloc(sizeof(struct key)); new_tuple->L = req->obj_id; new_tuple->Tmax = (req->hv) & ((1 << 24) - 1); params->prio_tree = insert_t(new_tuple, params->prio_tree); } - - // Update the priority tree and lookup hash when number of stored objects exceeds the threshold. - while (params->prio_tree != NULL && params->prio_tree->value >= params->threshold) { + + // Update the priority tree and lookup hash when number of stored objects + // exceeds the threshold. + while (params->prio_tree != NULL && + params->prio_tree->value >= params->threshold) { struct key *max = find_max_t(params->prio_tree)->key; uint64_t last_max = 0; params->rate = (double)max->Tmax / (double)(1 << 24); - //printf("rate: %f\n", params->rate); - // Update the sampler ratio when stored object overflows + // printf("rate: %f\n", params->rate); + // Update the sampler ratio when stored object overflows params->reader->sampler->sampling_ratio = params->rate; while ((last_max == max->Tmax) || (last_max == 0)) { obj_id_t id = max->L; - if (id==req->obj_id) distance = -2; + if (id == req->obj_id) distance = -2; last_max = max->Tmax; // Remove the key from prio_tree and update lookup and distance_tree. params->prio_tree = splay_delete_t(max, params->prio_tree); - gpointer hash_value_inner = g_hash_table_lookup(params->lookup_hash, GSIZE_TO_POINTER((gsize)id)); - g_hash_table_remove(params->lookup_hash, GSIZE_TO_POINTER((gsize)id)); - params->distance_tree = splay_delete((long long)hash_value_inner, params->distance_tree); + void *hash_value_inner = + hashmap_get(params->lookup_hash, (const void *)id, sizeof(obj_id_t)); + hashmap_remove(params->lookup_hash, (const void *)id, sizeof(obj_id_t)); + + params->distance_tree = + splay_delete((long long)hash_value_inner, params->distance_tree); if (params->prio_tree) max = find_max_t(params->prio_tree)->key; else @@ -72,7 +82,7 @@ uint64_t simulate_shards_mrc(struct PARAM *params) { reader_t *reader = params->reader; request_t *req = new_request(); uint64_t timestamp = 0; - uint64_t n_req=0; + uint64_t n_req = 0; read_one_req(reader, req); while (req->valid) { int64_t distance = params->compute_distance(params, req, timestamp); @@ -82,7 +92,7 @@ uint64_t simulate_shards_mrc(struct PARAM *params) { n_req++; continue; } - update_histogram(params->data, distance, params->rate); + update_histogram(params->data, distance, params->rate); read_one_req(reader, req); timestamp++; n_req++; @@ -107,21 +117,27 @@ void generate_shards_mrc(struct PARAM *params, char *path) { params->data = init_histogram(); params->prio_tree = NULL; params->distance_tree = NULL; - params->lookup_hash = g_hash_table_new(g_direct_hash, g_direct_equal); + hashmap_create_options_t lookup_hash_create_options = { + .initial_capacity = 16, + .comparer = obj_id_comparer, + .hasher = obj_id_hasher}; + params->lookup_hash = malloc(sizeof(hashmap_t)); + hashmap_create_ex(lookup_hash_create_options, params->lookup_hash); // Start the simulation. - uint64_t read_req=simulate_shards_mrc(params); + uint64_t read_req = simulate_shards_mrc(params); // In fixed-size mode, perform additional post-processing. if (params->ver == true) { wrap_up_histogram(params->data, params->rate); } - //SHARDS-adj + // SHARDS-adj adjust_histogram(params->data, n_req, params->rate); - + export_histogram_to_csv(params->data, params->rate, path); - g_hash_table_destroy(params->lookup_hash); + hashmap_destroy(params->lookup_hash); + free(params->lookup_hash); free_sTree_t(params->prio_tree); free_sTree(params->distance_tree); close_reader(params->reader); diff --git a/libCacheSim/bin/MRC/mrc_internal.h b/libCacheSim/bin/MRC/mrc_internal.h index aa0c7a54..e563ac7c 100644 --- a/libCacheSim/bin/MRC/mrc_internal.h +++ b/libCacheSim/bin/MRC/mrc_internal.h @@ -4,15 +4,17 @@ #include #include #include + #include "../../dataStructure/histogram.h" #include "../../dataStructure/splay.h" #include "../../dataStructure/splay_tuple.h" -#include "../../include/libCacheSim/reader.h" -#include "../../include/libCacheSim/enum.h" +#include "../../include/libCacheSim/admissionAlgo.h" #include "../../include/libCacheSim/cache.h" +#include "../../include/libCacheSim/enum.h" #include "../../include/libCacheSim/evictionAlgo.h" +#include "../../include/libCacheSim/hashmap.h" +#include "../../include/libCacheSim/hashmap_defs.in" #include "../../include/libCacheSim/reader.h" -#include "../../include/libCacheSim/admissionAlgo.h" #define N_ARGS 4 #define N_MAX_ALGO 16 @@ -22,7 +24,6 @@ // Forward declaration of struct Params struct PARAM; - #ifdef __cplusplus extern "C" { #endif @@ -31,11 +32,11 @@ struct SHARD_arguments { bool verver; long size; float rate; - char* mrc_algo; - char* trace_file; + char *mrc_algo; + char *trace_file; char *trace_type_str; trace_type_e trace_type; - char* trace_type_params; + char *trace_type_params; bool ignore_obj_size; int64_t n_req; }; @@ -43,16 +44,16 @@ struct SHARD_arguments { struct PARAM { float rate; bool ver; // 0 means fixed rate, 1 means fixed size - + int64_t threshold; - //GHashTable* prio_hash; - sTree_tuple* prio_tree; // root of the splay tree - sTree* distance_tree; - ReuseHistogram* data; - GHashTable* lookup_hash; + // GHashTable* prio_hash; + sTree_tuple *prio_tree; // root of the splay tree + sTree *distance_tree; + ReuseHistogram *data; + hashmap_t *lookup_hash; reader_t *reader; int64_t (*compute_distance)(struct PARAM *, request_t *, uint64_t); - void (*mrc_algo)(struct PARAM*, char* path); + void (*mrc_algo)(struct PARAM *, char *path); }; struct MINI_arguments { @@ -88,20 +89,21 @@ struct MINI_arguments { /* arguments generated */ reader_t *reader; cache_t *caches[N_MAX_ALGO * N_MAX_CACHE_SIZE]; - }; -int64_t compute_distance_fixed_rate(struct PARAM *params, request_t *req, uint64_t timestamp); +int64_t compute_distance_fixed_rate(struct PARAM *params, request_t *req, + uint64_t timestamp); -int64_t compute_distance_fixed_size(struct PARAM *params, request_t *req, uint64_t timestamp); +int64_t compute_distance_fixed_size(struct PARAM *params, request_t *req, + uint64_t timestamp); -void generate_shards_mrc(struct PARAM* params, char* path); +void generate_shards_mrc(struct PARAM *params, char *path); -cache_stat_t * generate_mini_mrc(struct MINI_arguments* args); +cache_stat_t *generate_mini_mrc(struct MINI_arguments *args); void parse_mrc_cmd(int argc, char **argv, struct PARAM *args); -void parse_mini_cmd(int argc, char* argv[], struct MINI_arguments* args); +void parse_mini_cmd(int argc, char *argv[], struct MINI_arguments *args); #ifdef __cplusplus } diff --git a/libCacheSim/bin/cli_reader_utils.c b/libCacheSim/bin/cli_reader_utils.c index 260ee027..4e8c16e5 100644 --- a/libCacheSim/bin/cli_reader_utils.c +++ b/libCacheSim/bin/cli_reader_utils.c @@ -6,6 +6,8 @@ #include #include +#include "../include/libCacheSim/hashmap.h" +#include "../include/libCacheSim/hashmap_defs.in" #include "../include/libCacheSim/reader.h" #include "../utils/include/mystr.h" @@ -242,7 +244,12 @@ void cal_working_set_size(reader_t *reader, int64_t *wss_obj, int64_t *wss_byte) { reset_reader(reader); request_t *req = new_request(); - GHashTable *obj_table = g_hash_table_new(g_direct_hash, g_direct_equal); + hashmap_create_options_t obj_table_create_options = { + .initial_capacity = 16, + .comparer = obj_id_comparer, + .hasher = obj_id_hasher}; + struct hashmap_s new_obj_table; + hashmap_create_ex(obj_table_create_options, &new_obj_table); *wss_obj = 0; *wss_byte = 0; @@ -267,11 +274,12 @@ void cal_working_set_size(reader_t *reader, int64_t *wss_obj, continue; } - if (g_hash_table_contains(obj_table, (gconstpointer)req->obj_id)) { + if (hashmap_get(&new_obj_table, (const void *)(req->obj_id), + sizeof(obj_id_t)) != NULL) { continue; } - g_hash_table_add(obj_table, (gpointer)req->obj_id); + hashmap_put(&new_obj_table, (void *)(req->obj_id), sizeof(obj_id_t), req); *wss_obj += 1; *wss_byte += req->obj_size; @@ -289,7 +297,7 @@ void cal_working_set_size(reader_t *reader, int64_t *wss_obj, (long long)*wss_byte); } - g_hash_table_destroy(obj_table); + hashmap_destroy(&new_obj_table); free_request(req); reset_reader(reader); } diff --git a/libCacheSim/include/libCacheSim/hashmap.h b/libCacheSim/include/libCacheSim/hashmap.h new file mode 100644 index 00000000..a14e3761 --- /dev/null +++ b/libCacheSim/include/libCacheSim/hashmap.h @@ -0,0 +1,742 @@ +/* + The latest version of this library is available on GitHub; + https://github.com/sheredom/hashmap.h +*/ + +/* + This is free and unencumbered software released into the public domain. + + Anyone is free to copy, modify, publish, use, compile, sell, or + distribute this software, either in source code form or as a compiled + binary, for any purpose, commercial or non-commercial, and by any + means. + + In jurisdictions that recognize copyright laws, the author or authors + of this software dedicate any and all copyright interest in the + software to the public domain. We make this dedication for the benefit + of the public at large and to the detriment of our heirs and + successors. We intend this dedication to be an overt act of + relinquishment in perpetuity of all present and future rights to this + software under copyright law. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + For more information, please refer to +*/ +#ifndef SHEREDOM_HASHMAP_H_INCLUDED +#define SHEREDOM_HASHMAP_H_INCLUDED + +#if defined(_MSC_VER) +// Workaround a bug in the MSVC runtime where it uses __cplusplus when not +// defined. +#pragma warning(push, 0) +#pragma warning(disable : 4668) +#endif + +#include +#include + +#if (defined(_MSC_VER) && defined(__AVX__)) || \ + (!defined(_MSC_VER) && defined(__SSE4_2__)) +#define HASHMAP_X86_SSE42 +#endif + +#if defined(HASHMAP_X86_SSE42) +#include +#endif + +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) +#define HASHMAP_ARM_CRC32 +#endif + +#if defined(HASHMAP_ARM_CRC32) +#include +#endif + +#if defined(_MSC_VER) +#include +#endif + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#if defined(_MSC_VER) +#pragma warning(push) +/* Stop MSVC complaining about unreferenced functions */ +#pragma warning(disable : 4505) +/* Stop MSVC complaining about not inlining functions */ +#pragma warning(disable : 4710) +/* Stop MSVC complaining about inlining functions! */ +#pragma warning(disable : 4711) +#endif + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wstatic-in-inline" + +#if __has_warning("-Wunsafe-buffer-usage") +#pragma clang diagnostic ignored "-Wunsafe-buffer-usage" +#endif +#endif + +#if defined(__TINYC__) +#define HASHMAP_ATTRIBUTE(a) __attribute((a)) +#else +#define HASHMAP_ATTRIBUTE(a) __attribute__((a)) +#endif + +#if defined(_MSC_VER) +#define HASHMAP_WEAK __inline +#elif defined(__MINGW32__) || defined(__MINGW64__) +#define HASHMAP_WEAK static HASHMAP_ATTRIBUTE(used) +#elif defined(__clang__) || defined(__GNUC__) || defined(__TINYC__) +#define HASHMAP_WEAK HASHMAP_ATTRIBUTE(weak) +#else +#error Non clang, non gcc, non MSVC, non tcc compiler found! +#endif + +#if defined(_MSC_VER) +#define HASHMAP_ALWAYS_INLINE __forceinline +#elif (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + defined(__cplusplus) +#define HASHMAP_ALWAYS_INLINE __attribute__((always_inline)) inline +#else +/* If we cannot use inline, its not safe to use always_inline, so we mark the + * function weak. */ +#define HASHMAP_ALWAYS_INLINE HASHMAP_WEAK +#endif + +#if defined(_MSC_VER) && (_MSC_VER < 1920) +typedef unsigned __int8 hashmap_uint8_t; +typedef unsigned __int32 hashmap_uint32_t; +typedef unsigned __int64 hashmap_uint64_t; +#else +#include +typedef uint8_t hashmap_uint8_t; +typedef uint32_t hashmap_uint32_t; +typedef uint64_t hashmap_uint64_t; +#endif + +typedef struct hashmap_element_s { + const void *key; + hashmap_uint32_t key_len; + int in_use; + void *data; +} hashmap_element_t; + +typedef hashmap_uint32_t (*hashmap_hasher_t)(hashmap_uint32_t seed, + const void *key, + hashmap_uint32_t key_len); +typedef int (*hashmap_comparer_t)(const void *a, hashmap_uint32_t a_len, + const void *b, hashmap_uint32_t b_len); + +typedef struct hashmap_s { + hashmap_uint32_t log2_capacity; + hashmap_uint32_t size; + hashmap_hasher_t hasher; + hashmap_comparer_t comparer; + struct hashmap_element_s *data; +} hashmap_t; + +#define HASHMAP_LINEAR_PROBE_LENGTH (8) + +typedef struct hashmap_create_options_s { + hashmap_hasher_t hasher; + hashmap_comparer_t comparer; + hashmap_uint32_t initial_capacity; + hashmap_uint32_t _; +} hashmap_create_options_t; + +#if defined(__cplusplus) +extern "C" { +#endif + +/// @brief Create a hashmap. +/// @param initial_capacity The initial capacity of the hashmap. +/// @param out_hashmap The storage for the created hashmap. Can be on stack. +/// @return On success 0 is returned. +/// @note The `out_hashmap` doens't need to be on heap. +HASHMAP_WEAK int hashmap_create(const hashmap_uint32_t initial_capacity, + struct hashmap_s *const out_hashmap); + +/// @brief Create a hashmap. +/// @param options The options to create the hashmap with. +/// @param out_hashmap The storage for the created hashmap. +/// @return On success 0 is returned. +/// +/// The `options`' members work as follows: +/// `initial_capacity`: The initial capacity of the hashmap. +/// `hasher`: Which hashing function to use with the hashmap (by default the +/// crc32 with Robert Jenkins' mix is used). +/// `comparer`: Which comparison function to use with the hashmap (by default +/// `memcmp` is used). +HASHMAP_WEAK int hashmap_create_ex(struct hashmap_create_options_s options, + struct hashmap_s *const out_hashmap); + +/// @brief Put an element into the hashmap. If the key already exists, its value +/// is updated. +/// @param hashmap The hashmap to insert into. +/// @param key The string key to use. +/// @param len The length of the string key. +/// @param value The value to insert. +/// @return On success 0 is returned. +/// +/// The key string slice is not copied when creating the hashmap entry, and thus +/// must remain a valid pointer until the hashmap entry is removed or the +/// hashmap is destroyed. +HASHMAP_WEAK int hashmap_put(struct hashmap_s *const hashmap, + const void *const key, const hashmap_uint32_t len, + void *const value); + +/// @brief Get an element from the hashmap. +/// @param hashmap The hashmap to get from. +/// @param key The string key to use. +/// @param len The length of the string key. +/// @return The previously set element, or NULL if none exists. +HASHMAP_WEAK void *hashmap_get(const struct hashmap_s *const hashmap, + const void *const key, + const hashmap_uint32_t len); + +/// @brief Remove an element from the hashmap. +/// @param hashmap The hashmap to remove from. +/// @param key The string key to use. +/// @param len The length of the string key. +/// @return On success 0 is returned. +HASHMAP_WEAK int hashmap_remove(struct hashmap_s *const hashmap, + const void *const key, + const hashmap_uint32_t len); + +/// @brief Remove an element from the hashmap. +/// @param hashmap The hashmap to remove from. +/// @param key The string key to use. +/// @param len The length of the string key. +/// @return On success the original stored key pointer is returned, on failure +/// NULL is returned. +HASHMAP_WEAK const void *hashmap_remove_and_return_key( + struct hashmap_s *const hashmap, const void *const key, + const hashmap_uint32_t len); + +/// @brief Iterate over all the elements in a hashmap. +/// @param hashmap The hashmap to iterate over. +/// @param iterator The function pointer to call on each element. +/// @param context The context to pass as the first argument to f. +/// @return If the entire hashmap was iterated then 0 is returned. Otherwise if +/// the callback function f returned non-zero then non-zero is returned. +HASHMAP_WEAK int hashmap_iterate(const struct hashmap_s *const hashmap, + int (*iterator)(void *const context, + void *const value), + void *const context); + +/// @brief Iterate over all the elements in a hashmap. +/// @param hashmap The hashmap to iterate over. +/// @param iterator The function pointer to call on each element. +/// @param context The context to pass as the first argument to f. +/// @return If the entire hashmap was iterated then 0 is returned. +/// Otherwise if the callback function f returned positive then the positive +/// value is returned. If the callback function returns -1, the current item +/// is removed and iteration continues. +HASHMAP_WEAK int hashmap_iterate_pairs( + struct hashmap_s *const hashmap, + int (*iterator)(void *const, struct hashmap_element_s *const), + void *const context); + +/// @brief Get the size of the hashmap. +/// @param hashmap The hashmap to get the size of. +/// @return The size of the hashmap. +HASHMAP_ALWAYS_INLINE hashmap_uint32_t +hashmap_num_entries(const struct hashmap_s *const hashmap); + +/// @brief Get the capacity of the hashmap. +/// @param hashmap The hashmap to get the size of. +/// @return The capacity of the hashmap. +HASHMAP_ALWAYS_INLINE hashmap_uint32_t +hashmap_capacity(const struct hashmap_s *const hashmap); + +/// @brief Destroy the hashmap. +/// @param hashmap The hashmap to destroy. +/// @note If the `hashmap` is created on heap, it's NOT automatically freed. +/// Care need to be taken to not leak memory. +HASHMAP_WEAK void hashmap_destroy(struct hashmap_s *const hashmap); + +static hashmap_uint32_t hashmap_crc32_hasher(const hashmap_uint32_t seed, + const void *const s, + const hashmap_uint32_t len); +static int hashmap_memcmp_comparer(const void *const a, + const hashmap_uint32_t a_len, + const void *const b, + const hashmap_uint32_t b_len); +HASHMAP_ALWAYS_INLINE hashmap_uint32_t hashmap_hash_helper_int_helper( + const struct hashmap_s *const m, const void *const key, + const hashmap_uint32_t len); +HASHMAP_ALWAYS_INLINE int hashmap_hash_helper( + const struct hashmap_s *const m, const void *const key, + const hashmap_uint32_t len, hashmap_uint32_t *const out_index); +HASHMAP_WEAK int hashmap_rehash_iterator(void *const new_hash, + struct hashmap_element_s *const e); +HASHMAP_ALWAYS_INLINE int hashmap_rehash_helper(struct hashmap_s *const m); +HASHMAP_ALWAYS_INLINE hashmap_uint32_t hashmap_clz(const hashmap_uint32_t x); + +#if defined(__cplusplus) +} +#endif + +#if defined(__cplusplus) +#define HASHMAP_CAST(type, x) static_cast(x) +#define HASHMAP_PTR_CAST(type, x) reinterpret_cast(x) +#define HASHMAP_NULL NULL +#else +#define HASHMAP_CAST(type, x) ((type)(x)) +#define HASHMAP_PTR_CAST(type, x) ((type)(x)) +#define HASHMAP_NULL 0 +#endif + +int hashmap_create(const hashmap_uint32_t initial_capacity, + struct hashmap_s *const out_hashmap) { + struct hashmap_create_options_s options; + memset(&options, 0, sizeof(options)); + options.initial_capacity = initial_capacity; + + return hashmap_create_ex(options, out_hashmap); +} + +int hashmap_create_ex(struct hashmap_create_options_s options, + struct hashmap_s *const out_hashmap) { + if (2 > options.initial_capacity) { + options.initial_capacity = 2; + } else if (0 != (options.initial_capacity & (options.initial_capacity - 1))) { + options.initial_capacity = 1u + << (32 - hashmap_clz(options.initial_capacity)); + } + + if (HASHMAP_NULL == options.hasher) { + options.hasher = &hashmap_crc32_hasher; + } + + if (HASHMAP_NULL == options.comparer) { + options.comparer = &hashmap_memcmp_comparer; + } + + out_hashmap->data = HASHMAP_CAST( + struct hashmap_element_s *, + calloc(options.initial_capacity + HASHMAP_LINEAR_PROBE_LENGTH, + sizeof(struct hashmap_element_s))); + + out_hashmap->log2_capacity = 31 - hashmap_clz(options.initial_capacity); + out_hashmap->size = 0; + out_hashmap->hasher = options.hasher; + out_hashmap->comparer = options.comparer; + + return 0; +} + +int hashmap_put(struct hashmap_s *const m, const void *const key, + const hashmap_uint32_t len, void *const value) { + hashmap_uint32_t index; + + if ((HASHMAP_NULL == key) || (0 == len)) { + return 1; + } + + /* Find a place to put our value. */ + while (!hashmap_hash_helper(m, key, len, &index)) { + if (hashmap_rehash_helper(m)) { + return 1; + } + } + + /* Set the data. */ + m->data[index].data = value; + m->data[index].key = key; + m->data[index].key_len = len; + + /* If the hashmap element was not already in use, set that it is being used + * and bump our size. */ + if (0 == m->data[index].in_use) { + m->data[index].in_use = 1; + m->size++; + } + + return 0; +} + +void *hashmap_get(const struct hashmap_s *const m, const void *const key, + const hashmap_uint32_t len) { + hashmap_uint32_t i, curr; + + if ((HASHMAP_NULL == key) || (0 == len)) { + return HASHMAP_NULL; + } + + curr = hashmap_hash_helper_int_helper(m, key, len); + + /* Linear probing, if necessary */ + for (i = 0; i < HASHMAP_LINEAR_PROBE_LENGTH; i++) { + const hashmap_uint32_t index = curr + i; + + if (m->data[index].in_use) { + if (m->comparer(m->data[index].key, m->data[index].key_len, key, len)) { + return m->data[index].data; + } + } + } + + /* Not found */ + return HASHMAP_NULL; +} + +int hashmap_remove(struct hashmap_s *const m, const void *const key, + const hashmap_uint32_t len) { + hashmap_uint32_t i, curr; + + if ((HASHMAP_NULL == key) || (0 == len)) { + return 1; + } + + curr = hashmap_hash_helper_int_helper(m, key, len); + + /* Linear probing, if necessary */ + for (i = 0; i < HASHMAP_LINEAR_PROBE_LENGTH; i++) { + const hashmap_uint32_t index = curr + i; + + if (m->data[index].in_use) { + if (m->comparer(m->data[index].key, m->data[index].key_len, key, len)) { + /* Blank out the fields including in_use */ + memset(&m->data[index], 0, sizeof(struct hashmap_element_s)); + + /* Reduce the size */ + m->size--; + + return 0; + } + } + } + + return 1; +} + +const void *hashmap_remove_and_return_key(struct hashmap_s *const m, + const void *const key, + const hashmap_uint32_t len) { + hashmap_uint32_t i, curr; + + if ((HASHMAP_NULL == key) || (0 == len)) { + return HASHMAP_NULL; + } + + curr = hashmap_hash_helper_int_helper(m, key, len); + + /* Linear probing, if necessary */ + for (i = 0; i < HASHMAP_LINEAR_PROBE_LENGTH; i++) { + const hashmap_uint32_t index = curr + i; + + if (m->data[index].in_use) { + if (m->comparer(m->data[index].key, m->data[index].key_len, key, len)) { + const void *const stored_key = m->data[index].key; + + /* Blank out the fields */ + memset(&m->data[index], 0, sizeof(struct hashmap_element_s)); + + /* Reduce the size */ + m->size--; + + return stored_key; + } + } + } + + return HASHMAP_NULL; +} + +int hashmap_iterate(const struct hashmap_s *const m, + int (*f)(void *const, void *const), void *const context) { + hashmap_uint32_t i; + + for (i = 0; i < (hashmap_capacity(m) + HASHMAP_LINEAR_PROBE_LENGTH); i++) { + if (m->data[i].in_use) { + if (!f(context, m->data[i].data)) { + return 1; + } + } + } + + return 0; +} + +int hashmap_iterate_pairs(struct hashmap_s *const m, + int (*f)(void *const, + struct hashmap_element_s *const), + void *const context) { + hashmap_uint32_t i; + struct hashmap_element_s *p; + int r; + + for (i = 0; i < (hashmap_capacity(m) + HASHMAP_LINEAR_PROBE_LENGTH); i++) { + p = &m->data[i]; + if (p->in_use) { + r = f(context, p); + switch (r) { + case -1: /* remove item */ + memset(p, 0, sizeof(struct hashmap_element_s)); + m->size--; + break; + case 0: /* continue iterating */ + break; + default: /* early exit */ + return 1; + } + } + } + return 0; +} + +void hashmap_destroy(struct hashmap_s *const m) { + free(m->data); + memset(m, 0, sizeof(struct hashmap_s)); +} + +HASHMAP_ALWAYS_INLINE hashmap_uint32_t +hashmap_num_entries(const struct hashmap_s *const m) { + return m->size; +} + +HASHMAP_ALWAYS_INLINE hashmap_uint32_t +hashmap_capacity(const struct hashmap_s *const m) { + return 1u << m->log2_capacity; +} + +hashmap_uint32_t hashmap_crc32_hasher(const hashmap_uint32_t seed, + const void *const k, + const hashmap_uint32_t len) { + hashmap_uint32_t i = 0; + hashmap_uint32_t crc32val = seed; + const hashmap_uint8_t *const s = HASHMAP_PTR_CAST(const hashmap_uint8_t *, k); + +#if defined(HASHMAP_X86_SSE42) + for (; (i + sizeof(hashmap_uint32_t)) < len; i += sizeof(hashmap_uint32_t)) { + hashmap_uint32_t next; + memcpy(&next, &s[i], sizeof(next)); + crc32val = _mm_crc32_u32(crc32val, next); + } + + for (; i < len; i++) { + crc32val = _mm_crc32_u8(crc32val, s[i]); + } +#elif defined(HASHMAP_ARM_CRC32) + for (; (i + sizeof(hashmap_uint64_t)) < len; i += sizeof(hashmap_uint64_t)) { + hashmap_uint64_t next; + memcpy(&next, &s[i], sizeof(next)); + crc32val = __crc32d(crc32val, next); + } + + for (; i < len; i++) { + crc32val = __crc32b(crc32val, s[i]); + } +#else + // Using polynomial 0x11EDC6F41 to match SSE 4.2's crc function. + static const hashmap_uint32_t crc32_tab[] = { + 0x00000000U, 0xF26B8303U, 0xE13B70F7U, 0x1350F3F4U, 0xC79A971FU, + 0x35F1141CU, 0x26A1E7E8U, 0xD4CA64EBU, 0x8AD958CFU, 0x78B2DBCCU, + 0x6BE22838U, 0x9989AB3BU, 0x4D43CFD0U, 0xBF284CD3U, 0xAC78BF27U, + 0x5E133C24U, 0x105EC76FU, 0xE235446CU, 0xF165B798U, 0x030E349BU, + 0xD7C45070U, 0x25AFD373U, 0x36FF2087U, 0xC494A384U, 0x9A879FA0U, + 0x68EC1CA3U, 0x7BBCEF57U, 0x89D76C54U, 0x5D1D08BFU, 0xAF768BBCU, + 0xBC267848U, 0x4E4DFB4BU, 0x20BD8EDEU, 0xD2D60DDDU, 0xC186FE29U, + 0x33ED7D2AU, 0xE72719C1U, 0x154C9AC2U, 0x061C6936U, 0xF477EA35U, + 0xAA64D611U, 0x580F5512U, 0x4B5FA6E6U, 0xB93425E5U, 0x6DFE410EU, + 0x9F95C20DU, 0x8CC531F9U, 0x7EAEB2FAU, 0x30E349B1U, 0xC288CAB2U, + 0xD1D83946U, 0x23B3BA45U, 0xF779DEAEU, 0x05125DADU, 0x1642AE59U, + 0xE4292D5AU, 0xBA3A117EU, 0x4851927DU, 0x5B016189U, 0xA96AE28AU, + 0x7DA08661U, 0x8FCB0562U, 0x9C9BF696U, 0x6EF07595U, 0x417B1DBCU, + 0xB3109EBFU, 0xA0406D4BU, 0x522BEE48U, 0x86E18AA3U, 0x748A09A0U, + 0x67DAFA54U, 0x95B17957U, 0xCBA24573U, 0x39C9C670U, 0x2A993584U, + 0xD8F2B687U, 0x0C38D26CU, 0xFE53516FU, 0xED03A29BU, 0x1F682198U, + 0x5125DAD3U, 0xA34E59D0U, 0xB01EAA24U, 0x42752927U, 0x96BF4DCCU, + 0x64D4CECFU, 0x77843D3BU, 0x85EFBE38U, 0xDBFC821CU, 0x2997011FU, + 0x3AC7F2EBU, 0xC8AC71E8U, 0x1C661503U, 0xEE0D9600U, 0xFD5D65F4U, + 0x0F36E6F7U, 0x61C69362U, 0x93AD1061U, 0x80FDE395U, 0x72966096U, + 0xA65C047DU, 0x5437877EU, 0x4767748AU, 0xB50CF789U, 0xEB1FCBADU, + 0x197448AEU, 0x0A24BB5AU, 0xF84F3859U, 0x2C855CB2U, 0xDEEEDFB1U, + 0xCDBE2C45U, 0x3FD5AF46U, 0x7198540DU, 0x83F3D70EU, 0x90A324FAU, + 0x62C8A7F9U, 0xB602C312U, 0x44694011U, 0x5739B3E5U, 0xA55230E6U, + 0xFB410CC2U, 0x092A8FC1U, 0x1A7A7C35U, 0xE811FF36U, 0x3CDB9BDDU, + 0xCEB018DEU, 0xDDE0EB2AU, 0x2F8B6829U, 0x82F63B78U, 0x709DB87BU, + 0x63CD4B8FU, 0x91A6C88CU, 0x456CAC67U, 0xB7072F64U, 0xA457DC90U, + 0x563C5F93U, 0x082F63B7U, 0xFA44E0B4U, 0xE9141340U, 0x1B7F9043U, + 0xCFB5F4A8U, 0x3DDE77ABU, 0x2E8E845FU, 0xDCE5075CU, 0x92A8FC17U, + 0x60C37F14U, 0x73938CE0U, 0x81F80FE3U, 0x55326B08U, 0xA759E80BU, + 0xB4091BFFU, 0x466298FCU, 0x1871A4D8U, 0xEA1A27DBU, 0xF94AD42FU, + 0x0B21572CU, 0xDFEB33C7U, 0x2D80B0C4U, 0x3ED04330U, 0xCCBBC033U, + 0xA24BB5A6U, 0x502036A5U, 0x4370C551U, 0xB11B4652U, 0x65D122B9U, + 0x97BAA1BAU, 0x84EA524EU, 0x7681D14DU, 0x2892ED69U, 0xDAF96E6AU, + 0xC9A99D9EU, 0x3BC21E9DU, 0xEF087A76U, 0x1D63F975U, 0x0E330A81U, + 0xFC588982U, 0xB21572C9U, 0x407EF1CAU, 0x532E023EU, 0xA145813DU, + 0x758FE5D6U, 0x87E466D5U, 0x94B49521U, 0x66DF1622U, 0x38CC2A06U, + 0xCAA7A905U, 0xD9F75AF1U, 0x2B9CD9F2U, 0xFF56BD19U, 0x0D3D3E1AU, + 0x1E6DCDEEU, 0xEC064EEDU, 0xC38D26C4U, 0x31E6A5C7U, 0x22B65633U, + 0xD0DDD530U, 0x0417B1DBU, 0xF67C32D8U, 0xE52CC12CU, 0x1747422FU, + 0x49547E0BU, 0xBB3FFD08U, 0xA86F0EFCU, 0x5A048DFFU, 0x8ECEE914U, + 0x7CA56A17U, 0x6FF599E3U, 0x9D9E1AE0U, 0xD3D3E1ABU, 0x21B862A8U, + 0x32E8915CU, 0xC083125FU, 0x144976B4U, 0xE622F5B7U, 0xF5720643U, + 0x07198540U, 0x590AB964U, 0xAB613A67U, 0xB831C993U, 0x4A5A4A90U, + 0x9E902E7BU, 0x6CFBAD78U, 0x7FAB5E8CU, 0x8DC0DD8FU, 0xE330A81AU, + 0x115B2B19U, 0x020BD8EDU, 0xF0605BEEU, 0x24AA3F05U, 0xD6C1BC06U, + 0xC5914FF2U, 0x37FACCF1U, 0x69E9F0D5U, 0x9B8273D6U, 0x88D28022U, + 0x7AB90321U, 0xAE7367CAU, 0x5C18E4C9U, 0x4F48173DU, 0xBD23943EU, + 0xF36E6F75U, 0x0105EC76U, 0x12551F82U, 0xE03E9C81U, 0x34F4F86AU, + 0xC69F7B69U, 0xD5CF889DU, 0x27A40B9EU, 0x79B737BAU, 0x8BDCB4B9U, + 0x988C474DU, 0x6AE7C44EU, 0xBE2DA0A5U, 0x4C4623A6U, 0x5F16D052U, + 0xAD7D5351U}; + + for (; i < len; i++) { + crc32val = crc32_tab[(HASHMAP_CAST(hashmap_uint8_t, crc32val) ^ s[i])] ^ + (crc32val >> 8); + } +#endif + + // Use the mix function from murmur3. + crc32val ^= len; + + crc32val ^= crc32val >> 16; + crc32val *= 0x85ebca6b; + crc32val ^= crc32val >> 13; + crc32val *= 0xc2b2ae35; + crc32val ^= crc32val >> 16; + + return crc32val; +} + +int hashmap_memcmp_comparer(const void *const a, const hashmap_uint32_t a_len, + const void *const b, const hashmap_uint32_t b_len) { + return (a_len == b_len) && (0 == memcmp(a, b, a_len)); +} + +HASHMAP_ALWAYS_INLINE hashmap_uint32_t +hashmap_hash_helper_int_helper(const struct hashmap_s *const m, + const void *const k, const hashmap_uint32_t l) { + return (m->hasher(~0u, k, l) * 2654435769u) >> (32u - m->log2_capacity); +} + +HASHMAP_ALWAYS_INLINE int hashmap_hash_helper( + const struct hashmap_s *const m, const void *const key, + const hashmap_uint32_t len, hashmap_uint32_t *const out_index) { + hashmap_uint32_t curr; + hashmap_uint32_t i; + hashmap_uint32_t first_free; + + /* If full, return immediately */ + if (hashmap_num_entries(m) == hashmap_capacity(m)) { + return 0; + } + + /* Find the best index */ + curr = hashmap_hash_helper_int_helper(m, key, len); + first_free = ~0u; + + for (i = 0; i < HASHMAP_LINEAR_PROBE_LENGTH; i++) { + const hashmap_uint32_t index = curr + i; + + if (!m->data[index].in_use) { + first_free = (first_free < index) ? first_free : index; + } else if (m->comparer(m->data[index].key, m->data[index].key_len, key, + len)) { + *out_index = index; + return 1; + } + } + + // Couldn't find a free element in the linear probe. + if (~0u == first_free) { + return 0; + } + + *out_index = first_free; + return 1; +} + +int hashmap_rehash_iterator(void *const new_hash, + struct hashmap_element_s *const e) { + int temp = hashmap_put(HASHMAP_PTR_CAST(struct hashmap_s *, new_hash), e->key, + e->key_len, e->data); + + if (0 < temp) { + return 1; + } + + /* clear old value to avoid stale pointers */ + return -1; +} + +/* + * Doubles the size of the hashmap, and rehashes all the elements + */ +HASHMAP_ALWAYS_INLINE int hashmap_rehash_helper(struct hashmap_s *const m) { + struct hashmap_create_options_s options; + struct hashmap_s new_m; + int flag; + + memset(&options, 0, sizeof(options)); + options.initial_capacity = hashmap_capacity(m) * 2; + options.hasher = m->hasher; + // upstream PR: #38 + options.comparer = m->comparer; + + if (0 == options.initial_capacity) { + return 1; + } + + flag = hashmap_create_ex(options, &new_m); + + if (0 != flag) { + return flag; + } + + /* copy the old elements to the new table */ + flag = hashmap_iterate_pairs(m, hashmap_rehash_iterator, + HASHMAP_PTR_CAST(void *, &new_m)); + + if (0 != flag) { + return flag; + } + + hashmap_destroy(m); + + /* put new hash into old hash structure by copying */ + memcpy(m, &new_m, sizeof(struct hashmap_s)); + + return 0; +} + +HASHMAP_ALWAYS_INLINE hashmap_uint32_t hashmap_clz(const hashmap_uint32_t x) { +#if defined(_MSC_VER) + unsigned long result; + _BitScanReverse(&result, x); + return 31u - HASHMAP_CAST(hashmap_uint32_t, result); +#elif defined(__TINYC__) + union { + double d; + hashmap_uint64_t u; + } u; + u.d = HASHMAP_CAST(double, x) + 0.5; + return 1054u - HASHMAP_CAST(hashmap_uint32_t, u.u >> 52); +#else + return HASHMAP_CAST(hashmap_uint32_t, __builtin_clz(x)); +#endif +} + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +#endif \ No newline at end of file diff --git a/libCacheSim/include/libCacheSim/hashmap_defs.in b/libCacheSim/include/libCacheSim/hashmap_defs.in new file mode 100644 index 00000000..c34ecf66 --- /dev/null +++ b/libCacheSim/include/libCacheSim/hashmap_defs.in @@ -0,0 +1,21 @@ +#ifndef INCLUDE_HASHMAP_DEFS +#define INCLUDE_HASHMAP_DEFS + +#include "../config.h" +#include "hashmap.h" + +// copied from gLib 2.84.0 glib/ghash.c:2647 +// glib uses pointers everywhere, we do not +static hashmap_uint32_t obj_id_hasher(const hashmap_uint32_t seed, + const void *const s, + const hashmap_uint32_t len) { + return (hashmap_uint32_t)(((obj_id_t)s >> 32) ^ ((obj_id_t)s & 0xffffffffU)); +} + +// also, do direct comparisions instead of mangling with pointers +static int obj_id_comparer(const void *const a, const hashmap_uint32_t a_len, + const void *const b, const hashmap_uint32_t b_len) { + return (const obj_id_t)a == (const obj_id_t)b; +} + +#endif \ No newline at end of file diff --git a/libCacheSim/profiler/CMakeLists.txt b/libCacheSim/profiler/CMakeLists.txt index c2b5ff40..50966036 100644 --- a/libCacheSim/profiler/CMakeLists.txt +++ b/libCacheSim/profiler/CMakeLists.txt @@ -1,6 +1,6 @@ aux_source_directory(. DIR_LIB_SRCS) add_library (profiler ${DIR_LIB_SRCS}) -target_link_libraries(profiler traceReader) +target_link_libraries(profiler traceReader utils) #file(GLOB src *.c) #add_library (profiler ${src}) diff --git a/libCacheSim/profiler/dist.c b/libCacheSim/profiler/dist.c index 472c9adb..839c8f54 100644 --- a/libCacheSim/profiler/dist.c +++ b/libCacheSim/profiler/dist.c @@ -6,13 +6,16 @@ extern "C" { #endif +#include "../include/libCacheSim/dist.h" + #include #include #include #include #include "../dataStructure/splay.h" -#include "../include/libCacheSim/dist.h" +#include "../include/libCacheSim/hashmap.h" +#include "../include/libCacheSim/hashmap_defs.in" #include "../include/libCacheSim/macro.h" /*********************************************************** @@ -30,10 +33,11 @@ extern "C" { * @param dist_type DIST_SINCE_LAST_ACCESS or DIST_SINCE_FIRST_ACCESS * @return distance to last access */ -int64_t get_access_dist_add_req(const request_t *req, GHashTable *hash_table, +int64_t get_access_dist_add_req(const request_t *req, hashmap_t *hash_table, const int64_t curr_ts, const dist_type_e dist_type) { - gpointer gp = g_hash_table_lookup(hash_table, GSIZE_TO_POINTER(req->obj_id)); + void *gp = + hashmap_get(hash_table, (const void *)(req->obj_id), sizeof(obj_id_t)); int64_t ret = -1; if (gp == NULL) { // it has not been requested before @@ -46,8 +50,8 @@ int64_t get_access_dist_add_req(const request_t *req, GHashTable *hash_table, if (dist_type == DIST_SINCE_LAST_ACCESS) { /* update last access time */ - g_hash_table_insert(hash_table, GSIZE_TO_POINTER(req->obj_id), - GSIZE_TO_POINTER((gsize)curr_ts)); + hashmap_put(hash_table, (const void *)(req->obj_id), sizeof(obj_id_t), + (void *)curr_ts); } else if (dist_type == DIST_SINCE_FIRST_ACCESS) { /* do nothing */ } else { @@ -70,9 +74,10 @@ int64_t get_access_dist_add_req(const request_t *req, GHashTable *hash_table, * @return stack distance */ int64_t get_stack_dist_add_req(const request_t *req, sTree **splay_tree, - GHashTable *hash_table, const int64_t curr_ts, + hashmap_t *hash_table, const int64_t curr_ts, int64_t *last_access_ts) { - gpointer gp = g_hash_table_lookup(hash_table, GSIZE_TO_POINTER(req->obj_id)); + void *gp = + hashmap_get(hash_table, (const void *)(req->obj_id), sizeof(obj_id_t)); int64_t ret = -1; sTree *newtree; @@ -85,7 +90,7 @@ int64_t get_stack_dist_add_req(const request_t *req, sTree **splay_tree, newtree = insert(curr_ts, *splay_tree); } else { // not first time access - int64_t old_ts = (int64_t)GPOINTER_TO_SIZE(gp); + int64_t old_ts = (int64_t)gp; if (last_access_ts != NULL) { *last_access_ts = old_ts; } @@ -95,8 +100,8 @@ int64_t get_stack_dist_add_req(const request_t *req, sTree **splay_tree, newtree = insert(curr_ts, newtree); } - g_hash_table_insert(hash_table, GSIZE_TO_POINTER(req->obj_id), - (gpointer)GSIZE_TO_POINTER((gsize)curr_ts)); + hashmap_put(hash_table, (const void *)(req->obj_id), sizeof(obj_id_t), + (void *)curr_ts); *splay_tree = newtree; @@ -123,8 +128,11 @@ int32_t *get_stack_dist(reader_t *reader, const dist_type_e dist_type, } } - GHashTable *hash_table = - g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + hashmap_t *hash_table = malloc(sizeof(hashmap_t)); + hashmap_create_options_t options = {.initial_capacity = 16, + .comparer = obj_id_comparer, + .hasher = obj_id_hasher}; + hashmap_create_ex(options, hash_table); // create splay tree sTree *splay_tree = NULL; @@ -153,7 +161,8 @@ int32_t *get_stack_dist(reader_t *reader, const dist_type_e dist_type, // clean up free_request(req); - g_hash_table_destroy(hash_table); + hashmap_destroy(hash_table); + free(hash_table); free_sTree(splay_tree); reset_reader(reader); return stack_dist_array; @@ -167,8 +176,11 @@ int32_t *get_access_dist(reader_t *reader, const dist_type_e dist_type, *array_size = get_num_of_req(reader); int32_t *dist_array = malloc(sizeof(int32_t) * get_num_of_req(reader)); - GHashTable *hash_table = - g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + hashmap_t *hash_table = malloc(sizeof(hashmap_t)); + hashmap_create_options_t options = {.initial_capacity = 16, + .comparer = obj_id_comparer, + .hasher = obj_id_hasher}; + hashmap_create_ex(options, hash_table); read_one_req(reader, req); @@ -186,7 +198,8 @@ int32_t *get_access_dist(reader_t *reader, const dist_type_e dist_type, // clean up free_request(req); - g_hash_table_destroy(hash_table); + hashmap_destroy(hash_table); + free(hash_table); reset_reader(reader); return dist_array; @@ -203,8 +216,7 @@ void save_dist(reader_t *const reader, const int32_t *dist_array, free(file_path); } -void save_dist_txt(reader_t *const reader , - const int32_t *dist_array, +void save_dist_txt(reader_t *const reader, const int32_t *dist_array, int64_t array_size, const char *const ofilepath, const dist_type_e dist_type) { char *file_path = (char *)malloc(strlen(ofilepath) + 128); @@ -243,39 +255,55 @@ int32_t *load_dist(reader_t *const reader, const char *const ifilepath, } void cnt_dist(const int32_t *dist_array, const int64_t array_size, - GHashTable *hash_table) { + hashmap_t *hash_table) { for (int64_t i = 0; i < array_size; i++) { int64_t dist = dist_array[i] == -1 ? INT64_MAX : dist_array[i]; - gpointer gp_dist = GSIZE_TO_POINTER((gsize)dist); - int64_t old_cnt = (int64_t)g_hash_table_lookup(hash_table, gp_dist); - g_hash_table_replace(hash_table, gp_dist, GSIZE_TO_POINTER(old_cnt + 1)); + int64_t old_cnt = + (int64_t)hashmap_get(hash_table, (const void *)dist, sizeof(int64_t)); + hashmap_put(hash_table, (const void *)dist, sizeof(int64_t), + (void *)(old_cnt + 1)); } } -void _write_dist_cnt(gpointer k, gpointer v, gpointer user_data) { - int64_t dist = (int64_t)GPOINTER_TO_SIZE(k); - int64_t cnt = (int64_t)GPOINTER_TO_SIZE(v); +/** + * void _write_dist_cnt(gpointer k, gpointer v, gpointer user_data) { + * int64_t dist = (int64_t)GPOINTER_TO_SIZE(k); + * int64_t cnt = (int64_t)GPOINTER_TO_SIZE(v); + * FILE *file = (FILE *)user_data; + * fprintf(file, "%ld:%ld, ", (long)dist, (long)cnt); + * } + **/ + +static int _write_dist_cnt(void *const user_data, + struct hashmap_element_s *const e) { FILE *file = (FILE *)user_data; - fprintf(file, "%ld:%ld, ", (long)dist, (long)cnt); + int64_t dist = (long)(e->key); + int64_t cnt = (long)(e->data); + fprintf(file, "%ld:%ld, ", dist, cnt); + return 1; } void save_dist_as_cnt_txt(reader_t *const reader, const int32_t *dist_array, const int64_t array_size, const char *const ofilepath, const dist_type_e dist_type) { - assert((int64_t) get_num_of_req(reader) == array_size); + assert((int64_t)get_num_of_req(reader) == array_size); char *file_path = (char *)malloc(strlen(ofilepath) + 128); sprintf(file_path, "%s.%s.cnt", ofilepath, g_dist_type_name[dist_type]); FILE *file = fopen(file_path, "w"); - GHashTable *hash_table = - g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + hashmap_t *hash_table = malloc(sizeof(hashmap_t)); + hashmap_create_options_t options = {.initial_capacity = 16, + .comparer = obj_id_comparer, + .hasher = obj_id_hasher}; + hashmap_create_ex(options, hash_table); cnt_dist(dist_array, get_num_of_req(reader), hash_table); - g_hash_table_foreach(hash_table, (GHFunc)_write_dist_cnt, file); + hashmap_iterate_pairs(hash_table, _write_dist_cnt, file); - g_hash_table_destroy(hash_table); + hashmap_destroy(hash_table); + free(hash_table); fclose(file); free(file_path); diff --git a/libCacheSim/profiler/profilerLRU.c b/libCacheSim/profiler/profilerLRU.c index 503365c1..ed688765 100644 --- a/libCacheSim/profiler/profilerLRU.c +++ b/libCacheSim/profiler/profilerLRU.c @@ -6,15 +6,18 @@ // Copyright © 2016 Juncheng. All rights reserved. // -#include "../dataStructure/splay.h" #include "../include/libCacheSim/profilerLRU.h" +#include "../dataStructure/splay.h" +#include "../include/libCacheSim/hashmap.h" +#include "../include/libCacheSim/hashmap_defs.in" + #ifdef __cplusplus extern "C" { #endif int64_t get_stack_dist_add_req(const request_t *req, sTree **splay_tree, - GHashTable *hash_table, const int64_t curr_ts, + hashmap_t *hash_table, const int64_t curr_ts, int64_t *last_access_ts); int64_t *_get_lru_hit_cnt(reader_t *reader, int64_t size); @@ -61,8 +64,11 @@ int64_t *_get_lru_hit_cnt(reader_t *reader, int64_t size) { request_t *req = new_request(); // create hash table and splay tree - GHashTable *hash_table = - g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + hashmap_t *hash_table = malloc(sizeof(hashmap_t)); + hashmap_create_options_t options = {.initial_capacity = 16, + .comparer = obj_id_comparer, + .hasher = obj_id_hasher}; + hashmap_create_ex(options, hash_table); sTree *splay_tree = NULL; read_one_req(reader, req); @@ -89,7 +95,8 @@ int64_t *_get_lru_hit_cnt(reader_t *reader, int64_t size) { // clean up free_request(req); - g_hash_table_destroy(hash_table); + hashmap_destroy(hash_table); + free(hash_table); free_sTree(splay_tree); reset_reader(reader); return hit_count_array; diff --git a/libCacheSim/profiler/simulator.c b/libCacheSim/profiler/simulator.c index a1e712f2..e81391b5 100644 --- a/libCacheSim/profiler/simulator.c +++ b/libCacheSim/profiler/simulator.c @@ -17,6 +17,7 @@ extern "C" { #include "../cache/cacheUtils.h" #include "../include/libCacheSim/evictionAlgo.h" #include "../include/libCacheSim/plugin.h" +#include "../utils/include/threadpool.h" #include "../utils/include/myprint.h" #include "../utils/include/mystr.h" @@ -29,16 +30,18 @@ typedef struct simulator_multithreading_params { reader_t *warmup_reader; int warmup_sec; /* num of seconds of requests used for warming up cache */ cache_stat_t *result; - GMutex mtx; /* prevent simultaneous write to progress */ - gint *progress; - gpointer other_data; + pthread_mutex_t mtx; /* prevent simultaneous write to progress */ + pthread_cond_t cond; + int *progress; + void *other_data; bool free_cache_when_finish; bool use_random_seed; } sim_mt_params_t; -static void _simulate(gpointer data, gpointer user_data) { +static void _simulate(void *user_data, void *data) { sim_mt_params_t *params = (sim_mt_params_t *)user_data; - int idx = GPOINTER_TO_UINT(data) - 1; + // int idx = GPOINTER_TO_UINT(data) - 1; + int idx = ((unsigned int)(unsigned long)(data)) - 1; if (params->use_random_seed) { set_rand_seed(rand()); } else { @@ -46,12 +49,15 @@ static void _simulate(gpointer data, gpointer user_data) { } cache_stat_t *result = params->result; - /* If an array of readers is provided, use the one corresponding to this cache; otherwise, use the single reader */ - reader_t *source_reader = params->readers ? params->readers[idx] : params->reader; + /* If an array of readers is provided, use the one corresponding to this + * cache; otherwise, use the single reader */ + reader_t *source_reader = + params->readers ? params->readers[idx] : params->reader; reader_t *cloned_reader = clone_reader(source_reader); request_t *req = new_request(); cache_t *local_cache = params->caches[idx]; - strncpy(result[idx].cache_name, local_cache->cache_name, CACHE_NAME_ARRAY_LEN); + strncpy(result[idx].cache_name, local_cache->cache_name, + CACHE_NAME_ARRAY_LEN); /* warm up using warmup_reader */ if (params->warmup_reader) { @@ -66,7 +72,8 @@ static void _simulate(gpointer data, gpointer user_data) { INFO("cache %s (size %" PRIu64 ") finishes warm up using warmup reader " "with %" PRIu64 " requests\n", - local_cache->cache_name, local_cache->cache_size, result[idx].n_warmup_req); + local_cache->cache_name, local_cache->cache_size, + result[idx].n_warmup_req); } read_one_req(cloned_reader, req); @@ -75,7 +82,8 @@ static void _simulate(gpointer data, gpointer user_data) { /* using warmup_frac or warmup_sec of requests from reader to warm up */ if (params->n_warmup_req > 0 || params->warmup_sec > 0) { uint64_t n_warmup = 0; - while (req->valid && (n_warmup < params->n_warmup_req || req->clock_time - start_ts < params->warmup_sec)) { + while (req->valid && (n_warmup < params->n_warmup_req || + req->clock_time - start_ts < params->warmup_sec)) { req->clock_time -= start_ts; local_cache->get(local_cache, req); n_warmup += 1; @@ -85,7 +93,8 @@ static void _simulate(gpointer data, gpointer user_data) { INFO("cache %s (size %" PRIu64 ") finishes warm up using " "with %" PRIu64 " requests, %.2lf hour trace time\n", - local_cache->cache_name, local_cache->cache_size, n_warmup, (double)(req->clock_time - start_ts) / 3600.0); + local_cache->cache_name, local_cache->cache_size, n_warmup, + (double)(req->clock_time - start_ts) / 3600.0); } while (req->valid) { @@ -126,9 +135,11 @@ static void _simulate(gpointer data, gpointer user_data) { result[idx].occupied_byte = local_cache->occupied_byte; // report progress - g_mutex_lock(&(params->mtx)); + pthread_mutex_lock(&(params->mtx)); (*(params->progress))++; - g_mutex_unlock(&(params->mtx)); + if (*(params->progress) >= params->n_caches - 1) + pthread_cond_broadcast(&(params->cond)); + pthread_mutex_unlock(&(params->mtx)); // clean up if (params->free_cache_when_finish) { @@ -138,9 +149,10 @@ static void _simulate(gpointer data, gpointer user_data) { close_reader(cloned_reader); } -cache_stat_t *simulate_at_multi_sizes_with_step_size(reader_t *const reader, const cache_t *cache, uint64_t step_size, - reader_t *warmup_reader, double warmup_frac, int warmup_sec, - int num_of_threads, bool use_random_seed) { +cache_stat_t *simulate_at_multi_sizes_with_step_size( + reader_t *const reader, const cache_t *cache, uint64_t step_size, + reader_t *warmup_reader, double warmup_frac, int warmup_sec, + int num_of_threads, bool use_random_seed) { int num_of_sizes = (int)ceil((double)cache->cache_size / (double)step_size); get_num_of_req(reader); uint64_t *cache_sizes = my_malloc_n(uint64_t, num_of_sizes); @@ -148,8 +160,9 @@ cache_stat_t *simulate_at_multi_sizes_with_step_size(reader_t *const reader, con cache_sizes[i] = step_size * (i + 1); } - cache_stat_t *res = simulate_at_multi_sizes(reader, cache, num_of_sizes, cache_sizes, warmup_reader, warmup_frac, - warmup_sec, num_of_threads, use_random_seed); + cache_stat_t *res = simulate_at_multi_sizes( + reader, cache, num_of_sizes, cache_sizes, warmup_reader, warmup_frac, + warmup_sec, num_of_threads, use_random_seed); my_free(sizeof(uint64_t) * num_of_sizes, cache_sizes); return res; } @@ -169,9 +182,10 @@ cache_stat_t *simulate_at_multi_sizes_with_step_size(reader_t *const reader, con * note that warmup_reader, warmup_frac and warmup_sec are mutually exclusive * */ -cache_stat_t *simulate_at_multi_sizes(reader_t *reader, const cache_t *cache, int num_of_sizes, - const uint64_t *cache_sizes, reader_t *warmup_reader, double warmup_frac, - int warmup_sec, int num_of_threads, bool use_random_seed) { +cache_stat_t *simulate_at_multi_sizes( + reader_t *reader, const cache_t *cache, int num_of_sizes, + const uint64_t *cache_sizes, reader_t *warmup_reader, double warmup_frac, + int warmup_sec, int num_of_threads, bool use_random_seed) { int progress = 0; cache_stat_t *result = my_malloc_n(cache_stat_t, num_of_sizes); @@ -184,23 +198,35 @@ cache_stat_t *simulate_at_multi_sizes(reader_t *reader, const cache_t *cache, in params->warmup_reader = warmup_reader; params->warmup_sec = warmup_sec; params->n_caches = num_of_sizes; - params->n_warmup_req = (uint64_t)((double)get_num_of_req(reader) * warmup_frac); + params->n_warmup_req = + (uint64_t)((double)get_num_of_req(reader) * warmup_frac); params->result = result; params->free_cache_when_finish = true; params->progress = &progress; params->use_random_seed = use_random_seed; - g_mutex_init(&(params->mtx)); + pthread_mutex_init(&(params->mtx), NULL); + pthread_cond_init(&(params->cond), NULL); // build the thread pool - GThreadPool *gthread_pool = g_thread_pool_new((GFunc)_simulate, (gpointer)params, num_of_threads, TRUE, NULL); - ASSERT_NOT_NULL(gthread_pool, "cannot create thread pool in simulator\n"); + + // GThreadPool *gthread_pool = g_thread_pool_new((GFunc)_simulate, + // (gpointer)params, num_of_threads, TRUE, NULL); + // ASSERT_NOT_NULL(gthread_pool, "cannot create thread pool in simulator\n"); + threadpool_t *thread_pool = (threadpool_t *)malloc(sizeof(threadpool_t)); + ASSERT_NOT_NULL(thread_pool, "cannot create thread pool in simulator\n"); + ASSERT_TRUE(threadpool_create(thread_pool, num_of_threads), + "cannot create thread pool in simulator\n"); // start computation params->caches = my_malloc_n(cache_t *, num_of_sizes); for (int i = 1; i < num_of_sizes + 1; i++) { - params->caches[i - 1] = create_cache_with_new_size(cache, cache_sizes[i - 1]); + params->caches[i - 1] = + create_cache_with_new_size(cache, cache_sizes[i - 1]); result[i - 1].cache_size = cache_sizes[i - 1]; - ASSERT_TRUE(g_thread_pool_push(gthread_pool, GSIZE_TO_POINTER(i), NULL), + // ASSERT_TRUE(g_thread_pool_push(gthread_pool, GSIZE_TO_POINTER(i), NULL), + // "cannot push data into thread_pool in get_miss_ratio\n"); + ASSERT_TRUE(threadpool_push(thread_pool, _simulate, params, + (void *)((unsigned long)i)), "cannot push data into thread_pool in get_miss_ratio\n"); } @@ -211,17 +237,22 @@ cache_stat_t *simulate_at_multi_sizes(reader_t *reader, const cache_t *cache, in INFO( "%s starts computation %s, num_warmup_req %lld, start cache size %s, " "end cache size %s, %d sizes, %d threads, please wait\n", - __func__, cache->cache_name, (long long)(params->n_warmup_req), start_cache_size, end_cache_size, num_of_sizes, - num_of_threads); + __func__, cache->cache_name, (long long)(params->n_warmup_req), + start_cache_size, end_cache_size, num_of_sizes, num_of_threads); // wait for all simulations to finish - while (progress < num_of_sizes - 1) { - print_progress((double)progress / (double)(num_of_sizes - 1) * 100); + pthread_mutex_lock(&(params->mtx)); + while (progress < num_of_sizes) { + // print_progress((double)progress / (double)(num_of_sizes - 1) * 100); + pthread_cond_wait(&(params->cond), &(params->mtx)); } + pthread_mutex_unlock(&(params->mtx)); // clean up - g_thread_pool_free(gthread_pool, FALSE, TRUE); - g_mutex_clear(&(params->mtx)); + // g_thread_pool_free(gthread_pool, FALSE, TRUE); + threadpool_destroy(thread_pool); + pthread_mutex_destroy(&(params->mtx)); + pthread_cond_destroy(&(params->cond)); my_free(sizeof(cache_t *) * num_of_sizes, params->caches); my_free(sizeof(sim_mt_params_t), params); @@ -241,9 +272,10 @@ cache_stat_t *simulate_at_multi_sizes(reader_t *reader, const cache_t *cache, in * @param num_of_threads * @return cache_stat_t* */ -cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], int num_of_caches, - reader_t *warmup_reader, double warmup_frac, int warmup_sec, - int num_of_threads, bool free_cache_when_finish, bool use_random_seed) { +cache_stat_t *simulate_with_multi_caches( + reader_t *reader, cache_t *caches[], int num_of_caches, + reader_t *warmup_reader, double warmup_frac, int warmup_sec, + int num_of_threads, bool free_cache_when_finish, bool use_random_seed) { assert(num_of_caches > 0); int i, progress = 0; @@ -255,28 +287,38 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], in params->reader = reader; params->readers = NULL; params->caches = caches; + params->n_caches = num_of_caches; params->warmup_reader = warmup_reader; params->warmup_sec = warmup_sec; params->use_random_seed = use_random_seed; if (warmup_frac > 1e-6) { - params->n_warmup_req = (uint64_t)((double)get_num_of_req(reader) * warmup_frac); + params->n_warmup_req = + (uint64_t)((double)get_num_of_req(reader) * warmup_frac); } else { params->n_warmup_req = 0; } params->result = result; params->free_cache_when_finish = free_cache_when_finish; params->progress = &progress; - g_mutex_init(&(params->mtx)); + pthread_mutex_init(&(params->mtx), NULL); + pthread_cond_init(&(params->cond), NULL); // build the thread pool - GThreadPool *gthread_pool = g_thread_pool_new((GFunc)_simulate, (gpointer)params, num_of_threads, TRUE, NULL); - ASSERT_NOT_NULL(gthread_pool, "cannot create thread pool in simulator\n"); + // GThreadPool *gthread_pool = g_thread_pool_new((GFunc)_simulate, + // (gpointer)params, num_of_threads, TRUE, NULL); + // ASSERT_NOT_NULL(gthread_pool, "cannot create thread pool in simulator\n"); + threadpool_t *thread_pool = (threadpool_t *)malloc(sizeof(threadpool_t)); + ASSERT_NOT_NULL(thread_pool, "cannot create thread pool in simulator\n"); + ASSERT_TRUE(threadpool_create(thread_pool, num_of_threads), + "cannot create thread pool in simulator\n"); // start computation for (i = 1; i < num_of_caches + 1; i++) { result[i - 1].cache_size = caches[i - 1]->cache_size; - - ASSERT_TRUE(g_thread_pool_push(gthread_pool, GSIZE_TO_POINTER(i), NULL), + // ASSERT_TRUE(g_thread_pool_push(gthread_pool, GSIZE_TO_POINTER(i), NULL), + // "cannot push data into thread_pool in get_miss_ratio\n"); + ASSERT_TRUE(threadpool_push(thread_pool, _simulate, params, + (void *)((unsigned long)i)), "cannot push data into thread_pool in get_miss_ratio\n"); } @@ -287,26 +329,33 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], in INFO( "%s starts computation, num_warmup_req %lld, start cache %s size %s, " "end cache %s size %s, %d caches, %d threads, please wait\n", - __func__, (long long)(params->n_warmup_req), caches[0]->cache_name, start_cache_size, - caches[num_of_caches - 1]->cache_name, end_cache_size, num_of_caches, num_of_threads); + __func__, (long long)(params->n_warmup_req), caches[0]->cache_name, + start_cache_size, caches[num_of_caches - 1]->cache_name, end_cache_size, + num_of_caches, num_of_threads); // wait for all simulations to finish - while (progress < num_of_caches - 1) { - print_progress((double)progress / (double)(num_of_caches - 1) * 100); + pthread_mutex_lock(&(params->mtx)); + while (progress < num_of_caches) { + // print_progress((double)progress / (double)(num_of_caches - 1) * 100); + pthread_cond_wait(&(params->cond), &(params->mtx)); } + pthread_mutex_unlock(&(params->mtx)); // clean up - g_thread_pool_free(gthread_pool, FALSE, TRUE); - g_mutex_clear(&(params->mtx)); + // g_thread_pool_free(gthread_pool, FALSE, TRUE); + threadpool_destroy(thread_pool); + pthread_mutex_destroy(&(params->mtx)); + pthread_cond_destroy(&(params->cond)); my_free(sizeof(sim_mt_params_t), params); // user is responsible for free-ing the result return result; } -cache_stat_t *simulate_with_multi_caches_scaling(reader_t **readers, cache_t *caches[], int num_of_caches, - reader_t *warmup_reader, double warmup_frac, int warmup_sec, - int num_of_threads, bool free_cache_when_finish) { +cache_stat_t *simulate_with_multi_caches_scaling( + reader_t **readers, cache_t *caches[], int num_of_caches, + reader_t *warmup_reader, double warmup_frac, int warmup_sec, + int num_of_threads, bool free_cache_when_finish) { int progress = 0; cache_stat_t *result = my_malloc_n(cache_stat_t, num_of_caches); @@ -316,25 +365,36 @@ cache_stat_t *simulate_with_multi_caches_scaling(reader_t **readers, cache_t *ca params->readers = readers; // use multi-readers for scaling params->reader = NULL; // not used in scaling mode params->caches = caches; + params->n_caches = num_of_caches; params->warmup_reader = warmup_reader; params->warmup_sec = warmup_sec; params->use_random_seed = false; // or set as desired if (warmup_frac > 1e-6) { - params->n_warmup_req = (uint64_t)((double)get_num_of_req(readers[0]) * warmup_frac); + params->n_warmup_req = + (uint64_t)((double)get_num_of_req(readers[0]) * warmup_frac); } else { params->n_warmup_req = 0; } params->result = result; params->free_cache_when_finish = free_cache_when_finish; params->progress = &progress; - g_mutex_init(&(params->mtx)); + pthread_mutex_init(&(params->mtx), NULL); + pthread_cond_init(&(params->cond), NULL); - GThreadPool *gthread_pool = g_thread_pool_new((GFunc)_simulate, (gpointer)params, num_of_threads, TRUE, NULL); - ASSERT_NOT_NULL(gthread_pool, "cannot create thread pool in simulator\n"); + // GThreadPool *gthread_pool = g_thread_pool_new((GFunc)_simulate, + // (gpointer)params, num_of_threads, TRUE, NULL); + // ASSERT_NOT_NULL(gthread_pool, "cannot create thread pool in simulator\n"); + threadpool_t *thread_pool = (threadpool_t *)malloc(sizeof(threadpool_t)); + ASSERT_NOT_NULL(thread_pool, "cannot create thread pool in simulator\n"); + ASSERT_TRUE(threadpool_create(thread_pool, num_of_threads), + "cannot create thread pool in simulator\n"); for (int i = 1; i < num_of_caches + 1; i++) { result[i - 1].cache_size = caches[i - 1]->cache_size; - ASSERT_TRUE(g_thread_pool_push(gthread_pool, GSIZE_TO_POINTER(i), NULL), + // ASSERT_TRUE(g_thread_pool_push(gthread_pool, GSIZE_TO_POINTER(i), NULL), + // "cannot push data into thread_pool in get_miss_ratio\n"); + ASSERT_TRUE(threadpool_push(thread_pool, _simulate, params, + (void *)((unsigned long)i)), "cannot push data into thread_pool in get_miss_ratio\n"); } @@ -343,18 +403,25 @@ cache_stat_t *simulate_with_multi_caches_scaling(reader_t **readers, cache_t *ca convert_size_to_str(result[num_of_caches - 1].cache_size, end_cache_size); INFO( - "simulate_with_multi_caches_scaling starts computation, num_warmup_req %lld, start cache size %s, end cache size " + "simulate_with_multi_caches_scaling starts computation, num_warmup_req " + "%lld, start cache size %s, end cache size " "%s, %d caches, %d threads, please wait\n", - (long long)params->n_warmup_req, start_cache_size, end_cache_size, num_of_caches, num_of_threads); + (long long)params->n_warmup_req, start_cache_size, end_cache_size, + num_of_caches, num_of_threads); - while (progress < num_of_caches - 1) { - print_progress((double)progress / (double)(num_of_caches - 1) * 100); + pthread_mutex_lock(&(params->mtx)); + while (progress < num_of_caches) { + // print_progress((double)progress / (double)(num_of_caches - 1) * 100); + pthread_cond_wait(&(params->cond), &(params->mtx)); } + pthread_mutex_unlock(&(params->mtx)); - g_thread_pool_free(gthread_pool, FALSE, TRUE); - g_mutex_clear(&(params->mtx)); + // g_thread_pool_free(gthread_pool, FALSE, TRUE); + threadpool_destroy(thread_pool); + pthread_mutex_destroy(&(params->mtx)); + pthread_cond_destroy(&(params->cond)); my_free(sizeof(sim_mt_params_t), params); - for (int i=0; isampler->sampling_ratio; } return result; diff --git a/libCacheSim/traceReader/reader.c b/libCacheSim/traceReader/reader.c index 11a4a474..a9bc95df 100644 --- a/libCacheSim/traceReader/reader.c +++ b/libCacheSim/traceReader/reader.c @@ -41,7 +41,7 @@ extern "C" { reader_t *setup_reader(const char *const trace_path, const trace_type_e trace_type, const reader_init_param_t *const init_params) { - static bool _info_printed = false; + static volatile bool _info_printed = false; int fd; struct stat st; diff --git a/libCacheSim/utils/include/threadpool.h b/libCacheSim/utils/include/threadpool.h new file mode 100644 index 00000000..a223ee5a --- /dev/null +++ b/libCacheSim/utils/include/threadpool.h @@ -0,0 +1,68 @@ +// Adapted from: https://nachtimwald.com/2019/04/12/thread-pool-in-c/ +// Originally Copyright (c) 2019 John Schember +// License: MIT License + +#ifndef INCLUDE_LIBCACHESIM_THREADPOOL_H +#define INCLUDE_LIBCACHESIM_THREADPOOL_H + +#include +#include +#include + +typedef void (*func_t)(void *arg1, void *arg2); + +struct threadpool_job { + func_t func; + void *arg1; + void *arg2; + struct threadpool_job *next; +}; +// linked list of waiting threads +// similar to GThreadPool we make there 2 arguments +typedef struct threadpool_job threadpool_job_t; + +struct threadpool { + threadpool_job_t *job_first; + threadpool_job_t *job_last; + pthread_mutex_t job_mutex; + pthread_cond_t job_cond; + pthread_cond_t working_cond; + size_t working_cnt; + size_t thread_cnt; + bool stop; +}; +typedef struct threadpool threadpool_t; + +/** + * @brief Create a fixed size thread pool on heap. + * @param tm Pointer to the thread pool structure + * @param num Number of threads in the pool + * @note `tm` MUST be allocated on the *heap* before this call + */ +bool threadpool_create(threadpool_t *tm, size_t num); + +/** + * @brief Destroy the thread pool and free its associated memory. + * @param tm Pointer to the thread pool structure + * @note The memory is also freed, so `tm` MUST NOT be reused after this call. + */ +void threadpool_destroy(threadpool_t *tm); + + +/** + * @brief Push a job to the thread pool. There is no limit on the number of waiting jobs. + * @param tm Pointer to the thread pool structure + * @param func Function to be executed in thread pool + * @param arg1 First argument to `func` + * @param arg2 Second argument to `func` + * @note `func` MUST take 2 pointer arguments only. Prepare arguments in a struct if needed. + */ +bool threadpool_push(threadpool_t *tm, func_t func, void *arg, void *arg2); + +/** + * @brief Wait for all jobs to finish. Usually there is no need to manually call this function. + * @param tm Pointer to the thread pool structure + */ +void threadpool_wait(threadpool_t *tm); + +#endif \ No newline at end of file diff --git a/libCacheSim/utils/threadpool.c b/libCacheSim/utils/threadpool.c new file mode 100644 index 00000000..a273a856 --- /dev/null +++ b/libCacheSim/utils/threadpool.c @@ -0,0 +1,177 @@ +// Adapted from: https://nachtimwald.com/2019/04/12/thread-pool-in-c/ +// Originally Copyright (c) 2019 John Schember +// License: MIT License + +#include "include/threadpool.h" + +#include +#include + +static threadpool_job_t *threadpool_job_create(func_t func, void *arg1, + void *arg2) { + threadpool_job_t *job; + + if (func == NULL) return NULL; + + job = malloc(sizeof(*job)); + job->func = func; + job->arg1 = arg1; + job->arg2 = arg2; + job->next = NULL; + return job; +} + +static void threadpool_job_destroy(threadpool_job_t *job) { + if (job == NULL) return; + free(job); +} + +static threadpool_job_t *threadpool_job_get(threadpool_t *tm) { + threadpool_job_t *job; + + if (tm == NULL) return NULL; + + job = tm->job_first; + if (job == NULL) return NULL; + + if (job->next == NULL) { + tm->job_first = NULL; + tm->job_last = NULL; + } else { + tm->job_first = job->next; + } + + return job; +} + +static void *threadpool_worker(void *arg) { + threadpool_t *tm = arg; + threadpool_job_t *job; + + while (1) { + pthread_mutex_lock(&(tm->job_mutex)); + + while (tm->job_first == NULL && !tm->stop) + pthread_cond_wait(&(tm->job_cond), &(tm->job_mutex)); + + if (tm->stop) break; + + job = threadpool_job_get(tm); + tm->working_cnt++; + pthread_mutex_unlock(&(tm->job_mutex)); + + if (job != NULL) { + job->func(job->arg1, job->arg2); + threadpool_job_destroy(job); + } + + pthread_mutex_lock(&(tm->job_mutex)); + tm->working_cnt--; + if (!tm->stop && tm->working_cnt == 0 && tm->job_first == NULL) + pthread_cond_signal(&(tm->working_cond)); + pthread_mutex_unlock(&(tm->job_mutex)); + } + + tm->thread_cnt--; + pthread_cond_signal(&(tm->working_cond)); + pthread_mutex_unlock(&(tm->job_mutex)); + return NULL; +} + +bool threadpool_create(threadpool_t *tm, size_t num) { + pthread_t thread; + size_t i; + + if (num == 0) num = 2; + + tm->thread_cnt = num; + tm->stop = false; + + pthread_mutex_init(&(tm->job_mutex), NULL); + pthread_cond_init(&(tm->job_cond), NULL); + pthread_cond_init(&(tm->working_cond), NULL); + + tm->job_first = NULL; + tm->job_last = NULL; + + for (i = 0; i < num; i++) { + int res = pthread_create(&thread, NULL, threadpool_worker, tm); + if (res != 0) { + if (i == 0) + return false; + else { + tm->thread_cnt = i; + threadpool_destroy(tm); + return false; + } + } + pthread_detach(thread); + } + + return true; +} + +void threadpool_destroy(threadpool_t *tm) { + threadpool_job_t *job; + threadpool_job_t *job2; + + if (tm == NULL) return; + + pthread_mutex_lock(&(tm->job_mutex)); + job = tm->job_first; + while (job != NULL) { + job2 = job->next; + threadpool_job_destroy(job); + job = job2; + } + tm->job_first = NULL; + tm->stop = true; + pthread_cond_broadcast(&(tm->job_cond)); + pthread_mutex_unlock(&(tm->job_mutex)); + + threadpool_wait(tm); + + pthread_mutex_destroy(&(tm->job_mutex)); + pthread_cond_destroy(&(tm->job_cond)); + pthread_cond_destroy(&(tm->working_cond)); + + free(tm); +} + +bool threadpool_push(threadpool_t *tm, func_t func, void *arg1, void *arg2) { + threadpool_job_t *job; + + if (tm == NULL) return false; + + job = threadpool_job_create(func, arg1, arg2); + if (job == NULL) return false; + + pthread_mutex_lock(&(tm->job_mutex)); + if (tm->job_first == NULL) { + tm->job_first = job; + tm->job_last = tm->job_first; + } else { + tm->job_last->next = job; + tm->job_last = job; + } + + pthread_cond_broadcast(&(tm->job_cond)); + pthread_mutex_unlock(&(tm->job_mutex)); + + return true; +} + +void threadpool_wait(threadpool_t *tm) { + if (tm == NULL) return; + + pthread_mutex_lock(&(tm->job_mutex)); + while (1) { + if (tm->job_first != NULL || (!tm->stop && tm->working_cnt != 0) || + (tm->stop && tm->thread_cnt != 0)) { + pthread_cond_wait(&(tm->working_cond), &(tm->job_mutex)); + } else { + break; + } + } + pthread_mutex_unlock(&(tm->job_mutex)); +}