Mitigate stash overflow by resizing the hashtable.

Added better logging.

3 files changed, 94 insertions, 28 deletions

diff --git a/src/htsinthash.c b/src/htsinthash.c
index 72b250c..285b165 100644
--- a/src/htsinthash.c
+++ b/src/htsinthash.c
@@ -82,7 +82,7 @@ Please visit our Website: http://www.httrack.com
 #define STASH_SIZE 16
 
 /** Minimum value for lg_size. **/
-#define MIN_LG_SIZE 8
+#define MIN_LG_SIZE 4
 
 /** Minimum value for pool.capacity. **/
 #define MIN_POOL_CAPACITY 256
@@ -178,6 +178,8 @@ struct struct_inthash {
 
     /** How to handle fatal assertions (might be NULL). **/
     struct {
+      /** logging **/
+      t_inthash_loghandler log;
       /** abort() **/
       t_inthash_asserthandler fatal;
       /** opaque argument **/
@@ -203,22 +205,36 @@ struct struct_inthash {
 #endif
 
 /* Logging level. */
+static void inthash_log(const inthash hashtable, inthash_loglevel level,
+                        const char *format, va_list args);
+#define DECLARE_LOG_FUNCTION(NAME, LEVEL) \
+static void NAME(const inthash hashtable, const char *format, ...) \
+  HTS_PRINTF_FUN(2, 3); \
+static void NAME(const inthash hashtable, const char *format, ...) { \
+  va_list args; \
+  va_start(args, format); \
+  inthash_log(hashtable, LEVEL, format, args); \
+  va_end(args); \
+}
 #if 0
 /* Verbose. */
-#define inthash_crit inthash_log
-#define inthash_info inthash_log
-#define inthash_debug inthash_log
-#define inthash_trace inthash_log
+DECLARE_LOG_FUNCTION(inthash_crit, inthash_log_critical)
+DECLARE_LOG_FUNCTION(inthash_warning, inthash_log_warning)
+DECLARE_LOG_FUNCTION(inthash_info, inthash_log_info)
+DECLARE_LOG_FUNCTION(inthash_debug, inthash_log_debug)
+DECLARE_LOG_FUNCTION(inthash_trace, inthash_log_trace)
 #elif 0
 /* Info. */
-#define inthash_crit inthash_log
-#define inthash_info inthash_log
+DECLARE_LOG_FUNCTION(inthash_crit, inthash_log_critical)
+DECLARE_LOG_FUNCTION(inthash_warning, inthash_log_warning)
+DECLARE_LOG_FUNCTION(inthash_info, inthash_log_info)
 #define inthash_debug inthash_log
 #define inthash_trace inthash_nolog
 #else
 /* No logging except stats and critical. */
-#define inthash_crit inthash_log
-#define inthash_info inthash_log
+DECLARE_LOG_FUNCTION(inthash_crit, inthash_log_critical)
+DECLARE_LOG_FUNCTION(inthash_warning, inthash_log_warning)
+DECLARE_LOG_FUNCTION(inthash_info, inthash_log_info)
 #define inthash_debug inthash_nolog
 #define inthash_trace inthash_nolog
 #endif
@@ -232,6 +248,9 @@ static char the_empty_string[1] = { 0 };
 /* global assertion handler */
 static t_inthash_asserthandler global_assert_handler = NULL;
 
+/* global assertion handler */
+static t_inthash_loghandler global_log_handler = NULL;
+
 /* default assertion handler, if neither hashtable one nor global one 
    were defined */
 static void inthash_fail(const char* exp, const char* file, int line) {
@@ -252,16 +271,18 @@ static void inthash_assert_failed(const inthash hashtable, const char* exp, cons
 }
 
 /* Logging */
-static void inthash_log(const inthash hashtable, const char *format, ...)
-                        HTS_PRINTF_FUN(2, 3);
-static void inthash_log(const inthash hashtable, const char *format, ...) {
-  va_list args;
+static void inthash_log(const inthash hashtable, inthash_loglevel level,
+                        const char *format, va_list args) {
   inthash_assert(hashtable, format != NULL);
-  fprintf(stderr, "[%p] ", (void*) hashtable);
-  va_start(args, format);
-  (void) vfprintf(stderr, format, args);
-  va_end(args);
-  putc('\n', stderr);
+  if (hashtable != NULL && hashtable->custom.error.log != NULL) {
+    hashtable->custom.error.log(hashtable->custom.error.arg, level, format, args);
+  } else if (global_log_handler != NULL) {
+    global_log_handler(hashtable, level, format, args);
+  } else {
+    fprintf(stderr, "[%p] ", (void*) hashtable);
+    (void) vfprintf(stderr, format, args);
+    putc('\n', stderr);
+  }
 }
 
 /* No logging (should be dropped by the compiler) */
@@ -874,11 +895,15 @@ int inthash_write_value(inthash hashtable, const char *name,
     /* size of half of the table */
     const size_t half_size = POW2(hashtable->lg_size - 1);
 
+    /* size of half of the stash */
+    const size_t half_stash_size = STASH_SIZE / 2;
+
     /* item was added: increase count */
     hashtable->used++;
 
-    /* table is more than half-full */
-    if (hashtable->used >= half_size) {
+    /* table is more than half-full, or stash is more than half-full */
+    if (hashtable->used >= half_size
+      || hashtable->stash.size >= half_stash_size) {
       size_t i;
 
       /* size before  */
@@ -889,6 +914,15 @@ int inthash_write_value(inthash hashtable, const char *name,
       /* size after doubling it */
       const size_t alloc_size = prev_alloc_size * 2;
 
+      /* log stash issues */
+      if (hashtable->stash.size >= half_stash_size 
+        && half_size > POW2(16) 
+        && hashtable->used < half_size / 4) {
+          inthash_warning(hashtable, 
+            "stash size still full despite %"UINT_64_FORMAT" elements used out of %"UINT_64_FORMAT,
+            hashtable->used, half_size*2);
+      }
+
       /* statistics */
       hashtable->stats.rehash_count++;
 
@@ -1232,9 +1266,11 @@ void inthash_value_set_key_handler(inthash hashtable,
 }
 
 void inthash_set_assert_handler(inthash hashtable,
+                                t_inthash_loghandler log,
                                 t_inthash_asserthandler fatal,
                                 void *arg) {
   inthash_assert(hashtable, fatal != NULL);
+  hashtable->custom.error.log = log;
   hashtable->custom.error.fatal = fatal;
   hashtable->custom.error.arg = arg;
 }
@@ -1320,6 +1356,8 @@ inthash_item *inthash_enum_next(struct_inthash_enum * e) {
   }
 }
 
-void inthash_set_global_assert_handler(t_inthash_asserthandler fatal) {
+void inthash_set_global_assert_handler(t_inthash_loghandler log,
+                                       t_inthash_asserthandler fatal) {
+  global_log_handler = log;
   global_assert_handler = fatal;
 }
diff --git a/src/htsinthash.h b/src/htsinthash.h
index a4d8ea8..248a697 100644
--- a/src/htsinthash.h
+++ b/src/htsinthash.h
@@ -112,18 +112,31 @@ struct inthash_item {
   inthash_keys hashes;
 };
 
+/** Log level. **/
+typedef enum inthash_loglevel {
+  inthash_log_critical,
+  inthash_log_warning,
+  inthash_log_info,
+  inthash_log_debug,
+  inthash_log_trace
+} inthash_loglevel;
+
 /** Alias for legacy code. **/
 typedef inthash_item inthash_chain;
 
 /** Value free handler **/
-typedef void (*t_inthash_freehandler) (void *arg, void *value);
+typedef void (*t_inthash_freehandler)(void *arg, void *value);
 
 /** Name dup handler. **/
-typedef char* (*t_inthash_duphandler) (void *arg, const char *name);
+typedef char* (*t_inthash_duphandler)(void *arg, const char *name);
 
 /** Hash computation handler. **/
 typedef inthash_keys (*t_inthash_hasheshandler)(void *arg, const char *value);
 
+/** Hashtable logging handler. **/
+typedef void (*t_inthash_loghandler)(void *arg, inthash_loglevel level, 
+                                     const char* format, va_list args);
+
 /** Hashtable fatal assertion failure. **/
 typedef void (*t_inthash_asserthandler)(void *arg, const char* exp, const char* file, int line);
 
@@ -219,9 +232,11 @@ void inthash_value_set_key_handler(inthash hashtable,
 
 /**
  * Set assertion failure handler.
- * fatal: handled called upon serious programming error
+ * log: handler called upon serious programming error
+ * fatal: handler called upon serious programming error
  **/
 void inthash_set_assert_handler(inthash hashtable,
+                                t_inthash_loghandler log,
                                 t_inthash_asserthandler fatal,
                                 void *arg);
 
@@ -323,10 +338,13 @@ inthash_keys inthash_hash_value(const char *value);
  * Set default global assertion failure handler.
  * The handler will be used if no specific handler was defined in the
  * hashtable itself.
- * fatal: handled called upon serious programming error (opaque argument 
- * is the hashtable)
+ * log: handler called upon serious error log (opaque argument 
+ * is the hashtable itself)
+ * fatal: handler called upon serious programming error (opaque argument 
+ * is the hashtable itself)
  **/
-void inthash_set_global_assert_handler(t_inthash_asserthandler fatal);
+void inthash_set_global_assert_handler(t_inthash_loghandler log,
+                                       t_inthash_asserthandler fatal);
 
 #endif
 
diff --git a/src/htslib.c b/src/htslib.c
index 407f225..9248cf9 100644
--- a/src/htslib.c
+++ b/src/htslib.c
@@ -5116,6 +5116,15 @@ static void default_inthash_asserthandler(void *arg, const char* exp, const char
   abortf_(exp, file, line);
 }
 
+static void default_inthash_loghandler(void *arg, inthash_loglevel level, 
+                                       const char* format, va_list args) {
+  if (level <= inthash_log_warning) {
+    fprintf(stderr, "** warning: ");
+  }
+  vfprintf(stderr, format, args);
+  fprintf(stderr, "\n");
+}
+
 static int hts_init_ok = 0;
 HTSEXT_API int hts_init(void) {
   const char *dbg_env;
@@ -5138,7 +5147,8 @@ HTSEXT_API int hts_init(void) {
   hts_debug_log_print("entering hts_init()");   /* debug */
 
   /* Init hashtable default assertion handler. */
-  inthash_set_global_assert_handler(default_inthash_asserthandler);
+  inthash_set_global_assert_handler(default_inthash_loghandler, 
+    default_inthash_asserthandler);
 
   /* Init threads (lazy init) */
   htsthread_init();