Better "too many links" reporting.

3 files changed, 25 insertions, 39 deletions

diff --git a/src/htsalias.c b/src/htsalias.c
index b1a7962..3472021 100644
--- a/src/htsalias.c
+++ b/src/htsalias.c
@@ -189,10 +189,10 @@ const char *hts_optalias[][4] = {
   /* Badly documented */
   {"debug-testfilters", "-#0", "param1", "debug: test filters"},
   {"advanced-flushlogs", "-#f", "single", ""},
-  {"advanced-maxfilters", "-#F", "param", ""},
+  {"advanced-maxfilters", "-#F", "param", "maximum number of scan rules"},
   {"version", "-#h", "single", ""},
   {"debug-scanstdin", "-#K", "single", ""},
-  {"advanced-maxlinks", "-#L", "single", ""},
+  {"advanced-maxlinks", "-#L", "single", "maximum number of links (0 to disable limit)"},
   {"advanced-progressinfo", "-#p", "single", "deprecated"},
   {"catch-url", "-#P", "single", "catch complex URL through proxy"},
   /*{"debug-oldftp","-#R","single",""}, */
diff --git a/src/htscore.c b/src/htscore.c
index fc7f822..08e3902 100644
--- a/src/htscore.c
+++ b/src/htscore.c
@@ -366,10 +366,10 @@ void hts_record_free(httrackp *opt) {
 }
 
 // adds a new link and returns a non-zero value upon success
-int hts_record_link(httrackp * opt,
-                    const char *address, const char *file, const char *save,
-                    const char *ref_address, const char *ref_file,
-                    const char *codebase) {
+static int hts_record_link_(httrackp * opt,
+                            const char *address, const char *file, const char *save,
+                            const char *ref_address, const char *ref_file,
+                            const char *codebase) {
   // create a new entry
   const size_t lien_tot = hts_record_link_alloc(opt);
   lien_url*const link = lien_tot != (size_t) -1 ? opt->liens[lien_tot] : NULL;
@@ -404,6 +404,21 @@ int hts_record_link(httrackp * opt,
   return 1;
 }
 
+int hts_record_link(httrackp * opt,
+                    const char *address, const char *file, const char *save,
+                    const char *ref_address, const char *ref_file,
+                    const char *codebase) {
+  const int success = 
+    hts_record_link_(opt, address, file, save, ref_address, ref_file, codebase);
+  if (!success) {
+    hts_log_print(opt, LOG_PANIC, "Too many links (links=%ld, limit=%ld)", 
+                  (long int) heap_top_index(), (long int) opt->maxlink);
+    hts_log_print(opt, LOG_INFO,
+      "To avoid that: use #L option for more links (example: -#L1000000)");
+  }
+  return success;
+}
+
 #define HT_INDEX_END do { \
 if (!makeindex_done) { \
 if (makeindex_fp) { \
@@ -788,8 +803,6 @@ int httpmirror(char *url1, httrackp * opt) {
                         fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
                         StringBuff(opt->path_html_utf8), "index.html")),
                         "", "", NULL)) {
-      printf("PANIC! : Not enough memory [%d]\n", __LINE__);
-      hts_log_print(opt, LOG_PANIC, "Not enough memory");
       XH_extuninit;             // désallocation mémoire & buffers
       return 0;
     }
@@ -3901,8 +3914,6 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
 
               // enregistrer fichier (MACRO)
               if (!hts_record_link(opt, afs.af.adr, afs.af.fil, afs.save, "", "", "")) {    // erreur, pas de place réservée
-                printf("PANIC! : Not enough memory [%d]\n", __LINE__);
-                hts_log_print(opt, LOG_PANIC, "Not enough memory");
                 opt->state.exit_xh = -1;        /* fatal error -> exit */
                 return 0;
               }
diff --git a/src/htsparse.c b/src/htsparse.c
index f171702..37f0357 100644
--- a/src/htsparse.c
+++ b/src/htsparse.c
@@ -3059,15 +3059,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                     if ((afs.af.adr[0] != '\0') && (p_type != 2) && (p_type != -2) && (forbidden_url != 1)) {  // si le fichier n'existe pas, ajouter à la liste                            
                       // n'y a-t-il pas trop de liens?
-                      if (0) {
-                        // CLEANUP
-#if 0
-                      if (opt->lien_tot + 1 >= lien_max - 4) {       // trop de liens!
-                        printf("PANIC! : Too many URLs : >%d [%d]\n", lien_tot,
+                      if (opt->maxlink > 0 && opt->lien_tot + 1 >= opt->maxlink) {       // trop de liens!
+                        printf("PANIC! : Too many URLs : >%d [%d]\n", opt->lien_tot,
                                __LINE__);
-                        hts_log_print(opt, LOG_PANIC,
-                                      "Too many URLs, giving up..(>%d)",
-                                      lien_max);
+                        hts_log_print(opt, LOG_PANIC, "Too many URLs, giving up..(>%d)",
+                                      opt->maxlink);
                         hts_log_print(opt, LOG_INFO,
                                       "To avoid that: use #L option for more links (example: -#L1000000)");
                         if ((opt->getmode & 1) && (ptr > 0)) {
@@ -3078,7 +3074,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                         }
                         XH_uninit;      // désallocation mémoire & buffers
                         return -1;
-#endif
                       } else {  // noter le lien sur la listes des liens à charger
                         int pass_fix, dejafait = 0;
 
@@ -3149,10 +3144,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                     if (checkrobots(_ROBOTS, afs.af.adr, "") == -1) {  // robots.txt ?
                                       // enregistrer robots.txt (MACRO)
                                       if (!hts_record_link(opt, afs.af.adr, "/robots.txt", "", "", "", NULL)) {
-                                        printf
-                                          ("PANIC! : Not enough memory [%d]\n",
-                                           __LINE__);
-                                        hts_log_print(opt, LOG_PANIC, "Not enough memory");
                                         if ((opt->getmode & 1) && (ptr > 0)) {
                                           if (fp) {
                                             fclose(fp);
@@ -3192,9 +3183,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           // enregistrer
                           if (!hts_record_link(opt, afs.af.adr, afs.af.fil, afs.save, 
                                                former.adr, former.fil, codebase)) {
-                            printf("PANIC! : Not enough memory [%d]\n",
-                                   __LINE__);
-                            hts_log_print(opt, LOG_PANIC, "Not enough memory");
                             if ((opt->getmode & 1) && (ptr > 0)) {
                               if (fp) {
                                 fclose(fp);
@@ -3546,9 +3534,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
                     heap_top()->premier = heap(ptr)->premier;
                     heap_top()->precedent = heap(ptr)->precedent;
                   } else {      // oups erreur, plus de mémoire!!
-                    printf("PANIC! : Not enough memory [%d]\n", __LINE__);
-                    hts_log_print(opt, LOG_PANIC, "Not enough memory");
-                    //if (opt->getmode & 1) { if (fp) { fclose(fp); fp=NULL; } }
                     XH_uninit;  // désallocation mémoire & buffers
                     return 0;
                   }
@@ -3646,9 +3631,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
           hash_invalidate_entry(hashptr, ptr);  // invalidate hashtable entry
           //
         } else {              // oups erreur, plus de mémoire!!
-          printf("PANIC! : Not enough memory [%d]\n", __LINE__);
-          hts_log_print(opt, LOG_PANIC, "Not enough memory");
-          //if (opt->getmode & 1) { if (fp) { fclose(fp); fp=NULL; } }
           XH_uninit;          // désallocation mémoire & buffers
           return 0;
         }
@@ -3783,10 +3765,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
             heap_top()->premier = heap(ptr)->premier;
             heap_top()->precedent = heap(ptr)->precedent;
           } else {              // oups erreur, plus de mémoire!!
-            printf("PANIC! : Not enough memory [%d]\n", __LINE__);
-            hts_log_print(opt, LOG_PANIC, "Not enough memory");
-            //if (opt->getmode & 1) { if (fp) { fclose(fp); fp=NULL; } }
-            XH_uninit;          // désallocation mémoire & buffers
             return 0;
           }
         }
@@ -3950,9 +3928,6 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
                             add.af.fil);
               //
             } else {            // oups erreur, plus de mémoire!!
-              printf("PANIC! : Not enough memory [%d]\n", __LINE__);
-              hts_log_print(opt, LOG_PANIC, "Not enough memory");
-              //if (opt->getmode & 1) { if (fp) { fclose(fp); fp=NULL; } }
               XH_uninit;        // désallocation mémoire & buffers
               return;
             }