diff options
| author | Xavier Roche <xroche@users.noreply.github.com> | 2013-05-14 20:30:28 +0000 |
|---|---|---|
| committer | Xavier Roche <xroche@users.noreply.github.com> | 2013-05-14 20:30:28 +0000 |
| commit | 07f4b003675260004c49755dde413944a16ca2d3 (patch) | |
| tree | 11050b4c7fbe1d7c63a38f5f9a6b5ba6d8260ae1 | |
| parent | b09def438c5e47305e4a5ab18d8a646a3b004fdc (diff) | |
Indenting cleanup for all *.[ch] files, except htsparse.c (too ugly to be automatically indented for now) and /minizip/, /mmsrip/ (external files)
setup:
indent -l80 -lc80 -nhnl -nut -bad -bap -bbo -br -brf -bli2 -brs -bls -br -ss -sai -pmt -nsaw -nsaf -nprs -i2 -ce -npsl -npcs -cs -sob -cdw -nbc -lp
logs:
indent: ./src/htsback.c:157: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsback.c:1417: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsback.c:1826: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsback.c:1833: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsback.c:1981: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsback.c:2685: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsback.c:2747: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsback.c:2861: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsback.c:3128: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsback.c:3512: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htswizard.c:140: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htswizard.c:597: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htswizard.c:598: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htswizard.c:611: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htscoremain.c:97: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htscoremain.c:106: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htscoremain.c:106: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htscoremain.c:256: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htscoremain.c:262: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htscoremain.c:935: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:271: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:272: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:273: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:274: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:275: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:276: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:277: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:278: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:279: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:280: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:281: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:662: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/httrack.c:752: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:413: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:414: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:415: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:416: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:417: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:418: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:419: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:420: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:421: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:422: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsweb.c:423: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.c:188: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.c:507: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.c:508: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.c:509: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.c:510: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.c:511: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.c:512: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.c:580: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.c:581: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.h:115: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.h:246: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.h:247: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.h:248: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.h:249: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.h:250: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.h:251: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.h:314: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/proxy/proxytrack.h:315: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:151: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:158: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:161: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:166: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:174: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:177: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:188: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:243: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:245: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:248: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:257: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htstools.c:647: Warning:old style assignment ambiguity in "=*". Assuming "= *"
indent: ./src/htsbauth.c:364: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htsbauth.c:387: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htscache.c:473: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsserver.h:112: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsserver.h:187: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsjava.c:312: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsjava.c:379: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htsjava.c:407: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsjava.c:472: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htsjava.c:483: Warning:old style assignment ambiguity in "=*". Assuming "= *"
indent: ./src/htslib.c:593: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:809: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:1743: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:1874: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:1896: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:1984: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2085: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2604: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2605: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2606: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2607: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2608: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2609: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2672: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2673: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2906: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2928: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:2996: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:4802: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htslib.c:5353: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:5354: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:5355: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:5357: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:5429: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htslib.c:5435: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htscore.c:208: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htscore.c:277: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htscore.c:279: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htscore.c:357: Warning:old style assignment ambiguity in "=*". Assuming "= *"
indent: ./src/htscore.c:394: Warning:old style assignment ambiguity in "=*". Assuming "= *"
indent: ./src/htscore.c:1544: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htscore.c:3330: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htscore.c:3361: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htshash.c:140: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htshash.c:217: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsname.c:733: Warning:old style assignment ambiguity in "=*". Assuming "= *"
indent: ./src/htsname.c:749: Warning:old style assignment ambiguity in "=*". Assuming "= *"
indent: ./src/htsname.c:933: Warning:old style assignment ambiguity in "=*". Assuming "= *"
indent: ./src/htsname.c:1520: Warning:old style assignment ambiguity in "=*". Assuming "= *"
indent: ./src/htszlib.c:76: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htszlib.c:81: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htscatchurl.c:268: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsinthash.c:156: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htsinthash.c:159: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htsinthash.c:176: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htsinthash.c:179: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htsinthash.c:226: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htsinthash.c:253: Warning:old style assignment ambiguity in "=&". Assuming "= &"
indent: ./src/htsftp.c:169: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsftp.c:177: Warning:old style assignment ambiguity in "=-". Assuming "= -"
indent: ./src/htsfilters.c:67: Warning:old style assignment ambiguity in "=*". Assuming "= *"
indent: ./src/htsfilters.c:80: Warning:old style assignment ambiguity in "=*". Assuming "= *"
87 files changed, 22858 insertions, 19133 deletions
diff --git a/libtest/callbacks-example-baselinks.c b/libtest/callbacks-example-baselinks.c index 246c921..f6705e5 100755 --- a/libtest/callbacks-example-baselinks.c +++ b/libtest/callbacks-example-baselinks.c @@ -25,18 +25,21 @@ #include "htsdefines.h" /* Local function definitions */ -static int process_file(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file); -static int check_detectedlink(t_hts_callbackarg *carg, httrackp* opt, char* link); -static int check_detectedlink_end(t_hts_callbackarg *carg, httrackp *opt); +static int process_file(t_hts_callbackarg * carg, httrackp * opt, char *html, + int len, const char *url_address, const char *url_file); +static int check_detectedlink(t_hts_callbackarg * carg, httrackp * opt, + char *link); +static int check_detectedlink_end(t_hts_callbackarg * carg, httrackp * opt); /* external functions */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); /* module entry point */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { const char *arg = strchr(argv, ','); + if (arg != NULL) arg++; @@ -45,8 +48,9 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { if (arg == NULL || *arg == '\0' || strlen(arg) >= HTS_URLMAXSIZE / 2) { fprintf(stderr, "** callback error: arguments expected or bad arguments\n"); fprintf(stderr, "usage: httrack --wrapper modulename,base\n"); - fprintf(stderr, "example: httrack --wrapper callback,http://www.example.com/\n"); - return 0; /* failed */ + fprintf(stderr, + "example: httrack --wrapper callback,http://www.example.com/\n"); + return 0; /* failed */ } else { char *callbacks_userdef = strdup(arg); /* userdef */ @@ -58,50 +62,56 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { fprintf(stderr, "Using root '%s'\n", callbacks_userdef); } - return 1; /* success */ + return 1; /* success */ } -static int process_file(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file) { - char* prevBase; +static int process_file(t_hts_callbackarg * carg, httrackp * opt, char *html, + int len, const char *url_address, + const char *url_file) { + char *prevBase; /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) { - return 0; /* Abort */ + if (!CALLBACKARG_PREV_FUN(carg, check_html) + (CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) { + return 0; /* Abort */ } } /* Disable base href, if any */ - if ( ( prevBase = strstr(html, "<BASE HREF=\"") ) != NULL) { + if ((prevBase = strstr(html, "<BASE HREF=\"")) != NULL) { prevBase[1] = 'X'; } - return 1; /* success */ + return 1; /* success */ } -static int check_detectedlink(t_hts_callbackarg *carg, httrackp* opt, char* link) { - const char *base = (char*) CALLBACKARG_USERDEF(carg); +static int check_detectedlink(t_hts_callbackarg * carg, httrackp * opt, + char *link) { + const char *base = (char *) CALLBACKARG_USERDEF(carg); /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, linkdetected) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, linkdetected)(CALLBACKARG_PREV_CARG(carg), opt, link)) { - return 0; /* Abort */ + if (!CALLBACKARG_PREV_FUN(carg, linkdetected) + (CALLBACKARG_PREV_CARG(carg), opt, link)) { + return 0; /* Abort */ } } /* The incoming (read/write) buffer is at least HTS_URLMAXSIZE bytes long */ if (strncmp(link, "http://", 7) == 0 || strncmp(link, "https://", 8) == 0) { char temp[HTS_URLMAXSIZE * 2]; + strcpy(temp, base); strcat(temp, link); strcpy(link, temp); } - return 1; /* success */ + return 1; /* success */ } -static int check_detectedlink_end(t_hts_callbackarg *carg, httrackp *opt) { - char *base = (char*) CALLBACKARG_USERDEF(carg); +static int check_detectedlink_end(t_hts_callbackarg * carg, httrackp * opt) { + char *base = (char *) CALLBACKARG_USERDEF(carg); fprintf(stderr, "Unplugged ..\n"); if (base != NULL) { @@ -111,8 +121,8 @@ static int check_detectedlink_end(t_hts_callbackarg *carg, httrackp *opt) { /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, end) != NULL) { - return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt); + return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt); } - return 1; /* success */ + return 1; /* success */ } diff --git a/libtest/callbacks-example-changecontent.c b/libtest/callbacks-example-changecontent.c index e141789..dc6c51d 100755 --- a/libtest/callbacks-example-changecontent.c +++ b/libtest/callbacks-example-changecontent.c @@ -25,33 +25,36 @@ #include "htsdefines.h" /* Local function definitions */ -static int postprocess(t_hts_callbackarg *carg, httrackp *opt, - char** html, int* len, - const char* url_address, const char* url_file); +static int postprocess(t_hts_callbackarg * carg, httrackp * opt, char **html, + int *len, const char *url_address, const char *url_file); /* external functions */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); /* module entry point */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { const char *arg = strchr(argv, ','); + if (arg != NULL) arg++; /* Plug callback functions */ CHAIN_FUNCTION(opt, postprocess, postprocess, NULL); - return 1; /* success */ + return 1; /* success */ } -static int postprocess(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) { +static int postprocess(t_hts_callbackarg * carg, httrackp * opt, char **html, + int *len, const char *url_address, + const char *url_file) { char *old = *html; /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, postprocess) != NULL) { - if (CALLBACKARG_PREV_FUN(carg, postprocess)(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) { + if (CALLBACKARG_PREV_FUN(carg, postprocess) + (CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) { /* Modified *html */ old = *html; } diff --git a/libtest/callbacks-example-contentfilter.c b/libtest/callbacks-example-contentfilter.c index bbd2262..c98120e 100755 --- a/libtest/callbacks-example-contentfilter.c +++ b/libtest/callbacks-example-contentfilter.c @@ -25,11 +25,12 @@ #include "htsdefines.h" /* Local function definitions */ -static int process(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, const char* address, const char* filename); -static int end(t_hts_callbackarg *carg, httrackp *opt); +static int process(t_hts_callbackarg * carg, httrackp * opt, char *html, + int len, const char *address, const char *filename); +static int end(t_hts_callbackarg * carg, httrackp * opt); /* external functions */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); /* TOLOWER */ #define TOLOWER_(a) (a >= 'A' && a <= 'Z') ? (a + ('a' - 'A')) : a @@ -41,32 +42,37 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); typedef struct t_my_userdef { char stringfilter[8192]; - char* stringfilters[128]; + char *stringfilters[128]; } t_my_userdef; /* module entry point */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { const char *arg = strchr(argv, ','); + if (arg != NULL) arg++; /* Check args */ if (arg == NULL || *arg == '\0') { fprintf(stderr, "** callback error: arguments expected or bad arguments\n"); - fprintf(stderr, "usage: httrack --wrapper callback,stringtofind,stringtofind..\n"); + fprintf(stderr, + "usage: httrack --wrapper callback,stringtofind,stringtofind..\n"); fprintf(stderr, "example: httrack --wrapper callback,apple,orange,lemon\n"); return 0; } else { - t_my_userdef *userdef = (t_my_userdef*) malloc(sizeof(t_my_userdef)); /* userdef */ - char * const stringfilter = userdef->stringfilter; - char** const stringfilters = userdef->stringfilters; + t_my_userdef *userdef = (t_my_userdef *) malloc(sizeof(t_my_userdef)); /* userdef */ + char *const stringfilter = userdef->stringfilter; + char **const stringfilters = userdef->stringfilters; + /* */ - char* a = stringfilter; + char *a = stringfilter; int i = 0; + fprintf(stderr, "** info: wrapper_init(%s) called!\n", arg); - fprintf(stderr, "** callback example: crawling pages only if specific keywords are found\n"); + fprintf(stderr, + "** callback example: crawling pages only if specific keywords are found\n"); /* stringfilters = split(arg, ','); */ strcpy(stringfilter, arg); @@ -75,9 +81,10 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { a = strchr(a, ','); if (a != NULL) { *a = '\0'; - a ++; + a++; } - fprintf(stderr, "** callback info: will crawl pages with '%s' in them\n", stringfilters[i]); + fprintf(stderr, "** callback info: will crawl pages with '%s' in them\n", + stringfilters[i]); i++; } stringfilters[i++] = NULL; @@ -87,35 +94,42 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { CHAIN_FUNCTION(opt, end, end, userdef); } - return 1; /* success */ + return 1; /* success */ } -static int process(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, const char* address, const char* filename) { - t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg); - /*char * const stringfilter = userdef->stringfilter;*/ - char** const stringfilters = userdef->stringfilters; +static int process(t_hts_callbackarg * carg, httrackp * opt, char *html, + int len, const char *address, const char *filename) { + t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg); + + /*char * const stringfilter = userdef->stringfilter; */ + char **const stringfilters = userdef->stringfilters; + /* */ int i = 0; int getIt = 0; - char* pos; + char *pos; /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt, html, len, address, filename)) { - return 0; /* Abort */ + if (!CALLBACKARG_PREV_FUN(carg, check_html) + (CALLBACKARG_PREV_CARG(carg), opt, html, len, address, filename)) { + return 0; /* Abort */ } } /* Process */ - if (strcmp(address, "primary") == 0 && strcmp(filename, "/primary") == 0) /* primary page (list of links) */ + if (strcmp(address, "primary") == 0 && strcmp(filename, "/primary") == 0) /* primary page (list of links) */ return 1; - while(stringfilters[i] != NULL && ! getIt) { - if ( ( pos = strstr(html, stringfilters[i]) ) != NULL) { + while(stringfilters[i] != NULL && !getIt) { + if ((pos = strstr(html, stringfilters[i])) != NULL) { int j; + getIt = 1; - fprintf(stderr, "** callback info: found '%s' keyword in '%s%s', crawling this page!\n", stringfilters[i], address, filename); + fprintf(stderr, + "** callback info: found '%s' keyword in '%s%s', crawling this page!\n", + stringfilters[i], address, filename); fprintf(stderr, "** details:\n(..)"); - for(j = 0; j < 72 && pos[j] ; j++) { + for(j = 0; j < 72 && pos[j]; j++) { if (pos[j] > 32) fprintf(stderr, "%c", pos[j]); else @@ -126,15 +140,18 @@ static int process(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, i++; } if (getIt) { - return 1; /* success */ + return 1; /* success */ } else { - fprintf(stderr, "** callback info: won't parse '%s%s' (no specified keywords found)\n", address, filename); - return 0; /* this page sucks, don't parse it */ + fprintf(stderr, + "** callback info: won't parse '%s%s' (no specified keywords found)\n", + address, filename); + return 0; /* this page sucks, don't parse it */ } } -static int end(t_hts_callbackarg *carg, httrackp *opt) { - t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg); +static int end(t_hts_callbackarg * carg, httrackp * opt) { + t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg); + fprintf(stderr, "** info: wrapper_exit() called!\n"); if (userdef != NULL) { free(userdef); @@ -143,8 +160,8 @@ static int end(t_hts_callbackarg *carg, httrackp *opt) { /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, end) != NULL) { - return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt); + return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt); } - return 1; /* success */ + return 1; /* success */ } diff --git a/libtest/callbacks-example-displayheader.c b/libtest/callbacks-example-displayheader.c index ae1efd4..bb71e5f 100755 --- a/libtest/callbacks-example-displayheader.c +++ b/libtest/callbacks-example-displayheader.c @@ -25,42 +25,43 @@ #include "htsdefines.h" /* Local function definitions */ -static int process(t_hts_callbackarg *carg, httrackp *opt, - char* buff, const char* adr, const char* fil, - const char* referer_adr, const char* referer_fil, - htsblk* incoming); +static int process(t_hts_callbackarg * carg, httrackp * opt, char *buff, + const char *adr, const char *fil, const char *referer_adr, + const char *referer_fil, htsblk * incoming); /* external functions */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); /* module entry point */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { const char *arg = strchr(argv, ','); + if (arg != NULL) arg++; /* Plug callback functions */ CHAIN_FUNCTION(opt, receivehead, process, NULL); - return 1; /* success */ + return 1; /* success */ } -static int process(t_hts_callbackarg *carg, httrackp *opt, - char* buff, const char* adr, const char* fil, - const char* referer_adr, const char* referer_fil, - htsblk* incoming) { +static int process(t_hts_callbackarg * carg, httrackp * opt, char *buff, + const char *adr, const char *fil, const char *referer_adr, + const char *referer_fil, htsblk * incoming) { /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, receivehead) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, receivehead)(CALLBACKARG_PREV_CARG(carg), opt, buff, adr, fil, referer_adr, referer_fil, incoming)) { - return 0; /* Abort */ + if (!CALLBACKARG_PREV_FUN(carg, receivehead) + (CALLBACKARG_PREV_CARG(carg), opt, buff, adr, fil, referer_adr, + referer_fil, incoming)) { + return 0; /* Abort */ } } /* Process */ printf("[ %s%s ]\n%s\n", adr, fil, buff); - return 1; /* success */ + return 1; /* success */ } diff --git a/libtest/callbacks-example-filename.c b/libtest/callbacks-example-filename.c index 361ebb0..4e1fbc5 100755 --- a/libtest/callbacks-example-filename.c +++ b/libtest/callbacks-example-filename.c @@ -24,10 +24,13 @@ #include "htsdefines.h" /* Local function definitions */ -static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save); +static int mysavename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, const char *fil_complete, + const char *referer_adr, const char *referer_fil, + char *save); /* external functions */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); /* Options settings */ #include "htsopt.h" @@ -54,33 +57,40 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); /* module entry point */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { const char *arg = strchr(argv, ','); + if (arg != NULL) arg++; /* Plug callback functions */ CHAIN_FUNCTION(opt, savename, mysavename, NULL); - return 1; /* success */ + return 1; /* success */ } -static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save) { - char* a; +static int mysavename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, const char *fil_complete, + const char *referer_adr, const char *referer_fil, + char *save) { + char *a; /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, savename) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, savename)(CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete, referer_adr, referer_fil, save)) { - return 0; /* Abort */ + if (!CALLBACKARG_PREV_FUN(carg, savename) + (CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete, + referer_adr, referer_fil, save)) { + return 0; /* Abort */ } } /* Process */ - for(a = save ; *a != 0 ; a++) { + for(a = save; *a != 0; a++) { char c = TOLOWER(*a); + if (c >= 'a' && c <= 'z') - *a = ( ( ( c - 'a' ) + 13 ) % 26 ) + 'a'; // ROT-13 + *a = (((c - 'a') + 13) % 26) + 'a'; // ROT-13 } - - return 1; /* success */ + + return 1; /* success */ } diff --git a/libtest/callbacks-example-filename2.c b/libtest/callbacks-example-filename2.c index 2683b7b..fe2d964 100755 --- a/libtest/callbacks-example-filename2.c +++ b/libtest/callbacks-example-filename2.c @@ -21,11 +21,14 @@ #include "htsdefines.h" /* Function definitions */ -static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save); -static int myend(t_hts_callbackarg *carg, httrackp *opt); +static int mysavename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, const char *fil_complete, + const char *referer_adr, const char *referer_fil, + char *save); +static int myend(t_hts_callbackarg * carg, httrackp * opt); /* external functions */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); /* TOLOWER */ #define TOLOWER_(a) (a >= 'A' && a <= 'Z') ? (a + ('a' - 'A')) : a @@ -45,41 +48,46 @@ typedef struct t_my_userdef { /* module entry point */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { const char *arg = strchr(argv, ','); + if (arg != NULL) arg++; /* Check args */ if (arg == NULL || *arg == '\0' || strchr(arg, ',') == NULL) { fprintf(stderr, "** callback error: arguments expected or bad arguments\n"); - fprintf(stderr, "usage: httrack --wrapper save-name=callback:mysavename,string1,string2\n"); - fprintf(stderr, "example: httrack --wrapper save-name=callback:mysavename,foo,bar\n"); - return 0; /* failed */ + fprintf(stderr, + "usage: httrack --wrapper save-name=callback:mysavename,string1,string2\n"); + fprintf(stderr, + "example: httrack --wrapper save-name=callback:mysavename,foo,bar\n"); + return 0; /* failed */ } else { char *pos = strchr(arg, ','); - t_my_userdef *userdef = (t_my_userdef*) malloc(sizeof(t_my_userdef)); - char * const string1 = userdef->string1; - char * const string2 = userdef->string2; + t_my_userdef *userdef = (t_my_userdef *) malloc(sizeof(t_my_userdef)); + char *const string1 = userdef->string1; + char *const string2 = userdef->string2; /* Split args */ fprintf(stderr, "** info: wrapper_init(%s) called!\n", arg); - fprintf(stderr, "** callback example: changing destination filename word by another one\n"); + fprintf(stderr, + "** callback example: changing destination filename word by another one\n"); string1[0] = string1[1] = '\0'; strncat(string1, arg, pos - arg); strcpy(string2, pos + 1); - fprintf(stderr, "** callback info: will replace %s by %s in filenames!\n", string1, string2); + fprintf(stderr, "** callback info: will replace %s by %s in filenames!\n", + string1, string2); /* Plug callback functions */ CHAIN_FUNCTION(opt, savename, mysavename, userdef); CHAIN_FUNCTION(opt, end, myend, userdef); } - return 1; /* success */ + return 1; /* success */ } -static int myend(t_hts_callbackarg *carg, httrackp *opt) { - t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg); +static int myend(t_hts_callbackarg * carg, httrackp * opt) { + t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg); fprintf(stderr, "** info: wrapper_exit() called!\n"); if (userdef != NULL) { @@ -89,23 +97,29 @@ static int myend(t_hts_callbackarg *carg, httrackp *opt) { /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, end) != NULL) { - return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt); + return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt); } - return 1; /* success */ + return 1; /* success */ } -static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save) { - t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg); - char * const string1 = userdef->string1; - char * const string2 = userdef->string2; +static int mysavename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, const char *fil_complete, + const char *referer_adr, const char *referer_fil, + char *save) { + t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg); + char *const string1 = userdef->string1; + char *const string2 = userdef->string2; + /* */ char *buff, *a, *b; /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, savename) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, savename)(CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete, referer_adr, referer_fil, save)) { - return 0; /* Abort */ + if (!CALLBACKARG_PREV_FUN(carg, savename) + (CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete, + referer_adr, referer_fil, save)) { + return 0; /* Abort */ } } @@ -113,9 +127,9 @@ static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_co buff = strdup(save); a = buff; b = save; - *b = '\0'; /* the "save" variable points to a buffer with "sufficient" space */ + *b = '\0'; /* the "save" variable points to a buffer with "sufficient" space */ while(*a) { - if (strncmp(a, string1, (int)strlen(string1)) == 0) { + if (strncmp(a, string1, (int) strlen(string1)) == 0) { strcat(b, string2); b += strlen(b); a += strlen(string1); @@ -126,5 +140,5 @@ static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_co } free(buff); - return 1; /* success */ + return 1; /* success */ } diff --git a/libtest/callbacks-example-filenameiisbug.c b/libtest/callbacks-example-filenameiisbug.c index bd882cf..b615abc 100755 --- a/libtest/callbacks-example-filenameiisbug.c +++ b/libtest/callbacks-example-filenameiisbug.c @@ -21,49 +21,61 @@ #include "htsdefines.h" /* Function definitions */ -static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save); +static int mysavename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, const char *fil_complete, + const char *referer_adr, const char *referer_fil, + char *save); /* external functions */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); /* module entry point */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { const char *arg = strchr(argv, ','); + if (arg != NULL) arg++; CHAIN_FUNCTION(opt, savename, mysavename, NULL); - return 1; /* success */ + return 1; /* success */ } /* Replaces all "offending" IIS extensions (exe, dll..) with "nice" ones */ -static int mysavename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save) { - static const char* iisBogus[] = { ".com", ".exe", ".dll", ".sh", NULL }; - static const char* iisBogusReplace[] = { ".c0m", ".ex3", ".dl1", ".5h", NULL }; /* MUST be the same sizes */ - char* a; +static int mysavename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, const char *fil_complete, + const char *referer_adr, const char *referer_fil, + char *save) { + static const char *iisBogus[] = { ".com", ".exe", ".dll", ".sh", NULL }; + static const char *iisBogusReplace[] = { ".c0m", ".ex3", ".dl1", ".5h", NULL }; /* MUST be the same sizes */ + char *a; /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, savename) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, savename)(CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete, referer_adr, referer_fil, save)) { - return 0; /* Abort */ + if (!CALLBACKARG_PREV_FUN(carg, savename) + (CALLBACKARG_PREV_CARG(carg), opt, adr_complete, fil_complete, + referer_adr, referer_fil, save)) { + return 0; /* Abort */ } } /* Process */ - for(a = save ; *a != '\0' ; a++) { + for(a = save; *a != '\0'; a++) { int i; - for(i = 0 ; iisBogus[i] != NULL ; i++) { + + for(i = 0; iisBogus[i] != NULL; i++) { int j; - for(j = 0 ; iisBogus[i][j] == a[j] && iisBogus[i][j] != '\0' ; j++); - if (iisBogus[i][j] == '\0' && ( a[j] == '\0' || a[j] == '/' || a[j] == '\\' ) ) { + + for(j = 0; iisBogus[i][j] == a[j] && iisBogus[i][j] != '\0'; j++) ; + if (iisBogus[i][j] == '\0' + && (a[j] == '\0' || a[j] == '/' || a[j] == '\\')) { strncpy(a, iisBogusReplace[i], strlen(iisBogusReplace[i])); break; } } } - return 1; /* success */ + return 1; /* success */ } diff --git a/libtest/callbacks-example-listlinks.c b/libtest/callbacks-example-listlinks.c index 8828f3f..390d7aa 100755 --- a/libtest/callbacks-example-listlinks.c +++ b/libtest/callbacks-example-listlinks.c @@ -24,13 +24,17 @@ #include "htsdefines.h" /* Function definitions */ -static int process_file(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, const char* url_address, const char* url_file); -static int check_detectedlink(t_hts_callbackarg *carg, httrackp *opt, char* link); -static int check_loop(t_hts_callbackarg *carg, httrackp *opt, void* back,int back_max,int back_index,int lien_tot,int lien_ntot,int stat_time,hts_stat_struct* stats); -static int end(t_hts_callbackarg *carg, httrackp *opt); +static int process_file(t_hts_callbackarg * carg, httrackp * opt, char *html, + int len, const char *url_address, const char *url_file); +static int check_detectedlink(t_hts_callbackarg * carg, httrackp * opt, + char *link); +static int check_loop(t_hts_callbackarg * carg, httrackp * opt, void *back, + int back_max, int back_index, int lien_tot, int lien_ntot, + int stat_time, hts_stat_struct * stats); +static int end(t_hts_callbackarg * carg, httrackp * opt); /* external functions */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); /* This sample just lists all links in documents with the parent link: @@ -45,15 +49,17 @@ typedef struct t_my_userdef { /* module entry point */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { t_my_userdef *userdef; + /* */ const char *arg = strchr(argv, ','); + if (arg != NULL) arg++; /* Create user-defined structure */ - userdef = (t_my_userdef*) malloc(sizeof(t_my_userdef)); /* userdef */ + userdef = (t_my_userdef *) malloc(sizeof(t_my_userdef)); /* userdef */ userdef->currentURLBeingParsed[0] = '\0'; /* Plug callback functions */ @@ -62,17 +68,20 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { CHAIN_FUNCTION(opt, linkdetected, check_detectedlink, userdef); CHAIN_FUNCTION(opt, loop, check_loop, userdef); - return 1; /* success */ + return 1; /* success */ } -static int process_file(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, const char* url_address, const char* url_file) { - t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg); - char * const currentURLBeingParsed = userdef->currentURLBeingParsed; +static int process_file(t_hts_callbackarg * carg, httrackp * opt, char *html, + int len, const char *url_address, + const char *url_file) { + t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg); + char *const currentURLBeingParsed = userdef->currentURLBeingParsed; /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) { - return 0; /* Abort */ + if (!CALLBACKARG_PREV_FUN(carg, check_html) + (CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) { + return 0; /* Abort */ } } @@ -81,43 +90,50 @@ static int process_file(t_hts_callbackarg *carg, httrackp *opt, char* html, int strcpy(currentURLBeingParsed, url_address); strcat(currentURLBeingParsed, url_file); - return 1; /* success */ + return 1; /* success */ } -static int check_detectedlink(t_hts_callbackarg *carg, httrackp *opt, char* link) { - t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg); - char * const currentURLBeingParsed = userdef->currentURLBeingParsed; +static int check_detectedlink(t_hts_callbackarg * carg, httrackp * opt, + char *link) { + t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg); + char *const currentURLBeingParsed = userdef->currentURLBeingParsed; /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, linkdetected) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, linkdetected)(CALLBACKARG_PREV_CARG(carg), opt, link)) { - return 0; /* Abort */ + if (!CALLBACKARG_PREV_FUN(carg, linkdetected) + (CALLBACKARG_PREV_CARG(carg), opt, link)) { + return 0; /* Abort */ } } /* Process */ printf("[%s] -> [%s]\n", currentURLBeingParsed, link); - return 1; /* success */ + return 1; /* success */ } -static int check_loop(t_hts_callbackarg *carg, httrackp *opt, void* back,int back_max,int back_index,int lien_tot,int lien_ntot,int stat_time,hts_stat_struct* stats) { - static int fun_animation=0; +static int check_loop(t_hts_callbackarg * carg, httrackp * opt, void *back, + int back_max, int back_index, int lien_tot, int lien_ntot, + int stat_time, hts_stat_struct * stats) { + static int fun_animation = 0; /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, loop) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, loop)(CALLBACKARG_PREV_CARG(carg), opt, back, back_max, back_index, lien_tot, lien_ntot, stat_time, stats)) { - return 0; /* Abort */ + if (!CALLBACKARG_PREV_FUN(carg, loop) + (CALLBACKARG_PREV_CARG(carg), opt, back, back_max, back_index, lien_tot, + lien_ntot, stat_time, stats)) { + return 0; /* Abort */ } } /* Process */ - printf("%c\r", "/-\\|"[(fun_animation++)%4]); + printf("%c\r", "/-\\|"[(fun_animation++) % 4]); return 1; } -static int end(t_hts_callbackarg *carg, httrackp *opt) { - t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg); +static int end(t_hts_callbackarg * carg, httrackp * opt) { + t_my_userdef *userdef = (t_my_userdef *) CALLBACKARG_USERDEF(carg); + fprintf(stderr, "** info: wrapper_exit() called!\n"); if (userdef != NULL) { free(userdef); @@ -126,8 +142,8 @@ static int end(t_hts_callbackarg *carg, httrackp *opt) { /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, end) != NULL) { - return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt); + return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt); } - return 1; /* success */ + return 1; /* success */ } diff --git a/libtest/callbacks-example-log.c b/libtest/callbacks-example-log.c index 21881b7..d7d1465 100755 --- a/libtest/callbacks-example-log.c +++ b/libtest/callbacks-example-log.c @@ -24,23 +24,24 @@ #include "htsdefines.h" /* external functions */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); -EXTERNAL_FUNCTION int hts_unplug(httrackp *opt); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); +EXTERNAL_FUNCTION int hts_unplug(httrackp * opt); /* local function called as "check_html" callback */ -static int process_file(t_hts_callbackarg *carg, httrackp *opt, - char* html, int len, const char* url_address, const char* url_file) { - void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg); /*optional user-defined arg*/ +static int process_file(t_hts_callbackarg * carg, httrackp * opt, char *html, + int len, const char *url_address, + const char *url_file) { + void *ourDummyArg = (void *) CALLBACKARG_USERDEF(carg); /*optional user-defined arg */ char *fmt; (void) ourDummyArg; /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */ if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt, - html, len, url_address, url_file)) { - return 0; /* abort */ - } + if (!CALLBACKARG_PREV_FUN(carg, check_html) + (CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) { + return 0; /* abort */ + } } /* log */ @@ -50,11 +51,11 @@ static int process_file(t_hts_callbackarg *carg, httrackp *opt, hts_log(opt, "log-wrapper-info", fmt); free(fmt); - return 1; /* success */ + return 1; /* success */ } -static int start_of_mirror(t_hts_callbackarg *carg, httrackp *opt) { - const char *arginfo = (char*) CALLBACKARG_USERDEF(carg); +static int start_of_mirror(t_hts_callbackarg * carg, httrackp * opt) { + const char *arginfo = (char *) CALLBACKARG_USERDEF(carg); fprintf(stderr, "* mirror start\n"); hts_log(opt, arginfo, "mirror started"); @@ -62,15 +63,15 @@ static int start_of_mirror(t_hts_callbackarg *carg, httrackp *opt) { /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */ if (CALLBACKARG_PREV_FUN(carg, end) != NULL) { /* status is ok on our side, return other callabck's status */ - return CALLBACKARG_PREV_FUN(carg, start)(CALLBACKARG_PREV_CARG(carg), opt); + return CALLBACKARG_PREV_FUN(carg, start) (CALLBACKARG_PREV_CARG(carg), opt); } - return 1; /* success */ + return 1; /* success */ } /* local function called as "end" callback */ -static int end_of_mirror(t_hts_callbackarg *carg, httrackp *opt) { - const char *arginfo = (char*) CALLBACKARG_USERDEF(carg); +static int end_of_mirror(t_hts_callbackarg * carg, httrackp * opt) { + const char *arginfo = (char *) CALLBACKARG_USERDEF(carg); fprintf(stderr, "* mirror end\n"); hts_log(opt, arginfo, "mirror ended"); @@ -78,19 +79,20 @@ static int end_of_mirror(t_hts_callbackarg *carg, httrackp *opt) { /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */ if (CALLBACKARG_PREV_FUN(carg, end) != NULL) { /* status is ok on our side, return other callabck's status */ - return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt); + return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt); } - return 1; /* success */ + return 1; /* success */ } /* module entry point the function name and prototype MUST match this prototype */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { /* optional argument passed in the commandline we won't be using here */ const char *arg = strchr(argv, ','); + if (arg != NULL) arg++; @@ -98,19 +100,19 @@ EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { if (arg == NULL) arg = "log-wrapper-info"; hts_log(opt, arg, "* plugging functions"); - CHAIN_FUNCTION(opt, check_html, process_file, (char*) arg); - CHAIN_FUNCTION(opt, start, start_of_mirror, (char*) arg); - CHAIN_FUNCTION(opt, end, end_of_mirror, (char*) arg); + CHAIN_FUNCTION(opt, check_html, process_file, (char *) arg); + CHAIN_FUNCTION(opt, start, start_of_mirror, (char *) arg); + CHAIN_FUNCTION(opt, end, end_of_mirror, (char *) arg); hts_log(opt, arg, "* module successfully plugged"); - return 1; /* success */ + return 1; /* success */ } /* module exit point the function name and prototype MUST match this prototype */ -EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) { +EXTERNAL_FUNCTION int hts_unplug(httrackp * opt) { hts_log(opt, "log-wrapper-info", "* module successfully unplugged"); return 1; } diff --git a/libtest/callbacks-example-simple.c b/libtest/callbacks-example-simple.c index f481ed2..975f695 100755 --- a/libtest/callbacks-example-simple.c +++ b/libtest/callbacks-example-simple.c @@ -24,32 +24,37 @@ #include "htsdefines.h" /* external functions */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); -EXTERNAL_FUNCTION int hts_unplug(httrackp *opt); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); +EXTERNAL_FUNCTION int hts_unplug(httrackp * opt); /* local function called as "check_html" callback */ -static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt, - /*other parameters are callback-specific*/ - char* html, int len, const char* url_address, const char* url_file) { - void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg); /*optional user-defined arg*/ +static int process_file(t_hts_callbackarg + /*the carg structure, holding various information */ * + carg, /*the option settings */ httrackp * opt, + /*other parameters are callback-specific */ + char *html, int len, const char *url_address, + const char *url_file) { + void *ourDummyArg = (void *) CALLBACKARG_USERDEF(carg); /*optional user-defined arg */ (void) ourDummyArg; /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */ if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) { - if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt, - html, len, url_address, url_file)) { - return 0; /* abort */ - } + if (!CALLBACKARG_PREV_FUN(carg, check_html) + (CALLBACKARG_PREV_CARG(carg), opt, html, len, url_address, url_file)) { + return 0; /* abort */ + } } printf("file %s%s content: %s\n", url_address, url_file, html); - return 1; /* success */ + return 1; /* success */ } /* local function called as "end" callback */ -static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) { - void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg); /*optional user-defined arg*/ +static int end_of_mirror(t_hts_callbackarg + /*the carg structure, holding various information */ * + carg, /*the option settings */ httrackp * opt) { + void *ourDummyArg = (void *) CALLBACKARG_USERDEF(carg); /*optional user-defined arg */ (void) ourDummyArg; @@ -59,35 +64,37 @@ static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */ if (CALLBACKARG_PREV_FUN(carg, end) != NULL) { /* status is ok on our side, return other callabck's status */ - return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt); + return CALLBACKARG_PREV_FUN(carg, end) (CALLBACKARG_PREV_CARG(carg), opt); } - return 1; /* success */ + return 1; /* success */ } /* module entry point the function name and prototype MUST match this prototype */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { /* optional argument passed in the commandline we won't be using here */ const char *arg = strchr(argv, ','); + if (arg != NULL) arg++; /* plug callback functions */ - CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL); - CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL); + CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg */ + NULL); + CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg */ NULL); - return 1; /* success */ + return 1; /* success */ } /* module exit point the function name and prototype MUST match this prototype */ -EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) { +EXTERNAL_FUNCTION int hts_unplug(httrackp * opt) { fprintf(stderr, "Module unplugged"); - return 1; /* success */ + return 1; /* success */ } diff --git a/libtest/example.c b/libtest/example.c index 21f7254..2556830 100644 --- a/libtest/example.c +++ b/libtest/example.c @@ -38,34 +38,36 @@ int main(void) { /* First, ask for an URL Note: For the test, option r2 (mirror max depth=1) and --testscan (no index, no cache, do not store, no log files) - */ - char _argv[][256] = {"httrack_test" , "<URL>" , "-r3" , "--testscan" , "" }; - char* argv[] = {NULL , NULL , NULL , NULL , NULL}; - int argc = 0; + */ + char _argv[][256] = { "httrack_test", "<URL>", "-r3", "--testscan", "" }; + char *argv[] = { NULL, NULL, NULL, NULL, NULL }; + int argc = 0; httrackp *opt; - int ret; + int ret; + while(strlen(_argv[argc])) { - argv[argc]=_argv[argc]; + argv[argc] = _argv[argc]; argc++; } - argv[argc]=NULL; + argv[argc] = NULL; printf("HTTrackLib test program\n"); printf("Enter URL (example: www.foobar.com/index.html) :"); - scanf("%s",argv[1]); + scanf("%s", argv[1]); printf("Test: 1 depth\n"); - /* Initialize the library */ + /* Initialize the library */ #ifdef _WIN32 { - WORD wVersionRequested; // requested version WinSock API + WORD wVersionRequested; // requested version WinSock API WSADATA wsadata; // Windows Sockets API data int stat; + wVersionRequested = 0x0101; - stat = WSAStartup( wVersionRequested, &wsadata ); + stat = WSAStartup(wVersionRequested, &wsadata); if (stat != 0) { printf("Winsock not found!\n"); return; - } else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) { + } else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) { printf("WINSOCK.DLL does not support version 1.1\n"); WSACleanup(); return; @@ -74,8 +76,8 @@ int main(void) { #endif hts_init(); - /* Create option settings and set callbacks (wrappers) */ - opt = hts_create_opt(); + /* Create option settings and set callbacks (wrappers) */ + opt = hts_create_opt(); CHAIN_FUNCTION(opt, init, httrack_wrapper_init, NULL); CHAIN_FUNCTION(opt, uninit, httrack_wrapper_uninit, NULL); @@ -102,15 +104,15 @@ int main(void) { CHAIN_FUNCTION(opt, receivehead, httrack_wrapper_receiveheader, NULL); /* Then, launch the mirror */ - ret = hts_main2(argc, argv, opt); + ret = hts_main2(argc, argv, opt); /* Wait for a key */ printf("\nPress ENTER key to exit\n"); - scanf("%s",argv[1]); + scanf("%s", argv[1]); - /* Clear option state */ - hts_free_opt(opt); - hts_uninit(); + /* Clear option state */ + hts_free_opt(opt); + hts_uninit(); #ifdef _WIN32 WSACleanup(); #endif @@ -119,83 +121,132 @@ int main(void) { return 0; } - /* CALLBACK FUNCTIONS */ /* Initialize the Winsock */ -static void CDECL httrack_wrapper_init(t_hts_callbackarg *carg) { +static void CDECL httrack_wrapper_init(t_hts_callbackarg * carg) { printf("Engine started\n"); } -static void CDECL httrack_wrapper_uninit(t_hts_callbackarg *carg) { +static void CDECL httrack_wrapper_uninit(t_hts_callbackarg * carg) { printf("Engine exited\n"); } -static int CDECL httrack_wrapper_start(t_hts_callbackarg *carg, httrackp* opt) { +static int CDECL httrack_wrapper_start(t_hts_callbackarg * carg, httrackp * opt) { printf("Start of mirror\n"); - return 1; + return 1; } -static int CDECL httrack_wrapper_chopt(t_hts_callbackarg *carg, httrackp* opt) { +static int CDECL httrack_wrapper_chopt(t_hts_callbackarg * carg, httrackp * opt) { return 1; } -static int CDECL httrack_wrapper_end(t_hts_callbackarg *carg, httrackp* opt) { +static int CDECL httrack_wrapper_end(t_hts_callbackarg * carg, httrackp * opt) { printf("End of mirror\n"); - return 1; + return 1; } -static int CDECL httrack_wrapper_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_address,const char* url_file) { - printf("Parsing html file: http://%s%s\n",url_address,url_file); +static int CDECL httrack_wrapper_checkhtml(t_hts_callbackarg * carg, + httrackp * opt, char *html, int len, + const char *url_address, + const char *url_file) { + printf("Parsing html file: http://%s%s\n", url_address, url_file); return 1; } -static int CDECL httrack_wrapper_loop(t_hts_callbackarg *carg, httrackp *opt, void* _back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time,hts_stat_struct* stats) { +static int CDECL httrack_wrapper_loop(t_hts_callbackarg * carg, httrackp * opt, + void *_back, int back_max, int back_index, + int lien_n, int lien_tot, int stat_time, + hts_stat_struct * stats) { /* printf("..httrack_wrapper_loop called\n"); */ return 1; } -static const char* CDECL httrack_wrapper_query(t_hts_callbackarg *carg, httrackp *opt, const char* question) { +static const char *CDECL httrack_wrapper_query(t_hts_callbackarg * carg, + httrackp * opt, + const char *question) { /* Answer is No */ return "N"; } -static const char* CDECL httrack_wrapper_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question) { +static const char *CDECL httrack_wrapper_query2(t_hts_callbackarg * carg, + httrackp * opt, + const char *question) { /* Answer is No */ return "N"; } -static const char* CDECL httrack_wrapper_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question) { +static const char *CDECL httrack_wrapper_query3(t_hts_callbackarg * carg, + httrackp * opt, + const char *question) { /* Answer is "" */ return ""; } -static int CDECL httrack_wrapper_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status) { - printf("Link status tested: http://%s%s\n",adr,fil); +static int CDECL httrack_wrapper_check(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + int status) { + printf("Link status tested: http://%s%s\n", adr, fil); return -1; } -static void CDECL httrack_wrapper_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile) { +static void CDECL httrack_wrapper_pause(t_hts_callbackarg * carg, + httrackp * opt, const char *lockfile) { /* Wait until lockfile is removed.. */ } -static void CDECL httrack_wrapper_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file) { +static void CDECL httrack_wrapper_filesave(t_hts_callbackarg * carg, + httrackp * opt, const char *file) { } -static int CDECL httrack_wrapper_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link) { - printf("Link detected: %s\n",link); +static int CDECL httrack_wrapper_linkdetected(t_hts_callbackarg * carg, + httrackp * opt, char *link) { + printf("Link detected: %s\n", link); return 1; } -static int CDECL httrack_wrapper_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, void* back) { +static int CDECL httrack_wrapper_xfrstatus(t_hts_callbackarg * carg, + httrackp * opt, void *back) { return 1; } -static int CDECL httrack_wrapper_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) { +static int CDECL httrack_wrapper_preprocesshtml(t_hts_callbackarg * carg, + httrackp * opt, char **html, + int *len, + const char *url_address, + const char *url_file) { return 1; } -static int CDECL httrack_wrapper_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) { +static int CDECL httrack_wrapper_postprocesshtml(t_hts_callbackarg * carg, + httrackp * opt, char **html, + int *len, + const char *url_address, + const char *url_file) { return 1; } -static int CDECL httrack_wrapper_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status) { +static int CDECL httrack_wrapper_check_mime(t_hts_callbackarg * carg, + httrackp * opt, const char *adr, + const char *fil, const char *mime, + int status) { return -1; } -static void CDECL httrack_wrapper_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* fil, const char* save, int is_new, int is_modified,int not_updated) { +static void CDECL httrack_wrapper_filesave2(t_hts_callbackarg * carg, + httrackp * opt, const char *adr, + const char *fil, const char *save, + int is_new, int is_modified, + int not_updated) { } -static int CDECL httrack_wrapper_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag) { +static int CDECL httrack_wrapper_linkdetected2(t_hts_callbackarg * carg, + httrackp * opt, char *link, + const char *start_tag) { return 1; } -static int CDECL httrack_wrapper_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save) { +static int CDECL httrack_wrapper_savename(t_hts_callbackarg * carg, + httrackp * opt, + const char *adr_complete, + const char *fil_complete, + const char *referer_adr, + const char *referer_fil, char *save) { return 1; } -static int CDECL httrack_wrapper_sendheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing) { +static int CDECL httrack_wrapper_sendheader(t_hts_callbackarg * carg, + httrackp * opt, char *buff, + const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, + htsblk * outgoing) { return 1; } -static int CDECL httrack_wrapper_receiveheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming) { +static int CDECL httrack_wrapper_receiveheader(t_hts_callbackarg * carg, + httrackp * opt, char *buff, + const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, + htsblk * incoming) { return 1; } diff --git a/libtest/example.h b/libtest/example.h index b1d8e38..7228653 100644 --- a/libtest/example.h +++ b/libtest/example.h @@ -9,26 +9,78 @@ #define CDECL #endif -static void CDECL httrack_wrapper_init(t_hts_callbackarg *carg); -static void CDECL httrack_wrapper_uninit(t_hts_callbackarg *carg); -static int CDECL httrack_wrapper_start(t_hts_callbackarg *carg, httrackp* opt); -static int CDECL httrack_wrapper_chopt(t_hts_callbackarg *carg, httrackp* opt); -static int CDECL httrack_wrapper_end(t_hts_callbackarg *carg, httrackp* opt); -static int CDECL httrack_wrapper_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_address,const char* url_file); -static int CDECL httrack_wrapper_loop(t_hts_callbackarg *carg, httrackp *opt, void* _back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time,hts_stat_struct* stats); -static const char* CDECL httrack_wrapper_query(t_hts_callbackarg *carg, httrackp *opt, const char* question); -static const char* CDECL httrack_wrapper_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question); -static const char* CDECL httrack_wrapper_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question); -static int CDECL httrack_wrapper_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status); -static void CDECL httrack_wrapper_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile); -static void CDECL httrack_wrapper_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file); -static int CDECL httrack_wrapper_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link); -static int CDECL httrack_wrapper_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, void* back); -static int CDECL httrack_wrapper_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file); -static int CDECL httrack_wrapper_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file); -static int CDECL httrack_wrapper_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status); -static void CDECL httrack_wrapper_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* fil, const char* save, int is_new, int is_modified,int not_updated); -static int CDECL httrack_wrapper_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag); -static int CDECL httrack_wrapper_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save); -static int CDECL httrack_wrapper_sendheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing); -static int CDECL httrack_wrapper_receiveheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming); +static void CDECL httrack_wrapper_init(t_hts_callbackarg * carg); +static void CDECL httrack_wrapper_uninit(t_hts_callbackarg * carg); +static int CDECL httrack_wrapper_start(t_hts_callbackarg * carg, + httrackp * opt); +static int CDECL httrack_wrapper_chopt(t_hts_callbackarg * carg, + httrackp * opt); +static int CDECL httrack_wrapper_end(t_hts_callbackarg * carg, httrackp * opt); +static int CDECL httrack_wrapper_checkhtml(t_hts_callbackarg * carg, + httrackp * opt, char *html, int len, + const char *url_address, + const char *url_file); +static int CDECL httrack_wrapper_loop(t_hts_callbackarg * carg, httrackp * opt, + void *_back, int back_max, int back_index, + int lien_n, int lien_tot, int stat_time, + hts_stat_struct * stats); +static const char *CDECL httrack_wrapper_query(t_hts_callbackarg * carg, + httrackp * opt, + const char *question); +static const char *CDECL httrack_wrapper_query2(t_hts_callbackarg * carg, + httrackp * opt, + const char *question); +static const char *CDECL httrack_wrapper_query3(t_hts_callbackarg * carg, + httrackp * opt, + const char *question); +static int CDECL httrack_wrapper_check(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + int status); +static void CDECL httrack_wrapper_pause(t_hts_callbackarg * carg, + httrackp * opt, const char *lockfile); +static void CDECL httrack_wrapper_filesave(t_hts_callbackarg * carg, + httrackp * opt, const char *file); +static int CDECL httrack_wrapper_linkdetected(t_hts_callbackarg * carg, + httrackp * opt, char *link); +static int CDECL httrack_wrapper_xfrstatus(t_hts_callbackarg * carg, + httrackp * opt, void *back); +static int CDECL httrack_wrapper_preprocesshtml(t_hts_callbackarg * carg, + httrackp * opt, char **html, + int *len, + const char *url_address, + const char *url_file); +static int CDECL httrack_wrapper_postprocesshtml(t_hts_callbackarg * carg, + httrackp * opt, char **html, + int *len, + const char *url_address, + const char *url_file); +static int CDECL httrack_wrapper_check_mime(t_hts_callbackarg * carg, + httrackp * opt, const char *adr, + const char *fil, const char *mime, + int status); +static void CDECL httrack_wrapper_filesave2(t_hts_callbackarg * carg, + httrackp * opt, const char *adr, + const char *fil, const char *save, + int is_new, int is_modified, + int not_updated); +static int CDECL httrack_wrapper_linkdetected2(t_hts_callbackarg * carg, + httrackp * opt, char *link, + const char *start_tag); +static int CDECL httrack_wrapper_savename(t_hts_callbackarg * carg, + httrackp * opt, + const char *adr_complete, + const char *fil_complete, + const char *referer_adr, + const char *referer_fil, char *save); +static int CDECL httrack_wrapper_sendheader(t_hts_callbackarg * carg, + httrackp * opt, char *buff, + const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, + htsblk * outgoing); +static int CDECL httrack_wrapper_receiveheader(t_hts_callbackarg * carg, + httrackp * opt, char *buff, + const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, + htsblk * incoming); diff --git a/src/hts-indextmpl.h b/src/hts-indextmpl.h index 83eba46..a4eb715 100644 --- a/src/hts-indextmpl.h +++ b/src/hts-indextmpl.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Index.html templates file */ /* Author: Xavier Roche */ @@ -37,7 +34,6 @@ Please visit our Website: http://www.httrack.com #ifndef HTTRACK_DEFTMPL #define HTTRACK_DEFTMPL - /* Index for each project */ /* regen: @@ -350,7 +346,6 @@ regen: ""LF\ ""LF - /* Other files (fade and backblue images) */ #define HTS_LOG_SECURITY_WARNING "note:\tthe hts-log.txt file, and hts-cache folder, may contain sensitive information,"LF\ diff --git a/src/htsalias.c b/src/htsalias.c index 908bc13..1a55a86 100644 --- a/src/htsalias.c +++ b/src/htsalias.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsalias.c subroutines: */ /* alias for command-line options and config files */ @@ -42,8 +39,8 @@ Please visit our Website: http://www.httrack.com #include "htsalias.h" #include "htsglobal.h" -void linput(FILE* fp,char* s,int max); -void hts_lowcase(char* s); +void linput(FILE * fp, char *s, int max); +void hts_lowcase(char *s); #define _NOT_NULL(a) ( (a!=NULL) ? (a) : "" ) // #define is_realspace(c) (strchr(" \x0d\x0a\x09\x0b\x0c",(c))!=NULL) @@ -70,7 +67,6 @@ void hts_lowcase(char* s); argc++ // END OF COPY OF cmdl_ins in htsmain.c - /* Aliases for command-line and config file definitions These definitions can be used: @@ -90,174 +86,193 @@ void hts_lowcase(char* s); param1 : this option must be alone, and needs one distinct parameter (-P <path>) param0 : this option must be alone, but the parameter should be put together (+*.gif) */ -const char* hts_optalias[][4] = { +const char *hts_optalias[][4] = { /* {"","","",""}, */ - {"path","-O","param1","output path"}, - {"chroot","-%O","param1","default top path"}, - {"mirror","-w","single",""}, - {"mirror-wizard","-W","single",""}, - {"get-files","-g","single",""}, - {"quiet","-q","single",""}, - {"mirrorlinks","-Y","single",""}, - {"proxy","-P","param1","proxy name:port"}, - {"bind","-%b","param1","hostname to bind"}, - {"httpproxy-ftp","-%f","param",""}, - {"depth","-r","param",""},{"recurse-levels","-r","param",""}, - {"ext-depth","-%e","param",""}, - {"max-files","-m","param",""}, - {"max-size","-M","param",""}, - {"max-time","-E","param",""}, - {"max-mms-time","-%m","param",""}, - {"max-rate","-A","param",""}, - {"max-pause","-G","param",""}, - {"sockets","-c","param","number of simultaneous connections allowed"},{"socket","-c","param","number of simultaneous connections allowed"},{"connection","-c","param","number of simultaneous connections allowed"}, - {"connection-per-second","-%c","param","number of connection per second allowed"}, - {"timeout","-T","",""}, - {"retries","-R","param","number of retries for non-fatal errors"}, - {"min-rate","-J","param",""}, - {"host-control","-H","param",""}, - {"extended-parsing","-%P","param",""}, - {"near","-n","single",""}, - {"delayed-type-check","-%N","single",""}, - {"cached-delayed-type-check","-%D","single",""}, - {"delayed-type-check-always","-%N2","single",""}, - {"disable-security-limits","-%!","single",""}, - {"test","-t","single",""}, - {"list","-%L","param1",""}, - {"urllist","-%S","param1",""}, - {"language","-%l","param1",""}, {"lang","-%l","param1",""}, - {"structure","-N","param",""}, {"user-structure","-N","param1",""}, - {"long-names","-L","param",""}, - {"keep-links","-K","param",""}, - {"mime-html","-%M","single",""}, {"mht","-%M","single",""}, - {"replace-external","-x","single",""}, - {"disable-passwords","-%x","single",""},{"disable-password","-%x","single",""}, - {"include-query-string","-%q","single",""}, - {"generate-errors","-o","single",""}, - {"do-not-generate-errors","-o0","single",""}, - {"purge-old","-X","param",""}, - {"cookies","-b","param",""}, - {"check-type","-u","param",""}, - {"assume","-%A","param1",""}, {"mimetype","-%A","param1",""}, - {"parse-java","-j","param",""}, - {"protocol","-@i","param",""}, - {"robots","-s","param",""}, - {"http-10","-%h","single",""},{"http-1.0","-%h","single",""}, - {"keep-alive","-%k","single",""}, - {"build-top-index","-%i","single",""}, - {"disable-compression","-%z","single",""}, - {"tolerant","-%B","single",""}, - {"updatehack","-%s","single",""}, {"sizehack","-%s","single",""}, - {"urlhack","-%u","single",""}, - {"user-agent","-F","param1","user-agent identity"}, - {"referer","-%R","param1","default referer URL"}, - {"from","-%E","param1","from email address"}, - {"footer","-%F","param1",""}, - {"cache","-C","param","number of retries for non-fatal errors"}, - {"store-all-in-cache","-k","single",""}, - {"do-not-recatch","-%n","single",""}, - {"do-not-log","-Q","single",""}, - {"extra-log","-z","single",""}, - {"debug-log","-Z","single",""}, - {"verbose","-v","single",""}, - {"file-log","-f","single",""}, - {"single-log","-f2","single",""}, - {"index","-I","single",""}, - {"search-index","-%I","single",""}, - {"priority","-p","param",""}, - {"debug-headers","-%H","single",""}, - {"userdef-cmd","-V","param1",""}, - {"callback","-%W","param1","plug an external callback"}, {"wrapper","-%W","param1","plug an external callback"}, - {"structure","-N","param1","user-defined structure"}, - {"usercommand","-V","param1","user-defined command"}, - {"display","-%v","single","show files transfered and other funny realtime information"}, - {"dos83","-L0","single",""}, - {"iso9660","-L2","single",""}, - {"disable-module","-%w","param1",""}, - {"no-background-on-suspend","-y0","single",""}, - {"background-on-suspend","-y","single",""}, - {"utf8-conversion","-%T","single",""}, - {"no-utf8-conversion","-%T0","single",""}, + {"path", "-O", "param1", "output path"}, + {"chroot", "-%O", "param1", "default top path"}, + {"mirror", "-w", "single", ""}, + {"mirror-wizard", "-W", "single", ""}, + {"get-files", "-g", "single", ""}, + {"quiet", "-q", "single", ""}, + {"mirrorlinks", "-Y", "single", ""}, + {"proxy", "-P", "param1", "proxy name:port"}, + {"bind", "-%b", "param1", "hostname to bind"}, + {"httpproxy-ftp", "-%f", "param", ""}, + {"depth", "-r", "param", ""}, {"recurse-levels", "-r", "param", ""}, + {"ext-depth", "-%e", "param", ""}, + {"max-files", "-m", "param", ""}, + {"max-size", "-M", "param", ""}, + {"max-time", "-E", "param", ""}, + {"max-mms-time", "-%m", "param", ""}, + {"max-rate", "-A", "param", ""}, + {"max-pause", "-G", "param", ""}, + {"sockets", "-c", "param", "number of simultaneous connections allowed"}, + {"socket", "-c", "param", "number of simultaneous connections allowed"}, + {"connection", "-c", "param", "number of simultaneous connections allowed"}, + {"connection-per-second", "-%c", "param", + "number of connection per second allowed"}, + {"timeout", "-T", "", ""}, + {"retries", "-R", "param", "number of retries for non-fatal errors"}, + {"min-rate", "-J", "param", ""}, + {"host-control", "-H", "param", ""}, + {"extended-parsing", "-%P", "param", ""}, + {"near", "-n", "single", ""}, + {"delayed-type-check", "-%N", "single", ""}, + {"cached-delayed-type-check", "-%D", "single", ""}, + {"delayed-type-check-always", "-%N2", "single", ""}, + {"disable-security-limits", "-%!", "single", ""}, + {"test", "-t", "single", ""}, + {"list", "-%L", "param1", ""}, + {"urllist", "-%S", "param1", ""}, + {"language", "-%l", "param1", ""}, {"lang", "-%l", "param1", ""}, + {"structure", "-N", "param", ""}, {"user-structure", "-N", "param1", ""}, + {"long-names", "-L", "param", ""}, + {"keep-links", "-K", "param", ""}, + {"mime-html", "-%M", "single", ""}, {"mht", "-%M", "single", ""}, + {"replace-external", "-x", "single", ""}, + {"disable-passwords", "-%x", "single", ""}, {"disable-password", "-%x", + "single", ""}, + {"include-query-string", "-%q", "single", ""}, + {"generate-errors", "-o", "single", ""}, + {"do-not-generate-errors", "-o0", "single", ""}, + {"purge-old", "-X", "param", ""}, + {"cookies", "-b", "param", ""}, + {"check-type", "-u", "param", ""}, + {"assume", "-%A", "param1", ""}, {"mimetype", "-%A", "param1", ""}, + {"parse-java", "-j", "param", ""}, + {"protocol", "-@i", "param", ""}, + {"robots", "-s", "param", ""}, + {"http-10", "-%h", "single", ""}, {"http-1.0", "-%h", "single", ""}, + {"keep-alive", "-%k", "single", ""}, + {"build-top-index", "-%i", "single", ""}, + {"disable-compression", "-%z", "single", ""}, + {"tolerant", "-%B", "single", ""}, + {"updatehack", "-%s", "single", ""}, {"sizehack", "-%s", "single", ""}, + {"urlhack", "-%u", "single", ""}, + {"user-agent", "-F", "param1", "user-agent identity"}, + {"referer", "-%R", "param1", "default referer URL"}, + {"from", "-%E", "param1", "from email address"}, + {"footer", "-%F", "param1", ""}, + {"cache", "-C", "param", "number of retries for non-fatal errors"}, + {"store-all-in-cache", "-k", "single", ""}, + {"do-not-recatch", "-%n", "single", ""}, + {"do-not-log", "-Q", "single", ""}, + {"extra-log", "-z", "single", ""}, + {"debug-log", "-Z", "single", ""}, + {"verbose", "-v", "single", ""}, + {"file-log", "-f", "single", ""}, + {"single-log", "-f2", "single", ""}, + {"index", "-I", "single", ""}, + {"search-index", "-%I", "single", ""}, + {"priority", "-p", "param", ""}, + {"debug-headers", "-%H", "single", ""}, + {"userdef-cmd", "-V", "param1", ""}, + {"callback", "-%W", "param1", "plug an external callback"}, {"wrapper", "-%W", + "param1", + "plug an external callback"}, + {"structure", "-N", "param1", "user-defined structure"}, + {"usercommand", "-V", "param1", "user-defined command"}, + {"display", "-%v", "single", + "show files transfered and other funny realtime information"}, + {"dos83", "-L0", "single", ""}, + {"iso9660", "-L2", "single", ""}, + {"disable-module", "-%w", "param1", ""}, + {"no-background-on-suspend", "-y0", "single", ""}, + {"background-on-suspend", "-y", "single", ""}, + {"utf8-conversion", "-%T", "single", ""}, + {"no-utf8-conversion", "-%T0", "single", ""}, /* */ /* DEPRECATED */ - {"stay-on-same-dir","-S","single","stay on the same directory - DEPRECATED"}, - {"can-go-down","-D","single","can only go down into subdirs - DEPRECATED"}, - {"can-go-up","-U","single","can only go to upper directories- DEPRECATED"}, - {"can-go-up-and-down","-B","single","can both go up&down into the directory structure - DEPRECATED"}, - {"stay-on-same-address","-a","single","stay on the same address - DEPRECATED"}, - {"stay-on-same-domain","-d","single","stay on the same principal domain - DEPRECATED"}, - {"stay-on-same-tld","-l","single","stay on the same TLD (eg: .com) - DEPRECATED"}, - {"go-everywhere","-e","single","go everywhere on the web - DEPRECATED"}, + {"stay-on-same-dir", "-S", "single", + "stay on the same directory - DEPRECATED"}, + {"can-go-down", "-D", "single", "can only go down into subdirs - DEPRECATED"}, + {"can-go-up", "-U", "single", "can only go to upper directories- DEPRECATED"}, + {"can-go-up-and-down", "-B", "single", + "can both go up&down into the directory structure - DEPRECATED"}, + {"stay-on-same-address", "-a", "single", + "stay on the same address - DEPRECATED"}, + {"stay-on-same-domain", "-d", "single", + "stay on the same principal domain - DEPRECATED"}, + {"stay-on-same-tld", "-l", "single", + "stay on the same TLD (eg: .com) - DEPRECATED"}, + {"go-everywhere", "-e", "single", "go everywhere on the web - DEPRECATED"}, /* Badly documented */ - {"debug-testfilters","-#0","param1","debug: test filters"}, - {"advanced-flushlogs","-#f","single",""}, - {"advanced-maxfilters","-#F","param",""}, - {"version","-#h","single",""}, - {"debug-scanstdin","-#K","single",""}, - {"advanced-maxlinks","-#L","single",""}, - {"advanced-progressinfo","-#p","single","deprecated"}, - {"catch-url","-#P","single","catch complex URL through proxy"}, - /*{"debug-oldftp","-#R","single",""},*/ - {"debug-xfrstats","-#T","single",""}, - {"advanced-wait","-#u","single",""}, - {"debug-ratestats","-#Z","single",""}, - {"exec","-#!","param1",""}, - {"fast-engine","-#X","single","Enable fast routines"}, - {"debug-overflows","-#X0","single","Attempt to detect buffer overflows"}, - {"debug-cache","-#C","param1","List files in the cache"}, - {"extract-cache","-#C","single","Extract meta-data"}, - {"debug-parsing","-#d","single","debug: test parser"}, - {"repair-cache","-#R","single","repair the damaged cache ZIP file"}, {"repair","-#R","single",""}, + {"debug-testfilters", "-#0", "param1", "debug: test filters"}, + {"advanced-flushlogs", "-#f", "single", ""}, + {"advanced-maxfilters", "-#F", "param", ""}, + {"version", "-#h", "single", ""}, + {"debug-scanstdin", "-#K", "single", ""}, + {"advanced-maxlinks", "-#L", "single", ""}, + {"advanced-progressinfo", "-#p", "single", "deprecated"}, + {"catch-url", "-#P", "single", "catch complex URL through proxy"}, + /*{"debug-oldftp","-#R","single",""}, */ + {"debug-xfrstats", "-#T", "single", ""}, + {"advanced-wait", "-#u", "single", ""}, + {"debug-ratestats", "-#Z", "single", ""}, + {"exec", "-#!", "param1", ""}, + {"fast-engine", "-#X", "single", "Enable fast routines"}, + {"debug-overflows", "-#X0", "single", "Attempt to detect buffer overflows"}, + {"debug-cache", "-#C", "param1", "List files in the cache"}, + {"extract-cache", "-#C", "single", "Extract meta-data"}, + {"debug-parsing", "-#d", "single", "debug: test parser"}, + {"repair-cache", "-#R", "single", "repair the damaged cache ZIP file"}, + {"repair", "-#R", "single", ""}, /* STANDARD ALIASES */ - {"spider","-p0C0I0t","single",""}, - {"testsite","-p0C0I0t","single",""}, - {"testlinks","-r1p0C0I0t","single",""}, {"test","-r1p0C0I0t","single",""}, {"bookmark","-r1p0C0I0t","single",""}, - {"mirror","-w","single",""}, - {"testscan","-p0C0I0Q","single",""}, {"scan","-p0C0I0Q","single",""}, {"check","-p0C0I0Q","single",""}, - {"skeleton","-p1","single",""}, - {"preserve","-%p","single",""}, - {"get","-qg","single",""}, - {"update","-iC2","single",""}, - {"continue","-iC1","single",""}, {"restart","-iC1","single",""}, - {"continue","-i","single",""}, /* for help alias */ - {"sucker","-r999","single",""}, - {"help","-h","single",""}, {"documentation","-h","single",""}, {"doc","-h","single",""}, - {"wide","-c32","single",""}, - {"tiny","-c1","single",""}, - {"ultrawide","-c48","single",""}, - {"http10","-%h","single",""}, - {"filelist","-%L","single",""}, {"list","-%L","single",""}, - {"filterlist","-%S","single",""}, + {"spider", "-p0C0I0t", "single", ""}, + {"testsite", "-p0C0I0t", "single", ""}, + {"testlinks", "-r1p0C0I0t", "single", ""}, {"test", "-r1p0C0I0t", "single", + ""}, {"bookmark", "-r1p0C0I0t", + "single", ""}, + {"mirror", "-w", "single", ""}, + {"testscan", "-p0C0I0Q", "single", ""}, {"scan", "-p0C0I0Q", "single", ""}, + {"check", "-p0C0I0Q", "single", ""}, + {"skeleton", "-p1", "single", ""}, + {"preserve", "-%p", "single", ""}, + {"get", "-qg", "single", ""}, + {"update", "-iC2", "single", ""}, + {"continue", "-iC1", "single", ""}, {"restart", "-iC1", "single", ""}, + {"continue", "-i", "single", ""}, /* for help alias */ + {"sucker", "-r999", "single", ""}, + {"help", "-h", "single", ""}, {"documentation", "-h", "single", ""}, {"doc", + "-h", + "single", + ""}, + {"wide", "-c32", "single", ""}, + {"tiny", "-c1", "single", ""}, + {"ultrawide", "-c48", "single", ""}, + {"http10", "-%h", "single", ""}, + {"filelist", "-%L", "single", ""}, {"list", "-%L", "single", ""}, + {"filterlist", "-%S", "single", ""}, /* END OF ALIASES */ /* Filters */ - {"allow","+","param0","allow filter"}, - {"deny","-","param0","deny filter"}, + {"allow", "+", "param0", "allow filter"}, + {"deny", "-", "param0", "deny filter"}, /* */ /* URLs */ - {"add","","param0","add URLs"}, + {"add", "", "param0", "add URLs"}, /* */ /* Specific */ - {"user","-%U","param1","output path"}, + {"user", "-%U", "param1", "output path"}, /* */ /* Internal */ - {"catchurl","--catchurl","single","catch complex URL through proxy"}, - {"updatehttrack","--updatehttrack","single","update HTTrack Website Copier"}, - {"clean","--clean","single","clean up log files and cache"}, - {"tide","--clean","single","clean up log files and cache"}, - {"autotest","-#T","single",""}, + {"catchurl", "--catchurl", "single", "catch complex URL through proxy"}, + {"updatehttrack", "--updatehttrack", "single", + "update HTTrack Website Copier"}, + {"clean", "--clean", "single", "clean up log files and cache"}, + {"tide", "--clean", "single", "clean up log files and cache"}, + {"autotest", "-#T", "single", ""}, /* */ - {"","","",""} + {"", "", "", ""} }; - /* Check for alias in command-line argc,argv as in main() @@ -267,139 +282,146 @@ const char* hts_optalias[][4] = { return value: number of arguments treated (0 if error) */ -int optalias_check(int argc,const char * const * argv,int n_arg, - int* return_argc,char** return_argv, - char* return_error) { - return_error[0]='\0'; - *return_argc=1; - if (argv[n_arg][0]=='-') - if (argv[n_arg][1]=='-') { - char command[1000]; - char param[1000]; - char addcommand[256]; - /* */ - char* position; - int need_param=1; - //int return_param=0; - int pos; - command[0]=param[0]=addcommand[0]='\0'; - - /* --sockets=8 */ - if ( (position=strchr(argv[n_arg],'=')) ) { - /* Copy command */ - strncatbuff(command,argv[n_arg]+2,(int) (position - (argv[n_arg]+2)) ); - /* Copy parameter */ - strcpybuff(param,position+1); - } - /* --nocache */ - else if (strncmp(argv[n_arg]+2,"no",2)==0) { - strcpybuff(command,argv[n_arg]+4); - strcpybuff(param,"0"); - } - /* --sockets 8 */ - else { - if (strncmp(argv[n_arg]+2,"wide-",5)==0) { - strcpybuff(addcommand,"c32"); - strcpybuff(command,strchr(argv[n_arg]+2,'-')+1); - } else if (strncmp(argv[n_arg]+2,"tiny-",5)==0) { - strcpybuff(addcommand,"c1"); - strcpybuff(command,strchr(argv[n_arg]+2,'-')+1); - } else - strcpybuff(command,argv[n_arg]+2); - need_param=2; - } +int optalias_check(int argc, const char *const *argv, int n_arg, + int *return_argc, char **return_argv, char *return_error) { + return_error[0] = '\0'; + *return_argc = 1; + if (argv[n_arg][0] == '-') + if (argv[n_arg][1] == '-') { + char command[1000]; + char param[1000]; + char addcommand[256]; + + /* */ + char *position; + int need_param = 1; + + //int return_param=0; + int pos; + + command[0] = param[0] = addcommand[0] = '\0'; + + /* --sockets=8 */ + if ((position = strchr(argv[n_arg], '='))) { + /* Copy command */ + strncatbuff(command, argv[n_arg] + 2, + (int) (position - (argv[n_arg] + 2))); + /* Copy parameter */ + strcpybuff(param, position + 1); + } + /* --nocache */ + else if (strncmp(argv[n_arg] + 2, "no", 2) == 0) { + strcpybuff(command, argv[n_arg] + 4); + strcpybuff(param, "0"); + } + /* --sockets 8 */ + else { + if (strncmp(argv[n_arg] + 2, "wide-", 5) == 0) { + strcpybuff(addcommand, "c32"); + strcpybuff(command, strchr(argv[n_arg] + 2, '-') + 1); + } else if (strncmp(argv[n_arg] + 2, "tiny-", 5) == 0) { + strcpybuff(addcommand, "c1"); + strcpybuff(command, strchr(argv[n_arg] + 2, '-') + 1); + } else + strcpybuff(command, argv[n_arg] + 2); + need_param = 2; + } - /* Now solve the alias */ - pos=optalias_find(command); - if (pos>=0) { - /* Copy real name */ - strcpybuff(command,hts_optalias[pos][1]); - /* With parameters? */ - if (strncmp(hts_optalias[pos][2],"param",5)==0) { - /* Copy parameters? */ - if (need_param == 2) { - if ((n_arg+1>=argc) || (argv[n_arg+1][0]=='-')) { /* no supplemental parameter */ - sprintf(return_error, - "Syntax error:\n\tOption %s needs to be followed by a parameter: %s <param>\n\t%s\n", - command,command,_NOT_NULL(optalias_help(command))); - return 0; + /* Now solve the alias */ + pos = optalias_find(command); + if (pos >= 0) { + /* Copy real name */ + strcpybuff(command, hts_optalias[pos][1]); + /* With parameters? */ + if (strncmp(hts_optalias[pos][2], "param", 5) == 0) { + /* Copy parameters? */ + if (need_param == 2) { + if ((n_arg + 1 >= argc) || (argv[n_arg + 1][0] == '-')) { /* no supplemental parameter */ + sprintf(return_error, + "Syntax error:\n\tOption %s needs to be followed by a parameter: %s <param>\n\t%s\n", + command, command, _NOT_NULL(optalias_help(command))); + return 0; + } + strcpybuff(param, argv[n_arg + 1]); + need_param = 2; } - strcpybuff(param,argv[n_arg+1]); - need_param=2; + } else + need_param = 1; + + /* Final result */ + + /* Must be alone (-P /tmp) */ + if (strcmp(hts_optalias[pos][2], "param1") == 0) { + strcpybuff(return_argv[0], command); + strcpybuff(return_argv[1], param); + *return_argc = 2; /* 2 parameters returned */ } - } else - need_param=1; - - /* Final result */ - - /* Must be alone (-P /tmp) */ - if (strcmp(hts_optalias[pos][2],"param1")==0) { - strcpybuff(return_argv[0],command); - strcpybuff(return_argv[1],param); - *return_argc=2; /* 2 parameters returned */ - } - /* Alone with parameter (+*.gif) */ - else if (strcmp(hts_optalias[pos][2],"param0")==0) { - /* Command */ - strcpybuff(return_argv[0],command); - strcatbuff(return_argv[0],param); - } - /* Together (-c8) */ - else { - /* Command */ - strcpybuff(return_argv[0],command); - /* Parameters accepted */ - if (strncmp(hts_optalias[pos][2],"param",5)==0) { - /* --cache=off or --index=on */ - if (strcmp(param,"off")==0) - strcatbuff(return_argv[0],"0"); - else if (strcmp(param,"on")==0) { - // on is the default - // strcatbuff(return_argv[0],"1"); - } else - strcatbuff(return_argv[0],param); + /* Alone with parameter (+*.gif) */ + else if (strcmp(hts_optalias[pos][2], "param0") == 0) { + /* Command */ + strcpybuff(return_argv[0], command); + strcatbuff(return_argv[0], param); } - *return_argc=1; /* 1 parameter returned */ + /* Together (-c8) */ + else { + /* Command */ + strcpybuff(return_argv[0], command); + /* Parameters accepted */ + if (strncmp(hts_optalias[pos][2], "param", 5) == 0) { + /* --cache=off or --index=on */ + if (strcmp(param, "off") == 0) + strcatbuff(return_argv[0], "0"); + else if (strcmp(param, "on") == 0) { + // on is the default + // strcatbuff(return_argv[0],"1"); + } else + strcatbuff(return_argv[0], param); + } + *return_argc = 1; /* 1 parameter returned */ + } + } else { + sprintf(return_error, "Unknown option: %s\n", command); + return 0; } - } else { - sprintf(return_error,"Unknown option: %s\n",command); - return 0; + return need_param; } - return need_param; - } /* Check -O <path> */ { int pos; - if ((pos=optreal_find(argv[n_arg]))>=0) { - if ( (strcmp(hts_optalias[pos][2],"param1")==0) || (strcmp(hts_optalias[pos][2],"param0")==0)) { - if ((n_arg+1>=argc) || (argv[n_arg+1][0]=='-')) { /* no supplemental parameter */ + + if ((pos = optreal_find(argv[n_arg])) >= 0) { + if ((strcmp(hts_optalias[pos][2], "param1") == 0) + || (strcmp(hts_optalias[pos][2], "param0") == 0)) { + if ((n_arg + 1 >= argc) || (argv[n_arg + 1][0] == '-')) { /* no supplemental parameter */ sprintf(return_error, - "Syntax error:\n\tOption %s needs to be followed by a parameter: %s <param>\n\t%s\n", - argv[n_arg],argv[n_arg],_NOT_NULL(optalias_help(argv[n_arg]))); + "Syntax error:\n\tOption %s needs to be followed by a parameter: %s <param>\n\t%s\n", + argv[n_arg], argv[n_arg], + _NOT_NULL(optalias_help(argv[n_arg]))); return 0; } /* Copy parameters */ - strcpybuff(return_argv[0],argv[n_arg]); - strcpybuff(return_argv[1],argv[n_arg+1]); + strcpybuff(return_argv[0], argv[n_arg]); + strcpybuff(return_argv[1], argv[n_arg + 1]); /* And return */ - *return_argc=2; /* 2 parameters returned */ - return 2; /* 2 parameters used */ + *return_argc = 2; /* 2 parameters returned */ + return 2; /* 2 parameters used */ } } } - + /* Copy and return other unknown option */ - strcpybuff(return_argv[0],argv[n_arg]); + strcpybuff(return_argv[0], argv[n_arg]); return 1; } /* Finds the <token> option alias and returns the index, or -1 if failed */ -int optalias_find(const char* token) { +int optalias_find(const char *token) { if (token[0] != '\0') { - int i=0; + int i = 0; + while(hts_optalias[i][0][0] != '\0') { - if (strcmp(token,hts_optalias[i][0])==0) { + if (strcmp(token, hts_optalias[i][0]) == 0) { return i; } i++; @@ -409,11 +431,12 @@ int optalias_find(const char* token) { } /* Finds the <token> real option and returns the index, or -1 if failed */ -int optreal_find(const char* token) { +int optreal_find(const char *token) { if (token[0] != '\0') { - int i=0; + int i = 0; + while(hts_optalias[i][0][0] != '\0') { - if (strcmp(token,hts_optalias[i][1])==0) { + if (strcmp(token, hts_optalias[i][1]) == 0) { return i; } i++; @@ -422,23 +445,24 @@ int optreal_find(const char* token) { return -1; } -const char* optreal_value(int p) { +const char *optreal_value(int p) { return hts_optalias[p][1]; } -const char* optalias_value(int p) { +const char *optalias_value(int p) { return hts_optalias[p][0]; } -const char* opttype_value(int p) { +const char *opttype_value(int p) { return hts_optalias[p][2]; } -const char* opthelp_value(int p) { +const char *opthelp_value(int p) { return hts_optalias[p][3]; } /* Help for option <token>, empty if not available, or NULL if unknown <token> */ -const char* optalias_help(const char* token) { - int pos=optalias_find(token); - if (pos>=0) +const char *optalias_help(const char *token) { + int pos = optalias_find(token); + + if (pos >= 0) return hts_optalias[pos][3]; else return NULL; @@ -452,76 +476,89 @@ const char* optalias_help(const char* token) { deny ad.* */ /* Note: NOT utf-8 */ -int optinclude_file(const char* name, - int* argc,char** argv,char* x_argvblk,int* x_ptr) { - FILE* fp; - fp=fopen(name,"rb"); +int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk, + int *x_ptr) { + FILE *fp; + + fp = fopen(name, "rb"); if (fp) { char line[256]; - int insert_after=1; /* first, insert after program filename */ + int insert_after = 1; /* first, insert after program filename */ + while(!feof(fp)) { - char *a,*b; + char *a, *b; int result; - + /* read line */ - linput(fp,line,250); + linput(fp, line, 250); hts_lowcase(line); if (strnotempty(line)) { /* no comment line: # // ; */ - if (strchr("#/;",line[0])==NULL) { + if (strchr("#/;", line[0]) == NULL) { /* right trim */ - a=line+strlen(line)-1; - while(is_realspace(*a)) *(a--) = '\0'; + a = line + strlen(line) - 1; + while(is_realspace(*a)) + *(a--) = '\0'; /* jump "set " and spaces */ - a=line; - while(is_realspace(*a)) a++; - if (strncmp(a,"set",3)==0) { - if (is_realspace(*(a+3))) { - a+=4; + a = line; + while(is_realspace(*a)) + a++; + if (strncmp(a, "set", 3) == 0) { + if (is_realspace(*(a + 3))) { + a += 4; } } - while(is_realspace(*a)) a++; + while(is_realspace(*a)) + a++; /* delete = ("sockets=8") */ - if ( (b=strchr(a,'=')) ) - *b=' '; - + if ((b = strchr(a, '='))) + *b = ' '; + /* isolate option and parameter */ - b=a; - while( (!is_realspace(*b)) && (*b) ) b++; + b = a; + while((!is_realspace(*b)) && (*b)) + b++; if (*b) { - *b='\0'; + *b = '\0'; b++; } /* a is now the option, b the parameter */ - + { int return_argc; char return_error[256]; - char _tmp_argv[4][HTS_CDLMAXSIZE]; - char* tmp_argv[4]; - tmp_argv[0]=_tmp_argv[0]; tmp_argv[1]=_tmp_argv[1]; tmp_argv[2]=_tmp_argv[2]; tmp_argv[3]=_tmp_argv[3]; - strcpybuff(tmp_argv[0],"--"); - strcatbuff(tmp_argv[0],a); - strcpybuff(tmp_argv[1],b); - - result=optalias_check(2,(const char * const *)tmp_argv,0, - &return_argc,(tmp_argv+2), - return_error); + char _tmp_argv[4][HTS_CDLMAXSIZE]; + char *tmp_argv[4]; + + tmp_argv[0] = _tmp_argv[0]; + tmp_argv[1] = _tmp_argv[1]; + tmp_argv[2] = _tmp_argv[2]; + tmp_argv[3] = _tmp_argv[3]; + strcpybuff(tmp_argv[0], "--"); + strcatbuff(tmp_argv[0], a); + strcpybuff(tmp_argv[1], b); + + result = + optalias_check(2, (const char *const *) tmp_argv, 0, &return_argc, + (tmp_argv + 2), return_error); if (!result) { - printf("%s\n",return_error); + printf("%s\n", return_error); } else { int insert_after_argc; + /* Insert parameters BUT so that they can be in the same order */ /* temporary argc: Number of parameters after minus insert_after_argc */ - insert_after_argc=(*argc)-insert_after; - cmdl_ins((tmp_argv[2]),insert_after_argc,(argv+insert_after),x_argvblk,(*x_ptr)); - *argc=insert_after_argc+insert_after; + insert_after_argc = (*argc) - insert_after; + cmdl_ins((tmp_argv[2]), insert_after_argc, (argv + insert_after), + x_argvblk, (*x_ptr)); + *argc = insert_after_argc + insert_after; insert_after++; /* Second one */ - if (return_argc>1) { - insert_after_argc=(*argc)-insert_after; - cmdl_ins((tmp_argv[3]),insert_after_argc,(argv+insert_after),x_argvblk,(*x_ptr)); - *argc=insert_after_argc+insert_after; + if (return_argc > 1) { + insert_after_argc = (*argc) - insert_after; + cmdl_ins((tmp_argv[3]), insert_after_argc, + (argv + insert_after), x_argvblk, (*x_ptr)); + *argc = insert_after_argc + insert_after; insert_after++; } /* increment to nbr of used parameters */ @@ -529,7 +566,7 @@ int optinclude_file(const char* name, } } } - + } } fclose(fp); @@ -540,9 +577,10 @@ int optinclude_file(const char* name, /* Get home directory, '.' if failed */ /* example: /home/smith */ -char* hts_gethome(void) { +char *hts_gethome(void) { #ifndef _WIN32_WCE - char* home = getenv( "HOME" ); + char *home = getenv("HOME"); + if (home) return home; else @@ -551,9 +589,10 @@ char* hts_gethome(void) { } /* Convert ~/foo into /home/smith/foo */ -void expand_home(String *str) { +void expand_home(String * str) { if (StringSub(*str, 1) == '~') { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + strcpybuff(tempo, hts_gethome()); strcatbuff(tempo, StringBuff(*str) + 1); StringCopy(*str, tempo); diff --git a/src/htsalias.h b/src/htsalias.h index 7baa179..8350e2a 100644 --- a/src/htsalias.h +++ b/src/htsalias.h @@ -17,45 +17,40 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsalias.h subroutines: */ /* alias for command-line options and config files */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - #ifndef HTSALIAS_DEFH #define HTSALIAS_DEFH /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -extern const char* hts_optalias[][4]; -int optalias_check(int argc,const char * const * argv,int n_arg, - int* return_argc,char** return_argv, - char* return_error); -int optalias_find(const char* token); -const char* optalias_help(const char* token); -int optreal_find(const char* token); -int optinclude_file(const char* name, - int* argc,char** argv,char* x_argvblk,int* x_ptr); -const char* optreal_value(int p); -const char* optalias_value(int p); -const char* opttype_value(int p); -const char* opthelp_value(int p); -char* hts_gethome(void); -void expand_home(String *str); +extern const char *hts_optalias[][4]; +int optalias_check(int argc, const char *const *argv, int n_arg, + int *return_argc, char **return_argv, char *return_error); +int optalias_find(const char *token); +const char *optalias_help(const char *token); +int optreal_find(const char *token); +int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk, + int *x_ptr); +const char *optreal_value(int p); +const char *optalias_value(int p); +const char *opttype_value(int p); +const char *opthelp_value(int p); +char *hts_gethome(void); +void expand_home(String * str); #endif #endif diff --git a/src/htsback.c b/src/htsback.c index 0d4da0d..fafa8e4 100644 --- a/src/htsback.c +++ b/src/htsback.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* backing system (multiple socket download) */ @@ -60,7 +57,7 @@ Please visit our Website: http://www.httrack.com #ifndef __cplusplus // DOS #ifndef _WIN32_WCE -#include <process.h> /* _beginthread, _endthread */ +#include <process.h> /* _beginthread, _endthread */ #endif #endif #else @@ -73,21 +70,21 @@ Please visit our Website: http://www.httrack.com #define VT_CLREOL "\33[K" /* Slot operations */ -static int slot_can_be_cached_on_disk(const lien_back* back); -static int slot_can_be_cleaned(const lien_back* back); -static int slot_can_be_finalized(httrackp* opt, const lien_back* back); +static int slot_can_be_cached_on_disk(const lien_back * back); +static int slot_can_be_cleaned(const lien_back * back); +static int slot_can_be_finalized(httrackp * opt, const lien_back * back); - -struct_back* back_new(int back_max) { +struct_back *back_new(int back_max) { int i; - struct_back* sback = calloct(1, sizeof(struct_back)); + struct_back *sback = calloct(1, sizeof(struct_back)); + sback->count = back_max; - sback->lnk = (lien_back*) calloct((back_max + 1), sizeof(lien_back)); + sback->lnk = (lien_back *) calloct((back_max + 1), sizeof(lien_back)); sback->ready = inthash_new(32767); - sback->ready_size_bytes = 0; + sback->ready_size_bytes = 0; inthash_value_is_malloc(sback->ready, 1); // init - for(i = 0 ; i < sback->count ; i++){ + for(i = 0; i < sback->count; i++) { sback->lnk[i].r.location = sback->lnk[i].location_buffer; sback->lnk[i].status = STATUS_FREE; sback->lnk[i].r.soc = INVALID_SOCKET; @@ -95,7 +92,7 @@ struct_back* back_new(int back_max) { return sback; } -void back_free(struct_back** sback) { +void back_free(struct_back ** sback) { if (sback != NULL && *sback != NULL) { if ((*sback)->lnk != NULL) { freet((*sback)->lnk); @@ -103,39 +100,43 @@ void back_free(struct_back** sback) { } if ((*sback)->ready != NULL) { inthash_delete(&(*sback)->ready); - (*sback)->ready_size_bytes = 0; + (*sback)->ready_size_bytes = 0; } freet(*sback); *sback = NULL; } } -void back_delete_all(httrackp* opt, cache_back* cache, struct_back* sback) { +void back_delete_all(httrackp * opt, cache_back * cache, struct_back * sback) { if (sback != NULL) { int i; + // delete live slots - for(i = 0 ; i < sback->count ; i++) { + for(i = 0; i < sback->count; i++) { back_delete(opt, cache, sback, i); } // delete stored slots if (sback->ready != NULL) { struct_inthash_enum e = inthash_enum_new(sback->ready); - inthash_chain* item; + inthash_chain *item; + while((item = inthash_enum_next(&e))) { #ifndef HTS_NO_BACK_ON_DISK - char *filename = (char*) item->value.ptr; - if (filename != NULL) { - (void) UNLINK(filename); - } + char *filename = (char *) item->value.ptr; + + if (filename != NULL) { + (void) UNLINK(filename); + } #else - /* clear entry content (but not yet the entry) */ - lien_back *back = (lien_back*) item->value.ptr; - back_clear_entry(back); + /* clear entry content (but not yet the entry) */ + lien_back *back = (lien_back *) item->value.ptr; + + back_clear_entry(back); #endif } - /* delete hashtable & content */ - inthash_delete(&sback->ready); - sback->ready_size_bytes = 0; + /* delete hashtable & content */ + inthash_delete(&sback->ready); + sback->ready_size_bytes = 0; } } } @@ -143,28 +144,32 @@ void back_delete_all(httrackp* opt, cache_back* cache, struct_back* sback) { // --- // routines de backing -static int back_index_ready(httrackp* opt, struct_back* sback, char* adr, char* fil, char* sav, int getIndex); -static int back_index_fetch(httrackp* opt, struct_back* sback, char* adr, char* fil, char* sav, int getIndex); +static int back_index_ready(httrackp * opt, struct_back * sback, char *adr, + char *fil, char *sav, int getIndex); +static int back_index_fetch(httrackp * opt, struct_back * sback, char *adr, + char *fil, char *sav, int getIndex); // retourne l'index d'un lien dans un tableau de backing -int back_index(httrackp* opt, struct_back* sback,char* adr,char* fil,char* sav) { - return back_index_fetch(opt,sback, adr, fil, sav, 1); +int back_index(httrackp * opt, struct_back * sback, char *adr, char *fil, + char *sav) { + return back_index_fetch(opt, sback, adr, fil, sav, 1); } -static int back_index_fetch(httrackp* opt, struct_back* sback, char* adr, char* fil, char* sav, int getIndex) { - lien_back* const back = sback->lnk; +static int back_index_fetch(httrackp * opt, struct_back * sback, char *adr, + char *fil, char *sav, int getIndex) { + lien_back *const back = sback->lnk; const int back_max = sback->count; - int index=-1; + int index = -1; int i; - for( i = 0 ; i < back_max ; i++ ) { - if (back[i].status >= 0 /* not free or alive */ - && strfield2(back[i].url_adr,adr) - && strcmp(back[i].url_fil,fil)==0) - { - if (index==-1) /* first time we meet, store it */ - index=i; - else if (sav != NULL && strcmp(back[i].url_sav, sav) == 0) { /* oops, check sav too */ - index=i; + + for(i = 0; i < back_max; i++) { + if (back[i].status >= 0 /* not free or alive */ + && strfield2(back[i].url_adr, adr) + && strcmp(back[i].url_fil, fil) == 0) { + if (index == -1) /* first time we meet, store it */ + index = i; + else if (sav != NULL && strcmp(back[i].url_sav, sav) == 0) { /* oops, check sav too */ + index = i; return index; } } @@ -177,136 +182,168 @@ static int back_index_fetch(httrackp* opt, struct_back* sback, char* adr, char* } /* resurrect stored entry */ -static int back_index_ready(httrackp* opt, struct_back* sback, char* adr, char* fil, char* sav, int getIndex) { - lien_back* const back = sback->lnk; - void* ptr = NULL; +static int back_index_ready(httrackp * opt, struct_back * sback, char *adr, + char *fil, char *sav, int getIndex) { + lien_back *const back = sback->lnk; + void *ptr = NULL; + if (inthash_read_pvoid(sback->ready, sav, &ptr)) { - if (!getIndex) { /* don't "pagefault" the entry */ - if (ptr != NULL) { - return sback->count; /* (invalid but) positive result */ - } else { - return -1; /* not found */ + if (!getIndex) { /* don't "pagefault" the entry */ + if (ptr != NULL) { + return sback->count; /* (invalid but) positive result */ + } else { + return -1; /* not found */ } } else if (ptr != NULL) { - lien_back* itemback = NULL; + lien_back *itemback = NULL; + #ifndef HTS_NO_BACK_ON_DISK FILE *fp; - char* fileback = (char*) ptr; - char catbuff[CATBUFF_SIZE]; - if (( fp = FOPEN(fconv(catbuff, fileback), "rb") ) != NULL ) { + char *fileback = (char *) ptr; + char catbuff[CATBUFF_SIZE]; + + if ((fp = FOPEN(fconv(catbuff, fileback), "rb")) != NULL) { if (back_unserialize(fp, &itemback) != 0) { if (itemback != NULL) { back_clear_entry(itemback); freet(itemback); itemback = NULL; } - hts_log_print(opt, LOG_INFO | LOG_ERRNO, "engine: warning: unserialize error for %s%s (%s)",adr,fil,sav); + hts_log_print(opt, LOG_INFO | LOG_ERRNO, + "engine: warning: unserialize error for %s%s (%s)", adr, + fil, sav); } fclose(fp); } else { - hts_log_print(opt, LOG_INFO | LOG_ERRNO, "engine: warning: unserialize error for %s%s (%s), file disappeared",adr,fil,sav); + hts_log_print(opt, LOG_INFO | LOG_ERRNO, + "engine: warning: unserialize error for %s%s (%s), file disappeared", + adr, fil, sav); } (void) UNLINK(fileback); #else - itemback = (lien_back*) ptr; -#endif - if (itemback != NULL) { - // move from hashtable to fast repository - int q = back_search(opt, sback); - if (q != -1) { - deletehttp(&back[q].r); // security check - back_move(itemback, &back[q]); - back_clear_entry(itemback); /* delete entry content */ - freet(itemback); /* delete item */ - itemback = NULL; - inthash_remove(sback->ready, sav); // delete item - sback->ready_size_bytes -= back[q].r.size; /* substract for stats */ - back_set_locked(sback, q); /* locked */ - return q; + itemback = (lien_back *) ptr; +#endif + if (itemback != NULL) { + // move from hashtable to fast repository + int q = back_search(opt, sback); + + if (q != -1) { + deletehttp(&back[q].r); // security check + back_move(itemback, &back[q]); + back_clear_entry(itemback); /* delete entry content */ + freet(itemback); /* delete item */ + itemback = NULL; + inthash_remove(sback->ready, sav); // delete item + sback->ready_size_bytes -= back[q].r.size; /* substract for stats */ + back_set_locked(sback, q); /* locked */ + return q; } else { - hts_log_print(opt, LOG_INFO, "engine: warning: unserialize error for %s%s (%s): no more space to wakeup frozen slots",adr,fil,sav); + hts_log_print(opt, LOG_INFO, + "engine: warning: unserialize error for %s%s (%s): no more space to wakeup frozen slots", + adr, fil, sav); } - } - } + } + } } return -1; } -static int slot_can_be_cached_on_disk(const lien_back* back) { - return - (back->status == STATUS_READY && back->locked == 0 - && back->url_sav[0] != '\0' - && strcmp(back->url_sav, BACK_ADD_TEST) != 0 - ); +static int slot_can_be_cached_on_disk(const lien_back * back) { + return (back->status == STATUS_READY && back->locked == 0 + && back->url_sav[0] != '\0' + && strcmp(back->url_sav, BACK_ADD_TEST) != 0); /* Note: not checking !IS_DELAYED_EXT(back->url_sav) or it will quickly cause the slots to be filled! */ } /* Put all backing entries that are ready in the storage hashtable to spare space and CPU */ -int back_cleanup_background(httrackp* opt,cache_back* cache,struct_back* sback) { - lien_back* const back = sback->lnk; +int back_cleanup_background(httrackp * opt, cache_back * cache, + struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; int nclean = 0; int i; - for( i = 0 ; i < back_max ; i++ ) { + + for(i = 0; i < back_max; i++) { // ready, not locked and suitable if (slot_can_be_cached_on_disk(&back[i])) { #ifdef HTS_NO_BACK_ON_DISK - lien_back* itemback; + lien_back *itemback; #endif /* Security check */ - int checkIndex = back_index_ready(opt, sback, back[i].url_adr, back[i].url_fil, back[i].url_sav, 1); + int checkIndex = + back_index_ready(opt, sback, back[i].url_adr, back[i].url_fil, + back[i].url_sav, 1); if (checkIndex != -1) { - hts_log_print(opt, LOG_WARNING, "engine: unexpected duplicate file entry: %s%s -> %s (%d '%s') / %s%s -> %s (%d '%s')", - back[checkIndex].url_adr, back[checkIndex].url_fil, back[checkIndex].url_sav, back[checkIndex].r.statuscode, back[checkIndex].r.msg, - back[i].url_adr, back[i].url_fil, back[i].url_sav, back[i].r.statuscode, back[i].r.msg - ); + hts_log_print(opt, LOG_WARNING, + "engine: unexpected duplicate file entry: %s%s -> %s (%d '%s') / %s%s -> %s (%d '%s')", + back[checkIndex].url_adr, back[checkIndex].url_fil, + back[checkIndex].url_sav, back[checkIndex].r.statuscode, + back[checkIndex].r.msg, back[i].url_adr, back[i].url_fil, + back[i].url_sav, back[i].r.statuscode, back[i].r.msg); back_delete(NULL, NULL, sback, checkIndex); #ifdef _DEBUG /* This should NOT happend! */ - { int duplicateEntryInBacklog = 1; assertf(!duplicateEntryInBacklog); } + { + int duplicateEntryInBacklog = 1; + + assertf(!duplicateEntryInBacklog); + } #endif - } + } #ifndef HTS_NO_BACK_ON_DISK - /* temporarily serialize the entry on disk */ - { - int fsz = (int) strlen(back[i].url_sav); + /* temporarily serialize the entry on disk */ + { + int fsz = (int) strlen(back[i].url_sav); char *filename = malloc(fsz + 8 + 1); + if (filename != NULL) { FILE *fp; + if (opt->getmode != 0) { sprintf(filename, "%s.tmp", back[i].url_sav); } else { - sprintf(filename, "%stmpfile%d.tmp", StringBuff(opt->path_html_utf8), opt->state.tmpnameid++); + sprintf(filename, "%stmpfile%d.tmp", + StringBuff(opt->path_html_utf8), opt->state.tmpnameid++); } /* Security check */ if (fexist_utf8(filename)) { - hts_log_print(opt, LOG_INFO, "engine: warning: temporary file %s already exists", filename); + hts_log_print(opt, LOG_INFO, + "engine: warning: temporary file %s already exists", + filename); } /* Create file and serialize slot */ - if ((fp = filecreate(NULL, filename)) != NULL) - { - if (back_serialize(fp, &back[i]) == 0) - { + if ((fp = filecreate(NULL, filename)) != NULL) { + if (back_serialize(fp, &back[i]) == 0) { inthash_add_pvoid(sback->ready, back[i].url_sav, filename); filename = NULL; - sback->ready_size_bytes += back[i].r.size; /* add for stats */ + sback->ready_size_bytes += back[i].r.size; /* add for stats */ nclean++; - back_clear_entry(&back[i]); /* entry is now recycled */ + back_clear_entry(&back[i]); /* entry is now recycled */ } else { - hts_log_print(opt, LOG_INFO | LOG_ERRNO, "engine: warning: serialize error for %s%s to %s: write error",back[i].url_adr,back[i].url_fil,filename); + hts_log_print(opt, LOG_INFO | LOG_ERRNO, + "engine: warning: serialize error for %s%s to %s: write error", + back[i].url_adr, back[i].url_fil, filename); } fclose(fp); } else { - hts_log_print(opt, LOG_INFO | LOG_ERRNO, "engine: warning: serialize error for %s%s to %s: open error (%s, %s)", back[i].url_adr, back[i].url_fil, filename, dir_exists(filename) ? "directory exists" : "directory does NOT exist!", fexist_utf8(filename) ? "file already exists!" : "file does not exist"); + hts_log_print(opt, LOG_INFO | LOG_ERRNO, + "engine: warning: serialize error for %s%s to %s: open error (%s, %s)", + back[i].url_adr, back[i].url_fil, filename, + dir_exists(filename) ? "directory exists" : + "directory does NOT exist!", + fexist_utf8(filename) ? "file already exists!" : + "file does not exist"); } - if (filename != NULL) - free(filename); - } else { - hts_log_print(opt, LOG_INFO | LOG_ERRNO, "engine: warning: serialize error for %s%s to %s: memory full",back[i].url_adr,back[i].url_fil,filename); - } - } + if (filename != NULL) + free(filename); + } else { + hts_log_print(opt, LOG_INFO | LOG_ERRNO, + "engine: warning: serialize error for %s%s to %s: memory full", + back[i].url_adr, back[i].url_fil, filename); + } + } #else - itemback = calloct(1, sizeof(lien_back)); + itemback = calloct(1, sizeof(lien_back)); back_move(&back[i], itemback); inthash_add_pvoid(sback->ready, itemback->url_sav, itemback); nclean++; @@ -317,37 +354,41 @@ int back_cleanup_background(httrackp* opt,cache_back* cache,struct_back* sback) } // nombre d'entrées libres dans le backing -int back_available(struct_back* sback) { - lien_back* const back = sback->lnk; +int back_available(struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; int i; - int nb=0; - for(i=0;i<back_max;i++) - if (back[i].status==STATUS_FREE) /* libre */ + int nb = 0; + + for(i = 0; i < back_max; i++) + if (back[i].status == STATUS_FREE) /* libre */ nb++; return nb; } // retourne estimation de la taille des html et fichiers stockés en mémoire -LLint back_incache(struct_back* sback) { - lien_back* const back = sback->lnk; +LLint back_incache(struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; int i; - LLint sum=0; - for(i=0;i<back_max;i++) - if (back[i].status!=-1) - if (back[i].r.adr) // ne comptabilier que les blocs en mémoire - sum+=max(back[i].r.size,back[i].r.totalsize); + LLint sum = 0; + + for(i = 0; i < back_max; i++) + if (back[i].status != -1) + if (back[i].r.adr) // ne comptabilier que les blocs en mémoire + sum += max(back[i].r.size, back[i].r.totalsize); // stored (ready) slots #ifdef HTS_NO_BACK_ON_DISK if (sback->ready != NULL) { struct_inthash_enum e = inthash_enum_new(sback->ready); - inthash_chain* item; + inthash_chain *item; + while((item = inthash_enum_next(&e))) { - lien_back* ritem = (lien_back*) item->value.ptr; - if (ritem->status!=-1) + lien_back *ritem = (lien_back *) item->value.ptr; + + if (ritem->status != -1) if (ritem->r.adr) // ne comptabilier que les blocs en mémoire - sum+=max(ritem->r.size,ritem->r.totalsize); + sum += max(ritem->r.size, ritem->r.totalsize); } } #endif @@ -355,24 +396,27 @@ LLint back_incache(struct_back* sback) { } // retourne estimation de la taille des html et fichiers stockés en mémoire -int back_done_incache(struct_back* sback) { - lien_back* const back = sback->lnk; +int back_done_incache(struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; int i; int n = 0; - for(i = 0 ; i < back_max ; i++) + + for(i = 0; i < back_max; i++) if (back[i].status == STATUS_READY) n++; // stored (ready) slots if (sback->ready != NULL) { #ifndef HTS_NO_BACK_ON_DISK - n += inthash_nitems(sback->ready); + n += inthash_nitems(sback->ready); #else - struct_inthash_enum e = inthash_enum_new(sback->ready); - inthash_chain* item; + struct_inthash_enum e = inthash_enum_new(sback->ready); + inthash_chain *item; + while((item = inthash_enum_next(&e))) { - lien_back* ritem = (lien_back*) item->value.ptr; - if (ritem->status==STATUS_READY) + lien_back *ritem = (lien_back *) item->value.ptr; + + if (ritem->status == STATUS_READY) n++; } #endif @@ -380,30 +424,32 @@ int back_done_incache(struct_back* sback) { return n; } - // le lien a-t-il été mis en backing? -HTS_INLINE int back_exist(struct_back* sback,httrackp* opt,char* adr,char* fil,char* sav) { - return (back_index_fetch(opt, sback, adr, fil, sav, /*don't fetch*/0) >= 0); +HTS_INLINE int back_exist(struct_back * sback, httrackp * opt, char *adr, + char *fil, char *sav) { + return (back_index_fetch(opt, sback, adr, fil, sav, /*don't fetch */ 0) >= 0); } // nombre de sockets en tâche de fond -int back_nsoc(struct_back* sback) { - lien_back* const back = sback->lnk; +int back_nsoc(struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; - int n=0; + int n = 0; int i; - for(i=0;i<back_max;i++) - if (back[i].status > 0) // only receive + + for(i = 0; i < back_max; i++) + if (back[i].status > 0) // only receive n++; return n; } -int back_nsoc_overall(struct_back* sback) { - lien_back* const back = sback->lnk; +int back_nsoc_overall(struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; - int n=0; + int n = 0; int i; - for(i=0;i<back_max;i++) + + for(i = 0; i < back_max; i++) if (back[i].status > 0 || back[i].status == STATUS_ALIVE) n++; @@ -414,351 +460,394 @@ int back_nsoc_overall(struct_back* sback) { // // fermer les paramètres de transfert, // et notamment vérifier les fichiers compressés (décompresser), callback etc. -int back_finalize(httrackp* opt, cache_back* cache, struct_back* sback, const int p) { - char catbuff[CATBUFF_SIZE]; - lien_back* const back = sback->lnk; +int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, + const int p) { + char catbuff[CATBUFF_SIZE]; + lien_back *const back = sback->lnk; const int back_max = sback->count; + assertf(p >= 0 && p < back_max); - /* Store ? */ - if (!back[p].finalized) { - back[p].finalized = 1; + /* Store ? */ + if (!back[p].finalized) { + back[p].finalized = 1; /* Don't store broken files. Note: check is done before compression. If the file is partial, the next run will attempt to continue it with compression too. - */ - if (back[p].r.totalsize >= 0 && back[p].r.statuscode > 0 - && back[p].r.size != back[p].r.totalsize && ! opt->tolerant) { + */ + if (back[p].r.totalsize >= 0 && back[p].r.statuscode > 0 + && back[p].r.size != back[p].r.totalsize && !opt->tolerant) { if (back[p].status == STATUS_READY) { - hts_log_print(opt, LOG_WARNING, "file not stored in cache due to bogus state (broken size, expected "LLintP" got "LLintP"): %s%s", - back[p].r.totalsize, back[p].r.size, back[p].url_adr,back[p].url_fil); + hts_log_print(opt, LOG_WARNING, + "file not stored in cache due to bogus state (broken size, expected " + LLintP " got " LLintP "): %s%s", back[p].r.totalsize, + back[p].r.size, back[p].url_adr, back[p].url_fil); } else { - hts_log_print(opt, LOG_INFO, "incomplete file not yet stored in cache (expected "LLintP" got "LLintP"): %s%s", - back[p].r.totalsize, back[p].r.size, back[p].url_adr,back[p].url_fil); + hts_log_print(opt, LOG_INFO, + "incomplete file not yet stored in cache (expected " + LLintP " got " LLintP "): %s%s", back[p].r.totalsize, + back[p].r.size, back[p].url_adr, back[p].url_fil); } return -1; } - if ( - (back[p].status == STATUS_READY) // ready - && - (back[p].r.statuscode > 0) // not internal error - ) - { - if (!back[p].testmode) { // not test mode - char* state="unknown"; + if ((back[p].status == STATUS_READY) // ready + && (back[p].r.statuscode > 0) // not internal error + ) { + if (!back[p].testmode) { // not test mode + char *state = "unknown"; - /* décompression */ + /* décompression */ #if HTS_USEZLIB - if (gz_is_available && back[p].r.compressed) { - if (back[p].r.size > 0) { - //if ( (back[p].r.adr) && (back[p].r.size>0) ) { - // stats - back[p].compressed_size=back[p].r.size; - // en mémoire -> passage sur disque - if (!back[p].r.is_write) { + if (gz_is_available && back[p].r.compressed) { + if (back[p].r.size > 0) { + //if ( (back[p].r.adr) && (back[p].r.size>0) ) { + // stats + back[p].compressed_size = back[p].r.size; + // en mémoire -> passage sur disque + if (!back[p].r.is_write) { #if 1 #ifdef _WIN32 #undef tempnam #define tempnam _tempnam #endif char *const tmp = tempnam(NULL, "httrack_temporaryGzipFile_"); + if (tmp != NULL) { - strcpybuff(back[p].tmpfile_buffer, tmp); + strcpybuff(back[p].tmpfile_buffer, tmp); free(tmp); back[p].tmpfile = back[p].tmpfile_buffer; } else { back[p].tmpfile = NULL; } #else - back[p].tmpfile_buffer[0]='\0'; - back[p].tmpfile=tmpnam(back[p].tmpfile_buffer); + back[p].tmpfile_buffer[0] = '\0'; + back[p].tmpfile = tmpnam(back[p].tmpfile_buffer); #endif - if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') { + if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') { /* note: tmpfile is a local system filename */ - back[p].r.out=fopen(back[p].tmpfile, "wb"); - if (back[p].r.out) { - if ((back[p].r.adr) && (back[p].r.size>0)) { - if (fwrite(back[p].r.adr,1,(size_t)back[p].r.size,back[p].r.out) != back[p].r.size) { - back[p].r.statuscode=STATUSCODE_INVALID; - strcpybuff(back[p].r.msg,"Write error when decompressing"); - } - } else { - back[p].tmpfile[0]='\0'; - back[p].r.statuscode=STATUSCODE_INVALID; - strcpybuff(back[p].r.msg,"Empty compressed file"); - } - } else { - back[p].tmpfile[0]='\0'; - back[p].r.statuscode=STATUSCODE_INVALID; - strcpybuff(back[p].r.msg,"Open error when decompressing (can not create a temporary file)"); - } - } - } - // fermer fichier sortie - if (back[p].r.out!=NULL) { - fclose(back[p].r.out); - back[p].r.out=NULL; - } - // décompression - if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') { - if (back[p].url_sav[0]) { - LLint size; - file_notify(opt,back[p].url_adr, back[p].url_fil, back[p].url_sav, 1, 1, back[p].r.notmodified); - filecreateempty(&opt->state.strc, back[p].url_sav); // filenote & co - if ((size = hts_zunpack(back[p].tmpfile, back[p].url_sav))>=0) { - back[p].r.size=back[p].r.totalsize=size; - // fichier -> mémoire - if (!back[p].r.is_write) { - deleteaddr(&back[p].r); - back[p].r.adr = readfile_utf8(back[p].url_sav); - if (!back[p].r.adr) { - back[p].r.statuscode=STATUSCODE_INVALID; - strcpybuff(back[p].r.msg,"Read error when decompressing"); - } - UNLINK(back[p].url_sav); - } + back[p].r.out = fopen(back[p].tmpfile, "wb"); + if (back[p].r.out) { + if ((back[p].r.adr) && (back[p].r.size > 0)) { + if (fwrite + (back[p].r.adr, 1, (size_t) back[p].r.size, + back[p].r.out) != back[p].r.size) { + back[p].r.statuscode = STATUSCODE_INVALID; + strcpybuff(back[p].r.msg, + "Write error when decompressing"); + } + } else { + back[p].tmpfile[0] = '\0'; + back[p].r.statuscode = STATUSCODE_INVALID; + strcpybuff(back[p].r.msg, "Empty compressed file"); + } + } else { + back[p].tmpfile[0] = '\0'; + back[p].r.statuscode = STATUSCODE_INVALID; + strcpybuff(back[p].r.msg, + "Open error when decompressing (can not create a temporary file)"); + } + } + } + // fermer fichier sortie + if (back[p].r.out != NULL) { + fclose(back[p].r.out); + back[p].r.out = NULL; + } + // décompression + if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') { + if (back[p].url_sav[0]) { + LLint size; + + file_notify(opt, back[p].url_adr, back[p].url_fil, + back[p].url_sav, 1, 1, back[p].r.notmodified); + filecreateempty(&opt->state.strc, back[p].url_sav); // filenote & co + if ((size = hts_zunpack(back[p].tmpfile, back[p].url_sav)) >= 0) { + back[p].r.size = back[p].r.totalsize = size; + // fichier -> mémoire + if (!back[p].r.is_write) { + deleteaddr(&back[p].r); + back[p].r.adr = readfile_utf8(back[p].url_sav); + if (!back[p].r.adr) { + back[p].r.statuscode = STATUSCODE_INVALID; + strcpybuff(back[p].r.msg, + "Read error when decompressing"); + } + UNLINK(back[p].url_sav); + } } else { back[p].r.statuscode = STATUSCODE_INVALID; strcpybuff(back[p].r.msg, "Error when decompressing"); } - } - /* ensure that no remaining temporary file exists */ - unlink(back[p].tmpfile); - back[p].tmpfile = NULL; - } - // stats - HTS_STAT.total_packed+=back[p].compressed_size; - HTS_STAT.total_unpacked+=back[p].r.size; - HTS_STAT.total_packedfiles++; - // unflag - } - } -#endif - /* Write mode to disk */ - if (back[p].r.is_write && back[p].r.adr != NULL) { - freet(back[p].r.adr); - back[p].r.adr = NULL; - } + } + /* ensure that no remaining temporary file exists */ + unlink(back[p].tmpfile); + back[p].tmpfile = NULL; + } + // stats + HTS_STAT.total_packed += back[p].compressed_size; + HTS_STAT.total_unpacked += back[p].r.size; + HTS_STAT.total_packedfiles++; + // unflag + } + } +#endif + /* Write mode to disk */ + if (back[p].r.is_write && back[p].r.adr != NULL) { + freet(back[p].r.adr); + back[p].r.adr = NULL; + } /* remove reference file, if any */ if (back[p].r.is_write) { url_savename_refname_remove(opt, back[p].url_adr, back[p].url_fil); } - /* ************************************************************************ - REAL MEDIA HACK - Check if we have to load locally the file - ************************************************************************ */ - if (back[p].r.statuscode == HTTP_OK) { // OK (ou 304 en backing) - if (back[p].r.is_write) { // Written file - if (may_be_hypertext_mime(opt,back[p].r.contenttype, back[p].url_fil)) { // to parse! - off_t sz; - sz=fsize_utf8(back[p].url_sav); - if (sz>0) { // ok, exists! - if (sz < 8192) { // ok, small file --> to parse! - FILE* fp=FOPEN(back[p].url_sav,"rb"); - if (fp) { - back[p].r.adr=malloct((int)sz + 2); - if (back[p].r.adr) { - if (fread(back[p].r.adr,1,sz,fp) == sz) { - back[p].r.size=sz; - back[p].r.adr[sz] = '\0'; - back[p].r.is_write = 0; /* not anymore a direct-to-disk file */ - } else { - freet(back[p].r.adr); - back[p].r.size=0; - back[p].r.adr = NULL; - back[p].r.statuscode=STATUSCODE_INVALID; - strcpybuff(back[p].r.msg, ".RAM read error"); - } - fclose(fp); - fp=NULL; - // remove (temporary) file! - UNLINK(fconv(catbuff,back[p].url_sav)); - } - if (fp) - fclose(fp); - } - } - } - } - } - } - /* EN OF REAL MEDIA HACK */ - - /* Stats */ - if (cache->txt) { - char flags[32]; - char s[256]; - time_t tt; - struct tm* A; - tt=time(NULL); - A=localtime(&tt); - if (A == NULL) { - int localtime_returned_null=0; - assert(localtime_returned_null); - } - strftime(s,250,"%H:%M:%S",A); - - flags[0]='\0'; - /* input flags */ - if (back[p].is_update) - strcatbuff(flags, "U"); // update request - else - strcatbuff(flags, "-"); - if (back[p].range_req_size) - strcatbuff(flags, "R"); // range request - else - strcatbuff(flags, "-"); - /* state flags */ - if (back[p].r.is_file) // direct to disk - strcatbuff(flags, "F"); - else - strcatbuff(flags, "-"); - /* output flags */ - if (!back[p].r.notmodified) - strcatbuff(flags, "M"); // modified - else - strcatbuff(flags, "-"); - if (back[p].r.is_chunk) // chunked - strcatbuff(flags, "C"); - else - strcatbuff(flags, "-"); - if (back[p].r.compressed) - strcatbuff(flags, "Z"); // gzip - else - strcatbuff(flags, "-"); - /* Err I had to split these.. */ - fprintf(cache->txt,"%s\t", s); - fprintf(cache->txt,LLintP"/", (LLint)back[p].r.size); - fprintf(cache->txt,LLintP,(LLint)back[p].r.totalsize); - fprintf(cache->txt,"\t%s\t",flags); - } + /* ************************************************************************ + REAL MEDIA HACK + Check if we have to load locally the file + ************************************************************************ */ + if (back[p].r.statuscode == HTTP_OK) { // OK (ou 304 en backing) + if (back[p].r.is_write) { // Written file + if (may_be_hypertext_mime(opt, back[p].r.contenttype, back[p].url_fil)) { // to parse! + off_t sz; + + sz = fsize_utf8(back[p].url_sav); + if (sz > 0) { // ok, exists! + if (sz < 8192) { // ok, small file --> to parse! + FILE *fp = FOPEN(back[p].url_sav, "rb"); + + if (fp) { + back[p].r.adr = malloct((int) sz + 2); + if (back[p].r.adr) { + if (fread(back[p].r.adr, 1, sz, fp) == sz) { + back[p].r.size = sz; + back[p].r.adr[sz] = '\0'; + back[p].r.is_write = 0; /* not anymore a direct-to-disk file */ + } else { + freet(back[p].r.adr); + back[p].r.size = 0; + back[p].r.adr = NULL; + back[p].r.statuscode = STATUSCODE_INVALID; + strcpybuff(back[p].r.msg, ".RAM read error"); + } + fclose(fp); + fp = NULL; + // remove (temporary) file! + UNLINK(fconv(catbuff, back[p].url_sav)); + } + if (fp) + fclose(fp); + } + } + } + } + } + } + /* EN OF REAL MEDIA HACK */ + + /* Stats */ + if (cache->txt) { + char flags[32]; + char s[256]; + time_t tt; + struct tm *A; + + tt = time(NULL); + A = localtime(&tt); + if (A == NULL) { + int localtime_returned_null = 0; + + assert(localtime_returned_null); + } + strftime(s, 250, "%H:%M:%S", A); + flags[0] = '\0'; + /* input flags */ + if (back[p].is_update) + strcatbuff(flags, "U"); // update request + else + strcatbuff(flags, "-"); + if (back[p].range_req_size) + strcatbuff(flags, "R"); // range request + else + strcatbuff(flags, "-"); + /* state flags */ + if (back[p].r.is_file) // direct to disk + strcatbuff(flags, "F"); + else + strcatbuff(flags, "-"); + /* output flags */ + if (!back[p].r.notmodified) + strcatbuff(flags, "M"); // modified + else + strcatbuff(flags, "-"); + if (back[p].r.is_chunk) // chunked + strcatbuff(flags, "C"); + else + strcatbuff(flags, "-"); + if (back[p].r.compressed) + strcatbuff(flags, "Z"); // gzip + else + strcatbuff(flags, "-"); + /* Err I had to split these.. */ + fprintf(cache->txt, "%s\t", s); + fprintf(cache->txt, LLintP "/", (LLint) back[p].r.size); + fprintf(cache->txt, LLintP, (LLint) back[p].r.totalsize); + fprintf(cache->txt, "\t%s\t", flags); + } #if HTS_USEZLIB - back[p].r.compressed=0; -#endif - - if (back[p].r.statuscode == HTTP_OK) { - if (back[p].r.size>=0) { - if (strcmp(back[p].url_fil,"/robots.txt") !=0 ) { - HTS_STAT.stat_bytes+=back[p].r.size; - HTS_STAT.stat_files++; - } - if ( (!back[p].r.notmodified) && (opt->is_update) ) { - HTS_STAT.stat_updated_files++; // page modifiée - if (back[p].is_update) { - hts_log_print(opt, LOG_INFO, "engine: transfer-status: link updated: %s%s -> %s",back[p].url_adr,back[p].url_fil,back[p].url_sav); - } else { - hts_log_print(opt, LOG_INFO, "engine: transfer-status: link added: %s%s -> %s",back[p].url_adr,back[p].url_fil,back[p].url_sav); - } - if (cache->txt) { - if (back[p].is_update) { - state="updated"; - } else { - state="added"; - } - } - } else { - hts_log_print(opt, LOG_INFO, "engine: transfer-status: link recorded: %s%s -> %s",back[p].url_adr,back[p].url_fil,back[p].url_sav); - if (cache->txt) { - if (opt->is_update) - state="untouched"; - else - state="added"; - } - } - } else { - hts_log_print(opt, LOG_INFO, "engine: transfer-status: empty file? (%d, '%s'): %s%s",back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil); - if (cache->txt) { - state="empty"; - } - } - } else { - hts_log_print(opt, LOG_INFO, "engine: transfer-status: link error (%d, '%s'): %s%s",back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil); - if (cache->txt) { - state="error"; - } - } - if (cache->txt) { - fprintf(cache->txt, - "%d\t" - "%s ('%s')\t" - "%s\t" - "%s%s\t" - "%s%s%s\t%s\t" - "(from %s%s%s)" - LF, - back[p].r.statuscode, - state, escape_check_url_addr(OPT_GET_BUFF(opt),back[p].r.msg), - escape_check_url_addr(OPT_GET_BUFF(opt),back[p].r.contenttype), - ((back[p].r.etag[0])?"etag:":((back[p].r.lastmodified[0])?"date:":"")), escape_check_url_addr(OPT_GET_BUFF(opt),(back[p].r.etag[0])?back[p].r.etag:(back[p].r.lastmodified)), - (link_has_authority(back[p].url_adr) ? "" : "http://"),escape_check_url_addr(OPT_GET_BUFF(opt),back[p].url_adr),escape_check_url_addr(OPT_GET_BUFF(opt),back[p].url_fil),escape_check_url_addr(OPT_GET_BUFF(opt),back[p].url_sav), - (link_has_authority(back[p].referer_adr) || !back[p].referer_adr[0]) ? "" : "http://",escape_check_url_addr(OPT_GET_BUFF(opt),back[p].referer_adr),escape_check_url_addr(OPT_GET_BUFF(opt),back[p].referer_fil) - ); - if (opt->flush) - fflush(cache->txt); - } - - /* Cache */ - if (!IS_DELAYED_EXT(back[p].url_sav)) { - cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,back[p].url_sav); - } else { + back[p].r.compressed = 0; +#endif + + if (back[p].r.statuscode == HTTP_OK) { + if (back[p].r.size >= 0) { + if (strcmp(back[p].url_fil, "/robots.txt") != 0) { + HTS_STAT.stat_bytes += back[p].r.size; + HTS_STAT.stat_files++; + } + if ((!back[p].r.notmodified) && (opt->is_update)) { + HTS_STAT.stat_updated_files++; // page modifiée + if (back[p].is_update) { + hts_log_print(opt, LOG_INFO, + "engine: transfer-status: link updated: %s%s -> %s", + back[p].url_adr, back[p].url_fil, + back[p].url_sav); + } else { + hts_log_print(opt, LOG_INFO, + "engine: transfer-status: link added: %s%s -> %s", + back[p].url_adr, back[p].url_fil, + back[p].url_sav); + } + if (cache->txt) { + if (back[p].is_update) { + state = "updated"; + } else { + state = "added"; + } + } + } else { + hts_log_print(opt, LOG_INFO, + "engine: transfer-status: link recorded: %s%s -> %s", + back[p].url_adr, back[p].url_fil, back[p].url_sav); + if (cache->txt) { + if (opt->is_update) + state = "untouched"; + else + state = "added"; + } + } + } else { + hts_log_print(opt, LOG_INFO, + "engine: transfer-status: empty file? (%d, '%s'): %s%s", + back[p].r.statuscode, back[p].r.msg, back[p].url_adr, + back[p].url_fil); + if (cache->txt) { + state = "empty"; + } + } + } else { + hts_log_print(opt, LOG_INFO, + "engine: transfer-status: link error (%d, '%s'): %s%s", + back[p].r.statuscode, back[p].r.msg, back[p].url_adr, + back[p].url_fil); + if (cache->txt) { + state = "error"; + } + } + if (cache->txt) { + fprintf(cache->txt, + "%d\t" "%s ('%s')\t" "%s\t" "%s%s\t" "%s%s%s\t%s\t" + "(from %s%s%s)" LF, back[p].r.statuscode, state, + escape_check_url_addr(OPT_GET_BUFF(opt), back[p].r.msg), + escape_check_url_addr(OPT_GET_BUFF(opt), + back[p].r.contenttype), + ((back[p].r. + etag[0]) ? "etag:" : ((back[p].r. + lastmodified[0]) ? "date:" : "")), + escape_check_url_addr(OPT_GET_BUFF(opt), + (back[p].r.etag[0]) ? back[p].r. + etag : (back[p].r.lastmodified)), + (link_has_authority(back[p].url_adr) ? "" : "http://"), + escape_check_url_addr(OPT_GET_BUFF(opt), back[p].url_adr), + escape_check_url_addr(OPT_GET_BUFF(opt), back[p].url_fil), + escape_check_url_addr(OPT_GET_BUFF(opt), back[p].url_sav), + (link_has_authority(back[p].referer_adr) + || !back[p].referer_adr[0]) ? "" : "http://", + escape_check_url_addr(OPT_GET_BUFF(opt), back[p].referer_adr), + escape_check_url_addr(OPT_GET_BUFF(opt), back[p].referer_fil) + ); + if (opt->flush) + fflush(cache->txt); + } + + /* Cache */ + if (!IS_DELAYED_EXT(back[p].url_sav)) { + cache_mayadd(opt, cache, &back[p].r, back[p].url_adr, back[p].url_fil, + back[p].url_sav); + } else { /* error */ - if (!HTTP_IS_OK(back[p].r.statuscode)) { - hts_log_print(opt, LOG_DEBUG, "redirect to %s%s",back[p].url_adr,back[p].url_fil); - /* Store only header reference */ - cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,NULL); - } else { + if (!HTTP_IS_OK(back[p].r.statuscode)) { + hts_log_print(opt, LOG_DEBUG, "redirect to %s%s", back[p].url_adr, + back[p].url_fil); + /* Store only header reference */ + cache_mayadd(opt, cache, &back[p].r, back[p].url_adr, + back[p].url_fil, NULL); + } else { /* Partial file, but marked as "ok" ? */ - hts_log_print(opt, LOG_WARNING, "file not stored in cache due to bogus state (incomplete type with %s (%d), size "LLintP"): %s%s",back[p].r.msg,back[p].r.statuscode,(LLint)back[p].r.size,back[p].url_adr,back[p].url_fil); - } - } - - // status finished callback - RUN_CALLBACK1(opt, xfrstatus, &back[p]); - - return 0; - } else { // testmode - if (back[p].r.statuscode / 100 >= 3) { /* Store 3XX, 4XX, 5XX test response codes, but NOT 2XX */ - /* Cache */ - cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,NULL); - } - } - } - } - return -1; + hts_log_print(opt, LOG_WARNING, + "file not stored in cache due to bogus state (incomplete type with %s (%d), size " + LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode, + (LLint) back[p].r.size, back[p].url_adr, + back[p].url_fil); + } + } + + // status finished callback + RUN_CALLBACK1(opt, xfrstatus, &back[p]); + + return 0; + } else { // testmode + if (back[p].r.statuscode / 100 >= 3) { /* Store 3XX, 4XX, 5XX test response codes, but NOT 2XX */ + /* Cache */ + cache_mayadd(opt, cache, &back[p].r, back[p].url_adr, back[p].url_fil, + NULL); + } + } + } + } + return -1; } /* try to keep the connection alive */ -int back_letlive(httrackp* opt, cache_back* cache, struct_back* sback, const int p) { - lien_back* const back = sback->lnk; - const int back_max = sback->count; - int checkerror; - htsblk* src = &back[p].r; - assertf(p >= 0 && p < back_max); - if (src && !src->is_file - && src->soc != INVALID_SOCKET - && src->statuscode >= 0 /* no timeout errors & co */ - && src->keep_alive_trailers == 0 /* not yet supported (chunk trailers) */ - && ! ( checkerror = check_sockerror(src->soc) ) - /*&& !check_sockdata(src->soc)*/ /* no unexpected data */ - ) { - htsblk tmp; - memset(&tmp, 0, sizeof(tmp)); - /* clear everything but connection: switch, close, and reswitch */ - back_connxfr(src, &tmp); - back_delete(opt, cache, sback, p); - //deletehttp(src); - back_connxfr(&tmp, src); - src->req.flush_garbage=1; /* ignore CRLF garbage */ - return 1; - } - return 0; +int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback, + const int p) { + lien_back *const back = sback->lnk; + const int back_max = sback->count; + int checkerror; + htsblk *src = &back[p].r; + + assertf(p >= 0 && p < back_max); + if (src && !src->is_file && src->soc != INVALID_SOCKET && src->statuscode >= 0 /* no timeout errors & co */ + && src->keep_alive_trailers == 0 /* not yet supported (chunk trailers) */ + && !(checkerror = check_sockerror(src->soc)) + /*&& !check_sockdata(src->soc) *//* no unexpected data */ + ) { + htsblk tmp; + + memset(&tmp, 0, sizeof(tmp)); + /* clear everything but connection: switch, close, and reswitch */ + back_connxfr(src, &tmp); + back_delete(opt, cache, sback, p); + //deletehttp(src); + back_connxfr(&tmp, src); + src->req.flush_garbage = 1; /* ignore CRLF garbage */ + return 1; + } + return 0; } -void back_connxfr(htsblk* src, htsblk* dst) { +void back_connxfr(htsblk * src, htsblk * dst) { dst->soc = src->soc; src->soc = INVALID_SOCKET; #if HTS_USEOPENSSL @@ -777,113 +866,122 @@ void back_connxfr(htsblk* src, htsblk* dst) { src->debugid = 0; } -void back_move(lien_back* src, lien_back* dst) { +void back_move(lien_back * src, lien_back * dst) { memcpy(dst, src, sizeof(lien_back)); - memset(src, 0, sizeof(lien_back)); - src->r.soc=INVALID_SOCKET; - src->status=STATUS_FREE; + memset(src, 0, sizeof(lien_back)); + src->r.soc = INVALID_SOCKET; + src->status = STATUS_FREE; src->r.location = src->location_buffer; dst->r.location = dst->location_buffer; } -void back_copy_static(const lien_back* src, lien_back* dst) { +void back_copy_static(const lien_back * src, lien_back * dst) { memcpy(dst, src, sizeof(lien_back)); - dst->r.soc=INVALID_SOCKET; - dst->r.adr = NULL; - dst->r.headers = NULL; - dst->r.out = NULL; + dst->r.soc = INVALID_SOCKET; + dst->r.adr = NULL; + dst->r.headers = NULL; + dst->r.out = NULL; dst->r.location = dst->location_buffer; - dst->r.fp = NULL; + dst->r.fp = NULL; #if HTS_USEOPENSSL - dst->r.ssl_con = NULL; + dst->r.ssl_con = NULL; #endif } -static int back_data_serialize(FILE *fp, const void *data, size_t size) { - if ( fwrite(&size, 1, sizeof(size), fp) == sizeof(size) - && ( size == 0 || fwrite(data, 1, size, fp) == size ) - ) - return 0; - return 1; /* error */ +static int back_data_serialize(FILE * fp, const void *data, size_t size) { + if (fwrite(&size, 1, sizeof(size), fp) == sizeof(size) + && (size == 0 || fwrite(data, 1, size, fp) == size) + ) + return 0; + return 1; /* error */ } -static int back_string_serialize(FILE *fp, const char *str) { - size_t size = ( str != NULL ) ? ( strlen(str) + 1 ) : 0; - return back_data_serialize(fp, str, size); +static int back_string_serialize(FILE * fp, const char *str) { + size_t size = (str != NULL) ? (strlen(str) + 1) : 0; + + return back_data_serialize(fp, str, size); } -static int back_data_unserialize(FILE *fp, void **str, size_t *size) { - *str = NULL; - if (fread(size, 1, sizeof(*size), fp) == sizeof(*size)) { - if (*size == 0) /* serialized NULL ptr */ - return 0; - *str = malloct(*size + 1); - if (*str == NULL) - return 1; /* error */ - ((char*) *str)[*size] = 0; /* guard byte */ - if (fread(*str, 1, *size, fp) == *size) - return 0; - } - return 1; /* error */ +static int back_data_unserialize(FILE * fp, void **str, size_t * size) { + *str = NULL; + if (fread(size, 1, sizeof(*size), fp) == sizeof(*size)) { + if (*size == 0) /* serialized NULL ptr */ + return 0; + *str = malloct(*size + 1); + if (*str == NULL) + return 1; /* error */ + ((char *) *str)[*size] = 0; /* guard byte */ + if (fread(*str, 1, *size, fp) == *size) + return 0; + } + return 1; /* error */ } -static int back_string_unserialize(FILE *fp, char **str) { - size_t dummy; - return back_data_unserialize(fp, (void**) str, &dummy); +static int back_string_unserialize(FILE * fp, char **str) { + size_t dummy; + + return back_data_unserialize(fp, (void **) str, &dummy); } -int back_serialize(FILE *fp, const lien_back* src) { - if (back_data_serialize(fp, src, sizeof(lien_back)) == 0 - && back_data_serialize(fp, src->r.adr, src->r.adr ? (size_t)src->r.size : 0) == 0 - && back_string_serialize(fp, src->r.headers) == 0 - && fflush(fp) == 0) - return 0; - return 1; +int back_serialize(FILE * fp, const lien_back * src) { + if (back_data_serialize(fp, src, sizeof(lien_back)) == 0 + && back_data_serialize(fp, src->r.adr, + src->r.adr ? (size_t) src->r.size : 0) == 0 + && back_string_serialize(fp, src->r.headers) == 0 && fflush(fp) == 0) + return 0; + return 1; } -int back_unserialize(FILE *fp, lien_back** dst) { - size_t size; - *dst = NULL; +int back_unserialize(FILE * fp, lien_back ** dst) { + size_t size; + + *dst = NULL; errno = 0; - if (back_data_unserialize(fp, (void**) dst, &size) == 0 && size == sizeof(lien_back)) { - (*dst)->tmpfile = NULL; - (*dst)->chunk_adr = NULL; - (*dst)->r.adr = NULL; - (*dst)->r.out = NULL; - (*dst)->r.location = (*dst)->location_buffer; - (*dst)->r.fp = NULL; - (*dst)->r.soc = INVALID_SOCKET; + if (back_data_unserialize(fp, (void **) dst, &size) == 0 + && size == sizeof(lien_back)) { + (*dst)->tmpfile = NULL; + (*dst)->chunk_adr = NULL; + (*dst)->r.adr = NULL; + (*dst)->r.out = NULL; + (*dst)->r.location = (*dst)->location_buffer; + (*dst)->r.fp = NULL; + (*dst)->r.soc = INVALID_SOCKET; #if HTS_USEOPENSSL - (*dst)->r.ssl_con = NULL; -#endif - if (back_data_unserialize(fp, (void**) &(*dst)->r.adr, &size) == 0) - { - (*dst)->r.size = size; - (*dst)->r.headers = NULL; - if (back_string_unserialize(fp, &(*dst)->r.headers) == 0) - return 0; /* ok */ - if ((*dst)->r.headers != NULL) - freet((*dst)->r.headers); - } - if ((*dst)->r.adr != NULL) - freet((*dst)->r.adr); - } - if (dst != NULL) - freet(dst); - *dst = NULL; - return 1; /* error */ + (*dst)->r.ssl_con = NULL; +#endif + if (back_data_unserialize(fp, (void **) &(*dst)->r.adr, &size) == 0) { + (*dst)->r.size = size; + (*dst)->r.headers = NULL; + if (back_string_unserialize(fp, &(*dst)->r.headers) == 0) + return 0; /* ok */ + if ((*dst)->r.headers != NULL) + freet((*dst)->r.headers); + } + if ((*dst)->r.adr != NULL) + freet((*dst)->r.adr); + } + if (dst != NULL) + freet(dst); + *dst = NULL; + return 1; /* error */ } /* serialize a reference ; used to store references of files being downloaded in case of broken download */ /* Note: NOT utf-8 */ -int back_serialize_ref(httrackp* opt, const lien_back* src) { - char *filename = url_savename_refname_fullpath(opt, src->url_adr, src->url_fil); +int back_serialize_ref(httrackp * opt, const lien_back * src) { + char *filename = + url_savename_refname_fullpath(opt, src->url_adr, src->url_fil); FILE *fp = fopen(filename, "wb"); + if (fp == NULL) { #ifdef _WIN32 - if (mkdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME)) == 0) + if (mkdir + (fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME)) + == 0) #else - if (mkdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME), S_IRWXU | S_IRWXG | S_IRWXO) == 0) + if (mkdir + (fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME), + S_IRWXU | S_IRWXG | S_IRWXO) == 0) #endif { /* note: local filename */ @@ -893,6 +991,7 @@ int back_serialize_ref(httrackp* opt, const lien_back* src) { } if (fp != NULL) { int ser = back_serialize(fp, src); + fclose(fp); return ser; } @@ -900,15 +999,18 @@ int back_serialize_ref(httrackp* opt, const lien_back* src) { } /* unserialize a reference ; used to store references of files being downloaded in case of broken download */ -int back_unserialize_ref(httrackp* opt, const char *adr, const char *fil, lien_back** dst) { +int back_unserialize_ref(httrackp * opt, const char *adr, const char *fil, + lien_back ** dst) { char *filename = url_savename_refname_fullpath(opt, adr, fil); FILE *fp = FOPEN(filename, "rb"); + if (fp != NULL) { int ser = back_unserialize(fp, dst); + fclose(fp); - if (ser != 0) { /* back_unserialize_ref() != 0 does not need cleaning up */ - back_clear_entry(*dst); /* delete entry content */ - freet(*dst); /* delete item */ + if (ser != 0) { /* back_unserialize_ref() != 0 does not need cleaning up */ + back_clear_entry(*dst); /* delete entry content */ + freet(*dst); /* delete item */ *dst = NULL; } return ser; @@ -917,83 +1019,88 @@ int back_unserialize_ref(httrackp* opt, const char *adr, const char *fil, lien_b } // clear, or leave for keep-alive -int back_maydelete(httrackp* opt,cache_back* cache,struct_back* sback, const int p) { - lien_back* const back = sback->lnk; +int back_maydelete(httrackp * opt, cache_back * cache, struct_back * sback, + const int p) { + lien_back *const back = sback->lnk; const int back_max = sback->count; + assertf(p >= 0 && p < back_max); - if (p >= 0 && p < back_max) { // on sait jamais.. + if (p >= 0 && p < back_max) { // on sait jamais.. if ( - /* Keep-alive authorized by user */ - !opt->nokeepalive - /* Socket currently is keep-alive! */ - && back[p].r.keep_alive - /* Remaining authorized requests */ - && back[p].r.keep_alive_max > 1 - /* Known keep-alive start (security) */ - && back[p].ka_time_start - /* We're on time */ - && time_local() < back[p].ka_time_start + back[p].r.keep_alive_t - /* Connection delay must not exceed keep-alive timeout */ - && ( opt->maxconn <= 0 || ( back[p].r.keep_alive_t > ( 1.0 / opt->maxconn ) ) ) + /* Keep-alive authorized by user */ + !opt->nokeepalive + /* Socket currently is keep-alive! */ + && back[p].r.keep_alive + /* Remaining authorized requests */ + && back[p].r.keep_alive_max > 1 + /* Known keep-alive start (security) */ + && back[p].ka_time_start + /* We're on time */ + && time_local() < back[p].ka_time_start + back[p].r.keep_alive_t + /* Connection delay must not exceed keep-alive timeout */ + && (opt->maxconn <= 0 + || (back[p].r.keep_alive_t > (1.0 / opt->maxconn))) ) { lien_back tmp; + strcpybuff(tmp.url_adr, back[p].url_adr); if (back_letlive(opt, cache, sback, p)) { strcpybuff(back[p].url_adr, tmp.url_adr); back[p].status = STATUS_ALIVE; // alive & waiting - hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): successfully saved #%d (%s)", - back[p].r.debugid, - back[p].url_adr); + hts_log_print(opt, LOG_DEBUG, + "(Keep-Alive): successfully saved #%d (%s)", + back[p].r.debugid, back[p].url_adr); return 1; } } - back_delete(opt,cache,sback, p); + back_delete(opt, cache, sback, p); } return 0; } // clear, or leave for keep-alive -void back_maydeletehttp(httrackp* opt, cache_back* cache, struct_back* sback, const int p) { - lien_back* const back = sback->lnk; +void back_maydeletehttp(httrackp * opt, cache_back * cache, struct_back * sback, + const int p) { + lien_back *const back = sback->lnk; const int back_max = sback->count; TStamp lt = 0; + assertf(p >= 0 && p < back_max); - if (back[p].r.soc!=INVALID_SOCKET) { + if (back[p].r.soc != INVALID_SOCKET) { int q; - if ( - back[p].r.soc != INVALID_SOCKET /* security check */ - && back[p].r.statuscode >= 0 /* no timeout errors & co */ - && back[p].r.keep_alive_trailers == 0 /* not yet supported (chunk trailers) */ - /* Socket not in I/O error status */ - && !back[p].r.is_file - && !check_sockerror(back[p].r.soc) - /* Keep-alive authorized by user */ - && !opt->nokeepalive - /* Socket currently is keep-alive! */ - && back[p].r.keep_alive - /* Remaining authorized requests */ - && back[p].r.keep_alive_max > 1 - /* Known keep-alive start (security) */ - && back[p].ka_time_start - /* We're on time */ - && ( lt = time_local() ) < back[p].ka_time_start + back[p].r.keep_alive_t - /* Connection delay must not exceed keep-alive timeout */ - && ( opt->maxconn <= 0 || ( back[p].r.keep_alive_t > ( 1.0 / opt->maxconn ) ) ) - /* Available slot in backing */ - && ( q = back_search(opt, sback) ) >= 0 - ) - { + + if (back[p].r.soc != INVALID_SOCKET /* security check */ + && back[p].r.statuscode >= 0 /* no timeout errors & co */ + && back[p].r.keep_alive_trailers == 0 /* not yet supported (chunk trailers) */ + /* Socket not in I/O error status */ + && !back[p].r.is_file && !check_sockerror(back[p].r.soc) + /* Keep-alive authorized by user */ + && !opt->nokeepalive + /* Socket currently is keep-alive! */ + && back[p].r.keep_alive + /* Remaining authorized requests */ + && back[p].r.keep_alive_max > 1 + /* Known keep-alive start (security) */ + && back[p].ka_time_start + /* We're on time */ + && (lt = time_local()) < back[p].ka_time_start + back[p].r.keep_alive_t + /* Connection delay must not exceed keep-alive timeout */ + && (opt->maxconn <= 0 + || (back[p].r.keep_alive_t > (1.0 / opt->maxconn))) + /* Available slot in backing */ + && (q = back_search(opt, sback)) >= 0) { lien_back tmp; + strcpybuff(tmp.url_adr, back[p].url_adr); - deletehttp(&back[q].r); // security check - back_connxfr(&back[p].r, &back[q].r); // transfer live connection settings from p to q - back[q].ka_time_start = back[p].ka_time_start; // refresh + deletehttp(&back[q].r); // security check + back_connxfr(&back[p].r, &back[q].r); // transfer live connection settings from p to q + back[q].ka_time_start = back[p].ka_time_start; // refresh back[p].r.soc = INVALID_SOCKET; strcpybuff(back[q].url_adr, tmp.url_adr); // address - back[q].status = STATUS_ALIVE; // alive & waiting - hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): successfully preserved #%d (%s)", - back[q].r.debugid, - back[q].url_adr); + back[q].status = STATUS_ALIVE; // alive & waiting + hts_log_print(opt, LOG_DEBUG, + "(Keep-Alive): successfully preserved #%d (%s)", + back[q].r.debugid, back[q].url_adr); } else { deletehttp(&back[p].r); back[p].r.soc = INVALID_SOCKET; @@ -1001,35 +1108,37 @@ void back_maydeletehttp(httrackp* opt, cache_back* cache, struct_back* sback, co } } - /* attempt to attach a live connection to this slot */ -int back_trylive(httrackp* opt,cache_back* cache,struct_back* sback, const int p) { - lien_back* const back = sback->lnk; +int back_trylive(httrackp * opt, cache_back * cache, struct_back * sback, + const int p) { + lien_back *const back = sback->lnk; const int back_max = sback->count; + assertf(p >= 0 && p < back_max); - if (p>=0 && back[p].status != STATUS_ALIVE) { // we never know.. - int i = back_searchlive(opt,sback, back[p].url_adr); // search slot + if (p >= 0 && back[p].status != STATUS_ALIVE) { // we never know.. + int i = back_searchlive(opt, sback, back[p].url_adr); // search slot + if (i >= 0 && i != p) { - deletehttp(&back[p].r); // security check - back_connxfr(&back[i].r, &back[p].r); // transfer live connection settings from i to p - back_delete(opt,cache,sback, i); // delete old slot - back[p].status=STATUS_CONNECTING; // ready to connect - return 1; // success: will reuse live connection + deletehttp(&back[p].r); // security check + back_connxfr(&back[i].r, &back[p].r); // transfer live connection settings from i to p + back_delete(opt, cache, sback, i); // delete old slot + back[p].status = STATUS_CONNECTING; // ready to connect + return 1; // success: will reuse live connection } } return 0; } /* search for a live position, or, if not possible, try to return a new one */ -int back_searchlive(httrackp* opt, struct_back* sback, char* search_addr) { - lien_back* const back = sback->lnk; +int back_searchlive(httrackp * opt, struct_back * sback, char *search_addr) { + lien_back *const back = sback->lnk; const int back_max = sback->count; int i; /* search for a live socket */ - for(i = 0 ; i < back_max ; i++ ) { + for(i = 0; i < back_max; i++) { if (back[i].status == STATUS_ALIVE) { - if (strfield2(back[i].url_adr, search_addr)) { /* same location (xxc: check also virtual hosts?) */ + if (strfield2(back[i].url_adr, search_addr)) { /* same location (xxc: check also virtual hosts?) */ if (time_local() < back[i].ka_time_start + back[i].r.keep_alive_t) { return i; } @@ -1038,14 +1147,14 @@ int back_searchlive(httrackp* opt, struct_back* sback, char* search_addr) { } return -1; } - -int back_search_quick(struct_back* sback) { - lien_back* const back = sback->lnk; + +int back_search_quick(struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; int i; /* try to find an empty place */ - for(i = 0 ; i < back_max ; i++ ) { + for(i = 0; i < back_max; i++) { if (back[i].status == STATUS_FREE) { return i; } @@ -1055,22 +1164,23 @@ int back_search_quick(struct_back* sback) { return -1; } -int back_search(httrackp* opt,struct_back* sback) { - lien_back* const back = sback->lnk; +int back_search(httrackp * opt, struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; int i; /* try to find an empty place */ - if ( ( i = back_search_quick(sback) ) != -1) + if ((i = back_search_quick(sback)) != -1) return i; /* couldn't find an empty place, try to requisition a keep-alive place */ - for(i = 0 ; i < back_max ; i++ ) { + for(i = 0; i < back_max; i++) { if (back[i].status == STATUS_ALIVE) { - lien_back* const back = sback->lnk; + lien_back *const back = sback->lnk; + /* close this place */ - back_clear_entry(&back[i]); /* Already finalized (this is the night of the living dead) */ - /*back_delete(opt,cache,sback, i);*/ + back_clear_entry(&back[i]); /* Already finalized (this is the night of the living dead) */ + /*back_delete(opt,cache,sback, i); */ return i; } } @@ -1079,223 +1189,235 @@ int back_search(httrackp* opt,struct_back* sback) { return -1; } -void back_set_finished(struct_back* sback, const int p) { - lien_back* const back = sback->lnk; +void back_set_finished(struct_back * sback, const int p) { + lien_back *const back = sback->lnk; const int back_max = sback->count; + assertf(p >= 0 && p < back_max); - if (p >= 0 && p < sback->count) { // we never know.. + if (p >= 0 && p < sback->count) { // we never know.. /* status: finished (waiting to be validated) */ - back[p].status=STATUS_READY; /* finished */ + back[p].status = STATUS_READY; /* finished */ /* close open r/w streams, if any */ - if (back[p].r.fp!=NULL) { + if (back[p].r.fp != NULL) { fclose(back[p].r.fp); - back[p].r.fp=NULL; + back[p].r.fp = NULL; } - if (back[p].r.out!=NULL) { // fermer fichier sortie + if (back[p].r.out != NULL) { // fermer fichier sortie fclose(back[p].r.out); - back[p].r.out=NULL; + back[p].r.out = NULL; } } } -void back_set_locked(struct_back* sback, const int p) { - lien_back* const back = sback->lnk; +void back_set_locked(struct_back * sback, const int p) { + lien_back *const back = sback->lnk; const int back_max = sback->count; + assertf(p >= 0 && p < back_max); if (p >= 0 && p < sback->count) { /* status: locked (in process, do not swap on disk) */ - back[p].locked = 1; /* locked */ + back[p].locked = 1; /* locked */ } } -void back_set_unlocked(struct_back* sback, const int p) { - lien_back* const back = sback->lnk; +void back_set_unlocked(struct_back * sback, const int p) { + lien_back *const back = sback->lnk; const int back_max = sback->count; + assertf(p >= 0 && p < back_max); if (p >= 0 && p < sback->count) { /* status: unlocked (can be swapped on disk) */ - back[p].locked = 0; /* unlocked */ + back[p].locked = 0; /* unlocked */ } } -int back_flush_output(httrackp* opt, cache_back* cache, struct_back* sback, const int p) { - lien_back* const back = sback->lnk; +int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback, + const int p) { + lien_back *const back = sback->lnk; const int back_max = sback->count; + assertf(p >= 0 && p < back_max); - if (p >= 0 && p < sback->count) { // on sait jamais.. - /* close input file */ - if (back[p].r.fp!=NULL) { + if (p >= 0 && p < sback->count) { // on sait jamais.. + /* close input file */ + if (back[p].r.fp != NULL) { fclose(back[p].r.fp); - back[p].r.fp=NULL; + back[p].r.fp = NULL; } /* fichier de sortie */ - if (back[p].r.out!=NULL) { // fermer fichier sortie + if (back[p].r.out != NULL) { // fermer fichier sortie fclose(back[p].r.out); - back[p].r.out=NULL; + back[p].r.out = NULL; } - /* set file time */ - if (back[p].r.is_write) { // ecriture directe + /* set file time */ + if (back[p].r.is_write) { // ecriture directe /* écrire date "remote" */ if (strnotempty(back[p].url_sav) - && strnotempty(back[p].r.lastmodified) - && fexist_utf8(back[p].url_sav)) // normalement existe si on a un fichier de sortie - { - set_filetime_rfc822(back[p].url_sav,back[p].r.lastmodified); - } + && strnotempty(back[p].r.lastmodified) + && fexist_utf8(back[p].url_sav)) // normalement existe si on a un fichier de sortie + { + set_filetime_rfc822(back[p].url_sav, back[p].r.lastmodified); + } /* executer commande utilisateur après chargement du fichier */ //xx usercommand(opt,0,NULL,back[p].url_sav, back[p].url_adr, back[p].url_fil); - back[p].r.is_write=0; + back[p].r.is_write = 0; } - return 1; - } - return 0; + return 1; + } + return 0; } // effacer entrée -int back_delete(httrackp* opt, cache_back* cache, struct_back* sback, const int p) { - lien_back* const back = sback->lnk; +int back_delete(httrackp * opt, cache_back * cache, struct_back * sback, + const int p) { + lien_back *const back = sback->lnk; const int back_max = sback->count; + assertf(p >= 0 && p < back_max); - if (p >= 0 && p < sback->count) { // on sait jamais.. + if (p >= 0 && p < sback->count) { // on sait jamais.. // Vérificateur d'intégrité #if DEBUG_CHECKINT - _CHECKINT(&back[p],"Appel back_delete") + _CHECKINT(&back[p], "Appel back_delete") #endif #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_delete: #%d\n" _ (int) p); + DEBUG_W("back_delete: #%d\n" _(int) p); #endif // Finalize if (!back[p].finalized) { - if ( - (back[p].status == STATUS_READY) // ready - && - (!back[p].testmode) // not test mode - && - (back[p].r.statuscode>0) // not internal error + if ((back[p].status == STATUS_READY) // ready + && (!back[p].testmode) // not test mode + && (back[p].r.statuscode > 0) // not internal error ) { - hts_log_print(opt, LOG_DEBUG, "File '%s%s' -> %s not yet saved in cache - saving now", back[p].url_adr, back[p].url_fil, back[p].url_sav); - } - if (cache != NULL) { - back_finalize(opt, cache, sback, p); - } + hts_log_print(opt, LOG_DEBUG, + "File '%s%s' -> %s not yet saved in cache - saving now", + back[p].url_adr, back[p].url_fil, back[p].url_sav); + } + if (cache != NULL) { + back_finalize(opt, cache, sback, p); + } } back[p].finalized = 0; - // flush output buffers - (void) back_flush_output(opt, cache, sback, p); + // flush output buffers + (void) back_flush_output(opt, cache, sback, p); - return back_clear_entry(&back[p]); + return back_clear_entry(&back[p]); } return 0; } /* ensure that the entry is not locked */ -void back_index_unlock(struct_back* sback, const int p) { - lien_back* const back = sback->lnk; +void back_index_unlock(struct_back * sback, const int p) { + lien_back *const back = sback->lnk; + if (back[p].locked) { - back[p].locked = 0; /* not locked anymore */ + back[p].locked = 0; /* not locked anymore */ } } /* the entry is available again */ -static void back_set_free(lien_back* back) { +static void back_set_free(lien_back * back) { back->locked = 0; back->status = STATUS_FREE; } /* delete entry content (clear the entry), but don't unallocate the entry itself */ -int back_clear_entry(lien_back* back) { - if (back != NULL) { +int back_clear_entry(lien_back * back) { + if (back != NULL) { // Libérer tous les sockets, handles, buffers.. - if (back->r.soc!=INVALID_SOCKET) { + if (back->r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_delete: deletehttp\n"); #endif deletehttp(&back->r); - back->r.soc=INVALID_SOCKET; + back->r.soc = INVALID_SOCKET; } - - if (back->r.adr!=NULL) { // reste un bloc à désallouer + + if (back->r.adr != NULL) { // reste un bloc à désallouer freet(back->r.adr); - back->r.adr=NULL; + back->r.adr = NULL; } - if (back->chunk_adr!=NULL) { // reste un bloc à désallouer + if (back->chunk_adr != NULL) { // reste un bloc à désallouer freet(back->chunk_adr); - back->chunk_adr=NULL; - back->chunk_size=0; - back->chunk_blocksize=0; - back->is_chunk=0; + back->chunk_adr = NULL; + back->chunk_size = 0; + back->chunk_blocksize = 0; + back->is_chunk = 0; + } + // only for security + if (back->tmpfile && back->tmpfile[0] != '\0') { + (void) unlink(back->tmpfile); + back->tmpfile = NULL; } - - // only for security - if (back->tmpfile && back->tmpfile[0] != '\0') { - (void) unlink(back->tmpfile); - back->tmpfile = NULL; - } - // headers if (back->r.headers != NULL) { freet(back->r.headers); back->r.headers = NULL; } - // Tout nettoyer - memset(back, 0, sizeof(lien_back)); + memset(back, 0, sizeof(lien_back)); back->r.soc = INVALID_SOCKET; - back->r.location = back->location_buffer; - + back->r.location = back->location_buffer; + // Le plus important: libérer le champ back_set_free(back); - return 1; + return 1; } return 0; } /* Space left on backing stack */ -int back_stack_available(struct_back* sback) { - lien_back* const back = sback->lnk; +int back_stack_available(struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; - int p=0,n=0; - for( ; p < back_max ; p++ ) - if ( back[p].status == STATUS_FREE ) + int p = 0, n = 0; + + for(; p < back_max; p++) + if (back[p].status == STATUS_FREE) n++; return n; } // ajouter un lien en backing -int back_add_if_not_exists(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test) { - back_clean(opt, cache, sback); /* first cleanup the backlog to ensure that we have some entry left */ - if (!back_exist(sback,opt,adr,fil,save)) { - return back_add(sback, opt, cache, adr, fil, save, referer_adr, referer_fil, test); - } +int back_add_if_not_exists(struct_back * sback, httrackp * opt, + cache_back * cache, char *adr, char *fil, char *save, + char *referer_adr, char *referer_fil, int test) { + back_clean(opt, cache, sback); /* first cleanup the backlog to ensure that we have some entry left */ + if (!back_exist(sback, opt, adr, fil, save)) { + return back_add(sback, opt, cache, adr, fil, save, referer_adr, referer_fil, + test); + } return 0; } -int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test) { - lien_back* const back = sback->lnk; +int back_add(struct_back * sback, httrackp * opt, cache_back * cache, char *adr, + char *fil, char *save, char *referer_adr, char *referer_fil, + int test) { + lien_back *const back = sback->lnk; const int back_max = sback->count; - int p=0; - char catbuff[CATBUFF_SIZE]; - char catbuff2[CATBUFF_SIZE]; - lien_back* itemback = NULL; + int p = 0; + char catbuff[CATBUFF_SIZE]; + char catbuff2[CATBUFF_SIZE]; + lien_back *itemback = NULL; #if (defined(_DEBUG) || defined(DEBUG)) - if (!test && back_exist(sback,opt,adr,fil,save)) { + if (!test && back_exist(sback, opt, adr, fil, save)) { int already_there = 0; - hts_log_print(opt, LOG_ERROR, "error: back_add(%s,%s,%s) duplicate", adr, fil, save); + + hts_log_print(opt, LOG_ERROR, "error: back_add(%s,%s,%s) duplicate", adr, + fil, save); } #endif // vérifier cohérence de adr et fil (non vide!) - if (strnotempty(adr)==0) { + if (strnotempty(adr) == 0) { hts_log_print(opt, LOG_WARNING, "error: adr is empty for back_add"); - return -1; // erreur! + return -1; // erreur! } - if (strnotempty(fil)==0) { + if (strnotempty(fil) == 0) { hts_log_print(opt, LOG_WARNING, "error: fil is empty for back_add"); - return -1; // erreur! + return -1; // erreur! } // FIN vérifier cohérence de adr et fil (non vide!) @@ -1304,107 +1426,115 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* // rechercher emplacement back_clean(opt, cache, sback); - if ( ( p = back_search(opt, sback) ) >= 0) { - back[p].send_too[0]='\0'; // éventuels paramètres supplémentaires à transmettre au serveur + if ((p = back_search(opt, sback)) >= 0) { + back[p].send_too[0] = '\0'; // éventuels paramètres supplémentaires à transmettre au serveur // clear r - if (back[p].r.soc!=INVALID_SOCKET) { /* we never know */ + if (back[p].r.soc != INVALID_SOCKET) { /* we never know */ deletehttp(&back[p].r); } //memset(&(back[p].r), 0, sizeof(htsblk)); hts_init_htsblk(&back[p].r); - back[p].r.location=back[p].location_buffer; + back[p].r.location = back[p].location_buffer; // créer entrée - strcpybuff(back[p].url_adr,adr); - strcpybuff(back[p].url_fil,fil); - strcpybuff(back[p].url_sav,save); + strcpybuff(back[p].url_adr, adr); + strcpybuff(back[p].url_fil, fil); + strcpybuff(back[p].url_sav, save); //back[p].links_index = links_index; // copier referer si besoin - strcpybuff(back[p].referer_adr,""); - strcpybuff(back[p].referer_fil,""); + strcpybuff(back[p].referer_adr, ""); + strcpybuff(back[p].referer_fil, ""); if ((referer_adr) && (referer_fil)) { // existe if ((strnotempty(referer_adr)) && (strnotempty(referer_fil))) { // non vide - if (referer_adr[0]!='!') { // non détruit - if (strcmp(referer_adr,"file://")) { // PAS file:// - if (strcmp(referer_adr,"primary")) { // pas referer 1er lien - strcpybuff(back[p].referer_adr,referer_adr); - strcpybuff(back[p].referer_fil,referer_fil); + if (referer_adr[0] != '!') { // non détruit + if (strcmp(referer_adr, "file://")) { // PAS file:// + if (strcmp(referer_adr, "primary")) { // pas referer 1er lien + strcpybuff(back[p].referer_adr, referer_adr); + strcpybuff(back[p].referer_fil, referer_fil); } } } } } // sav ne sert à rien pour le moment - back[p].r.size=0; // rien n'a encore été chargé - back[p].r.adr=NULL; // pas de bloc de mémoire - back[p].r.is_write=0; // à priori stockage en mémoire - back[p].maxfile_html=opt->maxfile_html; - back[p].maxfile_nonhtml=opt->maxfile_nonhtml; - back[p].testmode=test; // mode test? - if (!opt->http10) // option "forcer 1.0" désactivée - back[p].http11=1; // autoriser http/1.1 - back[p].head_request=0; - if (strcmp(back[p].url_sav,BACK_ADD_TEST)==0) // HEAD - back[p].head_request=1; - else if (strcmp(back[p].url_sav,BACK_ADD_TEST2)==0) // test en GET - back[p].head_request=2; // test en get + back[p].r.size = 0; // rien n'a encore été chargé + back[p].r.adr = NULL; // pas de bloc de mémoire + back[p].r.is_write = 0; // à priori stockage en mémoire + back[p].maxfile_html = opt->maxfile_html; + back[p].maxfile_nonhtml = opt->maxfile_nonhtml; + back[p].testmode = test; // mode test? + if (!opt->http10) // option "forcer 1.0" désactivée + back[p].http11 = 1; // autoriser http/1.1 + back[p].head_request = 0; + if (strcmp(back[p].url_sav, BACK_ADD_TEST) == 0) // HEAD + back[p].head_request = 1; + else if (strcmp(back[p].url_sav, BACK_ADD_TEST2) == 0) // test en GET + back[p].head_request = 2; // test en get /* Stop requested - abort backing */ /* For update mode: second check after cache lookup not to lose all previous cache data ! */ if (opt->state.stop && !opt->is_update) { - back[p].r.statuscode=STATUSCODE_INVALID; // fatal - strcpybuff(back[p].r.msg,"mirror stopped by user"); - back[p].status=STATUS_READY; // terminé + back[p].r.statuscode = STATUSCODE_INVALID; // fatal + strcpybuff(back[p].r.msg, "mirror stopped by user"); + back[p].status = STATUS_READY; // terminé back_set_finished(sback, p); - hts_log_print(opt, LOG_WARNING, "File not added due to mirror cancel: %s%s",adr,fil); + hts_log_print(opt, LOG_WARNING, + "File not added due to mirror cancel: %s%s", adr, fil); return 0; } - // test "fast header" cache ; that is, tests we did that lead to 3XX/4XX/5XX response codes if (cache->cached_tests != NULL) { intptr_t ptr = 0; - if (inthash_read(cache->cached_tests, concat(OPT_GET_BUFF(opt), adr, fil), &ptr)) { // gotcha + + if (inthash_read(cache->cached_tests, concat(OPT_GET_BUFF(opt), adr, fil), &ptr)) { // gotcha if (ptr != 0) { - char* text = (char*) ptr; - char* lf = strchr(text, '\n'); + char *text = (char *) ptr; + char *lf = strchr(text, '\n'); int code = 0; - if (sscanf(text, "%d", &code) == 1) { // got code - back[p].r.statuscode=code; - back[p].status=STATUS_READY; // terminé - if (lf != NULL && *lf != '\0') { // got location ? - strcpybuff(back[p].r.location, lf + 1); - } - return 0; + + if (sscanf(text, "%d", &code) == 1) { // got code + back[p].r.statuscode = code; + back[p].status = STATUS_READY; // terminé + if (lf != NULL && *lf != '\0') { // got location ? + strcpybuff(back[p].r.location, lf + 1); + } + return 0; } } } } - // tester cache - if ((strcmp(adr,"file://")) /* pas fichier */ - && ( (!test) || (cache->type==1) ) /* cache prioritaire, laisser passer en test! */ - && ( (strnotempty(save)) || (strcmp(fil,"/robots.txt")==0) ) ) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302.. + if ((strcmp(adr, "file://")) /* pas fichier */ + &&((!test) || (cache->type == 1)) /* cache prioritaire, laisser passer en test! */ + &&((strnotempty(save)) || (strcmp(fil, "/robots.txt") == 0))) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302.. #if HTS_FAST_CACHE intptr_t hash_pos; - int hash_pos_return=0; + int hash_pos_return = 0; #else - char* a=NULL; + char *a = NULL; #endif #if HTS_FAST_CACHE - if (cache->hashtable) { + if (cache->hashtable) { #else - if (cache->use) { + if (cache->use) { #endif - char BIGSTK buff[HTS_URLMAXSIZE*4]; + char BIGSTK buff[HTS_URLMAXSIZE * 4]; + #if HTS_FAST_CACHE - strcpybuff(buff,adr); strcatbuff(buff,fil); - hash_pos_return=inthash_read(cache->hashtable,buff,&hash_pos); + strcpybuff(buff, adr); + strcatbuff(buff, fil); + hash_pos_return = inthash_read(cache->hashtable, buff, &hash_pos); #else - buff[0]='\0'; strcatbuff(buff,"\n"); strcatbuff(buff,adr); strcatbuff(buff,"\n"); strcatbuff(buff,fil); strcatbuff(buff,"\n"); - a=strstr(cache->use,buff); + buff[0] = '\0'; + strcatbuff(buff, "\n"); + strcatbuff(buff, adr); + strcatbuff(buff, "\n"); + strcatbuff(buff, fil); + strcatbuff(buff, "\n"); + a = strstr(cache->use, buff); #endif - + // Ok, noté en cache->. mais bien présent dans le cache ou sur disque? #if HTS_FAST_CACHE // negative values when data is not in cache @@ -1412,63 +1542,81 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* #else if (a) { #endif - if (!test) { // non mode test + if (!test) { // non mode test #if HTS_FAST_CACHE==0 - int pos=-1; - a+=strlen(buff); - sscanf(a,"%d",&pos); // lire position + int pos = -1; + + a += strlen(buff); + sscanf(a, "%d", &pos); // lire position #endif #if HTS_FAST_CACHE==0 - if (pos<0) { // pas de mise en cache data, vérifier existence + if (pos < 0) { // pas de mise en cache data, vérifier existence #endif /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */ - if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff,save)) <= 0) { // fichier final n'existe pas ou est vide! - int found=0; + if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff, save)) <= 0) { // fichier final n'existe pas ou est vide! + int found = 0; /* It is possible that the file has been moved due to changes in build structure */ { - char BIGSTK previous_save[HTS_URLMAXSIZE*2]; + char BIGSTK previous_save[HTS_URLMAXSIZE * 2]; htsblk r; + previous_save[0] = '\0'; - r = cache_readex(opt, cache, adr, fil, /*head*/NULL, /*bound to back[p] (temporary)*/back[p].location_buffer, previous_save, /*ro*/1); + r = + cache_readex(opt, cache, adr, fil, /*head */ NULL, + /*bound to back[p] (temporary) */ + back[p].location_buffer, previous_save, /*ro */ + 1); /* Is supposed to be on disk only */ if (r.is_write && previous_save[0] != '\0') { /* Exists, but with another (old) filename: rename (almost) silently */ - if (strcmp(previous_save, save) != 0 && fexist_utf8(fconv(catbuff, previous_save))) { - rename(fconv(catbuff, previous_save), fconv(catbuff2,save)); - if (fexist_utf8(fconv(catbuff,save))) { + if (strcmp(previous_save, save) != 0 + && fexist_utf8(fconv(catbuff, previous_save))) { + rename(fconv(catbuff, previous_save), + fconv(catbuff2, save)); + if (fexist_utf8(fconv(catbuff, save))) { found = 1; - hts_log_print(opt, LOG_DEBUG, "File '%s' has been renamed since last mirror to '%s' ; applying changes", previous_save, save); + hts_log_print(opt, LOG_DEBUG, + "File '%s' has been renamed since last mirror to '%s' ; applying changes", + previous_save, save); } else { - hts_log_print(opt, LOG_ERROR, "Could not rename '%s' to '%s' ; will have to retransfer it", previous_save, save); + hts_log_print(opt, LOG_ERROR, + "Could not rename '%s' to '%s' ; will have to retransfer it", + previous_save, save); } } } back[p].location_buffer[0] = '\0'; } - + /* Not found ? */ if (!found) { #if HTS_FAST_CACHE - hash_pos_return=0; + hash_pos_return = 0; #else - a=NULL; + a = NULL; #endif // dévalider car non présent sur disque dans structure originale!!! // sinon, le fichier est ok à priori, mais on renverra un if-modified-since pour // en être sûr - if (opt->norecatch) { // tester norecatch - if (!fexist_utf8(fconv(catbuff,save))) { // fichier existe pas mais déclaré: on l'a effacé - FILE* fp=FOPEN(fconv(catbuff,save),"wb"); - if (fp) fclose(fp); - hts_log_print(opt, LOG_WARNING, "Previous file '%s' not found (erased by user ?), ignoring: %s%s",save,back[p].url_adr,back[p].url_fil); + if (opt->norecatch) { // tester norecatch + if (!fexist_utf8(fconv(catbuff, save))) { // fichier existe pas mais déclaré: on l'a effacé + FILE *fp = FOPEN(fconv(catbuff, save), "wb"); + + if (fp) + fclose(fp); + hts_log_print(opt, LOG_WARNING, + "Previous file '%s' not found (erased by user ?), ignoring: %s%s", + save, back[p].url_adr, back[p].url_fil); } } else { - hts_log_print(opt, LOG_WARNING, "Previous file '%s' not found (erased by user ?), recatching: %s%s",save,back[p].url_adr,back[p].url_fil); + hts_log_print(opt, LOG_WARNING, + "Previous file '%s' not found (erased by user ?), recatching: %s%s", + save, back[p].url_adr, back[p].url_fil); } } - } // fsize() <= 0 + } // fsize() <= 0 #if HTS_FAST_CACHE==0 } #endif @@ -1477,271 +1625,306 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* // } else { #if HTS_FAST_CACHE - hash_pos_return=0; + hash_pos_return = 0; #else - a=NULL; + a = NULL; #endif } // Existe pas en cache, ou bien pas de cache présent #if HTS_FAST_CACHE - if (hash_pos_return) { // OK existe en cache (et données aussi)! + if (hash_pos_return) { // OK existe en cache (et données aussi)! #else - if (a!=NULL) { // OK existe en cache (et données aussi)! + if (a != NULL) { // OK existe en cache (et données aussi)! #endif - const int cache_is_prioritary = cache->type == 1 || opt->state.stop != 0; - if (cache_is_prioritary) { // cache prioritaire (pas de test if-modified..) - // dans ce cas on peut également lire des réponses cachées comme 404,302... + const int cache_is_prioritary = cache->type == 1 + || opt->state.stop != 0; + if (cache_is_prioritary) { // cache prioritaire (pas de test if-modified..) + // dans ce cas on peut également lire des réponses cachées comme 404,302... // lire dans le cache if (!test) - back[p].r = cache_read(opt,cache,adr,fil,save, back[p].location_buffer); + back[p].r = + cache_read(opt, cache, adr, fil, save, back[p].location_buffer); else - back[p].r = cache_read(opt,cache,adr,fil,NULL, back[p].location_buffer); // charger en tête uniquement du cache + back[p].r = cache_read(opt, cache, adr, fil, NULL, back[p].location_buffer); // charger en tête uniquement du cache /* ensure correct location buffer set */ - back[p].r.location=back[p].location_buffer; + back[p].r.location = back[p].location_buffer; /* Interdiction taille par le wizard? --> détruire */ - if (back[p].r.statuscode != -1) { // pas d'erreur de lecture - if (!back_checksize(opt,&back[p],0)) { - back[p].status=STATUS_READY; // FINI + if (back[p].r.statuscode != -1) { // pas d'erreur de lecture + if (!back_checksize(opt, &back[p], 0)) { + back[p].status = STATUS_READY; // FINI back_set_finished(sback, p); - back[p].r.statuscode=STATUSCODE_TOO_BIG; + back[p].r.statuscode = STATUSCODE_TOO_BIG; if (!back[p].testmode) - strcpybuff(back[p].r.msg,"Cached file skipped (too big)"); + strcpybuff(back[p].r.msg, "Cached file skipped (too big)"); else - strcpybuff(back[p].r.msg,"Test: Cached file skipped (too big)"); + strcpybuff(back[p].r.msg, + "Test: Cached file skipped (too big)"); return 0; } } - if (back[p].r.statuscode != -1 || IS_DELAYED_EXT(save)) { // pas d'erreur de lecture ou test retardé + if (back[p].r.statuscode != -1 || IS_DELAYED_EXT(save)) { // pas d'erreur de lecture ou test retardé if (!test) { - hts_log_print(opt, LOG_DEBUG, "File immediately loaded from cache: %s%s",back[p].url_adr,back[p].url_fil); + hts_log_print(opt, LOG_DEBUG, + "File immediately loaded from cache: %s%s", + back[p].url_adr, back[p].url_fil); } else { - hts_log_print(opt, LOG_DEBUG, "File immediately tested from cache: %s%s",back[p].url_adr,back[p].url_fil); + hts_log_print(opt, LOG_DEBUG, + "File immediately tested from cache: %s%s", + back[p].url_adr, back[p].url_fil); } - back[p].r.notmodified=1; // fichier non modifié - back[p].status=STATUS_READY; // OK prêt + back[p].r.notmodified = 1; // fichier non modifié + back[p].status = STATUS_READY; // OK prêt //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified); // not modified back_set_finished(sback, p); // finalize transfer if (!test) { - if (back[p].r.statuscode>0) { - back_finalize(opt,cache,sback,p); + if (back[p].r.statuscode > 0) { + back_finalize(opt, cache, sback, p); } } return 0; - } else { // erreur + } else { // erreur // effacer r hts_init_htsblk(&back[p].r); //memset(&(back[p].r), 0, sizeof(htsblk)); - back[p].r.location=back[p].location_buffer; + back[p].r.location = back[p].location_buffer; // et continuer (chercher le fichier) } - - } else if (cache->type==2) { // si en cache, demander de tester If-Modified-Since + + } else if (cache->type == 2) { // si en cache, demander de tester If-Modified-Since htsblk r; - cache_header(opt,cache,adr,fil,&r); + + cache_header(opt, cache, adr, fil, &r); /* Interdiction taille par le wizard? */ { - LLint save_totalsize=back[p].r.totalsize; - back[p].r.totalsize=r.totalsize; - if (!back_checksize(opt,&back[p],1)) { + LLint save_totalsize = back[p].r.totalsize; + + back[p].r.totalsize = r.totalsize; + if (!back_checksize(opt, &back[p], 1)) { r.statuscode = STATUSCODE_INVALID; // - back[p].status=STATUS_READY; // FINI + back[p].status = STATUS_READY; // FINI back_set_finished(sback, p); - back[p].r.statuscode=STATUSCODE_TOO_BIG; - deletehttp(&back[p].r); back[p].r.soc=INVALID_SOCKET; + back[p].r.statuscode = STATUSCODE_TOO_BIG; + deletehttp(&back[p].r); + back[p].r.soc = INVALID_SOCKET; if (!back[p].testmode) - strcpybuff(back[p].r.msg,"File too big"); + strcpybuff(back[p].r.msg, "File too big"); else - strcpybuff(back[p].r.msg,"Test: File too big"); + strcpybuff(back[p].r.msg, "Test: File too big"); return 0; } - back[p].r.totalsize=save_totalsize; + back[p].r.totalsize = save_totalsize; } - + if (r.statuscode != -1) { - if (r.statuscode==HTTP_OK) { // uniquement des 200 (OK) - if (strnotempty(r.etag)) { // ETag (RFC2616) + if (r.statuscode == HTTP_OK) { // uniquement des 200 (OK) + if (strnotempty(r.etag)) { // ETag (RFC2616) /* - - If both an entity tag and a Last-Modified value have been - provided by the origin server, SHOULD use both validators in - cache-conditional requests. This allows both HTTP/1.0 and - HTTP/1.1 caches to respond appropriately. - */ + - If both an entity tag and a Last-Modified value have been + provided by the origin server, SHOULD use both validators in + cache-conditional requests. This allows both HTTP/1.0 and + HTTP/1.1 caches to respond appropriately. + */ if (strnotempty(r.lastmodified)) - sprintf(back[p].send_too,"If-None-Match: %s\r\nIf-Modified-Since: %s\r\n",r.etag,r.lastmodified); + sprintf(back[p].send_too, + "If-None-Match: %s\r\nIf-Modified-Since: %s\r\n", + r.etag, r.lastmodified); else - sprintf(back[p].send_too,"If-None-Match: %s\r\n",r.etag); - } - else if (strnotempty(r.lastmodified)) - sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",r.lastmodified); + sprintf(back[p].send_too, "If-None-Match: %s\r\n", r.etag); + } else if (strnotempty(r.lastmodified)) + sprintf(back[p].send_too, "If-Modified-Since: %s\r\n", + r.lastmodified); else if (strnotempty(cache->lastmodified)) - sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",cache->lastmodified); - + sprintf(back[p].send_too, "If-Modified-Since: %s\r\n", + cache->lastmodified); + /* this is an update of a file */ if (strnotempty(back[p].send_too)) - back[p].is_update=1; - back[p].r.req.nocompression=1; /* Do not compress when updating! */ - + back[p].is_update = 1; + back[p].r.req.nocompression = 1; /* Do not compress when updating! */ + } } #if DEBUGCA - printf("..is modified test %s\n",back[p].send_too); + printf("..is modified test %s\n", back[p].send_too); #endif - } - } + } + } /* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */ else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) { const long file_size = fsize_utf8(itemback->url_sav); + /* Found file on disk */ if (file_size > 0) { char *send_too = back[p].send_too; - sprintf(send_too, "Range: bytes="LLintP"-\r\n", (LLint) file_size); + + sprintf(send_too, "Range: bytes=" LLintP "-\r\n", (LLint) file_size); send_too += strlen(send_too); /* add etag information */ if (strnotempty(itemback->r.etag)) { - sprintf(send_too,"If-Match: %s\r\n", itemback->r.etag); + sprintf(send_too, "If-Match: %s\r\n", itemback->r.etag); send_too += strlen(send_too); } /* add date information */ if (strnotempty(itemback->r.lastmodified)) { - sprintf(send_too,"If-Unmodified-Since: %s\r\n", itemback->r.lastmodified); + sprintf(send_too, "If-Unmodified-Since: %s\r\n", + itemback->r.lastmodified); send_too += strlen(send_too); } - back[p].http11=1; /* 1.1 */ - back[p].range_req_size=(LLint) file_size; - back[p].r.req.range_used=1; - back[p].is_update=1; /* this is an update of a file */ - back[p].r.req.nocompression=1; /* Do not compress when updating! */ + back[p].http11 = 1; /* 1.1 */ + back[p].range_req_size = (LLint) file_size; + back[p].r.req.range_used = 1; + back[p].is_update = 1; /* this is an update of a file */ + back[p].r.req.nocompression = 1; /* Do not compress when updating! */ } else { /* broken ; remove */ url_savename_refname_remove(opt, adr, fil); } /* cleanup */ - back_clear_entry(itemback); /* delete entry content */ - freet(itemback); /* delete item */ + back_clear_entry(itemback); /* delete entry content */ + freet(itemback); /* delete item */ itemback = NULL; } /* Not in cache or temporary cache ; found on disk ? (hack) */ else if (fexist_utf8(save)) { - off_t sz=fsize_utf8(save); + off_t sz = fsize_utf8(save); + // Bon, là il est possible que le fichier ait été partiellement transféré // (s'il l'avait été en totalité il aurait été inscrit dans le cache ET existerait sur disque) // PAS de If-Modified-Since, on a pas connaissance des données à la date du cache // On demande juste les données restantes si le date est valide (206), tout sinon (200) - if ((ishtml(opt,save) != 1) && (ishtml(opt,back[p].url_fil)!=1)) { // NON HTML (liens changés!!) - if (sz>0) { // Fichier non vide? (question bête, sinon on transfert tout!) + if ((ishtml(opt, save) != 1) && (ishtml(opt, back[p].url_fil) != 1)) { // NON HTML (liens changés!!) + if (sz > 0) { // Fichier non vide? (question bête, sinon on transfert tout!) char lastmodified[256]; + get_filetime_rfc822(save, lastmodified); - if (strnotempty(lastmodified)) { /* pas de If-.. possible */ + if (strnotempty(lastmodified)) { /* pas de If-.. possible */ #if DEBUGCA - printf("..if unmodified since %s size "LLintP"\n", lastmodified, (LLint)sz); + printf("..if unmodified since %s size " LLintP "\n", lastmodified, + (LLint) sz); #endif - hts_log_print(opt, LOG_DEBUG, "File partially present ("LLintP" bytes): %s%s",(LLint)sz,back[p].url_adr,back[p].url_fil); + hts_log_print(opt, LOG_DEBUG, + "File partially present (" LLintP " bytes): %s%s", + (LLint) sz, back[p].url_adr, back[p].url_fil); /* impossible - don't have etag or date - if (strnotempty(back[p].r.etag)) { // ETag (RFC2616) - sprintf(back[p].send_too,"If-None-Match: %s\r\n",back[p].r.etag); - back[p].http11=1; // En tête 1.1 - } else if (strnotempty(back[p].r.lastmodified)) { - sprintf(back[p].send_too,"If-Unmodified-Since: %s\r\n",back[p].r.lastmodified); - back[p].http11=1; // En tête 1.1 - } else - */ + if (strnotempty(back[p].r.etag)) { // ETag (RFC2616) + sprintf(back[p].send_too,"If-None-Match: %s\r\n",back[p].r.etag); + back[p].http11=1; // En tête 1.1 + } else if (strnotempty(back[p].r.lastmodified)) { + sprintf(back[p].send_too,"If-Unmodified-Since: %s\r\n",back[p].r.lastmodified); + back[p].http11=1; // En tête 1.1 + } else + */ if (strlen(lastmodified)) { sprintf(back[p].send_too, - "If-Unmodified-Since: %s\r\nRange: bytes="LLintP"-\r\n" - , lastmodified, (LLint)sz); - back[p].http11=1; // En tête 1.1 - back[p].is_update=1; /* this is an update of a file */ - back[p].range_req_size=sz; - back[p].r.req.range_used=1; - back[p].r.req.nocompression=1; + "If-Unmodified-Since: %s\r\nRange: bytes=" LLintP + "-\r\n", lastmodified, (LLint) sz); + back[p].http11 = 1; // En tête 1.1 + back[p].is_update = 1; /* this is an update of a file */ + back[p].range_req_size = sz; + back[p].r.req.range_used = 1; + back[p].r.req.nocompression = 1; } else { - hts_log_print(opt, LOG_WARNING, "Could not find timestamp for partially present file, restarting (lost "LLintP" bytes): %s%s",(LLint)sz,back[p].url_adr,back[p].url_fil); + hts_log_print(opt, LOG_WARNING, + "Could not find timestamp for partially present file, restarting (lost " + LLintP " bytes): %s%s", (LLint) sz, + back[p].url_adr, back[p].url_fil); } - } else { - hts_log_print(opt, LOG_NOTICE, "File partially present ("LLintP" bytes) retransfered due to lack of cache: %s%s",(LLint)sz,back[p].url_adr,back[p].url_fil); + } else { + hts_log_print(opt, LOG_NOTICE, + "File partially present (" LLintP + " bytes) retransfered due to lack of cache: %s%s", + (LLint) sz, back[p].url_adr, back[p].url_fil); /* Sinon requête normale... */ - back[p].http11=0; + back[p].http11 = 0; } - } else if (opt->norecatch) { // tester norecatch - filenote(&opt->state.strc,save,NULL); // ne pas purger tout de même - file_notify(opt,back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified); - back[p].status=STATUS_READY; // OK prêt + } else if (opt->norecatch) { // tester norecatch + filenote(&opt->state.strc, save, NULL); // ne pas purger tout de même + file_notify(opt, back[p].url_adr, back[p].url_fil, back[p].url_sav, + 0, 0, back[p].r.notmodified); + back[p].status = STATUS_READY; // OK prêt back_set_finished(sback, p); - back[p].r.statuscode=STATUSCODE_INVALID; // erreur - strcpybuff(back[p].r.msg,"Null-size file not recaught"); + back[p].r.statuscode = STATUSCODE_INVALID; // erreur + strcpybuff(back[p].r.msg, "Null-size file not recaught"); return 0; } } else { - hts_log_print(opt, LOG_NOTICE, "HTML file ("LLintP" bytes) retransfered due to lack of cache: %s%s",(LLint)sz,back[p].url_adr,back[p].url_fil); + hts_log_print(opt, LOG_NOTICE, + "HTML file (" LLintP + " bytes) retransfered due to lack of cache: %s%s", + (LLint) sz, back[p].url_adr, back[p].url_fil); /* Sinon requête normale... */ - back[p].http11=0; + back[p].http11 = 0; } } } /* Stop requested - abort backing */ if (opt->state.stop) { - back[p].r.statuscode=STATUSCODE_INVALID; // fatal - strcpybuff(back[p].r.msg,"mirror stopped by user"); - back[p].status=STATUS_READY; // terminé + back[p].r.statuscode = STATUSCODE_INVALID; // fatal + strcpybuff(back[p].r.msg, "mirror stopped by user"); + back[p].status = STATUS_READY; // terminé back_set_finished(sback, p); - hts_log_print(opt, LOG_WARNING, "File not added due to mirror cancel: %s%s",adr,fil); + hts_log_print(opt, LOG_WARNING, + "File not added due to mirror cancel: %s%s", adr, fil); return 0; } { ///htsblk r; non directement dans la structure-réponse! T_SOC soc; - + // ouvrir liaison, envoyer requète // ne pas traiter ou recevoir l'en tête immédiatement hts_init_htsblk(&back[p].r); //memset(&(back[p].r), 0, sizeof(htsblk)); - back[p].r.location=back[p].location_buffer; + back[p].r.location = back[p].location_buffer; // recopier proxy if ((back[p].r.req.proxy.active = opt->proxy.active)) { if (StringBuff(opt->proxy.bindhost) != NULL) - strcpybuff(back[p].r.req.proxy.bindhost, StringBuff(opt->proxy.bindhost)); + strcpybuff(back[p].r.req.proxy.bindhost, + StringBuff(opt->proxy.bindhost)); if (StringBuff(opt->proxy.name) != NULL) strcpybuff(back[p].r.req.proxy.name, StringBuff(opt->proxy.name)); back[p].r.req.proxy.port = opt->proxy.port; } // et user-agent - strcpy(back[p].r.req.user_agent,StringBuff(opt->user_agent)); - strcpy(back[p].r.req.referer,StringBuff(opt->referer)); - strcpy(back[p].r.req.from,StringBuff(opt->from)); - strcpy(back[p].r.req.lang_iso,StringBuff(opt->lang_iso)); - back[p].r.req.user_agent_send=opt->user_agent_send; + strcpy(back[p].r.req.user_agent, StringBuff(opt->user_agent)); + strcpy(back[p].r.req.referer, StringBuff(opt->referer)); + strcpy(back[p].r.req.from, StringBuff(opt->from)); + strcpy(back[p].r.req.lang_iso, StringBuff(opt->lang_iso)); + back[p].r.req.user_agent_send = opt->user_agent_send; // et http11 - back[p].r.req.http11=back[p].http11; - back[p].r.req.nocompression=opt->nocompression; - back[p].r.req.nokeepalive=opt->nokeepalive; + back[p].r.req.http11 = back[p].http11; + back[p].r.req.nocompression = opt->nocompression; + back[p].r.req.nokeepalive = opt->nokeepalive; // mode ftp, court-circuit! - if (strfield(back[p].url_adr,"ftp://")) { + if (strfield(back[p].url_adr, "ftp://")) { if (back[p].testmode) { - hts_log_print(opt, LOG_DEBUG, "error: forbidden test with ftp link for back_add"); - return -1; // erreur pas de test permis + hts_log_print(opt, LOG_DEBUG, + "error: forbidden test with ftp link for back_add"); + return -1; // erreur pas de test permis } - if (!(back[p].r.req.proxy.active && opt->ftp_proxy)) { // connexion directe, gérée en thread - FTPDownloadStruct *str = (FTPDownloadStruct*) malloc(sizeof(FTPDownloadStruct)); - str->pBack = &back[p]; - str->pOpt = opt; - /* */ - back[p].status=STATUS_FTP_TRANSFER; // connexion ftp + if (!(back[p].r.req.proxy.active && opt->ftp_proxy)) { // connexion directe, gérée en thread + FTPDownloadStruct *str = + (FTPDownloadStruct *) malloc(sizeof(FTPDownloadStruct)); + str->pBack = &back[p]; + str->pOpt = opt; + /* */ + back[p].status = STATUS_FTP_TRANSFER; // connexion ftp #if USE_BEGINTHREAD - launch_ftp(str); + launch_ftp(str); #else #error Must have pthreads #endif @@ -1749,45 +1932,49 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* } } #if HTS_USEMMS - else if (strfield(back[p].url_adr,"mms://")) { - MMSDownloadStruct str; + else if (strfield(back[p].url_adr, "mms://")) { + MMSDownloadStruct str; + if (back[p].testmode) { - hts_log_print(opt, LOG_DEBUG, "error: forbidden test with mms link for back_add"); - return -1; // erreur pas de test permis + hts_log_print(opt, LOG_DEBUG, + "error: forbidden test with mms link for back_add"); + return -1; // erreur pas de test permis } if (back[p].r.req.proxy.active) { - hts_log_print(opt, LOG_WARNING, "warning: direct connection for mms links (proxy settings ignored)"); - } - back[p].status=STATUS_FTP_TRANSFER; // connexion externe - str.pBack = &back[p]; - str.pOpt = opt; - launch_mms(&str); - return 0; + hts_log_print(opt, LOG_WARNING, + "warning: direct connection for mms links (proxy settings ignored)"); + } + back[p].status = STATUS_FTP_TRANSFER; // connexion externe + str.pBack = &back[p]; + str.pOpt = opt; + launch_mms(&str); + return 0; } #endif #if HTS_USEOPENSSL - else if (SSL_is_available && strfield(back[p].url_adr,"https://")) { // let's rock + else if (SSL_is_available && strfield(back[p].url_adr, "https://")) { // let's rock back[p].r.ssl = 1; // back[p].r.ssl_soc = NULL; back[p].r.ssl_con = NULL; } #endif - + if (!back_trylive(opt, cache, sback, p)) { #if HTS_XGETHOST #if HDEBUG printf("back_solve..\n"); #endif - back[p].status=STATUS_WAIT_DNS; // tentative de résolution du nom de host - soc=INVALID_SOCKET; // pas encore ouverte - back_solve(opt, &back[p]); // préparer - if (host_wait(opt, &back[p])) { // prêt, par ex fichier ou dispo dans dns + back[p].status = STATUS_WAIT_DNS; // tentative de résolution du nom de host + soc = INVALID_SOCKET; // pas encore ouverte + back_solve(opt, &back[p]); // préparer + if (host_wait(opt, &back[p])) { // prêt, par ex fichier ou dispo dans dns #if HDEBUG printf("ok, dns cache ready..\n"); #endif - soc=http_xfopen(opt,0,0,0,back[p].send_too,adr,fil,&(back[p].r)); - if (soc==INVALID_SOCKET) { - back[p].status=STATUS_READY; // fini, erreur + soc = + http_xfopen(opt, 0, 0, 0, back[p].send_too, adr, fil, &(back[p].r)); + if (soc == INVALID_SOCKET) { + back[p].status = STATUS_READY; // fini, erreur back_set_finished(sback, p); } } @@ -1797,40 +1984,40 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* #if CNXDEBUG printf("XFopen..\n"); #endif - - if (strnotempty(back[p].send_too)) // envoyer un if-modified-since + + if (strnotempty(back[p].send_too)) // envoyer un if-modified-since #if HTS_XCONN - soc=http_xfopen(0,0,0,back[p].send_too,adr,fil,&(back[p].r)); + soc = http_xfopen(0, 0, 0, back[p].send_too, adr, fil, &(back[p].r)); #else - soc=http_xfopen(0,0,1,back[p].send_too,adr,fil,&(back[p].r)); + soc = http_xfopen(0, 0, 1, back[p].send_too, adr, fil, &(back[p].r)); #endif else #if HTS_XCONN - soc=http_xfopen(test,0,0,NULL,adr,fil,&(back[p].r)); + soc = http_xfopen(test, 0, 0, NULL, adr, fil, &(back[p].r)); #else - soc=http_xfopen(test,0,1,NULL,adr,fil,&(back[p].r)); + soc = http_xfopen(test, 0, 1, NULL, adr, fil, &(back[p].r)); #endif #endif } else { soc = back[p].r.soc; - hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): successfully linked #%d (for %s%s)", - back[p].r.debugid, - back[p].url_adr, back[p].url_fil); + hts_log_print(opt, LOG_DEBUG, + "(Keep-Alive): successfully linked #%d (for %s%s)", + back[p].r.debugid, back[p].url_adr, back[p].url_fil); } - - if (opt->timeout>0) { // gestion du opt->timeout - back[p].timeout=opt->timeout; - back[p].timeout_refresh=time_local(); + + if (opt->timeout > 0) { // gestion du opt->timeout + back[p].timeout = opt->timeout; + back[p].timeout_refresh = time_local(); } else { - back[p].timeout=-1; // pas de gestion (default) + back[p].timeout = -1; // pas de gestion (default) } - - if (opt->rateout>0) { // gestion d'un taux minimum de transfert toléré - back[p].rateout=opt->rateout; - back[p].rateout_time=time_local(); + + if (opt->rateout > 0) { // gestion d'un taux minimum de transfert toléré + back[p].rateout = opt->rateout; + back[p].rateout_time = time_local(); } else { - back[p].rateout=-1; // pas de gestion (default) + back[p].rateout = -1; // pas de gestion (default) } // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est @@ -1838,41 +2025,39 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* //if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre #if CNXDEBUG -printf("Xfopen ok, poll..\n"); + printf("Xfopen ok, poll..\n"); #endif #if HTS_XGETHOST - if (soc!=INVALID_SOCKET) - if (back[p].status==STATUS_WAIT_DNS) { // pas d'erreur - if (!back[p].r.is_file) - back[p].status=STATUS_CONNECTING; // connexion en cours - else - back[p].status=1; // fichier - } - + if (soc != INVALID_SOCKET) + if (back[p].status == STATUS_WAIT_DNS) { // pas d'erreur + if (!back[p].r.is_file) + back[p].status = STATUS_CONNECTING; // connexion en cours + else + back[p].status = 1; // fichier + } #else - if (soc==INVALID_SOCKET) { // erreur socket - back[p].status=STATUS_READY; // FINI + if (soc == INVALID_SOCKET) { // erreur socket + back[p].status = STATUS_READY; // FINI back_set_finished(sback, p); //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc); - back[p].r.soc=INVALID_SOCKET; + back[p].r.soc = INVALID_SOCKET; } else { if (!back[p].r.is_file) #if HTS_XCONN - back[p].status=STATUS_CONNECTING; // connexion en cours + back[p].status = STATUS_CONNECTING; // connexion en cours #else - back[p].status=99; // chargement en tête en cours + back[p].status = 99; // chargement en tête en cours #endif else - back[p].status=1; // chargement fichier + back[p].status = 1; // chargement fichier #if BDEBUG==1 printf("..loading header\n"); #endif } #endif - - } + } // note: si il y a erreur (404,etc) status=2 (terminé/échec) mais // le lien est considéré comme traité @@ -1881,121 +2066,125 @@ printf("Xfopen ok, poll..\n"); return 0; } else { - if (opt->log!=NULL) { - hts_log_print(opt, LOG_WARNING, "error: no space left in stack for back_add"); - if ( ( opt->state.debug_state & 1 ) == 0 ) { /* debug_state<0> == debug 'no space left in stack' */ + if (opt->log != NULL) { + hts_log_print(opt, LOG_WARNING, + "error: no space left in stack for back_add"); + if ((opt->state.debug_state & 1) == 0) { /* debug_state<0> == debug 'no space left in stack' */ int i; + hts_log_print(opt, LOG_WARNING, "debug: DUMPING %d BLOCKS", back_max); opt->state.debug_state |= 1; /* once */ /* OUTPUT FULL DEBUG INFORMATION THE FIRST TIME WE SEE THIS VERY ANNOYING BUG, - HOPING THAT SOME USER REPORT WILL QUICKLY SOLVE THIS PROBLEM :p */ - for(i = 0 ; i < back_max ; i++ ) { + HOPING THAT SOME USER REPORT WILL QUICKLY SOLVE THIS PROBLEM :p */ + for(i = 0; i < back_max; i++) { if (back[i].status != -1) { int may_clean = slot_can_be_cleaned(&back[i]); - int may_finalize = may_clean && slot_can_be_finalized(opt, &back[i]); + int may_finalize = may_clean + && slot_can_be_finalized(opt, &back[i]); int may_serialize = slot_can_be_cached_on_disk(&back[i]); - hts_log_print(opt, LOG_DEBUG, - "back[%03d]: may_clean=%d, may_finalize_disk=%d, may_serialize=%d:"LF - "\t" "finalized(%d), status(%d), locked(%d), delayed(%d), test(%d), "LF - "\t" "statuscode(%d), size(%d), is_write(%d), may_hypertext(%d), "LF - "\t" "contenttype(%s), url(%s%s), save(%s)", - i, - may_clean, may_finalize, may_serialize, - back[i].finalized, - back[i].status, - back[i].locked, - IS_DELAYED_EXT(back[i].url_sav), - back[i].testmode, - back[i].r.statuscode, - (int) back[i].r.size, - back[i].r.is_write, - may_be_hypertext_mime(opt,back[i].r.contenttype, back[i].url_fil), - /* */ - back[i].r.contenttype, - back[i].url_adr, back[i].url_fil, - back[i].url_sav ? back[i].url_sav : "<null>" - ); + + hts_log_print(opt, LOG_DEBUG, + "back[%03d]: may_clean=%d, may_finalize_disk=%d, may_serialize=%d:" + LF "\t" + "finalized(%d), status(%d), locked(%d), delayed(%d), test(%d), " + LF "\t" + "statuscode(%d), size(%d), is_write(%d), may_hypertext(%d), " + LF "\t" "contenttype(%s), url(%s%s), save(%s)", i, + may_clean, may_finalize, may_serialize, + back[i].finalized, back[i].status, back[i].locked, + IS_DELAYED_EXT(back[i].url_sav), back[i].testmode, + back[i].r.statuscode, (int) back[i].r.size, + back[i].r.is_write, may_be_hypertext_mime(opt, + back[i].r. + contenttype, + back[i]. + url_fil), + /* */ + back[i].r.contenttype, back[i].url_adr, + back[i].url_fil, + back[i].url_sav ? back[i].url_sav : "<null>"); } } } } - return -1; // plus de place + return -1; // plus de place } } - - #if HTS_XGETHOST #if USE_BEGINTHREAD // lancement multithread du robot typedef struct { - char iadr_p[HTS_URLMAXSIZE]; - httrackp *opt; + char iadr_p[HTS_URLMAXSIZE]; + httrackp *opt; } HostlookupStruct; -void Hostlookup(void* pP) { - HostlookupStruct *str = (HostlookupStruct*) pP; +void Hostlookup(void *pP) { + HostlookupStruct *str = (HostlookupStruct *) pP; char iadr[256]; - t_dnscache* cache=_hts_cache(str->opt); // adresse du cache - t_hostent* hp; - int error_found=0; + t_dnscache *cache = _hts_cache(str->opt); // adresse du cache + t_hostent *hp; + int error_found = 0; // recopier (après id:pass) -#if DEBUGDNS - printf("resolv in background: %s\n",jump_identification(iadr_p)); +#if DEBUGDNS + printf("resolv in background: %s\n", jump_identification(iadr_p)); #endif - strcpybuff(iadr,jump_identification(str->iadr_p)); + strcpybuff(iadr, jump_identification(str->iadr_p)); // couper éventuel : { char *a; - if ( (a=jump_toport(iadr)) ) - *a='\0'; // get rid of it + + if ((a = jump_toport(iadr))) + *a = '\0'; // get rid of it } freet(pP); hts_mutexlock(&dns_lock); while(cache->n) { - if (strcmp(cache->iadr,iadr)==0) { - error_found=1; + if (strcmp(cache->iadr, iadr) == 0) { + error_found = 1; } - cache=cache->n; // calculer queue + cache = cache->n; // calculer queue } - if (strcmp(cache->iadr,iadr)==0) { - error_found=1; + if (strcmp(cache->iadr, iadr) == 0) { + error_found = 1; } if (!error_found) { // en gros copie de hts_gethostbyname sans le return - cache->n=(t_dnscache*) calloct(1,sizeof(t_dnscache)); - if (cache->n!=NULL) { + cache->n = (t_dnscache *) calloct(1, sizeof(t_dnscache)); + if (cache->n != NULL) { t_fullhostent fullhostent_buffer; - strcpybuff(cache->n->iadr,iadr); - cache->n->host_length=0; /* pour le moment rien */ - cache->n->n=NULL; - + + strcpybuff(cache->n->iadr, iadr); + cache->n->host_length = 0; /* pour le moment rien */ + cache->n->n = NULL; + /* resolve */ -#if DEBUGDNS - printf("gethostbyname() in progress for %s\n",iadr); +#if DEBUGDNS + printf("gethostbyname() in progress for %s\n", iadr); #endif - cache->n->host_length=-1; + cache->n->host_length = -1; memset(cache->n->host_addr, 0, sizeof(cache->n->host_addr)); - hp=vxgethostbyname(iadr, &fullhostent_buffer); - if (hp!=NULL) { + hp = vxgethostbyname(iadr, &fullhostent_buffer); + if (hp != NULL) { memcpy(cache->n->host_addr, hp->h_addr, hp->h_length); cache->n->host_length = hp->h_length; } } } else { -#if DEBUGDNS - printf("aborting resolv for %s (found)\n",iadr); +#if DEBUGDNS + printf("aborting resolv for %s (found)\n", iadr); #endif } hts_mutexrelease(&dns_lock); -#if DEBUGDNS - printf("quitting resolv for %s (result: %d)\n",iadr,(cache->n!=NULL)?cache->n->host_length:(-999)); +#if DEBUGDNS + printf("quitting resolv for %s (result: %d)\n", iadr, + (cache->n != NULL) ? cache->n->host_length : (-999)); #endif } #endif @@ -2003,48 +2192,51 @@ void Hostlookup(void* pP) { // attendre que le host (ou celui du proxy) ait été résolu // si c'est un fichier, la résolution est immédiate // idem pour ftp:// -void back_solve(httrackp *opt, lien_back* back) { - if ((!strfield(back->url_adr,"file://")) - && ! strfield(back->url_adr,"ftp://") +void back_solve(httrackp * opt, lien_back * back) { + if ((!strfield(back->url_adr, "file://")) + && !strfield(back->url_adr, "ftp://") #if HTS_USEMMS - && ! strfield(back->url_adr,"mms://") + && !strfield(back->url_adr, "mms://") #endif - ) { - //## if (back->url_adr[0]!=lOCAL_CHAR) { // qq chose à préparer - const char* a; + ) { + //## if (back->url_adr[0]!=lOCAL_CHAR) { // qq chose à préparer + const char *a; + if (!(back->r.req.proxy.active)) - a=back->url_adr; + a = back->url_adr; else - a=back->r.req.proxy.name; + a = back->r.req.proxy.name; a = jump_protocol(a); - if (!hts_dnstest(opt, a)) { // non encore testé!.. + if (!hts_dnstest(opt, a)) { // non encore testé!.. // inscire en thread #ifdef _WIN32 // Windows #if USE_BEGINTHREAD { - HostlookupStruct *str = (HostlookupStruct*)malloct(sizeof(HostlookupStruct)); + HostlookupStruct *str = + (HostlookupStruct *) malloct(sizeof(HostlookupStruct)); if (str) { - strcpybuff(str->iadr_p, a); - str->opt = opt; + strcpybuff(str->iadr_p, a); + str->opt = opt; hts_newthread(Hostlookup, str); } } #else - /*t_hostent* h=*/ - /*hts_gethostbyname(a);*/ // calcul + /*t_hostent* h= */ + /*hts_gethostbyname(a); */// calcul #endif #else #if USE_BEGINTHREAD - char* p = calloct(strlen(a)+2,1); - if (p) { - strcpybuff(p,a); - hts_newthread( Hostlookup , p ); - } + char *p = calloct(strlen(a) + 2, 1); + + if (p) { + strcpybuff(p, a); + hts_newthread(Hostlookup, p); + } #else // Sous Unix, le gethostbyname() est bloquant.. - /*t_hostent* h=*/ - /*hts_gethostbyname(a);*/ // calcul + /*t_hostent* h= */ + /*hts_gethostbyname(a); */// calcul #endif #endif } @@ -2052,60 +2244,60 @@ void back_solve(httrackp *opt, lien_back* back) { } // détermine si le host a pu être résolu -int host_wait(httrackp *opt, lien_back* back) { - if ((!strfield(back->url_adr,"file://")) - && (!strfield(back->url_adr,"ftp://")) +int host_wait(httrackp * opt, lien_back * back) { + if ((!strfield(back->url_adr, "file://")) + && (!strfield(back->url_adr, "ftp://")) #if HTS_USEMMS - && (!strfield(back->url_adr,"mms://")) + && (!strfield(back->url_adr, "mms://")) #endif - ) { - //## if (back->url_adr[0]!=lOCAL_CHAR) { + ) { + //## if (back->url_adr[0]!=lOCAL_CHAR) { if (!(back->r.req.proxy.active)) { return (hts_dnstest(opt, back->url_adr)); } else { - return (hts_dnstest(opt, back->r.req.proxy.name)); + return (hts_dnstest(opt, back->r.req.proxy.name)); } - } else return 1; // prêt, fichier local + } else + return 1; // prêt, fichier local } #endif - // élimine les fichiers non html en backing (anticipation) // cleanup non-html files in backing to save backing space // and allow faster "save in cache" operation // also cleanup keep-alive sockets and ensure that not too many sockets are being opened -static int slot_can_be_cleaned(const lien_back* back) { - return - (back->status == STATUS_READY) // ready +static int slot_can_be_cleaned(const lien_back * back) { + return (back->status == STATUS_READY) // ready /* Check autoclean */ - && (!back->testmode) // not test mode - && (strnotempty(back->url_sav)) // filename exists - && (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK" - && (back->r.size >= 0) // size>=0 + && (!back->testmode) // not test mode + && (strnotempty(back->url_sav)) // filename exists + && (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK" + && (back->r.size >= 0) // size>=0 ; } -static int slot_can_be_finalized(httrackp* opt, const lien_back* back) { - return - back->r.is_write // not in memory (on disk, ready) - && !is_hypertext_mime(opt,back->r.contenttype, back->url_fil) // not HTML/hypertext - && !may_be_hypertext_mime(opt,back->r.contenttype, back->url_fil) // may NOT be parseable mime type +static int slot_can_be_finalized(httrackp * opt, const lien_back * back) { + return back->r.is_write // not in memory (on disk, ready) + && !is_hypertext_mime(opt, back->r.contenttype, back->url_fil) // not HTML/hypertext + && !may_be_hypertext_mime(opt, back->r.contenttype, back->url_fil) // may NOT be parseable mime type /* Has not been added before the heap saw the link, or now exists on heap */ - && ( !back->early_add || hash_read(opt->hash,back->url_sav,"",0,opt->urlhack) >= 0 ) - ; + && (!back->early_add + || hash_read(opt->hash, back->url_sav, "", 0, opt->urlhack) >= 0); } -void back_clean(httrackp* opt,cache_back* cache,struct_back* sback) { - lien_back* const back = sback->lnk; +void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; - int oneMore = ( (opt->state._hts_in_html_parsing == 2 && opt->maxsoc >= 2) || (opt->state._hts_in_html_parsing == 1 && opt->maxsoc >= 4) ) ? 1 : 0; // testing links + int oneMore = ((opt->state._hts_in_html_parsing == 2 && opt->maxsoc >= 2) || (opt->state._hts_in_html_parsing == 1 && opt->maxsoc >= 4)) ? 1 : 0; // testing links int i; - for(i=0;i<back_max;i++) { + + for(i = 0; i < back_max; i++) { if (slot_can_be_cleaned(&back[i])) { if (slot_can_be_finalized(opt, &back[i])) { - (void) back_flush_output(opt, cache, sback, i); // flush output buffers - usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr, back[i].url_fil); + (void) back_flush_output(opt, cache, sback, i); // flush output buffers + usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr, + back[i].url_fil); //if (back[i].links_index >= 0) { // assertf(back[i].links_index < opt->hash->max_lien); // opt->hash->liens[back[i].links_index]->pass2 = -1; @@ -2113,27 +2305,34 @@ void back_clean(httrackp* opt,cache_back* cache,struct_back* sback) { //} /* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */ { - int index = hash_read(opt->hash,back[i].url_sav,"",0,opt->urlhack); // lecture type 0 (sav) + int index = hash_read(opt->hash, back[i].url_sav, "", 0, opt->urlhack); // lecture type 0 (sav) + if (index >= 0) { - opt->hash->liens[index]->pass2 = -1; /* DONE! */ + opt->hash->liens[index]->pass2 = -1; /* DONE! */ } else { - hts_log_print(opt, LOG_INFO, "engine: warning: entry cleaned up, but no trace on heap: %s%s (%s)",back[i].url_adr, back[i].url_fil,back[i].url_sav); + hts_log_print(opt, LOG_INFO, + "engine: warning: entry cleaned up, but no trace on heap: %s%s (%s)", + back[i].url_adr, back[i].url_fil, back[i].url_sav); } } HTS_STAT.stat_background++; - hts_log_print(opt, LOG_INFO, "File successfully written in background: %s",back[i].url_sav); - back_maydelete(opt,cache,sback,i); // May delete backing entry + hts_log_print(opt, LOG_INFO, + "File successfully written in background: %s", + back[i].url_sav); + back_maydelete(opt, cache, sback, i); // May delete backing entry } else { if (!back[i].finalized) { if (1) { /* Ensure deleted or recycled socket */ /* BUT DO NOT YET WIPE back[i].r.adr */ - hts_log_print(opt, LOG_DEBUG, "file %s%s validated (cached, left in memory)",back[i].url_adr,back[i].url_fil); + hts_log_print(opt, LOG_DEBUG, + "file %s%s validated (cached, left in memory)", + back[i].url_adr, back[i].url_fil); back_maydeletehttp(opt, cache, sback, i); } else { /* - NOT YET HANDLED CORRECTLY (READ IN NEW CACHE TO DO) - */ + NOT YET HANDLED CORRECTLY (READ IN NEW CACHE TO DO) + */ /* Lock the entry but do not keep the html data in memory (in cache) */ if (opt->cache) { htsblk r; @@ -2143,41 +2342,41 @@ void back_clean(httrackp* opt,cache_back* cache,struct_back* sback) { assertf(back[i].r.soc == INVALID_SOCKET); /* Check header */ - cache_header(opt,cache,back[i].url_adr,back[i].url_fil,&r); + cache_header(opt, cache, back[i].url_adr, back[i].url_fil, &r); if (r.statuscode == HTTP_OK) { if (back[i].r.soc == INVALID_SOCKET) { /* Delete buffer and sockets */ deleteaddr(&back[i].r); deletehttp(&back[i].r); - hts_log_print(opt, LOG_DEBUG, "file %s%s temporarily left in cache to spare memory",back[i].url_adr,back[i].url_fil); + hts_log_print(opt, LOG_DEBUG, + "file %s%s temporarily left in cache to spare memory", + back[i].url_adr, back[i].url_fil); } } else { - hts_log_print(opt, LOG_WARNING, "Unexpected html cache lookup error during back clean"); + hts_log_print(opt, LOG_WARNING, + "Unexpected html cache lookup error during back clean"); } // xxc xxc } } } } - } else if (back[i].status == STATUS_ALIVE) { // waiting (keep-alive) - if ( - ! back[i].r.keep_alive - || back[i].r.soc == INVALID_SOCKET - || back[i].r.keep_alive_max < 1 - || time_local() >= back[i].ka_time_start + back[i].r.keep_alive_t - ) { - hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): live socket closed #%d (%s)", - back[i].r.debugid, - back[i].url_adr); - back_delete(opt,cache,sback, i); // delete backing entry + } else if (back[i].status == STATUS_ALIVE) { // waiting (keep-alive) + if (!back[i].r.keep_alive || back[i].r.soc == INVALID_SOCKET + || back[i].r.keep_alive_max < 1 + || time_local() >= back[i].ka_time_start + back[i].r.keep_alive_t) { + hts_log_print(opt, LOG_DEBUG, + "(Keep-Alive): live socket closed #%d (%s)", + back[i].r.debugid, back[i].url_adr); + back_delete(opt, cache, sback, i); // delete backing entry } } } /* switch connections to live ones */ - for(i=0;i<back_max;i++) { - if (back[i].status == STATUS_READY) { // ready + for(i = 0; i < back_max; i++) { + if (back[i].status == STATUS_READY) { // ready if (back[i].r.soc != INVALID_SOCKET) { - back_maydeletehttp(opt,cache,sback, i); + back_maydeletehttp(opt, cache, sback, i); } } } @@ -2185,177 +2384,182 @@ void back_clean(httrackp* opt,cache_back* cache,struct_back* sback) { if (opt->maxsoc > 0) { int max = opt->maxsoc + oneMore; int curr = back_nsoc_overall(sback); + if (curr > max) { - hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): deleting #%d sockets", - curr - max); + hts_log_print(opt, LOG_DEBUG, "(Keep-Alive): deleting #%d sockets", + curr - max); } - for(i = 0 ; i < back_max && curr > max ; i++) { + for(i = 0; i < back_max && curr > max; i++) { if (back[i].status == STATUS_ALIVE) { - back_delete(opt,cache,sback, i); // delete backing entry + back_delete(opt, cache, sback, i); // delete backing entry curr--; } } } /* transfer ready slots to the storage hashtable */ { - int nxfr = back_cleanup_background(opt,cache,sback); + int nxfr = back_cleanup_background(opt, cache, sback); + if (nxfr > 0) { - hts_log_print(opt, LOG_DEBUG, "(htsback): %d slots ready moved to background", nxfr); + hts_log_print(opt, LOG_DEBUG, + "(htsback): %d slots ready moved to background", nxfr); } } } - // attente (gestion des buffers des sockets) -void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_timestart) { - char catbuff[CATBUFF_SIZE]; - lien_back* const back = sback->lnk; +void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, + TStamp stat_timestart) { + char catbuff[CATBUFF_SIZE]; + lien_back *const back = sback->lnk; const int back_max = sback->count; unsigned int i_mod; - T_SOC nfds=INVALID_SOCKET; - fd_set fds,fds_c,fds_e; // fds pour lecture, connect (write), et erreur - int nsockets; // nbre sockets - LLint max_read_bytes; // max bytes read per sockets + T_SOC nfds = INVALID_SOCKET; + fd_set fds, fds_c, fds_e; // fds pour lecture, connect (write), et erreur + int nsockets; // nbre sockets + LLint max_read_bytes; // max bytes read per sockets struct timeval tv; - int do_wait=0; - int gestion_timeout=0; - int busy_recv=0; // pas de données pour le moment - int busy_state=0; // pas de connexions - int max_loop; // nombre de boucles max à parcourir.. - int max_loop_chk=0; - unsigned int mod_random = (unsigned int) ( time_local() + HTS_STAT.HTS_TOTAL_RECV ); + int do_wait = 0; + int gestion_timeout = 0; + int busy_recv = 0; // pas de données pour le moment + int busy_state = 0; // pas de connexions + int max_loop; // nombre de boucles max à parcourir.. + int max_loop_chk = 0; + unsigned int mod_random = + (unsigned int) (time_local() + HTS_STAT.HTS_TOTAL_RECV); // max. number of loops - max_loop=8; + max_loop = 8; #if 1 // Cleanup the stack to save space! - back_clean(opt,cache,sback); + back_clean(opt, cache, sback); #endif // recevoir tant qu'il y a des données (avec un maximum de max_loop boucles) - do_wait=0; - gestion_timeout=0; + do_wait = 0; + gestion_timeout = 0; do { int max_c; - busy_state=busy_recv=0; + + busy_state = busy_recv = 0; #if 0 - check_rate(stat_timestart,opt->maxrate); // vérifier taux de transfert + check_rate(stat_timestart, opt->maxrate); // vérifier taux de transfert #endif // inscrire les sockets actuelles, et rechercher l'ID la plus élevée FD_ZERO(&fds); FD_ZERO(&fds_c); FD_ZERO(&fds_e); - nsockets=0; - max_read_bytes=TAILLE_BUFFER; // maximum bytes that can be read - nfds=INVALID_SOCKET; + nsockets = 0; + max_read_bytes = TAILLE_BUFFER; // maximum bytes that can be read + nfds = INVALID_SOCKET; - max_c=1; - for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) { - // for(i=0;i<back_max;i++) { - unsigned int i = ( i_mod + mod_random ) % ( back_max ); + max_c = 1; + for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) { + // for(i=0;i<back_max;i++) { + unsigned int i = (i_mod + mod_random) % (back_max); // en cas de gestion du connect préemptif #if HTS_XCONN - if (back[i].status==STATUS_CONNECTING) { // connexion - do_wait=1; + if (back[i].status == STATUS_CONNECTING) { // connexion + do_wait = 1; // noter socket write - FD_SET(back[i].r.soc,&fds_c); - + FD_SET(back[i].r.soc, &fds_c); + // noter socket erreur - FD_SET(back[i].r.soc,&fds_e); + FD_SET(back[i].r.soc, &fds_e); // calculer max if (max_c) { - max_c=0; - nfds=back[i].r.soc; - } else if (back[i].r.soc>nfds) { + max_c = 0; + nfds = back[i].r.soc; + } else if (back[i].r.soc > nfds) { // ID socket la plus élevée - nfds=back[i].r.soc; + nfds = back[i].r.soc; } - + } else #endif #if HTS_XGETHOST - if (back[i].status==STATUS_WAIT_DNS) { // attente + if (back[i].status == STATUS_WAIT_DNS) { // attente // rien à faire.. } else #endif - // poll pour la lecture sur les sockets - if ((back[i].status>0) && (back[i].status<100)) { // en réception http - + // poll pour la lecture sur les sockets + if ((back[i].status > 0) && (back[i].status < 100)) { // en réception http + #if BDEBUG==1 //printf("....socket in progress: %d\n",back[i].r.soc); #endif // non local et non ftp if (!back[i].r.is_file) { - //## if (back[i].url_adr[0]!=lOCAL_CHAR) { - + //## if (back[i].url_adr[0]!=lOCAL_CHAR) { + // vérification de sécurité - if (back[i].r.soc!=INVALID_SOCKET) { // hey, you never know.. + if (back[i].r.soc != INVALID_SOCKET) { // hey, you never know.. // Do not endlessly wait when receiving SSL http data (Patrick Pfeifer) if (!(back[i].r.ssl && back[i].status > 0 && back[i].status < 1000)) { - do_wait=1; + do_wait = 1; // noter socket read - FD_SET(back[i].r.soc,&fds); + FD_SET(back[i].r.soc, &fds); // noter socket error - FD_SET(back[i].r.soc,&fds_e); + FD_SET(back[i].r.soc, &fds_e); // incrémenter nombre de sockets nsockets++; // calculer max if (max_c) { - max_c=0; - nfds=back[i].r.soc; - } else if (back[i].r.soc>nfds) { + max_c = 0; + nfds = back[i].r.soc; + } else if (back[i].r.soc > nfds) { // ID socket la plus élevée - nfds=back[i].r.soc; + nfds = back[i].r.soc; } } } else { - back[i].r.statuscode=STATUSCODE_CONNERROR; - if (back[i].status==STATUS_CONNECTING) - strcpybuff(back[i].r.msg,"Connect Error"); + back[i].r.statuscode = STATUSCODE_CONNERROR; + if (back[i].status == STATUS_CONNECTING) + strcpybuff(back[i].r.msg, "Connect Error"); else - strcpybuff(back[i].r.msg,"Receive Error"); - back[i].status=STATUS_READY; // terminé + strcpybuff(back[i].r.msg, "Receive Error"); + back[i].status = STATUS_READY; // terminé back_set_finished(sback, i); - hts_log_print(opt, LOG_WARNING, "Unexpected socket error during pre-loop"); + hts_log_print(opt, LOG_WARNING, + "Unexpected socket error during pre-loop"); } } - + } - } + } nfds++; - - if (do_wait) { // attendre + + if (do_wait) { // attendre // temps d'attente max: 2.5 seconde - tv.tv_sec=HTS_SOCK_SEC; - tv.tv_usec=HTS_SOCK_MS; - + tv.tv_sec = HTS_SOCK_SEC; + tv.tv_usec = HTS_SOCK_MS; + #if BDEBUG==1 printf("..select\n"); #endif - + // poller les sockets-attention au noyau sous Unix.. -#if HTS_WIDE_DEBUG +#if HTS_WIDE_DEBUG DEBUG_W("select\n"); #endif - select(nfds,&fds,&fds_c,&fds_e,&tv); -#if HTS_WIDE_DEBUG + select(nfds, &fds, &fds_c, &fds_e, &tv); +#if HTS_WIDE_DEBUG DEBUG_W("select done\n"); -#endif +#endif } - // maximum data which can be received for a socket, if limited if (nsockets) { - if (opt->maxrate>0) { - max_read_bytes = ( check_downloadable_bytes(opt->maxrate) / nsockets ); + if (opt->maxrate > 0) { + max_read_bytes = (check_downloadable_bytes(opt->maxrate) / nsockets); if (max_read_bytes > TAILLE_BUFFER) { /* limit size */ max_read_bytes = TAILLE_BUFFER; @@ -2366,363 +2570,398 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti } } if (!max_read_bytes) - busy_recv=0; - + busy_recv = 0; + // recevoir les données arrivées - for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) { - // for(i=0;i<back_max;i++) { - unsigned int i = ( i_mod + mod_random ) % ( back_max ); - if (back[i].status>0) { - if (!back[i].r.is_file) { // not file.. - if (back[i].r.soc!=INVALID_SOCKET) { // hey, you never know.. - int err=FD_ISSET(back[i].r.soc,&fds_e); + for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) { + // for(i=0;i<back_max;i++) { + unsigned int i = (i_mod + mod_random) % (back_max); + + if (back[i].status > 0) { + if (!back[i].r.is_file) { // not file.. + if (back[i].r.soc != INVALID_SOCKET) { // hey, you never know.. + int err = FD_ISSET(back[i].r.soc, &fds_e); + if (err) { - if (back[i].r.soc!=INVALID_SOCKET) { + if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait: deletehttp\n"); #endif deletehttp(&back[i].r); } - back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=STATUSCODE_CONNERROR; - if (back[i].status==STATUS_CONNECTING) - strcpybuff(back[i].r.msg,"Connect Error"); + back[i].r.soc = INVALID_SOCKET; + back[i].r.statuscode = STATUSCODE_CONNERROR; + if (back[i].status == STATUS_CONNECTING) + strcpybuff(back[i].r.msg, "Connect Error"); else - strcpybuff(back[i].r.msg,"Receive Error"); + strcpybuff(back[i].r.msg, "Receive Error"); if (back[i].status == STATUS_ALIVE) { /* Keep-alive socket */ - back_delete(opt,cache,sback, i); + back_delete(opt, cache, sback, i); } else { - back[i].status=STATUS_READY; // terminé + back[i].status = STATUS_READY; // terminé back_set_finished(sback, i); } } } } } - // ---- FLAG WRITE MIS A UN?: POUR LE CONNECT - if (back[i].status==STATUS_CONNECTING) { // attendre connect - int dispo=0; + if (back[i].status == STATUS_CONNECTING) { // attendre connect + int dispo = 0; + // vérifier l'existance de timeout-check if (!gestion_timeout) - if (back[i].timeout>0) - gestion_timeout=1; - - // connecté? - dispo=FD_ISSET(back[i].r.soc,&fds_c); - if (dispo) { // ok connected!! - busy_state=1; - + if (back[i].timeout > 0) + gestion_timeout = 1; + + // connecté? + dispo = FD_ISSET(back[i].r.soc, &fds_c); + if (dispo) { // ok connected!! + busy_state = 1; + #if HTS_USEOPENSSL - /* SSL mode */ - if (SSL_is_available && back[i].r.ssl) { - // handshake not yet launched - if (!back[i].r.ssl_con) { - SSL_CTX_set_options(openssl_ctx, SSL_OP_ALL); - // new session - back[i].r.ssl_con = SSL_new(openssl_ctx); - if (back[i].r.ssl_con) { - SSL_clear(back[i].r.ssl_con); - if (SSL_set_fd(back[i].r.ssl_con, back[i].r.soc) == 1) { - SSL_set_connect_state(back[i].r.ssl_con); - back[i].status = STATUS_SSL_WAIT_HANDSHAKE; /* handshake wait */ - } else - back[i].r.statuscode=STATUSCODE_SSL_HANDSHAKE; + /* SSL mode */ + if (SSL_is_available && back[i].r.ssl) { + // handshake not yet launched + if (!back[i].r.ssl_con) { + SSL_CTX_set_options(openssl_ctx, SSL_OP_ALL); + // new session + back[i].r.ssl_con = SSL_new(openssl_ctx); + if (back[i].r.ssl_con) { + SSL_clear(back[i].r.ssl_con); + if (SSL_set_fd(back[i].r.ssl_con, back[i].r.soc) == 1) { + SSL_set_connect_state(back[i].r.ssl_con); + back[i].status = STATUS_SSL_WAIT_HANDSHAKE; /* handshake wait */ } else - back[i].r.statuscode=STATUSCODE_SSL_HANDSHAKE; - } - /* Error */ - if (back[i].r.statuscode == STATUSCODE_SSL_HANDSHAKE) { - strcpybuff(back[i].r.msg, "bad SSL/TLS handshake"); - deletehttp(&back[i].r); - back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=STATUSCODE_NON_FATAL; - back[i].status=STATUS_READY; - back_set_finished(sback, i); - } + back[i].r.statuscode = STATUSCODE_SSL_HANDSHAKE; + } else + back[i].r.statuscode = STATUSCODE_SSL_HANDSHAKE; + } + /* Error */ + if (back[i].r.statuscode == STATUSCODE_SSL_HANDSHAKE) { + strcpybuff(back[i].r.msg, "bad SSL/TLS handshake"); + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; + back[i].r.statuscode = STATUSCODE_NON_FATAL; + back[i].status = STATUS_READY; + back_set_finished(sback, i); } - + } #endif #if BDEBUG==1 - printf("..connect ok on socket %d\n",back[i].r.soc); + printf("..connect ok on socket %d\n", back[i].r.soc); #endif - - if ((back[i].r.soc != INVALID_SOCKET) && (back[i].status==STATUS_CONNECTING)) { + + if ((back[i].r.soc != INVALID_SOCKET) + && (back[i].status == STATUS_CONNECTING)) { /* limit nb. connections/seconds to avoid server overload */ /*if (opt->maxconn>0) { - Sleep(1000/opt->maxconn); - }*/ - - back[i].ka_time_start=time_local(); - if (back[i].timeout>0) { // refresh timeout si besoin est - back[i].timeout_refresh=back[i].ka_time_start; + Sleep(1000/opt->maxconn); + } */ + + back[i].ka_time_start = time_local(); + if (back[i].timeout > 0) { // refresh timeout si besoin est + back[i].timeout_refresh = back[i].ka_time_start; } - if (back[i].rateout>0) { // le taux de transfert de base sur le début de la connexion - back[i].rateout_time=back[i].ka_time_start; + if (back[i].rateout > 0) { // le taux de transfert de base sur le début de la connexion + back[i].rateout_time = back[i].ka_time_start; } // envoyer header //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0) // vrai get HTS_STAT.stat_nrequests++; if (!back[i].head_request) - http_sendhead(opt, opt->cookie,0,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r); - else if (back[i].head_request==2) // test en GET! - http_sendhead(opt, opt->cookie,0,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r); - else // test! - http_sendhead(opt, opt->cookie,1,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r); - back[i].status=99; // attendre en tête maintenant + http_sendhead(opt, opt->cookie, 0, back[i].send_too, + back[i].url_adr, back[i].url_fil, + back[i].referer_adr, back[i].referer_fil, + &back[i].r); + else if (back[i].head_request == 2) // test en GET! + http_sendhead(opt, opt->cookie, 0, back[i].send_too, + back[i].url_adr, back[i].url_fil, + back[i].referer_adr, back[i].referer_fil, + &back[i].r); + else // test! + http_sendhead(opt, opt->cookie, 1, back[i].send_too, + back[i].url_adr, back[i].url_fil, + back[i].referer_adr, back[i].referer_fil, + &back[i].r); + back[i].status = 99; // attendre en tête maintenant } } - // attente gethostbyname } #if HTS_USEOPENSSL - else if (SSL_is_available && back[i].status == STATUS_SSL_WAIT_HANDSHAKE) { // wait for SSL handshake + else if (SSL_is_available && back[i].status == STATUS_SSL_WAIT_HANDSHAKE) { // wait for SSL handshake /* SSL mode */ if (back[i].r.ssl) { int conn_code; + if ((conn_code = SSL_connect(back[i].r.ssl_con)) <= 0) { /* non blocking I/O, will retry */ int err_code = SSL_get_error(back[i].r.ssl_con, conn_code); - if ( - (err_code != SSL_ERROR_WANT_READ) - && - (err_code != SSL_ERROR_WANT_WRITE) + + if ((err_code != SSL_ERROR_WANT_READ) + && (err_code != SSL_ERROR_WANT_WRITE) ) { char tmp[256]; - tmp[0]='\0'; + + tmp[0] = '\0'; ERR_error_string(err_code, tmp); - back[i].r.msg[0]='\0'; + back[i].r.msg[0] = '\0'; strncatbuff(back[i].r.msg, tmp, sizeof(back[i].r.msg) - 2); if (!strnotempty(back[i].r.msg)) { sprintf(back[i].r.msg, "SSL/TLS error %d", err_code); } deletehttp(&back[i].r); - back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=STATUSCODE_NON_FATAL; - back[i].status=STATUS_READY; + back[i].r.soc = INVALID_SOCKET; + back[i].r.statuscode = STATUSCODE_NON_FATAL; + back[i].status = STATUS_READY; back_set_finished(sback, i); - } - } else { /* got it! */ - back[i].status=STATUS_CONNECTING; // back to waitconnect + } + } else { /* got it! */ + back[i].status = STATUS_CONNECTING; // back to waitconnect } } else { strcpybuff(back[i].r.msg, "unexpected SSL/TLS error"); deletehttp(&back[i].r); - back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=STATUSCODE_NON_FATAL; - back[i].status=STATUS_READY; + back[i].r.soc = INVALID_SOCKET; + back[i].r.statuscode = STATUSCODE_NON_FATAL; + back[i].status = STATUS_READY; back_set_finished(sback, i); } - + } #endif #if HTS_XGETHOST - else if (back[i].status==STATUS_WAIT_DNS) { // attendre gethostbyname -#if DEBUGDNS + else if (back[i].status == STATUS_WAIT_DNS) { // attendre gethostbyname +#if DEBUGDNS //printf("status 101 for %s\n",back[i].url_adr); #endif if (!gestion_timeout) - if (back[i].timeout>0) - gestion_timeout=1; + if (back[i].timeout > 0) + gestion_timeout = 1; - if (host_wait(opt, &back[i])) { // prêt - back[i].status=STATUS_CONNECTING; // attente connexion - if (back[i].timeout>0) { // refresh timeout si besoin est - back[i].timeout_refresh=time_local(); + if (host_wait(opt, &back[i])) { // prêt + back[i].status = STATUS_CONNECTING; // attente connexion + if (back[i].timeout > 0) { // refresh timeout si besoin est + back[i].timeout_refresh = time_local(); } - if (back[i].rateout>0) { // le taux de transfert de base sur le début de la connexion - back[i].rateout_time=time_local(); + if (back[i].rateout > 0) { // le taux de transfert de base sur le début de la connexion + back[i].rateout_time = time_local(); } - back[i].r.soc=http_xfopen(opt,0,0,0,back[i].send_too,back[i].url_adr,back[i].url_fil,&(back[i].r)); - if (back[i].r.soc==INVALID_SOCKET) { - back[i].status=STATUS_READY; // fini, erreur + back[i].r.soc = + http_xfopen(opt, 0, 0, 0, back[i].send_too, back[i].url_adr, + back[i].url_fil, &(back[i].r)); + if (back[i].r.soc == INVALID_SOCKET) { + back[i].status = STATUS_READY; // fini, erreur back_set_finished(sback, i); - if (back[i].r.soc!=INVALID_SOCKET) { + if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(2): deletehttp\n"); #endif deletehttp(&back[i].r); } - back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=STATUSCODE_NON_FATAL; - if (strnotempty(back[i].r.msg)==0) - strcpybuff(back[i].r.msg,"Unable to resolve host name"); + back[i].r.soc = INVALID_SOCKET; + back[i].r.statuscode = STATUSCODE_NON_FATAL; + if (strnotempty(back[i].r.msg) == 0) + strcpybuff(back[i].r.msg, "Unable to resolve host name"); } } - - // ---- FLAG READ MIS A UN?: POUR LA RECEPTION + // ---- FLAG READ MIS A UN?: POUR LA RECEPTION } #endif #if USE_BEGINTHREAD // ..rien à faire, c'est magic les threads #else - else if (back[i].status==STATUS_FTP_TRANSFER) { // en réception ftp - if (!fexist(back[i].location_buffer)) { // terminé - FILE* fp; - fp=FOPEN(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok"),"rb"); + else if (back[i].status == STATUS_FTP_TRANSFER) { // en réception ftp + if (!fexist(back[i].location_buffer)) { // terminé + FILE *fp; + + fp = + FOPEN(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer, ".ok"), + "rb"); if (fp) { - int j=0; - fscanf(fp,"%d ",&(back[i].r.statuscode)); + int j = 0; + + fscanf(fp, "%d ", &(back[i].r.statuscode)); while(!feof(fp)) { int c = fgetc(fp); - if (c!=EOF) - back[i].r.msg[j++]=c; + + if (c != EOF) + back[i].r.msg[j++] = c; } - back[i].r.msg[j++]='\0'; + back[i].r.msg[j++] = '\0'; fclose(fp); - UNLINK(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok")); - strcpybuff(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok"),""); + UNLINK(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer, ".ok")); + strcpybuff(fconcat + (OPT_GET_BUFF(opt), back[i].location_buffer, ".ok"), ""); } else { - strcpybuff(back[i].r.msg,"Unknown ftp result, check if file is ok"); - back[i].r.statuscode=STATUSCODE_INVALID; + strcpybuff(back[i].r.msg, + "Unknown ftp result, check if file is ok"); + back[i].r.statuscode = STATUSCODE_INVALID; } - back[i].status=STATUS_READY; + back[i].status = STATUS_READY; back_set_finished(sback, i); // finalize transfer - if (back[i].r.statuscode>0) { - back_finalize(opt,cache,sback,i); + if (back[i].r.statuscode > 0) { + back_finalize(opt, cache, sback, i); } } } #endif - else if (back[i].status==STATUS_FTP_READY) { // ftp ready - back[i].status=STATUS_READY; + else if (back[i].status == STATUS_FTP_READY) { // ftp ready + back[i].status = STATUS_READY; back_set_finished(sback, i); // finalize transfer - if (back[i].r.statuscode>0) { - back_finalize(opt,cache,sback,i); + if (back[i].r.statuscode > 0) { + back_finalize(opt, cache, sback, i); } - } - else if ((back[i].status>0) && (back[i].status<1000)) { // en réception http - int dispo=0; - + } else if ((back[i].status > 0) && (back[i].status < 1000)) { // en réception http + int dispo = 0; + // vérifier l'existance de timeout-check if (!gestion_timeout) - if (back[i].timeout>0) - gestion_timeout=1; - - // données dispo? - //## if (back[i].url_adr[0]!=lOCAL_CHAR) - if (back[i].r.is_file) - dispo=1; - else if (back[i].r.ssl) - dispo=1; - else - dispo=FD_ISSET(back[i].r.soc,&fds); - - // Check transfer rate! - if (!max_read_bytes) - dispo=0; // limit transfer rate - - if (dispo) { // données dispo - LLint retour_fread; - busy_recv=1; // on récupère encore + if (back[i].timeout > 0) + gestion_timeout = 1; + + // données dispo? + //## if (back[i].url_adr[0]!=lOCAL_CHAR) + if (back[i].r.is_file) + dispo = 1; + else if (back[i].r.ssl) + dispo = 1; + else + dispo = FD_ISSET(back[i].r.soc, &fds); + + // Check transfer rate! + if (!max_read_bytes) + dispo = 0; // limit transfer rate + + if (dispo) { // données dispo + LLint retour_fread; + + busy_recv = 1; // on récupère encore #if BDEBUG==1 - printf("..data available on socket %d\n",back[i].r.soc); + printf("..data available on socket %d\n", back[i].r.soc); #endif - - // range size hack old location + // range size hack old location #if HTS_DIRECTDISK - // Court-circuit: - // Peut-on stocker le fichier directement sur disque? - // Ahh que ca serait vachement mieux et que ahh que la mémoire vous dit merci! - if (back[i].status) { - if (back[i].r.is_write==0) { // mode mémoire - if (back[i].r.adr==NULL) { // rien n'a été écrit - if (!back[i].testmode) { // pas mode test - if (strnotempty(back[i].url_sav)) { - if (strcmp(back[i].url_fil,"/robots.txt")) { - if (back[i].r.statuscode==HTTP_OK) { // 'OK' - if (!is_hypertext_mime(opt,back[i].r.contenttype, back[i].url_fil)) { // pas HTML - if (opt->getmode&2) { // on peut ecrire des non html - int fcheck=0; - int last_errno = 0; - back[i].r.is_write=1; // écrire - if (back[i].r.compressed - && + // Court-circuit: + // Peut-on stocker le fichier directement sur disque? + // Ahh que ca serait vachement mieux et que ahh que la mémoire vous dit merci! + if (back[i].status) { + if (back[i].r.is_write == 0) { // mode mémoire + if (back[i].r.adr == NULL) { // rien n'a été écrit + if (!back[i].testmode) { // pas mode test + if (strnotempty(back[i].url_sav)) { + if (strcmp(back[i].url_fil, "/robots.txt")) { + if (back[i].r.statuscode == HTTP_OK) { // 'OK' + if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) { // pas HTML + if (opt->getmode & 2) { // on peut ecrire des non html + int fcheck = 0; + int last_errno = 0; + + back[i].r.is_write = 1; // écrire + if (back[i].r.compressed && /* .gz are *NOT* depacked!! */ - (strfield(get_ext(catbuff,back[i].url_sav),"gz") == 0) - ) { + (strfield + (get_ext(catbuff, back[i].url_sav), "gz") == 0) + ) { #if 1 #ifdef _WIN32 #undef tempnam #define tempnam _tempnam #endif - char *const tmp = tempnam(NULL, "httrack_temporaryGzipFile_"); - if (tmp != NULL) { - strcpybuff(back[i].tmpfile_buffer, tmp); - free(tmp); - back[i].tmpfile = back[i].tmpfile_buffer; - } else { - back[i].tmpfile = NULL; - } + char *const tmp = + tempnam(NULL, "httrack_temporaryGzipFile_"); + if (tmp != NULL) { + strcpybuff(back[i].tmpfile_buffer, tmp); + free(tmp); + back[i].tmpfile = back[i].tmpfile_buffer; + } else { + back[i].tmpfile = NULL; + } #else - back[i].tmpfile_buffer[0]='\0'; - back[i].tmpfile=tmpnam(back[p].tmpfile_buffer); + back[i].tmpfile_buffer[0] = '\0'; + back[i].tmpfile = tmpnam(back[p].tmpfile_buffer); #endif - if (back[i].tmpfile != NULL && back[i].tmpfile[0]) { - if ((back[i].r.out=FOPEN(back[i].tmpfile,"wb")) == NULL) { - last_errno = errno; - } - } - } else { - file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 1, 1, back[i].r.notmodified); - back[i].r.compressed=0; - if ((back[i].r.out=filecreate(&opt->state.strc, back[i].url_sav)) == NULL) { + if (back[i].tmpfile != NULL && back[i].tmpfile[0]) { + if ((back[i].r.out = + FOPEN(back[i].tmpfile, "wb")) == NULL) { last_errno = errno; } } - if (back[i].r.out==NULL) { - errno = last_errno; - if ((fcheck=check_fatal_io_errno())) { - hts_log_print(opt, LOG_ERROR, "Mirror aborted: disk full or filesystem problems"); - opt->state.exit_xh=-1; /* fatal error */ - } + } else { + file_notify(opt, back[i].url_adr, back[i].url_fil, + back[i].url_sav, 1, 1, + back[i].r.notmodified); + back[i].r.compressed = 0; + if ((back[i].r.out = + filecreate(&opt->state.strc, + back[i].url_sav)) == NULL) { + last_errno = errno; } + } + if (back[i].r.out == NULL) { + errno = last_errno; + if ((fcheck = check_fatal_io_errno())) { + hts_log_print(opt, LOG_ERROR, + "Mirror aborted: disk full or filesystem problems"); + opt->state.exit_xh = -1; /* fatal error */ + } + } #if HDEBUG - printf("direct-disk: %s\n",back[i].url_sav); -#endif - hts_log_print(opt, LOG_DEBUG, "File received from net to disk: %s%s",back[i].url_adr,back[i].url_fil); - - if (back[i].r.out==NULL) { - hts_log_print(opt, LOG_ERROR | LOG_ERRNO, "Unable to save file %s",back[i].url_sav); - if (fcheck) { - hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up"); - } - back[i].r.is_write=0; // erreur, abandonner - } else { + printf("direct-disk: %s\n", back[i].url_sav); +#endif + hts_log_print(opt, LOG_DEBUG, + "File received from net to disk: %s%s", + back[i].url_adr, back[i].url_fil); + + if (back[i].r.out == NULL) { + hts_log_print(opt, LOG_ERROR | LOG_ERRNO, + "Unable to save file %s", + back[i].url_sav); + if (fcheck) { + hts_log_print(opt, LOG_ERROR, + "* * Fatal write error, giving up"); + } + back[i].r.is_write = 0; // erreur, abandonner + } else { #ifndef _WIN32 - chmod(back[i].url_sav, HTS_ACCESS_FILE); -#endif - /* create a temporary reference file in case of broken mirror */ - if (back[i].r.out != NULL) { - if (back_serialize_ref(opt, &back[i]) != 0) { - hts_log_print(opt, LOG_WARNING, "Could not create temporary reference file for %s%s",back[i].url_adr,back[i].url_fil); - } + chmod(back[i].url_sav, HTS_ACCESS_FILE); +#endif + /* create a temporary reference file in case of broken mirror */ + if (back[i].r.out != NULL) { + if (back_serialize_ref(opt, &back[i]) != 0) { + hts_log_print(opt, LOG_WARNING, + "Could not create temporary reference file for %s%s", + back[i].url_adr, + back[i].url_fil); } } - } else { // on coupe tout! - hts_log_print(opt, LOG_DEBUG, "File cancelled (non HTML): %s%s",back[i].url_adr,back[i].url_fil); - back[i].status=STATUS_READY; // terminé - back_set_finished(sback, i); - if (!back[i].testmode) - back[i].r.statuscode=STATUSCODE_INVALID; // EUHH CANCEL - else - back[i].r.statuscode=STATUSCODE_TEST_OK; // "TEST OK" - if (back[i].r.soc!=INVALID_SOCKET) { + } + } else { // on coupe tout! + hts_log_print(opt, LOG_DEBUG, + "File cancelled (non HTML): %s%s", + back[i].url_adr, back[i].url_fil); + back[i].status = STATUS_READY; // terminé + back_set_finished(sback, i); + if (!back[i].testmode) + back[i].r.statuscode = STATUSCODE_INVALID; // EUHH CANCEL + else + back[i].r.statuscode = STATUSCODE_TEST_OK; // "TEST OK" + if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_wait(3): deletehttp\n"); + DEBUG_W("back_wait(3): deletehttp\n"); #endif - deletehttp(&back[i].r); - } - back[i].r.soc=INVALID_SOCKET; + deletehttp(&back[i].r); } + back[i].r.soc = INVALID_SOCKET; } } } @@ -2731,541 +2970,636 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti } } } -#endif - - // réception de données depuis socket ou fichier - if (back[i].status) { - if (back[i].status==STATUS_WAIT_HEADERS) // recevoir par bloc de lignes - retour_fread=http_xfread1(&(back[i].r),0); - else if (back[i].status==STATUS_CHUNK_WAIT || back[i].status==STATUS_CHUNK_CR) { // recevoir longueur chunk en hexa caractère par caractère - // backuper pour lire dans le buffer chunk - htsblk r; - memcpy(&r, &(back[i].r), sizeof(htsblk)); - back[i].r.is_write=0; // mémoire - back[i].r.adr=back[i].chunk_adr; // adresse - back[i].r.size=back[i].chunk_size; // taille taille chunk - back[i].r.totalsize=-1; // total inconnu - back[i].r.out=NULL; - back[i].r.is_file=0; - // - // ligne par ligne - retour_fread=http_xfread1(&(back[i].r),-1); - // modifier et restaurer - back[i].chunk_adr=back[i].r.adr; // adresse - back[i].chunk_size=back[i].r.size; // taille taille chunk - memcpy(&(back[i].r), &r, sizeof(htsblk)); // restaurer véritable r - } - else if (back[i].is_chunk) { // attention chunk, limiter taille à lire + } +#endif + + // réception de données depuis socket ou fichier + if (back[i].status) { + if (back[i].status == STATUS_WAIT_HEADERS) // recevoir par bloc de lignes + retour_fread = http_xfread1(&(back[i].r), 0); + else if (back[i].status == STATUS_CHUNK_WAIT || back[i].status == STATUS_CHUNK_CR) { // recevoir longueur chunk en hexa caractère par caractère + // backuper pour lire dans le buffer chunk + htsblk r; + + memcpy(&r, &(back[i].r), sizeof(htsblk)); + back[i].r.is_write = 0; // mémoire + back[i].r.adr = back[i].chunk_adr; // adresse + back[i].r.size = back[i].chunk_size; // taille taille chunk + back[i].r.totalsize = -1; // total inconnu + back[i].r.out = NULL; + back[i].r.is_file = 0; + // + // ligne par ligne + retour_fread = http_xfread1(&(back[i].r), -1); + // modifier et restaurer + back[i].chunk_adr = back[i].r.adr; // adresse + back[i].chunk_size = back[i].r.size; // taille taille chunk + memcpy(&(back[i].r), &r, sizeof(htsblk)); // restaurer véritable r + } else if (back[i].is_chunk) { // attention chunk, limiter taille à lire #if CHUNKDEBUG==1 - printf("[%d] read %d bytes\n",(int)back[i].r.soc,(int)min(back[i].r.totalsize-back[i].r.size,max_read_bytes)); + printf("[%d] read %d bytes\n", (int) back[i].r.soc, + (int) min(back[i].r.totalsize - back[i].r.size, + max_read_bytes)); #endif - retour_fread=(int) http_xfread1(&(back[i].r),(int) min(back[i].r.totalsize-back[i].r.size,max_read_bytes)); - } else - retour_fread=(int) http_xfread1(&(back[i].r),(int) max_read_bytes); - // retour_fread=http_fread1(&(back[i].r)); + retour_fread = + (int) http_xfread1(&(back[i].r), + (int) min(back[i].r.totalsize - + back[i].r.size, max_read_bytes)); } else - retour_fread=READ_EOF; // interruption ou annulation interne (peut ne pas être une erreur) - - // Si réception chunk, tester si on est pas à la fin! - if (back[i].status==1) { - if (back[i].is_chunk) { // attendre prochain chunk - if (back[i].r.size==back[i].r.totalsize) { // fin chunk! - //printf("chunk end at %d\n",back[i].r.size); - back[i].status=STATUS_CHUNK_CR; /* fetch ending CRLF */ - if (back[i].chunk_adr!=NULL) { - freet(back[i].chunk_adr); - back[i].chunk_adr=NULL; - } - back[i].chunk_size=0; - retour_fread=0; // pas d'erreur + retour_fread = + (int) http_xfread1(&(back[i].r), (int) max_read_bytes); + // retour_fread=http_fread1(&(back[i].r)); + } else + retour_fread = READ_EOF; // interruption ou annulation interne (peut ne pas être une erreur) + + // Si réception chunk, tester si on est pas à la fin! + if (back[i].status == 1) { + if (back[i].is_chunk) { // attendre prochain chunk + if (back[i].r.size == back[i].r.totalsize) { // fin chunk! + //printf("chunk end at %d\n",back[i].r.size); + back[i].status = STATUS_CHUNK_CR; /* fetch ending CRLF */ + if (back[i].chunk_adr != NULL) { + freet(back[i].chunk_adr); + back[i].chunk_adr = NULL; + } + back[i].chunk_size = 0; + retour_fread = 0; // pas d'erreur #if CHUNKDEBUG==1 - printf("[%d] waiting for current chunk CRLF..\n",(int)back[i].r.soc); + printf("[%d] waiting for current chunk CRLF..\n", + (int) back[i].r.soc); #endif - } - } else if (back[i].r.keep_alive) { - if (back[i].r.size==back[i].r.totalsize) { // fin! - retour_fread=READ_EOF; // end - } + } + } else if (back[i].r.keep_alive) { + if (back[i].r.size == back[i].r.totalsize) { // fin! + retour_fread = READ_EOF; // end } } - - if (retour_fread < 0) { // fin réception - back[i].status=STATUS_READY; // terminé - back_set_finished(sback, i); - /*KA back[i].r.soc=INVALID_SOCKET; */ + } + + if (retour_fread < 0) { // fin réception + back[i].status = STATUS_READY; // terminé + back_set_finished(sback, i); + /*KA back[i].r.soc=INVALID_SOCKET; */ #if CHUNKDEBUG==1 - if (back[i].is_chunk) - printf("[%d] must be the last chunk for %s (connection closed) - %d/%d\n",(int)back[i].r.soc,back[i].url_fil,back[i].r.size,back[i].r.totalsize); + if (back[i].is_chunk) + printf + ("[%d] must be the last chunk for %s (connection closed) - %d/%d\n", + (int) back[i].r.soc, back[i].url_fil, back[i].r.size, + back[i].r.totalsize); #endif - if (retour_fread < 0 && retour_fread != READ_EOF) { - if (back[i].r.size > 0) - strcpybuff(back[i].r.msg, "Interrupted transfer"); - else - strcpybuff(back[i].r.msg, "No data (connection closed)"); - back[i].r.statuscode=STATUSCODE_CONNERROR; - } else if ((back[i].r.statuscode <= 0) && (strnotempty(back[i].r.msg)==0)) { + if (retour_fread < 0 && retour_fread != READ_EOF) { + if (back[i].r.size > 0) + strcpybuff(back[i].r.msg, "Interrupted transfer"); + else + strcpybuff(back[i].r.msg, "No data (connection closed)"); + back[i].r.statuscode = STATUSCODE_CONNERROR; + } else if ((back[i].r.statuscode <= 0) + && (strnotempty(back[i].r.msg) == 0)) { #if HDEBUG - printf("error interruped: %s\n",back[i].r.adr); -#endif - if (back[i].r.size>0) - strcpybuff(back[i].r.msg,"Interrupted transfer"); - else - strcpybuff(back[i].r.msg,"No data (connection closed)"); - back[i].r.statuscode=STATUSCODE_CONNERROR; - } - - // Close socket - if (back[i].r.soc!=INVALID_SOCKET) { + printf("error interruped: %s\n", back[i].r.adr); +#endif + if (back[i].r.size > 0) + strcpybuff(back[i].r.msg, "Interrupted transfer"); + else + strcpybuff(back[i].r.msg, "No data (connection closed)"); + back[i].r.statuscode = STATUSCODE_CONNERROR; + } + // Close socket + if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_wait(4): deletehttp\n"); + DEBUG_W("back_wait(4): deletehttp\n"); #endif - /*KA deletehttp(&back[i].r);*/ - back_maydeletehttp(opt, cache, sback, i); - } - - // finalize transfer - if (back[i].r.statuscode>0 - && !IS_DELAYED_EXT(back[i].url_sav) - ) { - back_finalize(opt,cache,sback,i); - } + /*KA deletehttp(&back[i].r); */ + back_maydeletehttp(opt, cache, sback, i); + } + // finalize transfer + if (back[i].r.statuscode > 0 && !IS_DELAYED_EXT(back[i].url_sav) + ) { + back_finalize(opt, cache, sback, i); + } - if (back[i].r.totalsize>=0) { // tester totalsize + if (back[i].r.totalsize >= 0) { // tester totalsize //if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) { // tester totalsize - if (back[i].r.totalsize!=back[i].r.size) { // pas la même! - if (!opt->tolerant) { - //#if HTS_CL_IS_FATAL - deleteaddr(&back[i].r); - if (back[i].r.size<back[i].r.totalsize) - back[i].r.statuscode=STATUSCODE_CONNERROR; // recatch - sprintf(back[i].r.msg,"Incorrect length ("LLintP" Bytes, "LLintP" expected)",(LLint)back[i].r.size,(LLint)back[i].r.totalsize); - } else { - //#else - // Un warning suffira.. - hts_log_print(opt, LOG_WARNING, "Incorrect length ("LLintP"!="LLintP" expected) for %s%s",(LLint)back[i].r.size,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil); - //#endif - } + if (back[i].r.totalsize != back[i].r.size) { // pas la même! + if (!opt->tolerant) { + //#if HTS_CL_IS_FATAL + deleteaddr(&back[i].r); + if (back[i].r.size < back[i].r.totalsize) + back[i].r.statuscode = STATUSCODE_CONNERROR; // recatch + sprintf(back[i].r.msg, + "Incorrect length (" LLintP " Bytes, " LLintP + " expected)", (LLint) back[i].r.size, + (LLint) back[i].r.totalsize); + } else { + //#else + // Un warning suffira.. + hts_log_print(opt, LOG_WARNING, + "Incorrect length (" LLintP "!=" LLintP + " expected) for %s%s", (LLint) back[i].r.size, + (LLint) back[i].r.totalsize, back[i].url_adr, + back[i].url_fil); + //#endif } } + } #if BDEBUG==1 - printf("transfer ok\n"); + printf("transfer ok\n"); #endif - } else if (retour_fread > 0) { // pas d'erreur de réception et data - if (back[i].timeout>0) { // refresh timeout si besoin est - back[i].timeout_refresh=time_local(); - } - - // Traitement des en têtes chunks ou en têtes - if (back[i].status==STATUS_CHUNK_WAIT || back[i].status==STATUS_CHUNK_CR) { // réception taille chunk en hexa ( après les en têtes, peut ne pas - if (back[i].chunk_size > 0 && back[i].chunk_adr[back[i].chunk_size-1]==10) { - int chunk_size=-1; - char chunk_data[64]; - if (back[i].chunk_size<32) { // pas trop gros - char* chstrip=back[i].chunk_adr; - back[i].chunk_adr[ back[i].chunk_size-1]='\0'; // octet nul - // skip leading spaces or cr - while(isspace(*chstrip)) chstrip++; - chunk_data[0] = '\0'; - strncatbuff(chunk_data, chstrip, sizeof(chunk_data) - 2); - // strip chunk-extension - while( (chstrip = strchr(chunk_data, ';'))) *chstrip='\0'; - while( (chstrip = strchr(chunk_data, ' '))) *chstrip='\0'; - while( (chstrip = strchr(chunk_data, '\r'))) *chstrip='\0'; -#if CHUNKDEBUG==1 - printf("[%d] chunk received and read: %s\n",(int)back[i].r.soc,chunk_data); -#endif - if (back[i].r.totalsize<0) - back[i].r.totalsize=0; // initialiser à 0 (-1 == unknown) - if (back[i].status==STATUS_CHUNK_WAIT) { // "real" chunk - if (sscanf(chunk_data,"%x",&chunk_size) == 1) { - if (chunk_size > 0) - back[i].chunk_blocksize = chunk_size; /* the data block chunk size */ - else - back[i].chunk_blocksize = -1; /* ending */ - back[i].r.totalsize+=chunk_size; // noter taille - if (back[i].r.adr != NULL || !back[i].r.is_write) { // Not to disk - back[i].r.adr=(char*) realloct(back[i].r.adr, (size_t)back[i].r.totalsize + 1); - if (!back[i].r.adr) { - if (cache->log!=NULL) { - hts_log_print(opt, LOG_ERROR, "not enough memory ("LLintP") for %s%s",(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil); - } - } - } + } else if (retour_fread > 0) { // pas d'erreur de réception et data + if (back[i].timeout > 0) { // refresh timeout si besoin est + back[i].timeout_refresh = time_local(); + } + // Traitement des en têtes chunks ou en têtes + if (back[i].status == STATUS_CHUNK_WAIT || back[i].status == STATUS_CHUNK_CR) { // réception taille chunk en hexa ( après les en têtes, peut ne pas + if (back[i].chunk_size > 0 + && back[i].chunk_adr[back[i].chunk_size - 1] == 10) { + int chunk_size = -1; + char chunk_data[64]; + + if (back[i].chunk_size < 32) { // pas trop gros + char *chstrip = back[i].chunk_adr; + + back[i].chunk_adr[back[i].chunk_size - 1] = '\0'; // octet nul + // skip leading spaces or cr + while(isspace(*chstrip)) + chstrip++; + chunk_data[0] = '\0'; + strncatbuff(chunk_data, chstrip, sizeof(chunk_data) - 2); + // strip chunk-extension + while((chstrip = strchr(chunk_data, ';'))) + *chstrip = '\0'; + while((chstrip = strchr(chunk_data, ' '))) + *chstrip = '\0'; + while((chstrip = strchr(chunk_data, '\r'))) + *chstrip = '\0'; #if CHUNKDEBUG==1 - printf("[%d] chunk length: %d - next total "LLintP":\n",(int)back[i].r.soc,(int)chunk_size,(LLint)back[i].r.totalsize); + printf("[%d] chunk received and read: %s\n", + (int) back[i].r.soc, chunk_data); #endif - } else { - hts_log_print(opt, LOG_WARNING, "Illegal chunk (%s) for %s%s",back[i].chunk_adr,back[i].url_adr,back[i].url_fil); + if (back[i].r.totalsize < 0) + back[i].r.totalsize = 0; // initialiser à 0 (-1 == unknown) + if (back[i].status == STATUS_CHUNK_WAIT) { // "real" chunk + if (sscanf(chunk_data, "%x", &chunk_size) == 1) { + if (chunk_size > 0) + back[i].chunk_blocksize = chunk_size; /* the data block chunk size */ + else + back[i].chunk_blocksize = -1; /* ending */ + back[i].r.totalsize += chunk_size; // noter taille + if (back[i].r.adr != NULL || !back[i].r.is_write) { // Not to disk + back[i].r.adr = + (char *) realloct(back[i].r.adr, + (size_t) back[i].r.totalsize + 1); + if (!back[i].r.adr) { + if (cache->log != NULL) { + hts_log_print(opt, LOG_ERROR, + "not enough memory (" LLintP + ") for %s%s", + (LLint) back[i].r.totalsize, + back[i].url_adr, back[i].url_fil); + } + } } - } else { /* back[i].status==STATUS_CHUNK_CR : just receiving ending CRLF after data */ - if (chunk_data[0] == '\0') { - if (back[i].chunk_blocksize > 0) - chunk_size=(int)back[i].chunk_blocksize; /* recent data chunk size */ - else if (back[i].chunk_blocksize == -1) - chunk_size=0; /* ending chunk */ - else - chunk_size=1; /* fake positive size for 1st chunk history */ #if CHUNKDEBUG==1 - printf("[%d] chunk CRLF seen\n", (int)back[i].r.soc); + printf("[%d] chunk length: %d - next total " LLintP ":\n", + (int) back[i].r.soc, (int) chunk_size, + (LLint) back[i].r.totalsize); #endif - } else { - hts_log_print(opt, LOG_WARNING, "illegal chunk CRLF (%s) for %s%s", back[i].chunk_adr,back[i].url_adr,back[i].url_fil); + } else { + hts_log_print(opt, LOG_WARNING, + "Illegal chunk (%s) for %s%s", + back[i].chunk_adr, back[i].url_adr, + back[i].url_fil); + } + } else { /* back[i].status==STATUS_CHUNK_CR : just receiving ending CRLF after data */ + if (chunk_data[0] == '\0') { + if (back[i].chunk_blocksize > 0) + chunk_size = (int) back[i].chunk_blocksize; /* recent data chunk size */ + else if (back[i].chunk_blocksize == -1) + chunk_size = 0; /* ending chunk */ + else + chunk_size = 1; /* fake positive size for 1st chunk history */ #if CHUNKDEBUG==1 - printf("[%d] chunk CRLF ERROR!! : '%s'\n", (int)back[i].r.soc, chunk_data); + printf("[%d] chunk CRLF seen\n", (int) back[i].r.soc); +#endif + } else { + hts_log_print(opt, LOG_WARNING, + "illegal chunk CRLF (%s) for %s%s", + back[i].chunk_adr, back[i].url_adr, + back[i].url_fil); +#if CHUNKDEBUG==1 + printf("[%d] chunk CRLF ERROR!! : '%s'\n", + (int) back[i].r.soc, chunk_data); #endif - } } - } else { - hts_log_print(opt, LOG_WARNING, "chunk too big ("LLintP") for %s%s",(LLint)back[i].chunk_size,back[i].url_adr,back[i].url_fil); } - - // ok, continuer sur le body - - // si chunk non nul continuer (ou commencer) - if (back[i].status==STATUS_CHUNK_CR && chunk_size > 0) { - back[i].status = STATUS_CHUNK_WAIT; /* waiting for next chunk (NN\r\n<data>\r\nNN\r\n<data>..\r\n0\r\n\r\n) */ + } else { + hts_log_print(opt, LOG_WARNING, + "chunk too big (" LLintP ") for %s%s", + (LLint) back[i].chunk_size, back[i].url_adr, + back[i].url_fil); + } + + // ok, continuer sur le body + + // si chunk non nul continuer (ou commencer) + if (back[i].status == STATUS_CHUNK_CR && chunk_size > 0) { + back[i].status = STATUS_CHUNK_WAIT; /* waiting for next chunk (NN\r\n<data>\r\nNN\r\n<data>..\r\n0\r\n\r\n) */ #if CHUNKDEBUG==1 - printf("[%d] waiting for next chunk\n", (int)back[i].r.soc); + printf("[%d] waiting for next chunk\n", (int) back[i].r.soc); #endif - } else if (back[i].status==STATUS_CHUNK_WAIT && chunk_size == 0) { /* final chunk */ - back[i].status=STATUS_CHUNK_CR; /* final CRLF */ + } else if (back[i].status == STATUS_CHUNK_WAIT && chunk_size == 0) { /* final chunk */ + back[i].status = STATUS_CHUNK_CR; /* final CRLF */ #if CHUNKDEBUG==1 - printf("[%d] waiting for final CRLF (chunk)\n", (int)back[i].r.soc); + printf("[%d] waiting for final CRLF (chunk)\n", + (int) back[i].r.soc); #endif - } else if (back[i].status==STATUS_CHUNK_WAIT && chunk_size >= 0) { /* will fetch data now */ - back[i].status=1; // continuer body + } else if (back[i].status == STATUS_CHUNK_WAIT && chunk_size >= 0) { /* will fetch data now */ + back[i].status = 1; // continuer body #if CHUNKDEBUG==1 - printf("[%d] waiting for body (chunk)\n", (int)back[i].r.soc); + printf("[%d] waiting for body (chunk)\n", + (int) back[i].r.soc); #endif - } else { /* zero-size-chunk-CRLF (end) or error */ + } else { /* zero-size-chunk-CRLF (end) or error */ #if CHUNKDEBUG==1 - printf("[%d] chunk end, total: %d\n",(int)back[i].r.soc,back[i].r.size); + printf("[%d] chunk end, total: %d\n", (int) back[i].r.soc, + back[i].r.size); #endif - /* End */ - //if (back[i].status==STATUS_CHUNK_CR) { - back[i].status=STATUS_READY; // fin - back_set_finished(sback, i); - //} + /* End */ + //if (back[i].status==STATUS_CHUNK_CR) { + back[i].status = STATUS_READY; // fin + back_set_finished(sback, i); + //} - // finalize transfer if not temporary - if (!IS_DELAYED_EXT(back[i].url_sav)) { - back_finalize(opt,cache,sback,i); - } else { - if (back[i].r.statuscode == HTTP_OK) { - hts_log_print(opt, LOG_WARNING, "unexpected incomplete type with 200 code at %s%s", back[i].url_adr, back[i].url_fil); - } + // finalize transfer if not temporary + if (!IS_DELAYED_EXT(back[i].url_sav)) { + back_finalize(opt, cache, sback, i); + } else { + if (back[i].r.statuscode == HTTP_OK) { + hts_log_print(opt, LOG_WARNING, + "unexpected incomplete type with 200 code at %s%s", + back[i].url_adr, back[i].url_fil); } - if (back[i].r.soc!=INVALID_SOCKET) { + } + if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_wait(5): deletehttp\n"); -#endif - /* Error */ - if (chunk_size < 0) { - deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - deleteaddr(&back[i].r); - back[i].r.statuscode=STATUSCODE_INVALID; - strcpybuff(back[i].r.msg,"Invalid chunk"); + DEBUG_W("back_wait(5): deletehttp\n"); +#endif + /* Error */ + if (chunk_size < 0) { + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; + deleteaddr(&back[i].r); + back[i].r.statuscode = STATUSCODE_INVALID; + strcpybuff(back[i].r.msg, "Invalid chunk"); #if CHUNKDEBUG==1 - printf("[%d] chunk error\n", (int)back[i].r.soc); + printf("[%d] chunk error\n", (int) back[i].r.soc); #endif - } else /* if chunk_size == 0 */ { + } else { /* if chunk_size == 0 */ + #if CHUNKDEBUG==1 - printf("[%d] all chunks now received\n", (int)back[i].r.soc); -#endif - - /* Tester totalsize en fin de chunk */ - if ((back[i].r.totalsize>=0)) { // tester totalsize - if (back[i].r.totalsize!=back[i].r.size) { // pas la même! - if (!opt->tolerant) { - deleteaddr(&back[i].r); - back[i].r.statuscode=STATUSCODE_INVALID; - strcpybuff(back[i].r.msg,"Incorrect length"); - } else { - // Un warning suffira.. - hts_log_print(opt, LOG_WARNING, "Incorrect length ("LLintP"!="LLintP" expected) for %s%s",(LLint)back[i].r.size,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil); - } + printf("[%d] all chunks now received\n", + (int) back[i].r.soc); +#endif + + /* Tester totalsize en fin de chunk */ + if ((back[i].r.totalsize >= 0)) { // tester totalsize + if (back[i].r.totalsize != back[i].r.size) { // pas la même! + if (!opt->tolerant) { + deleteaddr(&back[i].r); + back[i].r.statuscode = STATUSCODE_INVALID; + strcpybuff(back[i].r.msg, "Incorrect length"); + } else { + // Un warning suffira.. + hts_log_print(opt, LOG_WARNING, + "Incorrect length (" LLintP "!=" + LLintP " expected) for %s%s", + (LLint) back[i].r.size, + (LLint) back[i].r.totalsize, + back[i].url_adr, back[i].url_fil); } } - - /* Oops, trailers! */ - if (back[i].r.keep_alive_trailers) { - /* fixme (not yet supported) */ - } - } - - + + /* Oops, trailers! */ + if (back[i].r.keep_alive_trailers) { + /* fixme (not yet supported) */ + } + } + } - - // effacer buffer (chunk en tete) - if (back[i].chunk_adr!=NULL) { - freet(back[i].chunk_adr); - back[i].chunk_adr=NULL; - back[i].chunk_size=0; - // NO! xxback[i].chunk_blocksize = 0; - } - - } // taille buffer chunk > 1 && LF - // - } else if (back[i].status==STATUS_WAIT_HEADERS) { // en têtes (avant le chunk si il est présent) - // - if (back[i].r.size>=2) { - // double LF - if ( - ((back[i].r.adr[back[i].r.size-1]==10) && (back[i].r.adr[back[i].r.size-2]==10)) - || - (back[i].r.adr[0] == '<') /* bogus server */ - ) { - char rcvd[2048]; - int ptr=0; - int noFreebuff=0; - + } + + // effacer buffer (chunk en tete) + if (back[i].chunk_adr != NULL) { + freet(back[i].chunk_adr); + back[i].chunk_adr = NULL; + back[i].chunk_size = 0; + // NO! xxback[i].chunk_blocksize = 0; + } + + } // taille buffer chunk > 1 && LF + // + } else if (back[i].status == STATUS_WAIT_HEADERS) { // en têtes (avant le chunk si il est présent) + // + if (back[i].r.size >= 2) { + // double LF + if (((back[i].r.adr[back[i].r.size - 1] == 10) + && (back[i].r.adr[back[i].r.size - 2] == 10)) + || (back[i].r.adr[0] == '<') /* bogus server */ + ) { + char rcvd[2048]; + int ptr = 0; + int noFreebuff = 0; + #if BDEBUG==1 - printf("..ok, header received\n"); -#endif - - - /* Hack for zero-length headers */ - if (back[i].status != 0 && back[i].r.adr[0] != '<') { - - // ---------------------------------------- - // traiter en-tête! - // status-line à récupérer - ptr+=binput(back[i].r.adr+ptr,rcvd,2000); - if (strnotempty(rcvd)==0) { - /* Bogus CRLF, OR recycled connection and trailing chunk CRLF */ - ptr+=binput(back[i].r.adr+ptr,rcvd,2000); - } - - // traiter status-line - treatfirstline(&back[i].r,rcvd); - -#if HDEBUG - printf("(Buffer) Status-Code=%d\n",back[i].r.statuscode); -#endif - if (_DEBUG_HEAD) { - if (ioinfo) { - fprintf(ioinfo,"[%d] response for %s%s:\r\ncode=%d\r\n", - back[i].r.debugid, jump_identification(back[i].url_adr),back[i].url_fil,back[i].r.statuscode); - fprintfio(ioinfo,back[i].r.adr,">>> "); - fprintf(ioinfo,"\r\n"); - fflush(ioinfo); - } // en-tête - } - - // header // ** !attention! HTTP/0.9 non supporté - do { - ptr+=binput(back[i].r.adr+ptr,rcvd,2000); + printf("..ok, header received\n"); +#endif + + /* Hack for zero-length headers */ + if (back[i].status != 0 && back[i].r.adr[0] != '<') { + + // ---------------------------------------- + // traiter en-tête! + // status-line à récupérer + ptr += binput(back[i].r.adr + ptr, rcvd, 2000); + if (strnotempty(rcvd) == 0) { + /* Bogus CRLF, OR recycled connection and trailing chunk CRLF */ + ptr += binput(back[i].r.adr + ptr, rcvd, 2000); + } + // traiter status-line + treatfirstline(&back[i].r, rcvd); + #if HDEBUG - printf("(buffer)>%s\n",rcvd); + printf("(Buffer) Status-Code=%d\n", back[i].r.statuscode); #endif - /* - if (_DEBUG_HEAD) { - if (ioinfo) { - fprintf(ioinfo,"(buffer)>%s\r\n",rcvd); + if (_DEBUG_HEAD) { + if (ioinfo) { + fprintf(ioinfo, + "[%d] response for %s%s:\r\ncode=%d\r\n", + back[i].r.debugid, + jump_identification(back[i].url_adr), + back[i].url_fil, back[i].r.statuscode); + fprintfio(ioinfo, back[i].r.adr, ">>> "); + fprintf(ioinfo, "\r\n"); fflush(ioinfo); - } - } - */ - - if (strnotempty(rcvd)) - treathead(opt->cookie,back[i].url_adr,back[i].url_fil,&back[i].r,rcvd); // traiter - - // parfois les serveurs buggés renvoient un content-range avec un 200 - if (back[i].r.statuscode==HTTP_OK) // 'OK' - if (strfield(rcvd, "content-range:")) { // Avec un content-range: relisez les RFC.. - // Fake range (the file is complete) - if (!(back[i].r.crange_start == 0 && back[i].r.crange_end == back[i].r.crange - 1)) { - back[i].r.statuscode=HTTP_PARTIAL_CONTENT; // FORCER A 206 !!!!! - } + } // en-tête + } + // header // ** !attention! HTTP/0.9 non supporté + do { + ptr += binput(back[i].r.adr + ptr, rcvd, 2000); +#if HDEBUG + printf("(buffer)>%s\n", rcvd); +#endif + /* + if (_DEBUG_HEAD) { + if (ioinfo) { + fprintf(ioinfo,"(buffer)>%s\r\n",rcvd); + fflush(ioinfo); + } + } + */ + + if (strnotempty(rcvd)) + treathead(opt->cookie, back[i].url_adr, back[i].url_fil, &back[i].r, rcvd); // traiter + + // parfois les serveurs buggés renvoient un content-range avec un 200 + if (back[i].r.statuscode == HTTP_OK) // 'OK' + if (strfield(rcvd, "content-range:")) { // Avec un content-range: relisez les RFC.. + // Fake range (the file is complete) + if (! + (back[i].r.crange_start == 0 + && back[i].r.crange_end == + back[i].r.crange - 1)) { + back[i].r.statuscode = HTTP_PARTIAL_CONTENT; // FORCER A 206 !!!!! } - - } while(strnotempty(rcvd)); - // ---------------------------------------- + } - } else { - // assume text/html, OK - treatfirstline(&back[i].r, back[i].r.adr); - noFreebuff=1; - } - - // Callback - { - int test_head = RUN_CALLBACK6(opt, receivehead, - back[i].r.adr, back[i].url_adr, back[i].url_fil, back[i].referer_adr, back[i].referer_fil, &back[i].r); - if (test_head!=1) { - hts_log_print(opt, LOG_WARNING, "External wrapper aborted transfer, breaking connection: %s%s",back[i].url_adr,back[i].url_fil); - back[i].status=STATUS_READY; // FINI - back_set_finished(sback, i); - deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - strcpybuff(back[i].r.msg,"External wrapper aborted transfer"); - back[i].r.statuscode = STATUSCODE_INVALID; - } + } while(strnotempty(rcvd)); + // ---------------------------------------- + + } else { + // assume text/html, OK + treatfirstline(&back[i].r, back[i].r.adr); + noFreebuff = 1; + } + + // Callback + { + int test_head = RUN_CALLBACK6(opt, receivehead, + back[i].r.adr, + back[i].url_adr, + back[i].url_fil, + back[i].referer_adr, + back[i].referer_fil, + &back[i].r); + if (test_head != 1) { + hts_log_print(opt, LOG_WARNING, + "External wrapper aborted transfer, breaking connection: %s%s", + back[i].url_adr, back[i].url_fil); + back[i].status = STATUS_READY; // FINI + back_set_finished(sback, i); + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; + strcpybuff(back[i].r.msg, + "External wrapper aborted transfer"); + back[i].r.statuscode = STATUSCODE_INVALID; } + } - // Free headers memory now - // Actually, save them for informational purpose - if (!noFreebuff) { - char* block = back[i].r.adr; - back[i].r.adr = NULL; - deleteaddr(&back[i].r); - back[i].r.headers = block; - } - - /* - Status code and header-response hacks - */ - - - // Check response : 203 == 200 - if (back[i].r.statuscode==HTTP_NON_AUTHORITATIVE_INFORMATION) { - back[i].r.statuscode=HTTP_OK; // forcer "OK" - } else if (back[i].r.statuscode == HTTP_CONTINUE) { - back[i].status=STATUS_WAIT_HEADERS; - back[i].r.size=0; - back[i].r.totalsize=-1; - back[i].chunk_size=0; - back[i].r.statuscode=STATUSCODE_INVALID; - back[i].r.msg[0]='\0'; - hts_log_print(opt, LOG_DEBUG, "Status 100 detected for %s%s, continuing headers",back[i].url_adr,back[i].url_fil); - continue; + // Free headers memory now + // Actually, save them for informational purpose + if (!noFreebuff) { + char *block = back[i].r.adr; + + back[i].r.adr = NULL; + deleteaddr(&back[i].r); + back[i].r.headers = block; + } + + /* + Status code and header-response hacks + */ + + // Check response : 203 == 200 + if (back[i].r.statuscode == + HTTP_NON_AUTHORITATIVE_INFORMATION) { + back[i].r.statuscode = HTTP_OK; // forcer "OK" + } else if (back[i].r.statuscode == HTTP_CONTINUE) { + back[i].status = STATUS_WAIT_HEADERS; + back[i].r.size = 0; + back[i].r.totalsize = -1; + back[i].chunk_size = 0; + back[i].r.statuscode = STATUSCODE_INVALID; + back[i].r.msg[0] = '\0'; + hts_log_print(opt, LOG_DEBUG, + "Status 100 detected for %s%s, continuing headers", + back[i].url_adr, back[i].url_fil); + continue; + } + + /* + Solve "false" 416 problems + */ + if (back[i].r.statuscode == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE) { // 'Requested Range Not Satisfiable' + // Example: + // Range: bytes=2830- + // -> + // Content-Range: bytes */2830 + if (back[i].range_req_size == back[i].r.crange) { + filenote(&opt->state.strc, back[i].url_sav, NULL); + file_notify(opt, back[i].url_adr, back[i].url_fil, + back[i].url_sav, 0, 0, back[i].r.notmodified); + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; + back[i].status = STATUS_READY; // READY + back_set_finished(sback, i); + back[i].r.size = back[i].r.totalsize = + back[i].range_req_size; + back[i].r.statuscode = HTTP_NOT_MODIFIED; // NOT MODIFIED + hts_log_print(opt, LOG_DEBUG, + "File seems complete (good 416 message), breaking connection: %s%s", + back[i].url_adr, back[i].url_fil); } - - /* - Solve "false" 416 problems - */ - if (back[i].r.statuscode==HTTP_REQUESTED_RANGE_NOT_SATISFIABLE) { // 'Requested Range Not Satisfiable' - // Example: - // Range: bytes=2830- - // -> - // Content-Range: bytes */2830 - if (back[i].range_req_size == back[i].r.crange) { - filenote(&opt->state.strc,back[i].url_sav,NULL); - file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); - deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - back[i].status=STATUS_READY; // READY - back_set_finished(sback, i); - back[i].r.size=back[i].r.totalsize=back[i].range_req_size; - back[i].r.statuscode=HTTP_NOT_MODIFIED; // NOT MODIFIED - hts_log_print(opt, LOG_DEBUG, "File seems complete (good 416 message), breaking connection: %s%s",back[i].url_adr,back[i].url_fil); + } + // transform 406 into 200 ; we'll catch embedded links inside the choice page + if (back[i].r.statuscode == 406) { // 'Not Acceptable' + back[i].r.statuscode = HTTP_OK; + } + // 'do not erase already downloaded file' + // on an updated file + // with an error : consider a 304 error + if (!opt->delete_old) { + if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update + && !back[i].testmode) { + if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) { + hts_log_print(opt, LOG_DEBUG, + "Error ignored %d (%s) because of 'no purge' option for %s%s", + back[i].r.statuscode, back[i].r.msg, + back[i].url_adr, back[i].url_fil); + back[i].r.statuscode = HTTP_NOT_MODIFIED; + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; } } - - // transform 406 into 200 ; we'll catch embedded links inside the choice page - if (back[i].r.statuscode==406) { // 'Not Acceptable' - back[i].r.statuscode=HTTP_OK; - } - - // 'do not erase already downloaded file' - // on an updated file - // with an error : consider a 304 error - if (!opt->delete_old) { - if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update && !back[i].testmode) { - if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) { - hts_log_print(opt, LOG_DEBUG, "Error ignored %d (%s) because of 'no purge' option for %s%s",back[i].r.statuscode,back[i].r.msg,back[i].url_adr,back[i].url_fil); - back[i].r.statuscode = HTTP_NOT_MODIFIED; - deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - } - } - } - - // Various hacks to limit re-transfers when updating a mirror - // Force update if same size detected - if (opt->sizehack) { - // We already have the file - // and ask the remote server for an update - // Some servers, especially dynamic pages severs, always - // answer that the page has been modified since last visit - // And answer with a 200 (OK) response, and the same page - // If the size is the same, and the option has been set, we assume - // that the file is identical - and therefore let's break the connection - if (back[i].is_update) { // mise à jour - if (back[i].r.statuscode==HTTP_OK && !back[i].testmode) { // 'OK' - htsblk r = cache_read(opt,cache,back[i].url_adr,back[i].url_fil,NULL,NULL); // lire entrée cache - if (r.statuscode == HTTP_OK) { // OK pas d'erreur cache - LLint len1,len2; - len1=r.totalsize; - len2=back[i].r.totalsize; - if (r.size>0) - len1=r.size; - if (len1>=0) { - if (len1 == len2) { // tailles identiques - back[i].r.statuscode=HTTP_NOT_MODIFIED; // forcer NOT MODIFIED - deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - hts_log_print(opt, LOG_DEBUG, "File seems complete (same size), breaking connection: %s%s",back[i].url_adr,back[i].url_fil); - } + } + // Various hacks to limit re-transfers when updating a mirror + // Force update if same size detected + if (opt->sizehack) { + // We already have the file + // and ask the remote server for an update + // Some servers, especially dynamic pages severs, always + // answer that the page has been modified since last visit + // And answer with a 200 (OK) response, and the same page + // If the size is the same, and the option has been set, we assume + // that the file is identical - and therefore let's break the connection + if (back[i].is_update) { // mise à jour + if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) { // 'OK' + htsblk r = cache_read(opt, cache, back[i].url_adr, back[i].url_fil, NULL, NULL); // lire entrée cache + + if (r.statuscode == HTTP_OK) { // OK pas d'erreur cache + LLint len1, len2; + + len1 = r.totalsize; + len2 = back[i].r.totalsize; + if (r.size > 0) + len1 = r.size; + if (len1 >= 0) { + if (len1 == len2) { // tailles identiques + back[i].r.statuscode = HTTP_NOT_MODIFIED; // forcer NOT MODIFIED + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; + hts_log_print(opt, LOG_DEBUG, + "File seems complete (same size), breaking connection: %s%s", + back[i].url_adr, back[i].url_fil); } - } else { - hts_log_print(opt, LOG_WARNING, "File seems complete (same size), but there was a cache read error (%u): %s%s", r.statuscode, back[i].url_adr, back[i].url_fil); - } - if (r.adr) { - freet(r.adr); - r.adr = NULL; } + } else { + hts_log_print(opt, LOG_WARNING, + "File seems complete (same size), but there was a cache read error (%u): %s%s", + r.statuscode, back[i].url_adr, + back[i].url_fil); + } + if (r.adr) { + freet(r.adr); + r.adr = NULL; } } } - - // Various hacks to limit re-transfers when updating a mirror - // Detect already downloaded file (with another browser, for example) - if (opt->sizehack) { - if (!back[i].is_update) { // mise à jour - if (back[i].r.statuscode==HTTP_OK && !back[i].testmode) { // 'OK' - if (!is_hypertext_mime(opt,back[i].r.contenttype, back[i].url_fil)) { // not HTML - if (strnotempty(back[i].url_sav)) { // target found - int size = fsize_utf8(back[i].url_sav); // target size - if (size >= 0) { - if (back[i].r.totalsize == size) { // same size! - deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - back[i].status=STATUS_READY; // READY - back_set_finished(sback, i); - back[i].r.size=back[i].r.totalsize; - filenote(&opt->state.strc,back[i].url_sav,NULL); - file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); - back[i].r.statuscode=HTTP_NOT_MODIFIED; // NOT MODIFIED - hts_log_print(opt, LOG_DEBUG, "File seems complete (same size file discovered), breaking connection: %s%s",back[i].url_adr,back[i].url_fil); - } + } + // Various hacks to limit re-transfers when updating a mirror + // Detect already downloaded file (with another browser, for example) + if (opt->sizehack) { + if (!back[i].is_update) { // mise à jour + if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) { // 'OK' + if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) { // not HTML + if (strnotempty(back[i].url_sav)) { // target found + int size = fsize_utf8(back[i].url_sav); // target size + + if (size >= 0) { + if (back[i].r.totalsize == size) { // same size! + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; + back[i].status = STATUS_READY; // READY + back_set_finished(sback, i); + back[i].r.size = back[i].r.totalsize; + filenote(&opt->state.strc, back[i].url_sav, + NULL); + file_notify(opt, back[i].url_adr, + back[i].url_fil, back[i].url_sav, 0, + 0, back[i].r.notmodified); + back[i].r.statuscode = HTTP_NOT_MODIFIED; // NOT MODIFIED + hts_log_print(opt, LOG_DEBUG, + "File seems complete (same size file discovered), breaking connection: %s%s", + back[i].url_adr, back[i].url_fil); } } } } } } - - // Various hacks to limit re-transfers when updating a mirror - // Detect bad range: header - if (opt->sizehack) { - // We have request for a partial file (with a 'Range: NNN-' header) - // and received a complete file notification (200), with 'Content-length: NNN' - // it might be possible that we had the complete file - // this is the case in *most* cases, so break the connection - if (back[i].r.is_write==0) { // mode mémoire - if (back[i].r.adr==NULL) { // rien n'a été écrit - if (!back[i].testmode) { // pas mode test - if (strnotempty(back[i].url_sav)) { - if (strcmp(back[i].url_fil,"/robots.txt")) { - if (back[i].r.statuscode==HTTP_OK) { // 'OK' - if (!is_hypertext_mime(opt,back[i].r.contenttype, back[i].url_fil)) { // pas HTML - if (back[i].r.statuscode==HTTP_OK) { // "OK" - if (back[i].range_req_size>0) { // but Range: requested - if (back[i].range_req_size == back[i].r.totalsize) { // And same size + } + // Various hacks to limit re-transfers when updating a mirror + // Detect bad range: header + if (opt->sizehack) { + // We have request for a partial file (with a 'Range: NNN-' header) + // and received a complete file notification (200), with 'Content-length: NNN' + // it might be possible that we had the complete file + // this is the case in *most* cases, so break the connection + if (back[i].r.is_write == 0) { // mode mémoire + if (back[i].r.adr == NULL) { // rien n'a été écrit + if (!back[i].testmode) { // pas mode test + if (strnotempty(back[i].url_sav)) { + if (strcmp(back[i].url_fil, "/robots.txt")) { + if (back[i].r.statuscode == HTTP_OK) { // 'OK' + if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) { // pas HTML + if (back[i].r.statuscode == HTTP_OK) { // "OK" + if (back[i].range_req_size > 0) { // but Range: requested + if (back[i].range_req_size == back[i].r.totalsize) { // And same size #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_wait(skip_range): deletehttp\n"); -#endif - deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - back[i].status=STATUS_READY; // READY - back_set_finished(sback, i); - back[i].r.size=back[i].r.totalsize; - filenote(&opt->state.strc,back[i].url_sav,NULL); - file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); - back[i].r.statuscode=HTTP_NOT_MODIFIED; // NOT MODIFIED - hts_log_print(opt, LOG_DEBUG, "File seems complete (reget failed), breaking connection: %s%s",back[i].url_adr,back[i].url_fil); - } + DEBUG_W + ("back_wait(skip_range): deletehttp\n"); +#endif + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; + back[i].status = STATUS_READY; // READY + back_set_finished(sback, i); + back[i].r.size = back[i].r.totalsize; + filenote(&opt->state.strc, + back[i].url_sav, NULL); + file_notify(opt, back[i].url_adr, + back[i].url_fil, + back[i].url_sav, 0, 0, + back[i].r.notmodified); + back[i].r.statuscode = HTTP_NOT_MODIFIED; // NOT MODIFIED + hts_log_print(opt, LOG_DEBUG, + "File seems complete (reget failed), breaking connection: %s%s", + back[i].url_adr, + back[i].url_fil); } } - } + } } } @@ -3273,353 +3607,385 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti } } } - // END - Various hacks to limit re-transfers when updating a mirror - - /* - End of status code and header-response hacks - */ - - - - /* Interdiction taille par le wizard? */ - if (back[i].r.soc!=INVALID_SOCKET) { - if (!back_checksize(opt,&back[i],1)) { - back[i].status=STATUS_READY; // FINI - back_set_finished(sback, i); - back[i].r.statuscode=STATUSCODE_TOO_BIG; - deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - if (!back[i].testmode) - strcpybuff(back[i].r.msg,"File too big"); - else - strcpybuff(back[i].r.msg,"Test: File too big"); - } + } + // END - Various hacks to limit re-transfers when updating a mirror + + /* + End of status code and header-response hacks + */ + + /* Interdiction taille par le wizard? */ + if (back[i].r.soc != INVALID_SOCKET) { + if (!back_checksize(opt, &back[i], 1)) { + back[i].status = STATUS_READY; // FINI + back_set_finished(sback, i); + back[i].r.statuscode = STATUSCODE_TOO_BIG; + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; + if (!back[i].testmode) + strcpybuff(back[i].r.msg, "File too big"); + else + strcpybuff(back[i].r.msg, "Test: File too big"); } - - /* sinon, continuer */ - /* if (back[i].r.soc!=INVALID_SOCKET) { // ok récupérer body? */ - // head: terminé - if (back[i].head_request) { - hts_log_print(opt, LOG_DEBUG, "Tested file: %s%s",back[i].url_adr,back[i].url_fil); + } + + /* sinon, continuer */ + /* if (back[i].r.soc!=INVALID_SOCKET) { // ok récupérer body? */ + // head: terminé + if (back[i].head_request) { + hts_log_print(opt, LOG_DEBUG, "Tested file: %s%s", + back[i].url_adr, back[i].url_fil); #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_wait(head request): deletehttp\n"); + DEBUG_W("back_wait(head request): deletehttp\n"); #endif - // Couper connexion - if (!back[i].http11) { /* NO KA */ - deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - } - back[i].status=STATUS_READY; // terminé - back_set_finished(sback, i); + // Couper connexion + if (!back[i].http11) { /* NO KA */ + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; } - // traiter une éventuelle erreur 304 (cache à jour utilisable) - else if (back[i].r.statuscode==HTTP_NOT_MODIFIED) { // document à jour dans le cache - // lire dans le cache - // ** NOTE: pas de vérif de la taille ici!! + back[i].status = STATUS_READY; // terminé + back_set_finished(sback, i); + } + // traiter une éventuelle erreur 304 (cache à jour utilisable) + else if (back[i].r.statuscode == HTTP_NOT_MODIFIED) { // document à jour dans le cache + // lire dans le cache + // ** NOTE: pas de vérif de la taille ici!! #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_wait(file is not modified): deletehttp\n"); -#endif - /* clear everything but connection: switch, close, and reswitch */ - { - htsblk tmp; - memset(&tmp, 0, sizeof(tmp)); - back_connxfr(&back[i].r, &tmp); - back[i].r=cache_read(opt,cache,back[i].url_adr,back[i].url_fil,back[i].url_sav,back[i].location_buffer); - back[i].r.location=back[i].location_buffer; - back_connxfr(&tmp,&back[i].r); - } + DEBUG_W("back_wait(file is not modified): deletehttp\n"); +#endif + /* clear everything but connection: switch, close, and reswitch */ + { + htsblk tmp; + + memset(&tmp, 0, sizeof(tmp)); + back_connxfr(&back[i].r, &tmp); + back[i].r = + cache_read(opt, cache, back[i].url_adr, back[i].url_fil, + back[i].url_sav, back[i].location_buffer); + back[i].r.location = back[i].location_buffer; + back_connxfr(&tmp, &back[i].r); + } - // hack: - // In case of 'if-unmodified-since' hack, a 304 status can be sent - // then, force 'ok' status - if (back[i].r.statuscode == STATUSCODE_INVALID) { - if (fexist_utf8(back[i].url_sav)) { - back[i].r.statuscode=HTTP_OK; // OK - strcpybuff(back[i].r.msg, "OK (cached)"); - back[i].r.is_file=1; - back[i].r.totalsize = back[i].r.size = fsize_utf8(back[i].url_sav); - get_httptype(opt,back[i].r.contenttype, back[i].url_sav, 1); - hts_log_print(opt, LOG_DEBUG, "Not-modified status without cache guessed: %s%s",back[i].url_adr,back[i].url_fil); - } + // hack: + // In case of 'if-unmodified-since' hack, a 304 status can be sent + // then, force 'ok' status + if (back[i].r.statuscode == STATUSCODE_INVALID) { + if (fexist_utf8(back[i].url_sav)) { + back[i].r.statuscode = HTTP_OK; // OK + strcpybuff(back[i].r.msg, "OK (cached)"); + back[i].r.is_file = 1; + back[i].r.totalsize = back[i].r.size = + fsize_utf8(back[i].url_sav); + get_httptype(opt, back[i].r.contenttype, + back[i].url_sav, 1); + hts_log_print(opt, LOG_DEBUG, + "Not-modified status without cache guessed: %s%s", + back[i].url_adr, back[i].url_fil); } + } + // Status is okay? + if (back[i].r.statuscode != -1) { // pas d'erreur de lecture + back[i].status = STATUS_READY; // OK prêt + back_set_finished(sback, i); + back[i].r.notmodified = 1; // NON modifié! + hts_log_print(opt, LOG_DEBUG, + "File loaded after test from cache: %s%s", + back[i].url_adr, back[i].url_fil); - // Status is okay? - if (back[i].r.statuscode!=-1) { // pas d'erreur de lecture - back[i].status=STATUS_READY; // OK prêt - back_set_finished(sback, i); - back[i].r.notmodified=1; // NON modifié! - hts_log_print(opt, LOG_DEBUG, "File loaded after test from cache: %s%s",back[i].url_adr,back[i].url_fil); - - // finalize - //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); // not modified - if (back[i].r.statuscode>0) { - back_finalize(opt,cache,sback,i); - } - + // finalize + //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); // not modified + if (back[i].r.statuscode > 0) { + back_finalize(opt, cache, sback, i); + } #if DEBUGCA - printf("..document à jour après requète: %s%s\n",back[i].url_adr,back[i].url_fil); + printf("..document à jour après requète: %s%s\n", + back[i].url_adr, back[i].url_fil); #endif - - //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode); - } else { // erreur - back[i].status=STATUS_READY; // terminé - back_set_finished(sback, i); - //printf("erreur cache\n"); - - } - + + //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode); + } else { // erreur + back[i].status = STATUS_READY; // terminé + back_set_finished(sback, i); + //printf("erreur cache\n"); + + } + /********** NO - must complete the body! ********** */ #if 0 - } else if (HTTP_IS_REDIRECT(back[i].r.statuscode) - || (back[i].r.statuscode==412) - || (back[i].r.statuscode==416) - ) { // Ne pas prendre le html, erreurs connues et gérées + } else if (HTTP_IS_REDIRECT(back[i].r.statuscode) + || (back[i].r.statuscode == 412) + || (back[i].r.statuscode == 416) + ) { // Ne pas prendre le html, erreurs connues et gérées #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_wait(301,302,303,307,412,416..): deletehttp\n"); + DEBUG_W + ("back_wait(301,302,303,307,412,416..): deletehttp\n"); #endif - // Couper connexion - /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;*/ - back_maydeletehttp(opt, cache, sback, i); + // Couper connexion + /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */ + back_maydeletehttp(opt, cache, sback, i); - back[i].status=STATUS_READY; // terminé - back_set_finished(sback, i); - // finalize - if (back[i].r.statuscode>0) { - back_finalize(opt,cache,sback,i); - } + back[i].status = STATUS_READY; // terminé + back_set_finished(sback, i); + // finalize + if (back[i].r.statuscode > 0) { + back_finalize(opt, cache, sback, i); + } #endif /********** **************************** ********** */ - } else { // il faut aller le chercher - - // effacer buffer (requète) - if (!noFreebuff) { - deleteaddr(&back[i].r); - back[i].r.size=0; - } - - // traiter 206 (partial content) - // xxc SI CHUNK VERIFIER QUE CA MARCHE?? - if (back[i].r.statuscode==206) { // on nous envoie un morceau (la fin) coz une partie sur disque! - off_t sz=fsize_utf8(back[i].url_sav); -#if HDEBUG - printf("partial content: "LLintP" on disk..\n",(LLint)sz); -#endif - if (sz>=0) { - if (!is_hypertext_mime(opt,back[i].r.contenttype, back[i].url_sav)) { // pas HTML - if (opt->getmode&2) { // on peut ecrire des non html **sinon ben euhh sera intercepté plus loin, donc rap sur ce qui va sortir** - filenote(&opt->state.strc,back[i].url_sav,NULL); // noter fichier comme connu - file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 1, back[i].r.notmodified); - back[i].r.out=FOPEN(fconv(catbuff,back[i].url_sav),"ab"); // append - if (back[i].r.out) { - back[i].r.is_write=1; // écrire - back[i].r.size=sz; // déja écrit - back[i].r.statuscode=HTTP_OK; // Forcer 'OK' - if (back[i].r.totalsize>=0) - back[i].r.totalsize+=sz; // plus en fait - fseek(back[i].r.out,0,SEEK_END); // à la fin - /* create a temporary reference file in case of broken mirror */ - if (back_serialize_ref(opt, &back[i]) != 0) { - hts_log_print(opt, LOG_WARNING, "Could not create temporary reference file for %s%s",back[i].url_adr,back[i].url_fil); - } + } else { // il faut aller le chercher + + // effacer buffer (requète) + if (!noFreebuff) { + deleteaddr(&back[i].r); + back[i].r.size = 0; + } + // traiter 206 (partial content) + // xxc SI CHUNK VERIFIER QUE CA MARCHE?? + if (back[i].r.statuscode == 206) { // on nous envoie un morceau (la fin) coz une partie sur disque! + off_t sz = fsize_utf8(back[i].url_sav); + #if HDEBUG - printf("continue interrupted file\n"); + printf("partial content: " LLintP " on disk..\n", + (LLint) sz); #endif - } else { // On est dans la m** - back[i].status=STATUS_READY; // terminé (voir plus loin) - back_set_finished(sback, i); - strcpybuff(back[i].r.msg,"Can not open partial file"); + if (sz >= 0) { + if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_sav)) { // pas HTML + if (opt->getmode & 2) { // on peut ecrire des non html **sinon ben euhh sera intercepté plus loin, donc rap sur ce qui va sortir** + filenote(&opt->state.strc, back[i].url_sav, NULL); // noter fichier comme connu + file_notify(opt, back[i].url_adr, back[i].url_fil, + back[i].url_sav, 0, 1, + back[i].r.notmodified); + back[i].r.out = FOPEN(fconv(catbuff, back[i].url_sav), "ab"); // append + if (back[i].r.out) { + back[i].r.is_write = 1; // écrire + back[i].r.size = sz; // déja écrit + back[i].r.statuscode = HTTP_OK; // Forcer 'OK' + if (back[i].r.totalsize >= 0) + back[i].r.totalsize += sz; // plus en fait + fseek(back[i].r.out, 0, SEEK_END); // à la fin + /* create a temporary reference file in case of broken mirror */ + if (back_serialize_ref(opt, &back[i]) != 0) { + hts_log_print(opt, LOG_WARNING, + "Could not create temporary reference file for %s%s", + back[i].url_adr, back[i].url_fil); } - } - } else { // mémoire - FILE* fp=FOPEN(fconv(catbuff,back[i].url_sav),"rb"); - if (fp) { - LLint alloc_mem=sz + 1; - if (back[i].r.totalsize>=0) - alloc_mem+=back[i].r.totalsize; // AJOUTER RESTANT! - if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct((size_t)alloc_mem)) ) { - back[i].r.size=sz; - if (back[i].r.totalsize>=0) - back[i].r.totalsize+=sz; // plus en fait - if (( fread(back[i].r.adr,1,sz,fp)) != sz) { - back[i].status=STATUS_READY; // terminé (voir plus loin) - back_set_finished(sback, i); - strcpybuff(back[i].r.msg,"Can not read partial file"); - } else { - back[i].r.statuscode=HTTP_OK; // Forcer 'OK' #if HDEBUG - printf("continue in mem interrupted file\n"); + printf("continue interrupted file\n"); #endif - } - } else { - back[i].status=STATUS_READY; // terminé (voir plus loin) + } else { // On est dans la m** + back[i].status = STATUS_READY; // terminé (voir plus loin) + back_set_finished(sback, i); + strcpybuff(back[i].r.msg, + "Can not open partial file"); + } + } + } else { // mémoire + FILE *fp = + FOPEN(fconv(catbuff, back[i].url_sav), "rb"); + if (fp) { + LLint alloc_mem = sz + 1; + + if (back[i].r.totalsize >= 0) + alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT! + if (deleteaddr(&back[i].r) + && (back[i].r.adr = + (char *) malloct((size_t) alloc_mem))) { + back[i].r.size = sz; + if (back[i].r.totalsize >= 0) + back[i].r.totalsize += sz; // plus en fait + if ((fread(back[i].r.adr, 1, sz, fp)) != sz) { + back[i].status = STATUS_READY; // terminé (voir plus loin) back_set_finished(sback, i); - strcpybuff(back[i].r.msg,"No memory for partial file"); + strcpybuff(back[i].r.msg, + "Can not read partial file"); + } else { + back[i].r.statuscode = HTTP_OK; // Forcer 'OK' +#if HDEBUG + printf("continue in mem interrupted file\n"); +#endif } - fclose(fp); - } else { // Argh.. - back[i].status=STATUS_READY; // terminé (voir plus loin) + } else { + back[i].status = STATUS_READY; // terminé (voir plus loin) back_set_finished(sback, i); - strcpybuff(back[i].r.msg,"Can not open partial file"); + strcpybuff(back[i].r.msg, + "No memory for partial file"); } + fclose(fp); + } else { // Argh.. + back[i].status = STATUS_READY; // terminé (voir plus loin) + back_set_finished(sback, i); + strcpybuff(back[i].r.msg, + "Can not open partial file"); } - } else { // Non trouvé?? - back[i].status=STATUS_READY; // terminé (voir plus loin) - back_set_finished(sback, i); - strcpybuff(back[i].r.msg,"Can not find partial file"); } - // Erreur? - if (back[i].status==STATUS_READY) { - if (back[i].r.soc!=INVALID_SOCKET) { + } else { // Non trouvé?? + back[i].status = STATUS_READY; // terminé (voir plus loin) + back_set_finished(sback, i); + strcpybuff(back[i].r.msg, "Can not find partial file"); + } + // Erreur? + if (back[i].status == STATUS_READY) { + if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_wait(206 solve problems): deletehttp\n"); + DEBUG_W + ("back_wait(206 solve problems): deletehttp\n"); #endif - deletehttp(&back[i].r); - } - back[i].r.soc=INVALID_SOCKET; - //back[i].r.statuscode=206; ???????? - back[i].r.statuscode=STATUSCODE_NON_FATAL; - if (strnotempty(back[i].r.msg)) - strcpybuff(back[i].r.msg,"Error attempting to solve status 206 (partial file)"); + deletehttp(&back[i].r); } + back[i].r.soc = INVALID_SOCKET; + //back[i].r.statuscode=206; ???????? + back[i].r.statuscode = STATUSCODE_NON_FATAL; + if (strnotempty(back[i].r.msg)) + strcpybuff(back[i].r.msg, + "Error attempting to solve status 206 (partial file)"); } - - if (back[i].status!=0) { // non terminé (erreur) - if (!back[i].testmode) { // fichier normal - - if (back[i].r.empty /* ?? && back[i].r.statuscode==HTTP_OK */) { // empty response - // Couper connexion - back_maydeletehttp(opt, cache, sback, i); - /* KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */ - back[i].status=STATUS_READY; // terminé - back_set_finished(sback, i); - if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct( 2)) ) { - back[i].r.adr[0] = 0; - } - back_finalize(opt,cache,sback,i); + } + + if (back[i].status != 0) { // non terminé (erreur) + if (!back[i].testmode) { // fichier normal + + if (back[i].r.empty /* ?? && back[i].r.statuscode==HTTP_OK */ ) { // empty response + // Couper connexion + back_maydeletehttp(opt, cache, sback, i); + /* KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */ + back[i].status = STATUS_READY; // terminé + back_set_finished(sback, i); + if (deleteaddr(&back[i].r) + && (back[i].r.adr = (char *) malloct(2))) { + back[i].r.adr[0] = 0; } - else if (!back[i].r.is_chunk) { // pas de chunk - //if (back[i].r.http11!=2) { // pas de chunk - back[i].is_chunk=0; - back[i].status=1; // start body - } else { + back_finalize(opt, cache, sback, i); + } else if (!back[i].r.is_chunk) { // pas de chunk + //if (back[i].r.http11!=2) { // pas de chunk + back[i].is_chunk = 0; + back[i].status = 1; // start body + } else { #if CHUNKDEBUG==1 - printf("[%d] chunk encoding detected %s..\n",(int)back[i].r.soc, back[i].url_fil); -#endif - back[i].is_chunk=1; - back[i].chunk_adr=NULL; - back[i].chunk_size=0; - back[i].chunk_blocksize=0; - back[i].status=STATUS_CHUNK_WAIT; // start body wait chunk - back[i].r.totalsize=-1; /* devalidate size! (rfc) */ - } - if (back[i].rateout>0) { - back[i].rateout_time=time_local(); // refresh pour transfer rate - } + printf("[%d] chunk encoding detected %s..\n", + (int) back[i].r.soc, back[i].url_fil); +#endif + back[i].is_chunk = 1; + back[i].chunk_adr = NULL; + back[i].chunk_size = 0; + back[i].chunk_blocksize = 0; + back[i].status = STATUS_CHUNK_WAIT; // start body wait chunk + back[i].r.totalsize = -1; /* devalidate size! (rfc) */ + } + if (back[i].rateout > 0) { + back[i].rateout_time = time_local(); // refresh pour transfer rate + } #if HDEBUG - printf("(buffer) start body!\n"); + printf("(buffer) start body!\n"); #endif - } else { // mode test, ne pas passer en 1!! - back[i].status=STATUS_READY; // READY - back_set_finished(sback, i); + } else { // mode test, ne pas passer en 1!! + back[i].status = STATUS_READY; // READY + back_set_finished(sback, i); #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_wait(test ok): deletehttp\n"); + DEBUG_W("back_wait(test ok): deletehttp\n"); #endif - deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - if (back[i].r.statuscode==HTTP_OK) { - strcpybuff(back[i].r.msg,"Test: OK"); - back[i].r.statuscode=STATUSCODE_TEST_OK; // test réussi - } - else { // test a échoué, on ne change rien sauf que l'erreur est à titre indicatif - char tempo[1000]; - strcpybuff(tempo,back[i].r.msg); - strcpybuff(back[i].r.msg,"Test: "); - strcatbuff(back[i].r.msg,tempo); - } - + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; + if (back[i].r.statuscode == HTTP_OK) { + strcpybuff(back[i].r.msg, "Test: OK"); + back[i].r.statuscode = STATUSCODE_TEST_OK; // test réussi + } else { // test a échoué, on ne change rien sauf que l'erreur est à titre indicatif + char tempo[1000]; + + strcpybuff(tempo, back[i].r.msg); + strcpybuff(back[i].r.msg, "Test: "); + strcatbuff(back[i].r.msg, tempo); } + } - - } - - /*}*/ - - } // si LF - } // r.size>2 - } // si == 99 - - } // si pas d'erreurs + } + + } + + /*} */ + + } // si LF + } // r.size>2 + } // si == 99 + + } // si pas d'erreurs #if BDEBUG==1 - printf("bytes overall: %d\n",back[i].r.size); + printf("bytes overall: %d\n", back[i].r.size); #endif - } // données dispo - - // en cas d'erreur cl, supprimer éventuel fichier sur disque + } // données dispo + + // en cas d'erreur cl, supprimer éventuel fichier sur disque #if HTS_REMOVE_BAD_FILES - if (back[i].status<0) { - if (!back[i].testmode) { // pas en test - UNLINK(back[i].url_sav); // éliminer fichier (endommagé) - //printf("&& %s\n",back[i].url_sav); - } + if (back[i].status < 0) { + if (!back[i].testmode) { // pas en test + UNLINK(back[i].url_sav); // éliminer fichier (endommagé) + //printf("&& %s\n",back[i].url_sav); } + } #endif - /* funny log for commandline users */ - //if (!opt->quiet) { - // petite animation - if (opt->verbosedisplay==1) { - if (back[i].status==STATUS_READY) { - if (back[i].r.statuscode==HTTP_OK) - printf("* %s%s ("LLintP" bytes) - OK"VT_CLREOL"\r",back[i].url_adr,back[i].url_fil,(LLint)back[i].r.size); - else - printf("* %s%s ("LLintP" bytes) - %d"VT_CLREOL"\r",back[i].url_adr,back[i].url_fil,(LLint)back[i].r.size,back[i].r.statuscode); - fflush(stdout); - } + /* funny log for commandline users */ + //if (!opt->quiet) { + // petite animation + if (opt->verbosedisplay == 1) { + if (back[i].status == STATUS_READY) { + if (back[i].r.statuscode == HTTP_OK) + printf("* %s%s (" LLintP " bytes) - OK" VT_CLREOL "\r", + back[i].url_adr, back[i].url_fil, (LLint) back[i].r.size); + else + printf("* %s%s (" LLintP " bytes) - %d" VT_CLREOL "\r", + back[i].url_adr, back[i].url_fil, (LLint) back[i].r.size, + back[i].r.statuscode); + fflush(stdout); } - //} - + } + //} + + } // status>0 + } // for - } // status>0 - } // for - // vérifier timeouts if (gestion_timeout) { TStamp act; - act=time_local(); // temps en secondes - for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) { - // for(i=0;i<back_max;i++) { - unsigned int i = ( i_mod + mod_random ) % ( back_max ); - if (back[i].status>0) { // réception/connexion/.. - if (back[i].timeout>0) { + + act = time_local(); // temps en secondes + for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) { + // for(i=0;i<back_max;i++) { + unsigned int i = (i_mod + mod_random) % (back_max); + + if (back[i].status > 0) { // réception/connexion/.. + if (back[i].timeout > 0) { //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout); - if (((int) (act-back[i].timeout_refresh))>=back[i].timeout) { - if (back[i].r.soc!=INVALID_SOCKET) { + if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) { + if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(timeout): deletehttp\n"); #endif deletehttp(&back[i].r); } - back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=STATUSCODE_TIMEOUT; - if (back[i].status==STATUS_CONNECTING) - strcpybuff(back[i].r.msg,"Connect Time Out"); - else if (back[i].status==STATUS_WAIT_DNS) - strcpybuff(back[i].r.msg,"DNS Time Out"); + back[i].r.soc = INVALID_SOCKET; + back[i].r.statuscode = STATUSCODE_TIMEOUT; + if (back[i].status == STATUS_CONNECTING) + strcpybuff(back[i].r.msg, "Connect Time Out"); + else if (back[i].status == STATUS_WAIT_DNS) + strcpybuff(back[i].r.msg, "DNS Time Out"); else - strcpybuff(back[i].r.msg,"Receive Time Out"); - back[i].status=STATUS_READY; // terminé + strcpybuff(back[i].r.msg, "Receive Time Out"); + back[i].status = STATUS_READY; // terminé back_set_finished(sback, i); - } else if ((back[i].rateout>0) && (back[i].status<99)) { - if (((int) (act-back[i].rateout_time))>=HTS_WATCHRATE) { // checker au bout de 15s - if ( (int) ((back[i].r.size)/(act-back[i].rateout_time)) < back[i].rateout ) { // trop lent - back[i].status=STATUS_READY; // terminé + } else if ((back[i].rateout > 0) && (back[i].status < 99)) { + if (((int) (act - back[i].rateout_time)) >= HTS_WATCHRATE) { // checker au bout de 15s + if ((int) ((back[i].r.size) / (act - back[i].rateout_time)) < back[i].rateout) { // trop lent + back[i].status = STATUS_READY; // terminé back_set_finished(sback, i); - if (back[i].r.soc!=INVALID_SOCKET) { + if (back[i].r.soc != INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(rateout): deletehttp\n"); #endif deletehttp(&back[i].r); } - back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=STATUSCODE_SLOW; - strcpybuff(back[i].r.msg,"Transfer Rate Too Low"); + back[i].r.soc = INVALID_SOCKET; + back[i].r.statuscode = STATUSCODE_SLOW; + strcpybuff(back[i].r.msg, "Transfer Rate Too Low"); } } } @@ -3629,177 +3995,201 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti } max_loop--; max_loop_chk++; - } while((busy_state) && (busy_recv) && (max_loop>0)); + } while((busy_state) && (busy_recv) && (max_loop > 0)); if ((!busy_recv) && (!busy_state)) { - if (max_loop_chk>=1) { - Sleep(10); // un tite pause pour éviter les lag.. + if (max_loop_chk >= 1) { + Sleep(10); // un tite pause pour éviter les lag.. } } } -int back_checksize(httrackp* opt,lien_back* eback,int check_only_totalsize) { +int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize) { LLint size_to_test; + if (check_only_totalsize) - size_to_test=eback->r.totalsize; + size_to_test = eback->r.totalsize; else - size_to_test=max(eback->r.totalsize,eback->r.size); - if (size_to_test>=0) { - + size_to_test = max(eback->r.totalsize, eback->r.size); + if (size_to_test >= 0) { + /* Interdiction taille par le wizard? */ - if (hts_testlinksize(opt,eback->url_adr,eback->url_fil,eback->r.totalsize/1024)==-1) { - return 0; /* interdit */ - } - + if (hts_testlinksize + (opt, eback->url_adr, eback->url_fil, + eback->r.totalsize / 1024) == -1) { + return 0; /* interdit */ + } + /* vérifier taille classique (heml et non html) */ - if ((istoobig(opt,size_to_test,eback->maxfile_html,eback->maxfile_nonhtml,eback->r.contenttype))) { - return 0; /* interdit */ + if ((istoobig + (opt, size_to_test, eback->maxfile_html, eback->maxfile_nonhtml, + eback->r.contenttype))) { + return 0; /* interdit */ } } return 1; } -int back_checkmirror(httrackp* opt) { +int back_checkmirror(httrackp * opt) { // Check max size - if ((opt->maxsite>0) && (HTS_STAT.stat_bytes >= opt->maxsite)) { - if (!opt->state.stop) { /* not yet stopped */ - hts_log_print(opt, LOG_ERROR, "More than "LLintP" bytes have been transfered.. giving up",(LLint)opt->maxsite); - /* cancel mirror smoothly */ + if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) { + if (!opt->state.stop) { /* not yet stopped */ + hts_log_print(opt, LOG_ERROR, + "More than " LLintP + " bytes have been transfered.. giving up", + (LLint) opt->maxsite); + /* cancel mirror smoothly */ hts_request_stop(opt, 0); - } - return 1; /* don'k break mirror too sharply for size limits, but stop requested */ + } + return 1; /* don'k break mirror too sharply for size limits, but stop requested */ /*return 0; - */ + */ } // Check max time - if ((opt->maxtime>0) && ((time_local()-HTS_STAT.stat_timestart)>opt->maxtime)) { - if (!opt->state.stop) { /* not yet stopped */ - hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",opt->maxtime); + if ((opt->maxtime > 0) + && ((time_local() - HTS_STAT.stat_timestart) > opt->maxtime)) { + if (!opt->state.stop) { /* not yet stopped */ + hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up", + opt->maxtime); /* cancel mirror smoothly */ hts_request_stop(opt, 0); } } - return 1; /* Ok, go on */ + return 1; /* Ok, go on */ } - // octets transférés + add -LLint back_transfered(LLint nb,struct_back* sback) { - lien_back* const back = sback->lnk; +LLint back_transfered(LLint nb, struct_back * sback) { + lien_back *const back = sback->lnk; const int back_max = sback->count; int i; + // ajouter octets en instance - for(i=0;i<back_max;i++) - if ((back[i].status>0) && (back[i].status<99 || back[i].status>=1000)) + for(i = 0; i < back_max; i++) + if ((back[i].status > 0) && (back[i].status < 99 || back[i].status >= 1000)) nb += back[i].r.size; // stored (ready) slots if (sback->ready != NULL) { #ifndef HTS_NO_BACK_ON_DISK - nb += sback->ready_size_bytes; + nb += sback->ready_size_bytes; #else struct_inthash_enum e = inthash_enum_new(sback->ready); - inthash_chain* item; + inthash_chain *item; + while((item = inthash_enum_next(&e))) { - lien_back* ritem = (lien_back*) item->value.ptr; - if ((ritem->status>0) && (ritem->status<99 || ritem->status>=1000)) + lien_back *ritem = (lien_back *) item->value.ptr; + + if ((ritem->status > 0) && (ritem->status < 99 || ritem->status >= 1000)) nb += ritem->r.size; } #endif } - return nb; + return nb; } // infos backing // j: 1 afficher sockets 2 afficher autres 3 tout afficher -void back_info(struct_back* sback,int i,int j,FILE* fp) { - lien_back* const back = sback->lnk; +void back_info(struct_back * sback, int i, int j, FILE * fp) { + lien_back *const back = sback->lnk; const int back_max = sback->count; + assertf(i >= 0 && i < back_max); - if (back[i].status>=0) { - char BIGSTK s[HTS_URLMAXSIZE*2+1024]; - s[0]='\0'; - back_infostr(sback,i,j,s); - strcatbuff(s,LF); - fprintf(fp,"%s",s); + if (back[i].status >= 0) { + char BIGSTK s[HTS_URLMAXSIZE * 2 + 1024]; + + s[0] = '\0'; + back_infostr(sback, i, j, s); + strcatbuff(s, LF); + fprintf(fp, "%s", s); } } // infos backing // j: 1 afficher sockets 2 afficher autres 3 tout afficher -void back_infostr(struct_back* sback,int i,int j,char* s) { - lien_back* const back = sback->lnk; +void back_infostr(struct_back * sback, int i, int j, char *s) { + lien_back *const back = sback->lnk; const int back_max = sback->count; + assertf(i >= 0 && i < back_max); - if (back[i].status>=0) { - int aff=0; + if (back[i].status >= 0) { + int aff = 0; + if (j & 1) { - if (back[i].status==STATUS_CONNECTING) { - strcatbuff(s,"CONNECT "); - } else if (back[i].status==STATUS_WAIT_HEADERS) { - strcatbuff(s,"INFOS "); - aff=1; - } else if (back[i].status==STATUS_CHUNK_WAIT || back[i].status==STATUS_CHUNK_CR) { - strcatbuff(s,"INFOSC"); // infos chunk - aff=1; + if (back[i].status == STATUS_CONNECTING) { + strcatbuff(s, "CONNECT "); + } else if (back[i].status == STATUS_WAIT_HEADERS) { + strcatbuff(s, "INFOS "); + aff = 1; + } else if (back[i].status == STATUS_CHUNK_WAIT + || back[i].status == STATUS_CHUNK_CR) { + strcatbuff(s, "INFOSC"); // infos chunk + aff = 1; + } else if (back[i].status > 0) { + strcatbuff(s, "RECEIVE "); + aff = 1; } - else if (back[i].status>0) { - strcatbuff(s,"RECEIVE "); - aff=1; - } - } + } if (j & 2) { - if (back[i].status==STATUS_READY) { + if (back[i].status == STATUS_READY) { switch (back[i].r.statuscode) { case 200: - strcatbuff(s,"READY "); - aff=1; + strcatbuff(s, "READY "); + aff = 1; break; case -1: - strcatbuff(s,"ERROR "); - aff=1; + strcatbuff(s, "ERROR "); + aff = 1; break; case -2: - strcatbuff(s,"TIMEOUT "); - aff=1; + strcatbuff(s, "TIMEOUT "); + aff = 1; break; case -3: - strcatbuff(s,"TOOSLOW "); - aff=1; + strcatbuff(s, "TOOSLOW "); + aff = 1; break; case 400: - strcatbuff(s,"BADREQUEST "); - aff=1; + strcatbuff(s, "BADREQUEST "); + aff = 1; break; - case 401: case 403: - strcatbuff(s,"FORBIDDEN "); - aff=1; + case 401: + case 403: + strcatbuff(s, "FORBIDDEN "); + aff = 1; break; case 404: - strcatbuff(s,"NOT FOUND "); - aff=1; + strcatbuff(s, "NOT FOUND "); + aff = 1; break; case 500: - strcatbuff(s,"SERVERROR "); - aff=1; + strcatbuff(s, "SERVERROR "); + aff = 1; break; default: { char s2[256]; - sprintf(s2,"ERROR(%d)",back[i].r.statuscode); - strcatbuff(s,s2); + + sprintf(s2, "ERROR(%d)", back[i].r.statuscode); + strcatbuff(s, s2); } - aff=1; + aff = 1; } } } - + if (aff) { { - char BIGSTK s2[HTS_URLMAXSIZE*2+1024]; - sprintf(s2,"\"%s",back[i].url_adr); strcatbuff(s,s2); - - if (back[i].url_fil[0]!='/') strcatbuff(s,"/"); - sprintf(s2,"%s\" ",back[i].url_fil); strcatbuff(s,s2); - sprintf(s,LLintP" "LLintP" ",(LLint)back[i].r.size,(LLint)back[i].r.totalsize); strcatbuff(s,s2); + char BIGSTK s2[HTS_URLMAXSIZE * 2 + 1024]; + + sprintf(s2, "\"%s", back[i].url_adr); + strcatbuff(s, s2); + + if (back[i].url_fil[0] != '/') + strcatbuff(s, "/"); + sprintf(s2, "%s\" ", back[i].url_fil); + strcatbuff(s, s2); + sprintf(s, LLintP " " LLintP " ", (LLint) back[i].r.size, + (LLint) back[i].r.totalsize); + strcatbuff(s, s2); } } } diff --git a/src/htsback.h b/src/htsback.h index 12f627c..a59dafc 100644 --- a/src/htsback.h +++ b/src/htsback.h @@ -17,14 +17,12 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ @@ -34,9 +32,8 @@ Please visit our Website: http://www.httrack.com /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - #ifndef HTSBACK_DEFH -#define HTSBACK_DEFH +#define HTSBACK_DEFH #include "htsglobal.h" @@ -72,62 +69,79 @@ typedef struct htsblk htsblk; #ifdef HTS_INTERNAL_BYTECODE // create/destroy -struct_back* back_new(int back_max); -void back_free(struct_back** sback); +struct_back *back_new(int back_max); +void back_free(struct_back ** sback); // backing #define BACK_ADD_TEST "(dummy)" #define BACK_ADD_TEST2 "(dummy2)" -int back_index(httrackp* opt, struct_back* sback,char* adr,char* fil,char* sav); -int back_available(struct_back* sback); -LLint back_incache(struct_back* sback); -int back_done_incache(struct_back* sback); -HTS_INLINE int back_exist(struct_back* sback,httrackp* opt,char* adr,char* fil,char* sav); -int back_nsoc(struct_back* sback); -int back_nsoc_overall(struct_back* sback); -int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test); -int back_add_if_not_exists(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test); -int back_stack_available(struct_back* sback); -int back_search(httrackp* opt, struct_back* sback); -int back_search_quick(struct_back* sback); -void back_clean(httrackp* opt,cache_back* cache,struct_back* sback); -int back_cleanup_background(httrackp* opt,cache_back* cache,struct_back* sback); -void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_timestart); -int back_letlive(httrackp* opt, cache_back* cache, struct_back* sback, const int p); -int back_searchlive(httrackp* opt, struct_back* sback, char* search_addr); -void back_connxfr(htsblk* src, htsblk* dst); -void back_move(lien_back* src, lien_back* dst); -void back_copy_static(const lien_back* src, lien_back* dst); -int back_serialize(FILE *fp, const lien_back* src); -int back_unserialize(FILE *fp, lien_back** dst); -int back_serialize_ref(httrackp* opt, const lien_back* src); -int back_unserialize_ref(httrackp* opt, const char *adr, const char *fil, lien_back** dst); -void back_set_finished(struct_back* sback, const int p); -void back_set_locked(struct_back* sback, const int p); -void back_set_unlocked(struct_back* sback, const int p); -int back_delete(httrackp* opt,cache_back* cache,struct_back* sback,const int p); -void back_index_unlock(struct_back* sback, const int p); -int back_clear_entry(lien_back* back); -int back_flush_output(httrackp* opt, cache_back* cache, struct_back* sback, const int p); -void back_delete_all(httrackp* opt, cache_back* cache, struct_back* sback); -int back_maydelete(httrackp* opt, cache_back* cache, struct_back* sback, const int p); -void back_maydeletehttp(httrackp* opt, cache_back* cache, struct_back* sback, const int p); -int back_trylive(httrackp* opt,cache_back* cache,struct_back* sback, const int p); -int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,const int p); -void back_info(struct_back* sback,int i,int j,FILE* fp); -void back_infostr(struct_back* sback,int i,int j,char* s); -LLint back_transfered(LLint add,struct_back* sback); +int back_index(httrackp * opt, struct_back * sback, char *adr, char *fil, + char *sav); +int back_available(struct_back * sback); +LLint back_incache(struct_back * sback); +int back_done_incache(struct_back * sback); +HTS_INLINE int back_exist(struct_back * sback, httrackp * opt, char *adr, + char *fil, char *sav); +int back_nsoc(struct_back * sback); +int back_nsoc_overall(struct_back * sback); +int back_add(struct_back * sback, httrackp * opt, cache_back * cache, char *adr, + char *fil, char *save, char *referer_adr, char *referer_fil, + int test); +int back_add_if_not_exists(struct_back * sback, httrackp * opt, + cache_back * cache, char *adr, char *fil, char *save, + char *referer_adr, char *referer_fil, int test); +int back_stack_available(struct_back * sback); +int back_search(httrackp * opt, struct_back * sback); +int back_search_quick(struct_back * sback); +void back_clean(httrackp * opt, cache_back * cache, struct_back * sback); +int back_cleanup_background(httrackp * opt, cache_back * cache, + struct_back * sback); +void back_wait(struct_back * sback, httrackp * opt, cache_back * cache, + TStamp stat_timestart); +int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback, + const int p); +int back_searchlive(httrackp * opt, struct_back * sback, char *search_addr); +void back_connxfr(htsblk * src, htsblk * dst); +void back_move(lien_back * src, lien_back * dst); +void back_copy_static(const lien_back * src, lien_back * dst); +int back_serialize(FILE * fp, const lien_back * src); +int back_unserialize(FILE * fp, lien_back ** dst); +int back_serialize_ref(httrackp * opt, const lien_back * src); +int back_unserialize_ref(httrackp * opt, const char *adr, const char *fil, + lien_back ** dst); +void back_set_finished(struct_back * sback, const int p); +void back_set_locked(struct_back * sback, const int p); +void back_set_unlocked(struct_back * sback, const int p); +int back_delete(httrackp * opt, cache_back * cache, struct_back * sback, + const int p); +void back_index_unlock(struct_back * sback, const int p); +int back_clear_entry(lien_back * back); +int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback, + const int p); +void back_delete_all(httrackp * opt, cache_back * cache, struct_back * sback); +int back_maydelete(httrackp * opt, cache_back * cache, struct_back * sback, + const int p); +void back_maydeletehttp(httrackp * opt, cache_back * cache, struct_back * sback, + const int p); +int back_trylive(httrackp * opt, cache_back * cache, struct_back * sback, + const int p); +int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback, + const int p); +void back_info(struct_back * sback, int i, int j, FILE * fp); +void back_infostr(struct_back * sback, int i, int j, char *s); +LLint back_transfered(LLint add, struct_back * sback); + // hostback #if HTS_XGETHOST -void back_solve(httrackp *opt,lien_back* sback); -int host_wait(httrackp *opt, lien_back* sback); +void back_solve(httrackp * opt, lien_back * sback); +int host_wait(httrackp * opt, lien_back * sback); #endif -int back_checksize(httrackp* opt,lien_back* eback,int check_only_totalsize); -int back_checkmirror(httrackp* opt); +int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize); +int back_checkmirror(httrackp * opt); #if HTS_XGETHOST #if USE_BEGINTHREAD -void Hostlookup(void* iadr_p); +void Hostlookup(void *iadr_p); #endif #endif diff --git a/src/htsbase.h b/src/htsbase.h index d850bd9..e301b05 100644 --- a/src/htsbase.h +++ b/src/htsbase.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Basic definitions */ /* Used in .c files for basic (malloc() ..) definitions */ @@ -98,11 +95,9 @@ extern "C" { // caractère maj #define isUpperLetter(a) ( ((a) >= 'A') && ((a) <= 'Z') ) - /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE - // functions #ifdef _WIN32 #define DynamicGet(handle, sym) GetProcAddress(handle, sym) @@ -111,8 +106,8 @@ extern "C" { #endif // emergency log -typedef void (*t_abortLog)(char* msg, char* file, int line); -extern HTSEXT_API t_abortLog abortLog__; + typedef void (*t_abortLog) (char *msg, char *file, int line); + extern HTSEXT_API t_abortLog abortLog__; #define abortLog(a) abortLog__(a, __FILE__, __LINE__) #define _ , #ifndef _WIN32_WCE @@ -173,14 +168,14 @@ extern HTSEXT_API t_abortLog abortLog__; #define strdupt(A) hts_strdup(A) #define freet(A) do { hts_free(A); (A) = NULL; } while(0) #define realloct(A,B) hts_realloc(A,B) -void hts_freeall(); -void* hts_malloc (size_t); -void* hts_calloc(size_t,size_t); -char* hts_strdup(char*); -void* hts_xmalloc(size_t,size_t); -void hts_free (void*); -void* hts_realloc (void*,size_t); -mlink* hts_find(char* adr); + void hts_freeall(); + void *hts_malloc(size_t); + void *hts_calloc(size_t, size_t); + char *hts_strdup(char *); + void *hts_xmalloc(size_t, size_t); + void hts_free(void *); + void *hts_realloc(void *, size_t); + mlink *hts_find(char *adr); /* protected memcpy */ #define memcpybuff(A, B, N) do { \ mlink* lnk = hts_find((void*)(A)); \ @@ -195,9 +190,9 @@ mlink* hts_find(char* adr); #endif -typedef void (* htsErrorCallback)(char* msg, char* file, int line); -extern HTSEXT_API htsErrorCallback htsCallbackErr; -extern HTSEXT_API int htsMemoryFastXfr; + typedef void (*htsErrorCallback) (char *msg, char *file, int line); + extern HTSEXT_API htsErrorCallback htsCallbackErr; + extern HTSEXT_API int htsMemoryFastXfr; /* */ diff --git a/src/htsbasenet.h b/src/htsbasenet.h index 32462c7..723da97 100644 --- a/src/htsbasenet.h +++ b/src/htsbasenet.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Basic net definitions */ /* Used in .c and .h files that needs hostent and so */ @@ -41,7 +38,7 @@ Please visit our Website: http://www.httrack.com #ifdef _WIN32 #if HTS_INET6==0 - #include <winsock2.h> +#include <winsock2.h> #else #ifndef _WIN32_WCE @@ -57,8 +54,8 @@ Please visit our Website: http://www.httrack.com //#include <winsock2.h> //#include <tpipv6.h> #else - #include <winsock2.h> - #include <socket.h> +#include <winsock2.h> +#include <socket.h> #endif #endif @@ -67,8 +64,8 @@ typedef struct hostent FAR t_hostent; #else #define HTS_USESCOPEID - #define INVALID_SOCKET -1 - typedef struct hostent t_hostent; +#define INVALID_SOCKET -1 +typedef struct hostent t_hostent; #endif #if HTS_USEOPENSSL @@ -113,26 +110,26 @@ typedef struct hostent FAR t_hostent; /* */ typedef void SSL_CTX; -typedef void* SSL; +typedef void *SSL; typedef void SSL_METHOD; -typedef int (*t_SSL_shutdown)(SSL *ssl); -typedef void (*t_SSL_free)(SSL *ssl); -typedef SSL (*t_SSL_new)(SSL_CTX *ctx); -typedef int (*t_SSL_clear)(SSL *ssl); -typedef int (*t_SSL_set_fd)(SSL *ssl, int fd); -typedef void (*t_SSL_set_connect_state)(SSL *ssl); -typedef int (*t_SSL_connect)(SSL *ssl); -typedef int (*t_SSL_get_error)(SSL *ssl, int ret); -typedef int (*t_SSL_write)(SSL *ssl, const void *buf, int num); -typedef int (*t_SSL_read)(SSL *ssl, void *buf, int num); -typedef int (*t_SSL_library_init)(void); -typedef void (*t_ERR_load_crypto_strings)(void); -typedef void (*t_ERR_load_SSL_strings)(void); -typedef SSL_METHOD * (*t_SSLv23_client_method)(void); -typedef SSL_CTX * (*t_SSL_CTX_new)(SSL_METHOD *method); -typedef char * (*t_ERR_error_string)(unsigned long e, char *buf); -typedef void (*t_SSL_load_error_strings)(void); -typedef long (*t_SSL_CTX_ctrl)(SSL_CTX *ctx, int cmd, long larg, char *parg); +typedef int (*t_SSL_shutdown) (SSL * ssl); +typedef void (*t_SSL_free) (SSL * ssl); +typedef SSL(*t_SSL_new) (SSL_CTX * ctx); +typedef int (*t_SSL_clear) (SSL * ssl); +typedef int (*t_SSL_set_fd) (SSL * ssl, int fd); +typedef void (*t_SSL_set_connect_state) (SSL * ssl); +typedef int (*t_SSL_connect) (SSL * ssl); +typedef int (*t_SSL_get_error) (SSL * ssl, int ret); +typedef int (*t_SSL_write) (SSL * ssl, const void *buf, int num); +typedef int (*t_SSL_read) (SSL * ssl, void *buf, int num); +typedef int (*t_SSL_library_init) (void); +typedef void (*t_ERR_load_crypto_strings) (void); +typedef void (*t_ERR_load_SSL_strings) (void); +typedef SSL_METHOD *(*t_SSLv23_client_method) (void); +typedef SSL_CTX *(*t_SSL_CTX_new) (SSL_METHOD * method); +typedef char *(*t_ERR_error_string) (unsigned long e, char *buf); +typedef void (*t_SSL_load_error_strings) (void); +typedef long (*t_SSL_CTX_ctrl) (SSL_CTX * ctx, int cmd, long larg, char *parg); /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE @@ -167,7 +164,7 @@ From /usr/include/openssl/ssl.h #define SSL_ERROR_WANT_READ 2 #define SSL_ERROR_WANT_WRITE 3 #define SSL_ERROR_WANT_X509_LOOKUP 4 -#define SSL_ERROR_SYSCALL 5 /* look at error stack/return value/errno */ +#define SSL_ERROR_SYSCALL 5 /* look at error stack/return value/errno */ #define SSL_ERROR_ZERO_RETURN 6 #define SSL_ERROR_WANT_CONNECT 7 #define SSL_OP_ALL 0x000FFFFFL diff --git a/src/htsbauth.c b/src/htsbauth.c index 3f04c9b..2d469e5 100644 --- a/src/htsbauth.c +++ b/src/htsbauth.c @@ -16,18 +16,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* basic authentication: password storage */ @@ -48,73 +45,82 @@ Please visit our Website: http://www.httrack.com // gestion des cookie // ajoute, dans l'ordre // !=0 : erreur -int cookie_add(t_cookie* cookie,char* cook_name,char* cook_value,char* domain,char* path) { - char buffer[8192]; - char* a=cookie->data; - char* insert; +int cookie_add(t_cookie * cookie, char *cook_name, char *cook_value, + char *domain, char *path) { + char buffer[8192]; + char *a = cookie->data; + char *insert; char cook[16384]; + // effacer éventuel cookie en double - cookie_del(cookie,cook_name,domain,path); - if ((int)strlen(cook_value)>1024) return -1; // trop long - if ((int)strlen(cook_name)>256) return -1; // trop long - if ((int)strlen(domain)>256) return -1; // trop long - if ((int)strlen(path)>256) return -1; // trop long - if ((int)( - strlen(cookie->data) - +strlen(cook_value) - +strlen(cook_name) - +strlen(domain) - +strlen(path) - +256 - ) > cookie->max_len) return -1; // impossible d'ajouter - - insert=a; // insérer ici - while (*a) { - if ( strlen(cookie_get(buffer, a,2)) < strlen(path) ) // long. path (le + long est prioritaire) - a=cookie->data+strlen(cookie->data); // fin + cookie_del(cookie, cook_name, domain, path); + if ((int) strlen(cook_value) > 1024) + return -1; // trop long + if ((int) strlen(cook_name) > 256) + return -1; // trop long + if ((int) strlen(domain) > 256) + return -1; // trop long + if ((int) strlen(path) > 256) + return -1; // trop long + if ((int) (strlen(cookie->data) + + strlen(cook_value) + + strlen(cook_name) + + strlen(domain) + + strlen(path) + + 256) > cookie->max_len) + return -1; // impossible d'ajouter + + insert = a; // insérer ici + while(*a) { + if (strlen(cookie_get(buffer, a, 2)) < strlen(path)) // long. path (le + long est prioritaire) + a = cookie->data + strlen(cookie->data); // fin else { - a=strchr(a,'\n'); // prochain champ - if (a==NULL) - a=cookie->data+strlen(cookie->data); // fin + a = strchr(a, '\n'); // prochain champ + if (a == NULL) + a = cookie->data + strlen(cookie->data); // fin else a++; - while(*a=='\n') a++; - insert=a; // insérer ici + while(*a == '\n') + a++; + insert = a; // insérer ici } } // construction du cookie - strcpybuff(cook,domain); - strcatbuff(cook,"\t"); - strcatbuff(cook,"TRUE"); - strcatbuff(cook,"\t"); - strcatbuff(cook,path); - strcatbuff(cook,"\t"); - strcatbuff(cook,"FALSE"); - strcatbuff(cook,"\t"); - strcatbuff(cook,"1999999999"); - strcatbuff(cook,"\t"); - strcatbuff(cook,cook_name); - strcatbuff(cook,"\t"); - strcatbuff(cook,cook_value); - strcatbuff(cook,"\n"); - if (!( ((int) strlen(cookie->data) + (int) strlen(cook)) < cookie->max_len)) return -1; // impossible d'ajouter - cookie_insert(insert,cook); + strcpybuff(cook, domain); + strcatbuff(cook, "\t"); + strcatbuff(cook, "TRUE"); + strcatbuff(cook, "\t"); + strcatbuff(cook, path); + strcatbuff(cook, "\t"); + strcatbuff(cook, "FALSE"); + strcatbuff(cook, "\t"); + strcatbuff(cook, "1999999999"); + strcatbuff(cook, "\t"); + strcatbuff(cook, cook_name); + strcatbuff(cook, "\t"); + strcatbuff(cook, cook_value); + strcatbuff(cook, "\n"); + if (!(((int) strlen(cookie->data) + (int) strlen(cook)) < cookie->max_len)) + return -1; // impossible d'ajouter + cookie_insert(insert, cook); #if DEBUG_COOK - printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",cook_name,cook_value,domain,path); + printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n", + cook_name, cook_value, domain, path); //printf(">>>cook: %s<<<\n",cookie->data); #endif return 0; } // effacer cookie si existe -int cookie_del(t_cookie* cookie,char* cook_name,char* domain,char* path) { - char *a,*b; - b=cookie_find(cookie->data,cook_name,domain,path); +int cookie_del(t_cookie * cookie, char *cook_name, char *domain, char *path) { + char *a, *b; + + b = cookie_find(cookie->data, cook_name, domain, path); if (b) { - a=cookie_nextfield(b); - cookie_delete(b,(int) (a - b)); + a = cookie_nextfield(b); + cookie_delete(b, (int) (a - b)); #if DEBUG_COOK - printf("deleted old cookie: %s %s %s\n",cook_name,domain,path); + printf("deleted old cookie: %s %s %s\n", cook_name, domain, path); #endif } return 0; @@ -123,100 +129,112 @@ int cookie_del(t_cookie* cookie,char* cook_name,char* domain,char* path) { // rechercher cookie à partir de la position s (par exemple s=cookie.data) // renvoie pointeur sur ligne, ou NULL si introuvable // path est aligné à droite et cook_name peut être vide (chercher alors tout cookie) -// .doubleclick.net TRUE / FALSE 1999999999 id A -char* cookie_find(char* s,char* cook_name,char* domain,char* path) { - char buffer[8192]; - char* a=s; - while (*a) { +// .doubleclick.net TRUE / FALSE 1999999999 id A +char *cookie_find(char *s, char *cook_name, char *domain, char *path) { + char buffer[8192]; + char *a = s; + + while(*a) { int t; - if (strnotempty(cook_name)==0) - t=1; // accepter par défaut + + if (strnotempty(cook_name) == 0) + t = 1; // accepter par défaut else - t=( strcmp(cookie_get(buffer, a,5),cook_name)==0 ); // tester si même nom - if (t) { // même nom ou nom qualconque + t = (strcmp(cookie_get(buffer, a, 5), cook_name) == 0); // tester si même nom + if (t) { // même nom ou nom qualconque // - char* chk_dom=cookie_get(buffer,a,0); // domaine concerné par le cookie + char *chk_dom = cookie_get(buffer, a, 0); // domaine concerné par le cookie + if ((int) strlen(chk_dom) <= (int) strlen(domain)) { - if ( strcmp(chk_dom,domain+strlen(domain)-strlen(chk_dom))==0 ) { // même domaine + if (strcmp(chk_dom, domain + strlen(domain) - strlen(chk_dom)) == 0) { // même domaine // - char* chk_path=cookie_get(buffer,a,2); // chemin concerné par le cookie + char *chk_path = cookie_get(buffer, a, 2); // chemin concerné par le cookie + if ((int) strlen(chk_path) <= (int) strlen(path)) { - if (strncmp(path,chk_path,strlen(chk_path))==0 ) { // même chemin + if (strncmp(path, chk_path, strlen(chk_path)) == 0) { // même chemin return a; } } } } } - a=cookie_nextfield(a); + a = cookie_nextfield(a); } return NULL; } // renvoie prochain champ -char* cookie_nextfield(char* a) { - char* b=a; - a=strchr(a,'\n'); // prochain champ - if (a==NULL) - a=b+strlen(b); // fin +char *cookie_nextfield(char *a) { + char *b = a; + + a = strchr(a, '\n'); // prochain champ + if (a == NULL) + a = b + strlen(b); // fin else a++; - while(*a=='\n') a++; + while(*a == '\n') + a++; return a; } // lire cookies.txt // lire également (Windows seulement) les *@*.txt (cookies IE copiés) // !=0 : erreur -int cookie_load(t_cookie* cookie, const char* fpath, const char* name) { - char catbuff[CATBUFF_SIZE]; - char buffer[8192]; - // cookie->data[0]='\0'; +int cookie_load(t_cookie * cookie, const char *fpath, const char *name) { + char catbuff[CATBUFF_SIZE]; + char buffer[8192]; + + // cookie->data[0]='\0'; // Fusionner d'abord les éventuels cookies IE #ifdef _WIN32 { WIN32_FIND_DATAA find; HANDLE h; - char pth[MAX_PATH + 32]; - strcpybuff(pth,fpath); - strcatbuff(pth,"*@*.txt"); - h = FindFirstFileA((char*)pth,&find); + char pth[MAX_PATH + 32]; + + strcpybuff(pth, fpath); + strcatbuff(pth, "*@*.txt"); + h = FindFirstFileA((char *) pth, &find); if (h != INVALID_HANDLE_VALUE) { do { - if (!(find.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY )) - if (!(find.dwFileAttributes & FILE_ATTRIBUTE_SYSTEM )) { - FILE* fp=fopen(fconcat(catbuff, fpath, find.cFileName),"rb"); + if (!(find.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + if (!(find.dwFileAttributes & FILE_ATTRIBUTE_SYSTEM)) { + FILE *fp = fopen(fconcat(catbuff, fpath, find.cFileName), "rb"); + if (fp) { char cook_name[256]; char cook_value[1000]; char domainpathpath[512]; char dummy[512]; + // - char domain[256]; // domaine cookie (.netscape.com) - char path[256]; // chemin (/) - int cookie_merged=0; + char domain[256]; // domaine cookie (.netscape.com) + char path[256]; // chemin (/) + int cookie_merged = 0; + // // Read all cookies - while( ! feof(fp) ) { - cook_name[0] = cook_value[0] = domainpathpath[0] + while(!feof(fp)) { + cook_name[0] = cook_value[0] = domainpathpath[0] = dummy[0] = domain[0] = path[0] = '\0'; - linput(fp,cook_name,250); - if ( ! feof(fp) ) { - linput(fp,cook_value,250); - if ( ! feof(fp) ) { + linput(fp, cook_name, 250); + if (!feof(fp)) { + linput(fp, cook_value, 250); + if (!feof(fp)) { int i; - linput(fp,domainpathpath,500); + + linput(fp, domainpathpath, 500); /* Read 6 other useless values */ - for(i = 0 ; ! feof(fp) && i < 6 ; i++) { - linput(fp,dummy,500); + for(i = 0; !feof(fp) && i < 6; i++) { + linput(fp, dummy, 500); } - if (strnotempty(cook_name) - && strnotempty(cook_value) - && strnotempty(domainpathpath)) { - if (ident_url_absolute(domainpathpath,domain,path)>=0) { - cookie_add(cookie,cook_name,cook_value,domain,path); - cookie_merged=1; + if (strnotempty(cook_name) + && strnotempty(cook_value) + && strnotempty(domainpathpath)) { + if (ident_url_absolute(domainpathpath, domain, path) >= 0) { + cookie_add(cookie, cook_name, cook_value, domain, path); + cookie_merged = 1; } } } @@ -224,37 +242,40 @@ int cookie_load(t_cookie* cookie, const char* fpath, const char* name) { } fclose(fp); if (cookie_merged) - remove(fconcat(catbuff,fpath,find.cFileName)); - } // if fp + remove(fconcat(catbuff, fpath, find.cFileName)); + } // if fp } - } while(FindNextFileA(h,&find)); + } while(FindNextFileA(h, &find)); FindClose(h); } } #endif - + // Ensuite, cookies.txt { - FILE* fp = fopen(fconcat(catbuff, fpath, name),"rb"); + FILE *fp = fopen(fconcat(catbuff, fpath, name), "rb"); + if (fp) { char BIGSTK line[8192]; - while( (!feof(fp)) && (((int) strlen(cookie->data)) < cookie->max_len)) { - rawlinput(fp,line,8100); + + while((!feof(fp)) && (((int) strlen(cookie->data)) < cookie->max_len)) { + rawlinput(fp, line, 8100); if (strnotempty(line)) { - if (strlen(line)<8000) { - if (line[0]!='#') { - char domain[256]; // domaine cookie (.netscape.com) - char path[256]; // chemin (/) - char cook_name[1024]; // nom cookie (MYCOOK) - char BIGSTK cook_value[8192]; // valeur (ID=toto,S=1234) - strcpybuff(domain,cookie_get(buffer,line,0)); // host - strcpybuff(path,cookie_get(buffer,line,2)); // path - strcpybuff(cook_name,cookie_get(buffer,line,5)); // name - strcpybuff(cook_value,cookie_get(buffer,line,6)); // value + if (strlen(line) < 8000) { + if (line[0] != '#') { + char domain[256]; // domaine cookie (.netscape.com) + char path[256]; // chemin (/) + char cook_name[1024]; // nom cookie (MYCOOK) + char BIGSTK cook_value[8192]; // valeur (ID=toto,S=1234) + + strcpybuff(domain, cookie_get(buffer, line, 0)); // host + strcpybuff(path, cookie_get(buffer, line, 2)); // path + strcpybuff(cook_name, cookie_get(buffer, line, 5)); // name + strcpybuff(cook_value, cookie_get(buffer, line, 6)); // value #if DEBUG_COOK - printf("%s\n",line); + printf("%s\n", line); #endif - cookie_add(cookie,cook_name,cook_value,domain,path); + cookie_add(cookie, cook_name, cook_value, domain, path); } } } @@ -268,17 +289,22 @@ int cookie_load(t_cookie* cookie, const char* fpath, const char* name) { // écrire cookies.txt // !=0 : erreur -int cookie_save(t_cookie* cookie,char* name) { - char catbuff[CATBUFF_SIZE]; +int cookie_save(t_cookie * cookie, char *name) { + char catbuff[CATBUFF_SIZE]; + if (strnotempty(cookie->data)) { char BIGSTK line[8192]; - FILE* fp = fopen(fconv(catbuff,name),"wb"); + FILE *fp = fopen(fconv(catbuff, name), "wb"); + if (fp) { - char* a=cookie->data; - fprintf(fp,"# HTTrack Website Copier Cookie File"LF"# This file format is compatible with Netscape cookies"LF); + char *a = cookie->data; + + fprintf(fp, + "# HTTrack Website Copier Cookie File" LF + "# This file format is compatible with Netscape cookies" LF); do { - a+=binput(a,line,8000); - fprintf(fp,"%s"LF,line); + a += binput(a, line, 8000); + fprintf(fp, "%s" LF, line); } while(strnotempty(line)); fclose(fp); return 0; @@ -289,30 +315,33 @@ int cookie_save(t_cookie* cookie,char* name) { } // insertion chaine ins avant s -void cookie_insert(char* s,char* ins) { - char* buff; - if (strnotempty(s)==0) { // rien à faire, juste concat - strcatbuff(s,ins); +void cookie_insert(char *s, char *ins) { + char *buff; + + if (strnotempty(s) == 0) { // rien à faire, juste concat + strcatbuff(s, ins); } else { - buff=(char*) malloct(strlen(s)+2); + buff = (char *) malloct(strlen(s) + 2); if (buff) { - strcpybuff(buff,s); // copie temporaire - strcpybuff(s,ins); // insérer - strcatbuff(s,buff); // copier + strcpybuff(buff, s); // copie temporaire + strcpybuff(s, ins); // insérer + strcatbuff(s, buff); // copier freet(buff); } } } + // destruction chaine dans s position pos -void cookie_delete(char* s,int pos) { - char* buff; - if (strnotempty(s+pos)==0) { // rien à faire, effacer - s[0]='\0'; +void cookie_delete(char *s, int pos) { + char *buff; + + if (strnotempty(s + pos) == 0) { // rien à faire, effacer + s[0] = '\0'; } else { - buff=(char*) malloct(strlen(s+pos)+2); + buff = (char *) malloct(strlen(s + pos) + 2); if (buff) { - strcpybuff(buff,s+pos); // copie temporaire - strcpybuff(s,buff); // copier + strcpybuff(buff, s + pos); // copie temporaire + strcpybuff(s, buff); // copier freet(buff); } } @@ -320,28 +349,34 @@ void cookie_delete(char* s,int pos) { // renvoie champ param de la chaine cookie_base // ex: cookie_get("ceci est<tab>un<tab>exemple",1) renvoi "un" -char* cookie_get(char *buffer,char* cookie_base,int param) { - char * limit; +char *cookie_get(char *buffer, char *cookie_base, int param) { + char *limit; - while(*cookie_base=='\n') cookie_base++; - limit = strchr(cookie_base,'\n'); - if (!limit) limit=cookie_base+strlen(cookie_base); + while(*cookie_base == '\n') + cookie_base++; + limit = strchr(cookie_base, '\n'); + if (!limit) + limit = cookie_base + strlen(cookie_base); if (limit) { if (param) { int i; - for(i=0;i<param;i++) { + + for(i = 0; i < param; i++) { if (cookie_base) { - cookie_base=strchr(cookie_base,'\t'); // prochain tab - if (cookie_base) cookie_base++; + cookie_base = strchr(cookie_base, '\t'); // prochain tab + if (cookie_base) + cookie_base++; } } } if (cookie_base) { - if ( cookie_base < limit) { - char* a = cookie_base; - while( (*a) && (*a!='\t') && (*a!='\n')) a++; - buffer[0]='\0'; - strncatbuff(buffer,cookie_base,(int) (a - cookie_base)); + if (cookie_base < limit) { + char *a = cookie_base; + + while((*a) && (*a != '\t') && (*a != '\n')) + a++; + buffer[0] = '\0'; + strncatbuff(buffer, cookie_base, (int) (a - cookie_base)); return buffer; } else return ""; @@ -350,28 +385,29 @@ char* cookie_get(char *buffer,char* cookie_base,int param) { } else return ""; } -// fin cookies - +// fin cookies // -- basic auth -- /* déclarer un répertoire comme possédant une authentification propre */ -int bauth_add(t_cookie* cookie,char* adr,char* fil,char* auth) { - char buffer[HTS_URLMAXSIZE*2]; - if (cookie) { - if (!bauth_check(cookie,adr,fil)) { // n'existe pas déja - bauth_chain* chain=&cookie->auth; - char* prefix=bauth_prefix(buffer,adr,fil); +int bauth_add(t_cookie * cookie, char *adr, char *fil, char *auth) { + char buffer[HTS_URLMAXSIZE * 2]; + + if (cookie) { + if (!bauth_check(cookie, adr, fil)) { // n'existe pas déja + bauth_chain *chain = &cookie->auth; + char *prefix = bauth_prefix(buffer, adr, fil); + /* fin de la chaine */ while(chain->next) - chain=chain->next; - chain->next=(bauth_chain*) calloc(sizeof(bauth_chain),1); + chain = chain->next; + chain->next = (bauth_chain *) calloc(sizeof(bauth_chain), 1); if (chain->next) { - chain=chain->next; - chain->next=NULL; - strcpybuff(chain->auth,auth); - strcpybuff(chain->prefix,prefix); + chain = chain->next; + chain->next = NULL; + strcpybuff(chain->auth, auth); + strcpybuff(chain->prefix, prefix); return 1; } } @@ -381,33 +417,38 @@ int bauth_add(t_cookie* cookie,char* adr,char* fil,char* auth) { /* tester adr et fil, et retourner authentification si nécessaire */ /* sinon, retourne NULL */ -char* bauth_check(t_cookie* cookie,char* adr,char* fil) { - char buffer[HTS_URLMAXSIZE*2]; +char *bauth_check(t_cookie * cookie, char *adr, char *fil) { + char buffer[HTS_URLMAXSIZE * 2]; + if (cookie) { - bauth_chain* chain=&cookie->auth; - char* prefix=bauth_prefix(buffer,adr,fil); + bauth_chain *chain = &cookie->auth; + char *prefix = bauth_prefix(buffer, adr, fil); + while(chain) { if (strnotempty(chain->prefix)) { - if (strncmp(prefix,chain->prefix,strlen(chain->prefix))==0) { + if (strncmp(prefix, chain->prefix, strlen(chain->prefix)) == 0) { return chain->auth; } } - chain=chain->next; + chain = chain->next; } } return NULL; } -char* bauth_prefix(char *prefix,char* adr,char* fil) { - char* a; - strcpybuff(prefix,jump_identification(adr)); - strcatbuff(prefix,fil); - a=strchr(prefix,'?'); - if (a) *a='\0'; - if (strchr(prefix,'/')) { - a=prefix+strlen(prefix)-1; - while(*a != '/') a--; - *(a+1)='\0'; +char *bauth_prefix(char *prefix, char *adr, char *fil) { + char *a; + + strcpybuff(prefix, jump_identification(adr)); + strcatbuff(prefix, fil); + a = strchr(prefix, '?'); + if (a) + *a = '\0'; + if (strchr(prefix, '/')) { + a = prefix + strlen(prefix) - 1; + while(*a != '/') + a--; + *(a + 1) = '\0'; } return prefix; } diff --git a/src/htsbauth.h b/src/htsbauth.h index 4664c9c..272b59e 100644 --- a/src/htsbauth.h +++ b/src/htsbauth.h @@ -17,14 +17,12 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ @@ -34,10 +32,8 @@ Please visit our Website: http://www.httrack.com /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - - #ifndef HTSBAUTH_DEFH -#define HTSBAUTH_DEFH +#define HTSBAUTH_DEFH // robots wizard #ifndef HTS_DEF_FWSTRUCT_bauth_chain @@ -45,12 +41,11 @@ Please visit our Website: http://www.httrack.com typedef struct bauth_chain bauth_chain; #endif struct bauth_chain { - char prefix[1024]; /* www.foo.com/secure/ */ - char auth[1024]; /* base-64 encoded user:pass */ - struct bauth_chain* next; /* next element */ + char prefix[1024]; /* www.foo.com/secure/ */ + char auth[1024]; /* base-64 encoded user:pass */ + struct bauth_chain *next; /* next element */ }; - // buffer pour les cookies et authentification #ifndef HTS_DEF_FWSTRUCT_t_cookie #define HTS_DEF_FWSTRUCT_t_cookie @@ -62,25 +57,25 @@ struct t_cookie { bauth_chain auth; }; - /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE // cookies -int cookie_add(t_cookie* cookie,char* cook_name,char* cook_value,char* domain,char* path); -int cookie_del(t_cookie* cookie,char* cook_name,char* domain,char* path); -int cookie_load(t_cookie* cookie, const char* path, const char* name); -int cookie_save(t_cookie* cookie,char* name); -void cookie_insert(char* s,char* ins); -void cookie_delete(char* s,int pos); -char* cookie_get(char *buffer, char* cookie_base,int param); -char* cookie_find(char* s,char* cook_name,char* domain,char* path); -char* cookie_nextfield(char* a); +int cookie_add(t_cookie * cookie, char *cook_name, char *cook_value, + char *domain, char *path); +int cookie_del(t_cookie * cookie, char *cook_name, char *domain, char *path); +int cookie_load(t_cookie * cookie, const char *path, const char *name); +int cookie_save(t_cookie * cookie, char *name); +void cookie_insert(char *s, char *ins); +void cookie_delete(char *s, int pos); +char *cookie_get(char *buffer, char *cookie_base, int param); +char *cookie_find(char *s, char *cook_name, char *domain, char *path); +char *cookie_nextfield(char *a); // basic auth -int bauth_add(t_cookie* cookie,char* adr,char* fil,char* auth); -char* bauth_check(t_cookie* cookie,char* adr,char* fil); -char* bauth_prefix(char *buffer, char* adr,char* fil); +int bauth_add(t_cookie * cookie, char *adr, char *fil, char *auth); +char *bauth_check(t_cookie * cookie, char *adr, char *fil); +char *bauth_prefix(char *buffer, char *adr, char *fil); #endif diff --git a/src/htscache.c b/src/htscache.c index 22f57e6..edfe296 100644 --- a/src/htscache.c +++ b/src/htscache.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* cache system (index and stores files in cache) */ @@ -71,7 +68,6 @@ with with <string>(name) = <length in ascii>+<lf>+<data> - VERSION 1.1/1.2 : --------------- @@ -106,56 +102,66 @@ with // Nouveau: si != text/html ne stocke que la taille - -void cache_mayadd(httrackp* opt,cache_back* cache,htsblk* r,const char* url_adr,const char* url_fil,const char* url_save) { - hts_log_print(opt, LOG_DEBUG, "File checked by cache: %s",url_adr); +void cache_mayadd(httrackp * opt, cache_back * cache, htsblk * r, + const char *url_adr, const char *url_fil, + const char *url_save) { + hts_log_print(opt, LOG_DEBUG, "File checked by cache: %s", url_adr); // ---stockage en cache--- // stocker dans le cache? if (opt->cache) { if (cache_writable(cache)) { // ensure not a temporary filename (should not happend ?!) if (IS_DELAYED_EXT(url_save)) { - hts_log_print(opt, LOG_WARNING, "aborted cache validation: %s%s still has temporary name %s", url_adr, url_fil, url_save); - return ; + hts_log_print(opt, LOG_WARNING, + "aborted cache validation: %s%s still has temporary name %s", + url_adr, url_fil, url_save); + return; } - // c'est le seul endroit ou l'on ajoute des elements dans le cache (fichier entier ou header) // on stocke tout fichier "ok", mais également les réponses 404,301,302... if ( #if 1 - r->statuscode > 0 + r->statuscode > 0 #else - /* We don't store 5XX errors, because it might be a server problem */ - (r->statuscode==HTTP_OK) /* stocker réponse standard, plus */ - || (r->statuscode==204) /* no content */ - || HTTP_IS_REDIRECT(r->statuscode) /* redirect */ - || (r->statuscode==401) /* authorization */ - || (r->statuscode==403) /* unauthorized */ - || (r->statuscode==404) /* not found */ - || (r->statuscode==410) /* gone */ + /* We don't store 5XX errors, because it might be a server problem */ + (r->statuscode == HTTP_OK) /* stocker réponse standard, plus */ + ||(r->statuscode == 204) /* no content */ + ||HTTP_IS_REDIRECT(r->statuscode) /* redirect */ + ||(r->statuscode == 401) /* authorization */ + ||(r->statuscode == 403) /* unauthorized */ + ||(r->statuscode == 404) /* not found */ + ||(r->statuscode == 410) /* gone */ #endif - ) - { /* ne pas stocker si la page générée est une erreur */ + ) { /* ne pas stocker si la page générée est une erreur */ if (!r->is_file) { // stocker fichiers (et robots.txt) - if ( url_save == NULL || (strnotempty(url_save)) || (strcmp(url_fil,"/robots.txt")==0)) { + if (url_save == NULL || (strnotempty(url_save)) + || (strcmp(url_fil, "/robots.txt") == 0)) { // ajouter le fichier au cache - cache_add(opt,cache,r,url_adr,url_fil,url_save,opt->all_in_cache,StringBuff(opt->path_html_utf8)); + cache_add(opt, cache, r, url_adr, url_fil, url_save, + opt->all_in_cache, StringBuff(opt->path_html_utf8)); // // store a reference NOT to redo the same test zillions of times! // (problem reported by Lars Clausen) // we just store statuscode + location (if any) if (url_save == NULL && r->statuscode / 100 >= 3) { // cached "fast" header doesn't yet exists - if (inthash_read(cache->cached_tests, concat(OPT_GET_BUFF(opt), url_adr, url_fil), NULL) == 0) { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - sprintf(tempo, "%d", (int)r->statuscode); + if (inthash_read + (cache->cached_tests, + concat(OPT_GET_BUFF(opt), url_adr, url_fil), NULL) == 0) { + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + + sprintf(tempo, "%d", (int) r->statuscode); if (r->location != NULL && r->location[0] != '\0') { strcatbuff(tempo, "\n"); strcatbuff(tempo, r->location); } - hts_log_print(opt, LOG_DEBUG, "Cached fast-header response: %s%s is %d", url_adr, url_fil, (int)r->statuscode); - inthash_add(cache->cached_tests, concat(OPT_GET_BUFF(opt), url_adr, url_fil), (intptr_t)strdupt(tempo)); + hts_log_print(opt, LOG_DEBUG, + "Cached fast-header response: %s%s is %d", + url_adr, url_fil, (int) r->statuscode); + inthash_add(cache->cached_tests, + concat(OPT_GET_BUFF(opt), url_adr, url_fil), + (intptr_t) strdupt(tempo)); } } } @@ -214,46 +220,47 @@ struct cache_back_zip_entry { } \ } while(0) - /* Ajout d'un fichier en cache */ -void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,const char* url_adr,const char* url_fil,const char* url_save,int all_in_cache,const char* path_prefix) { - char BIGSTK filename[HTS_URLMAXSIZE*4]; - char catbuff[CATBUFF_SIZE]; - int dataincache=0; // put data in cache ? +void cache_add(httrackp * opt, cache_back * cache, const htsblk * r, + const char *url_adr, const char *url_fil, const char *url_save, + int all_in_cache, const char *path_prefix) { + char BIGSTK filename[HTS_URLMAXSIZE * 4]; + char catbuff[CATBUFF_SIZE]; + int dataincache = 0; // put data in cache ? char BIGSTK headers[8192]; int headersSize = 0; + //int entryBodySize = 0; //int entryFilenameSize = 0; zip_fileinfo fi; - const char* url_save_suffix = url_save; - int zErr; + const char *url_save_suffix = url_save; + int zErr; // robots.txt hack if (url_save == NULL) { - dataincache=0; // testing links - } - else { - if ( (strnotempty(url_save)==0) ) { - if (strcmp(url_fil,"/robots.txt")==0) // robots.txt - dataincache=1; + dataincache = 0; // testing links + } else { + if ((strnotempty(url_save) == 0)) { + if (strcmp(url_fil, "/robots.txt") == 0) // robots.txt + dataincache = 1; else - return; // error (except robots.txt) + return; // error (except robots.txt) } /* Data in cache ? */ - if (is_hypertext_mime(opt,r->contenttype, url_fil) - || (may_be_hypertext_mime(opt,r->contenttype, url_fil) && r->adr != NULL) - /* store error messages! */ - || !HTTP_IS_OK(r->statuscode) - ) - { - dataincache=1; - } else if (all_in_cache) { - dataincache=1; - } + if (is_hypertext_mime(opt, r->contenttype, url_fil) + || (may_be_hypertext_mime(opt, r->contenttype, url_fil) + && r->adr != NULL) + /* store error messages! */ + || !HTTP_IS_OK(r->statuscode) + ) { + dataincache = 1; + } else if (all_in_cache) { + dataincache = 1; + } } - if (r->size < 0) // error + if (r->size < 0) // error return; // data in cache @@ -267,41 +274,45 @@ void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,const char* url_a headersSize = 0; /* */ { - const char* message; + const char *message; + if (strlen(r->msg) < 32) { message = r->msg; } else { message = "(See X-StatusMessage)"; } /* 64 characters MAX for first line */ - sprintf(headers + headersSize, "HTTP/1.%c %d %s\r\n", '1', r->statuscode, message); + sprintf(headers + headersSize, "HTTP/1.%c %d %s\r\n", '1', r->statuscode, + message); } headersSize += (int) strlen(headers + headersSize); - if (path_prefix != NULL && path_prefix[0] != '\0' && url_save != NULL && url_save[0] != '\0') { - int prefixLen = (int) strlen(path_prefix); - if (strncmp(url_save, path_prefix, prefixLen) == 0) { - url_save_suffix += prefixLen; - } - } + if (path_prefix != NULL && path_prefix[0] != '\0' && url_save != NULL + && url_save[0] != '\0') { + int prefixLen = (int) strlen(path_prefix); + + if (strncmp(url_save, path_prefix, prefixLen) == 0) { + url_save_suffix += prefixLen; + } + } /* Second line MUST ALWAYS be X-In-Cache */ ZIP_FIELD_INT_FORCE(headers, headersSize, "X-In-Cache", dataincache); ZIP_FIELD_INT(headers, headersSize, "X-StatusCode", r->statuscode); ZIP_FIELD_STRING(headers, headersSize, "X-StatusMessage", r->msg); - ZIP_FIELD_INT(headers, headersSize, "X-Size", r->size); // size - ZIP_FIELD_STRING(headers, headersSize, "Content-Type", r->contenttype); // contenttype - ZIP_FIELD_STRING(headers, headersSize, "X-Charset", r->charset); // contenttype + ZIP_FIELD_INT(headers, headersSize, "X-Size", r->size); // size + ZIP_FIELD_STRING(headers, headersSize, "Content-Type", r->contenttype); // contenttype + ZIP_FIELD_STRING(headers, headersSize, "X-Charset", r->charset); // contenttype ZIP_FIELD_STRING(headers, headersSize, "Last-Modified", r->lastmodified); // last-modified - ZIP_FIELD_STRING(headers, headersSize, "Etag", r->etag); // Etag - ZIP_FIELD_STRING(headers, headersSize, "Location", r->location); // 'location' pour moved - ZIP_FIELD_STRING(headers, headersSize, "Content-Disposition", r->cdispo); // Content-disposition - ZIP_FIELD_STRING(headers, headersSize, "X-Addr", url_adr); // Original address - ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil); // Original URI filename - ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save_suffix); // Original save filename - + ZIP_FIELD_STRING(headers, headersSize, "Etag", r->etag); // Etag + ZIP_FIELD_STRING(headers, headersSize, "Location", r->location); // 'location' pour moved + ZIP_FIELD_STRING(headers, headersSize, "Content-Disposition", r->cdispo); // Content-disposition + ZIP_FIELD_STRING(headers, headersSize, "X-Addr", url_adr); // Original address + ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil); // Original URI filename + ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save_suffix); // Original save filename + //entryFilenameSize = (int) ( strlen(url_adr) + strlen(url_fil)); - + /* Filename */ if (!link_has_authority(url_adr)) { strcpybuff(filename, "http://"); @@ -314,8 +325,9 @@ void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,const char* url_a /* Time */ memset(&fi, 0, sizeof(fi)); if (r->lastmodified[0] != '\0') { - struct tm buffer; - struct tm* tm_s=convert_time_rfc822(&buffer, r->lastmodified); + struct tm buffer; + struct tm *tm_s = convert_time_rfc822(&buffer, r->lastmodified); + if (tm_s) { fi.tmz_date.tm_sec = (uInt) tm_s->tm_sec; fi.tmz_date.tm_min = (uInt) tm_s->tm_min; @@ -325,71 +337,75 @@ void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,const char* url_a fi.tmz_date.tm_year = (uInt) tm_s->tm_year; } } - + /* Open file - NOTE: headers in "comment" */ - if ((zErr = zipOpenNewFileInZip((zipFile) cache->zipOutput, - filename, - &fi, - /* - Store headers in realtime in the local file directory as extra field - In case of crash, we'll be able to recover the whole ZIP file by rescanning it - */ - headers, - (uInt) strlen(headers), - NULL, - 0, - NULL, /* comment */ - Z_DEFLATED, - Z_DEFAULT_COMPRESSION)) != Z_OK) - { + if ((zErr = zipOpenNewFileInZip((zipFile) cache->zipOutput, filename, &fi, + /* + Store headers in realtime in the local file directory as extra field + In case of crash, we'll be able to recover the whole ZIP file by rescanning it + */ + headers, (uInt) strlen(headers), NULL, 0, NULL, /* comment */ + Z_DEFLATED, Z_DEFAULT_COMPRESSION)) != Z_OK) { int zip_zipOpenNewFileInZip_failed = 0; + assertf(zip_zipOpenNewFileInZip_failed); } - + /* Write data in cache */ if (dataincache) { if (r->is_write == 0) { if (r->size > 0 && r->adr != NULL) { - if ((zErr = zipWriteInFileInZip((zipFile) cache->zipOutput, r->adr, (int) r->size)) != Z_OK) { + if ((zErr = + zipWriteInFileInZip((zipFile) cache->zipOutput, r->adr, + (int) r->size)) != Z_OK) { int zip_zipWriteInFileInZip_failed = 0; + assertf(zip_zipWriteInFileInZip_failed); } } } else { - FILE* fp; + FILE *fp; + // On recopie le fichier->. - off_t file_size=fsize_utf8(fconv(catbuff, url_save)); - if (file_size>=0) { - fp=FOPEN(fconv(catbuff, url_save),"rb"); - if (fp!=NULL) { + off_t file_size = fsize_utf8(fconv(catbuff, url_save)); + + if (file_size >= 0) { + fp = FOPEN(fconv(catbuff, url_save), "rb"); + if (fp != NULL) { char BIGSTK buff[32768]; size_t nl; + do { - nl=fread(buff,1,32768,fp); - if (nl>0) { - if ((zErr = zipWriteInFileInZip((zipFile) cache->zipOutput, buff, (int) nl)) != Z_OK) { + nl = fread(buff, 1, 32768, fp); + if (nl > 0) { + if ((zErr = + zipWriteInFileInZip((zipFile) cache->zipOutput, buff, + (int) nl)) != Z_OK) { int zip_zipWriteInFileInZip_failed = 0; + assertf(zip_zipWriteInFileInZip_failed); } } - } while(nl>0); + } while(nl > 0); fclose(fp); } else { /* Err FIXME - lost file */ } - } /* Empty files are OK */ + } /* Empty files are OK */ } } - + /* Close */ if ((zErr = zipCloseFileInZip((zipFile) cache->zipOutput)) != Z_OK) { int zip_zipCloseFileInZip_failed = 0; + assertf(zip_zipCloseFileInZip_failed); } /* Flush */ if ((zErr = zipFlush((zipFile) cache->zipOutput)) != 0) { int zip_zipFlush_failed = 0; + assertf(zip_zipFlush_failed); } } @@ -397,162 +413,181 @@ void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,const char* url_a #else /* Ajout d'un fichier en cache */ -void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,char* url_adr,char* url_fil,char* url_save,int all_in_cache) { +void cache_add(httrackp * opt, cache_back * cache, const htsblk * r, + char *url_adr, char *url_fil, char *url_save, int all_in_cache) { int pos; char s[256]; - char BIGSTK buff[HTS_URLMAXSIZE*4]; - int ok=1; - int dataincache=0; // donnée en cache? - FILE* cache_ndx = cache->ndx; - FILE* cache_dat = cache->dat; - /*char digest[32+2];*/ - /*digest[0]='\0';*/ + char BIGSTK buff[HTS_URLMAXSIZE * 4]; + int ok = 1; + int dataincache = 0; // donnée en cache? + FILE *cache_ndx = cache->ndx; + FILE *cache_dat = cache->dat; + + /*char digest[32+2]; */ + /*digest[0]='\0'; */ // Longueur url_save==0? - if ( (strnotempty(url_save)==0) ) { - if (strcmp(url_fil,"/robots.txt")==0) // robots.txt - dataincache=1; - else if (strcmp(url_fil,"/test")==0) // testing links - dataincache=0; + if ((strnotempty(url_save) == 0)) { + if (strcmp(url_fil, "/robots.txt") == 0) // robots.txt + dataincache = 1; + else if (strcmp(url_fil, "/test") == 0) // testing links + dataincache = 0; else - return; // erreur (sauf robots.txt) + return; // erreur (sauf robots.txt) } /* - if (r->size <= 0) // taille <= 0 - return; // refusé.. - */ + if (r->size <= 0) // taille <= 0 + return; // refusé.. + */ // Mettre les *donées* en cache ? - if (is_hypertext_mime(opt,r->contenttype, url_fil)) // html, mise en cache des données et - dataincache=1; // pas uniquement de l'en tête + if (is_hypertext_mime(opt, r->contenttype, url_fil)) // html, mise en cache des données et + dataincache = 1; // pas uniquement de l'en tête else if (all_in_cache) - dataincache=1; // forcer tout en cache + dataincache = 1; // forcer tout en cache /* calcul md5 ? */ /* - if (is_hypertext_mime(opt,r->contenttype)) { // html, calcul MD5 - if (r->adr) { - domd5mem(r->adr,r->size,digest,1); - } - }*/ + if (is_hypertext_mime(opt,r->contenttype)) { // html, calcul MD5 + if (r->adr) { + domd5mem(r->adr,r->size,digest,1); + } + } */ // Position - fflush(cache_dat); fflush(cache_ndx); - pos=ftell(cache_dat); + fflush(cache_dat); + fflush(cache_ndx); + pos = ftell(cache_dat); // écrire pointeur seek, adresse, fichier - if (dataincache) // patcher - sprintf(s,"%d\n",pos); // ecrire tel que (eh oui évite les \0..) + if (dataincache) // patcher + sprintf(s, "%d\n", pos); // ecrire tel que (eh oui évite les \0..) else - sprintf(s,"%d\n",-pos); // ecrire tel que (eh oui évite les \0..) + sprintf(s, "%d\n", -pos); // ecrire tel que (eh oui évite les \0..) // data // écrire données en-tête, données fichier /*if (!dataincache) { // patcher - r->size=-r->size; // négatif - }*/ + r->size=-r->size; // négatif + } */ // Construction header - ok=0; - if (cache_wint(cache_dat,r->statuscode) != -1 // statuscode - && cache_wLLint(cache_dat,r->size) != -1 // size - && cache_wstr(cache_dat,r->msg) != -1 // msg - && cache_wstr(cache_dat,r->contenttype) != -1 // contenttype - && cache_wstr(cache_dat,r->charset) != -1 // contenttype - && cache_wstr(cache_dat,r->lastmodified) != -1 // last-modified - && cache_wstr(cache_dat,r->etag) != -1 // Etag - && cache_wstr(cache_dat,(r->location!=NULL)?r->location:"") != -1 // 'location' pour moved - && cache_wstr(cache_dat,r->cdispo) != -1 // Content-disposition - && cache_wstr(cache_dat,url_adr) != -1 // Original address - && cache_wstr(cache_dat,url_fil) != -1 // Original URI filename - && cache_wstr(cache_dat,url_save) != -1 // Original save filename - && cache_wstr(cache_dat,r->headers) != -1 // Full HTTP Headers - && cache_wstr(cache_dat,"HTS") != -1 // end of header - ) { - ok=1; /* ok */ + ok = 0; + if (cache_wint(cache_dat, r->statuscode) != -1 // statuscode + && cache_wLLint(cache_dat, r->size) != -1 // size + && cache_wstr(cache_dat, r->msg) != -1 // msg + && cache_wstr(cache_dat, r->contenttype) != -1 // contenttype + && cache_wstr(cache_dat, r->charset) != -1 // contenttype + && cache_wstr(cache_dat, r->lastmodified) != -1 // last-modified + && cache_wstr(cache_dat, r->etag) != -1 // Etag + && cache_wstr(cache_dat, (r->location != NULL) ? r->location : "") != -1 // 'location' pour moved + && cache_wstr(cache_dat, r->cdispo) != -1 // Content-disposition + && cache_wstr(cache_dat, url_adr) != -1 // Original address + && cache_wstr(cache_dat, url_fil) != -1 // Original URI filename + && cache_wstr(cache_dat, url_save) != -1 // Original save filename + && cache_wstr(cache_dat, r->headers) != -1 // Full HTTP Headers + && cache_wstr(cache_dat, "HTS") != -1 // end of header + ) { + ok = 1; /* ok */ } // Fin construction header - /*if ((int) fwrite((char*) &r,1,sizeof(htsblk),cache_dat) == sizeof(htsblk)) {*/ + /*if ((int) fwrite((char*) &r,1,sizeof(htsblk),cache_dat) == sizeof(htsblk)) { */ if (ok) { - if (dataincache) { // mise en cache? - if (!r->adr) { /* taille nulle (parfois en cas de 301 */ - if (cache_wLLint(cache_dat,0)==-1) /* 0 bytes */ - ok=0; - } else if (r->is_write==0) { // en mémoire, recopie directe - if (cache_wLLint(cache_dat,r->size)!=-1) { - if (r->size>0) { // taille>0 - if (fwrite(r->adr,1,r->size,cache_dat)!=r->size) - ok=0; - } else // taille=0, ne rien écrire - ok=0; + if (dataincache) { // mise en cache? + if (!r->adr) { /* taille nulle (parfois en cas de 301 */ + if (cache_wLLint(cache_dat, 0) == -1) /* 0 bytes */ + ok = 0; + } else if (r->is_write == 0) { // en mémoire, recopie directe + if (cache_wLLint(cache_dat, r->size) != -1) { + if (r->size > 0) { // taille>0 + if (fwrite(r->adr, 1, r->size, cache_dat) != r->size) + ok = 0; + } else // taille=0, ne rien écrire + ok = 0; } else - ok=0; - } else { // recopier fichier dans cache - FILE* fp; + ok = 0; + } else { // recopier fichier dans cache + FILE *fp; + // On recopie le fichier->. - off_t file_size=fsize_utf8(fconv(catbuff, url_save)); - if (file_size>=0) { - if (cache_wLLint(cache_dat,file_size)!=-1) { - fp=FOPEN(fconv(catbuff, url_save),"rb"); - if (fp!=NULL) { + off_t file_size = fsize_utf8(fconv(catbuff, url_save)); + + if (file_size >= 0) { + if (cache_wLLint(cache_dat, file_size) != -1) { + fp = FOPEN(fconv(catbuff, url_save), "rb"); + if (fp != NULL) { char BIGSTK buff[32768]; ssize_t nl; + do { - nl=fread(buff,1,32768,fp); - if (nl>0) { - if (fwrite(buff,1,nl,cache_dat)!=nl) { // erreur - nl=-1; - ok=0; + nl = fread(buff, 1, 32768, fp); + if (nl > 0) { + if (fwrite(buff, 1, nl, cache_dat) != nl) { // erreur + nl = -1; + ok = 0; } } - } while(nl>0); + } while(nl > 0); fclose(fp); - } else ok=0; - } else ok=0; - } else ok=0; + } else + ok = 0; + } else + ok = 0; + } else + ok = 0; } } else { - if (cache_wLLint(cache_dat,0)==-1) /* 0 bytes */ - ok=0; + if (cache_wLLint(cache_dat, 0) == -1) /* 0 bytes */ + ok = 0; } - } else ok=0; + } else + ok = 0; /*if (!dataincache) { // dépatcher - r->size=-r->size; - }*/ + r->size=-r->size; + } */ // index // adresse+cr+fichier+cr if (ok) { - buff[0]='\0'; strcatbuff(buff,url_adr); strcatbuff(buff,"\n"); strcatbuff(buff,url_fil); strcatbuff(buff,"\n"); - cache_wstr(cache_ndx,buff); - fwrite(s,1,strlen(s),cache_ndx); - } // si ok=0 on a peut être écrit des données pour rien mais on s'en tape - + buff[0] = '\0'; + strcatbuff(buff, url_adr); + strcatbuff(buff, "\n"); + strcatbuff(buff, url_fil); + strcatbuff(buff, "\n"); + cache_wstr(cache_ndx, buff); + fwrite(s, 1, strlen(s), cache_ndx); + } // si ok=0 on a peut être écrit des données pour rien mais on s'en tape + // en cas de plantage, on aura au moins le cache! - fflush(cache_dat); fflush(cache_ndx); + fflush(cache_dat); + fflush(cache_ndx); } #endif - -htsblk cache_read(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location) { - return cache_readex(opt,cache,adr,fil,save,location,NULL,0); +htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr, + const char *fil, const char *save, char *location) { + return cache_readex(opt, cache, adr, fil, save, location, NULL, 0); } -htsblk cache_read_ro(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location) { - return cache_readex(opt,cache,adr,fil,save,location,NULL,1); +htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr, + const char *fil, const char *save, char *location) { + return cache_readex(opt, cache, adr, fil, save, location, NULL, 1); } -htsblk cache_read_including_broken(httrackp* opt,cache_back* cache,const char* adr,const char* fil) { - htsblk r = cache_read(opt,cache,adr,fil,NULL,NULL); +htsblk cache_read_including_broken(httrackp * opt, cache_back * cache, + const char *adr, const char *fil) { + htsblk r = cache_read(opt, cache, adr, fil, NULL, NULL); + if (r.statuscode == -1) { lien_back *itemback = NULL; + if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) { r = itemback->r; /* cleanup */ - back_clear_entry(itemback); /* delete entry content */ - freet(itemback); /* delete item */ + back_clear_entry(itemback); /* delete entry content */ + freet(itemback); /* delete item */ itemback = NULL; return r; } @@ -560,48 +595,58 @@ htsblk cache_read_including_broken(httrackp* opt,cache_back* cache,const char* a return r; } -static htsblk cache_readex_old(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location, - char* return_save, int readonly); +static htsblk cache_readex_old(httrackp * opt, cache_back * cache, + const char *adr, const char *fil, + const char *save, char *location, + char *return_save, int readonly); -static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location, - char* return_save, int readonly); +static htsblk cache_readex_new(httrackp * opt, cache_back * cache, + const char *adr, const char *fil, + const char *save, char *location, + char *return_save, int readonly); // lecture d'un fichier dans le cache // si save==null alors test unqiquement -htsblk cache_readex(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location, - char* return_save, int readonly) { +htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr, + const char *fil, const char *save, char *location, + char *return_save, int readonly) { if (cache->zipInput != NULL) { - return cache_readex_new(opt, cache, adr, fil, save, location, return_save, readonly); + return cache_readex_new(opt, cache, adr, fil, save, location, return_save, + readonly); } else { - return cache_readex_old(opt, cache, adr, fil, save, location, return_save, readonly); + return cache_readex_old(opt, cache, adr, fil, save, location, return_save, + readonly); } } // lecture d'un fichier dans le cache // si save==null alors test unqiquement -static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* target_save,char* location, - char* return_save, int readonly) { - char BIGSTK location_default[HTS_URLMAXSIZE*2]; - char BIGSTK buff[HTS_URLMAXSIZE*2]; - char BIGSTK previous_save[HTS_URLMAXSIZE*2]; - char BIGSTK previous_save_[HTS_URLMAXSIZE*2]; - char catbuff[CATBUFF_SIZE]; +static htsblk cache_readex_new(httrackp * opt, cache_back * cache, + const char *adr, const char *fil, + const char *target_save, char *location, + char *return_save, int readonly) { + char BIGSTK location_default[HTS_URLMAXSIZE * 2]; + char BIGSTK buff[HTS_URLMAXSIZE * 2]; + char BIGSTK previous_save[HTS_URLMAXSIZE * 2]; + char BIGSTK previous_save_[HTS_URLMAXSIZE * 2]; + char catbuff[CATBUFF_SIZE]; intptr_t hash_pos; int hash_pos_return; htsblk r; + hts_init_htsblk(&r); //memset(&r, 0, sizeof(htsblk)); r.soc=INVALID_SOCKET; - location_default[0] = '\0'; - previous_save[0] = previous_save_[0] = '\0'; + location_default[0] = '\0'; + previous_save[0] = previous_save_[0] = '\0'; if (location) { r.location = location; } else { r.location = location_default; } - strcpybuff(r.location, ""); + strcpybuff(r.location, ""); strcpybuff(buff, adr); - strcatbuff(buff,fil); + strcatbuff(buff, fil); hash_pos_return = inthash_read(cache->hashtable, buff, &hash_pos); /* avoid errors on data entries */ if (adr[0] == '/' && adr[1] == '/' && adr[2] == '[') { @@ -614,36 +659,42 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,c if (hash_pos_return != 0) { uLong posInZip; + if (hash_pos > 0) { posInZip = (uLong) hash_pos; } else { - posInZip = (uLong) -hash_pos; + posInZip = (uLong) - hash_pos; } if (unzSetOffset((unzFile) cache->zipInput, posInZip) == Z_OK) { /* Read header (Max 8KiB) */ if (unzOpenCurrentFile((unzFile) cache->zipInput) == Z_OK) { char BIGSTK headerBuff[8192 + 2]; int readSizeHeader; + //int totalHeader = 0; int dataincache = 0; - + /* For BIG comments */ - headerBuff[0] - = headerBuff[sizeof(headerBuff) - 1] - = headerBuff[sizeof(headerBuff) - 2] + headerBuff[0] + = headerBuff[sizeof(headerBuff) - 1] + = headerBuff[sizeof(headerBuff) - 2] = headerBuff[sizeof(headerBuff) - 3] = '\0'; - if ( (readSizeHeader = unzGetLocalExtrafield((unzFile) cache->zipInput, headerBuff, sizeof(headerBuff) - 2)) > 0) - /*if (unzGetCurrentFileInfo((unzFile) cache->zipInput, NULL, - NULL, 0, NULL, 0, headerBuff, sizeof(headerBuff) - 2) == Z_OK ) */ + if ((readSizeHeader = + unzGetLocalExtrafield((unzFile) cache->zipInput, headerBuff, + sizeof(headerBuff) - 2)) > 0) + /*if (unzGetCurrentFileInfo((unzFile) cache->zipInput, NULL, + NULL, 0, NULL, 0, headerBuff, sizeof(headerBuff) - 2) == Z_OK ) */ { int offset = 0; char BIGSTK line[HTS_URLMAXSIZE + 2]; int lineEof = 0; - /*readSizeHeader = (int) strlen(headerBuff);*/ + + /*readSizeHeader = (int) strlen(headerBuff); */ headerBuff[readSizeHeader] = '\0'; do { - char* value; + char *value; + line[0] = '\0'; offset += binput(headerBuff + offset, line, sizeof(line) - 2); if (line[0] == '\0') { @@ -652,99 +703,112 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,c value = strchr(line, ':'); if (value != NULL) { *value++ = '\0'; - if (*value == ' ' || *value == '\t') value++; + if (*value == ' ' || *value == '\t') + value++; ZIP_READFIELD_INT(line, value, "X-In-Cache", dataincache); ZIP_READFIELD_INT(line, value, "X-Statuscode", r.statuscode); - ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r.msg); // msg - ZIP_READFIELD_LLINT(line, value, "X-Size", r.size); // size - ZIP_READFIELD_STRING(line, value, "Content-Type", r.contenttype); // contenttype - ZIP_READFIELD_STRING(line, value, "X-Charset", r.charset); // contenttype - ZIP_READFIELD_STRING(line, value, "Last-Modified", r.lastmodified); // last-modified - ZIP_READFIELD_STRING(line, value, "Etag", r.etag); // Etag - ZIP_READFIELD_STRING(line, value, "Location", r.location); // 'location' pour moved - ZIP_READFIELD_STRING(line, value, "Content-Disposition", r.cdispo); // Content-disposition + ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r.msg); // msg + ZIP_READFIELD_LLINT(line, value, "X-Size", r.size); // size + ZIP_READFIELD_STRING(line, value, "Content-Type", r.contenttype); // contenttype + ZIP_READFIELD_STRING(line, value, "X-Charset", r.charset); // contenttype + ZIP_READFIELD_STRING(line, value, "Last-Modified", r.lastmodified); // last-modified + ZIP_READFIELD_STRING(line, value, "Etag", r.etag); // Etag + ZIP_READFIELD_STRING(line, value, "Location", r.location); // 'location' pour moved + ZIP_READFIELD_STRING(line, value, "Content-Disposition", r.cdispo); // Content-disposition //ZIP_READFIELD_STRING(line, value, "X-Addr", ..); // Original address //ZIP_READFIELD_STRING(line, value, "X-Fil", ..); // Original URI filename - ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename + ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename } } while(offset < readSizeHeader && !lineEof); //totalHeader = offset; - /* Previous entry */ - if (previous_save_[0] != '\0') { - int pathLen = (int) strlen(StringBuff(opt->path_html_utf8)); - if (pathLen != 0 && strncmp(previous_save_, StringBuff(opt->path_html_utf8), pathLen) != 0) { // old (<3.40) buggy format - sprintf(previous_save, "%s%s", StringBuff(opt->path_html_utf8), previous_save_); - } else { - strcpy(previous_save, previous_save_); - } - } + /* Previous entry */ + if (previous_save_[0] != '\0') { + int pathLen = (int) strlen(StringBuff(opt->path_html_utf8)); + + if (pathLen != 0 && strncmp(previous_save_, StringBuff(opt->path_html_utf8), pathLen) != 0) { // old (<3.40) buggy format + sprintf(previous_save, "%s%s", StringBuff(opt->path_html_utf8), + previous_save_); + } else { + strcpy(previous_save, previous_save_); + } + } if (return_save != NULL) { strcpybuff(return_save, previous_save); } /* Complete fields */ - r.totalsize=r.size; - r.adr=NULL; - r.out=NULL; - r.fp=NULL; + r.totalsize = r.size; + r.adr = NULL; + r.out = NULL; + r.fp = NULL; /* Do not get data ? Do some final tests. */ if (target_save == NULL) { // si save==null, ne rien charger (juste en tête) - if (r.statuscode==HTTP_OK && !is_hypertext_mime(opt, r.contenttype, fil)) { // pas HTML, écrire sur disk directement - r.is_write = 1; /* supposed to be on disk (informational) */ + if (r.statuscode == HTTP_OK && !is_hypertext_mime(opt, r.contenttype, fil)) { // pas HTML, écrire sur disk directement + r.is_write = 1; /* supposed to be on disk (informational) */ } /* Ensure the file is present, because returning a reference to a missing file is useless! */ - if (!dataincache) { /* Data are supposed to be on disk */ - if (!fexist_utf8(fconv(catbuff, previous_save))) { // un fichier existe déja + if (!dataincache) { /* Data are supposed to be on disk */ + if (!fexist_utf8(fconv(catbuff, previous_save))) { // un fichier existe déja if (!opt->norecatch) { - hts_log_print(opt, LOG_DEBUG, "Cache: could not find %s", previous_save); - r.statuscode=STATUSCODE_INVALID; + hts_log_print(opt, LOG_DEBUG, "Cache: could not find %s", + previous_save); + r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Previous cache file not found"); } } - } // otherwise, the ZIP file is supposed to be consistent with data. + } // otherwise, the ZIP file is supposed to be consistent with data. } /* Read data ? */ - else { /* ne pas lire uniquement header */ + else { /* ne pas lire uniquement header */ int ok = 0; #if HTS_DIRECTDISK // Not ro, and pure data (not HTML and friends) to be saved now. - if (!readonly && r.statuscode==HTTP_OK && !is_hypertext_mime(opt,r.contenttype, fil) && strnotempty(target_save)) { // pas HTML, écrire sur disk directement - r.is_write=1; // écrire + if (!readonly && r.statuscode == HTTP_OK && !is_hypertext_mime(opt, r.contenttype, fil) && strnotempty(target_save)) { // pas HTML, écrire sur disk directement + r.is_write = 1; // écrire // Data is supposed to be on disk if (!dataincache) { r.msg[0] = '\0'; // File exists on disk with declared cache name (this is expected!) - if (fexist_utf8(fconv(catbuff, previous_save))) { // un fichier existe déja + if (fexist_utf8(fconv(catbuff, previous_save))) { // un fichier existe déja // Expected size ? - const size_t fsize = fsize_utf8(fconv(catbuff, previous_save)); + const size_t fsize = + fsize_utf8(fconv(catbuff, previous_save)); if (fsize == r.size) { // Target name is the previous name, and the file looks good: nothing to do! if (strcmp(previous_save, target_save) == 0) { // So far so good - ok=1; // plus rien à faire + ok = 1; // plus rien à faire } // Different filenames: rename now! else { char catbuff2[CATBUFF_SIZE]; - if (RENAME(fconv(catbuff, previous_save), fconv(catbuff2, target_save)) == 0) { + + if (RENAME + (fconv(catbuff, previous_save), + fconv(catbuff2, target_save)) == 0) { // So far so good - ok=1; // plus rien à faire + ok = 1; // plus rien à faire - hts_log_print(opt, LOG_DEBUG, "File '%s' has been renamed since last mirror to '%s' ; applying changes", previous_save, target_save); + hts_log_print(opt, LOG_DEBUG, + "File '%s' has been renamed since last mirror to '%s' ; applying changes", + previous_save, target_save); } else { - r.statuscode=STATUSCODE_INVALID; + r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "Unable to rename file on disk"); } } } else { - hts_log_print(opt, LOG_WARNING, "warning: file size on disk ("LLintP") does not have the expected size ("LLintP"))", (LLint) fsize, (LLint) r.size); + hts_log_print(opt, LOG_WARNING, + "warning: file size on disk (" LLintP + ") does not have the expected size (" LLintP + "))", (LLint) fsize, (LLint) r.size); } } // File exists with the target name and not previous one ? @@ -752,90 +816,102 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,c else if (fexist_utf8(fconv(catbuff, target_save))) { // Expected size ? const size_t fsize = fsize_utf8(fconv(catbuff, target_save)); + if (fsize == r.size) { // So far so good - ok=1; // plus rien à faire + ok = 1; // plus rien à faire } else { - hts_log_print(opt, LOG_WARNING, "warning: renamed file size on disk ("LLintP") does not have the expected size ("LLintP"))", (LLint) fsize, (LLint) r.size); + hts_log_print(opt, LOG_WARNING, + "warning: renamed file size on disk (" LLintP + ") does not have the expected size (" LLintP + "))", (LLint) fsize, (LLint) r.size); } } - // File is present and sane ? if (ok) { // Register file not to wipe it later - filenote(&opt->state.strc,target_save,NULL); // noter comme connu - file_notify(opt,adr, fil, target_save, 0, 0, 1); // data in cache + filenote(&opt->state.strc, target_save, NULL); // noter comme connu + file_notify(opt, adr, fil, target_save, 0, 0, 1); // data in cache } // Nope else { // Default behavior if (!opt->norecatch) { - r.statuscode=STATUSCODE_INVALID; + r.statuscode = STATUSCODE_INVALID; if (r.msg[0] == '\0') { strcpybuff(r.msg, "Previous cache file not found"); } } // Do not recatch broken/erased files else { - file_notify(opt,adr, fil, target_save, 1, 0, 0); + file_notify(opt, adr, fil, target_save, 1, 0, 0); filecreateempty(&opt->state.strc, target_save); // - r.statuscode=STATUSCODE_INVALID; + r.statuscode = STATUSCODE_INVALID; strcpybuff(r.msg, "File deleted by user not recaught"); } } } // Load data from cache (not from a disk file) else { - file_notify(opt,adr, fil, target_save, 1, 1, 1); // data in cache - r.out=filecreate(&opt->state.strc, target_save); + file_notify(opt, adr, fil, target_save, 1, 1, 1); // data in cache + r.out = filecreate(&opt->state.strc, target_save); #if HDEBUG - printf("direct-disk: %s\n",save); + printf("direct-disk: %s\n", save); #endif - if (r.out!=NULL) { - char BIGSTK buff[32768+4]; + if (r.out != NULL) { + char BIGSTK buff[32768 + 4]; LLint size = r.size; + if (size > 0) { size_t nl; + do { - nl = unzReadCurrentFile((unzFile) cache->zipInput, buff, (int)minimum(size, 32768)); - if (nl>0) { - size-=nl; - if (fwrite(buff,1,nl,r.out)!=nl) { // erreur + nl = + unzReadCurrentFile((unzFile) cache->zipInput, buff, + (int) minimum(size, 32768)); + if (nl > 0) { + size -= nl; + if (fwrite(buff, 1, nl, r.out) != nl) { // erreur int last_errno = errno; - r.statuscode=STATUSCODE_INVALID; - sprintf(r.msg,"Cache Read Error : Read To Disk: %s", strerror(last_errno)); + + r.statuscode = STATUSCODE_INVALID; + sprintf(r.msg, "Cache Read Error : Read To Disk: %s", + strerror(last_errno)); } } - } while((nl>0) && (size>0) && (r.statuscode!=-1)); + } while((nl > 0) && (size > 0) && (r.statuscode != -1)); } - + fclose(r.out); - r.out=NULL; + r.out = NULL; #ifndef _WIN32 - chmod(target_save, HTS_ACCESS_FILE); -#endif + chmod(target_save, HTS_ACCESS_FILE); +#endif } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Write Error : Unable to Create File"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, + "Cache Write Error : Unable to Create File"); //printf("%s\n",save); } } - + } else #endif - { // lire en mémoire - + { // lire en mémoire + // We need to get bytes on memory, but the previous version is (supposed to be) on disk. if (!dataincache) { // Empty previous save name, or file does not exist ? - if (!strnotempty(previous_save) || !fexist_utf8(previous_save)) { // Pas de donnée en cache, bizarre car html!!! + if (!strnotempty(previous_save) || !fexist_utf8(previous_save)) { // Pas de donnée en cache, bizarre car html!!! // Hack: if error page data is missing (pre-3.45 releases), create one. We won't use it anyway. if (!HTTP_IS_OK(r.statuscode)) { const int size = 512; + r.adr = malloct(size); - sprintf(r.adr, "<html><!-- Missing Error Page ; Generated by HTTrack Website Copier --><head><title>HTTP Error %u</title></head><body><h1>HTTP Error %u</h1></body></html>", - r.statuscode, r.statuscode); + sprintf(r.adr, + "<html><!-- Missing Error Page ; Generated by HTTrack Website Copier --><head><title>HTTP Error %u</title></head><body><h1>HTTP Error %u</h1></body></html>", + r.statuscode, r.statuscode); r.size = strlen(r.adr); assertf(r.size < size); strcpy(r.contenttype, "text/html"); @@ -847,93 +923,101 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,c } // Otherwise, this is a real error. else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Previous cache file not found (empty filename)"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, + "Previous cache file not found (empty filename)"); } - } else { /* Read in memory from disk */ - FILE*const fp = FOPEN(fconv(catbuff, previous_save), "rb"); + } else { /* Read in memory from disk */ + FILE *const fp = FOPEN(fconv(catbuff, previous_save), "rb"); + if (fp != NULL) { - r.adr = (char*) malloct((int) r.size + 4); + r.adr = (char *) malloct((int) r.size + 4); if (r.adr != NULL) { - if (r.size > 0 && fread(r.adr, 1, (int) r.size, fp) != r.size) { + if (r.size > 0 + && fread(r.adr, 1, (int) r.size, fp) != r.size) { int last_errno = errno; - r.statuscode=STATUSCODE_INVALID; - sprintf(r.msg,"Read error in cache disk data: %s", strerror(last_errno)); + + r.statuscode = STATUSCODE_INVALID; + sprintf(r.msg, "Read error in cache disk data: %s", + strerror(last_errno)); } } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Read error (memory exhausted) from cache"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, + "Read error (memory exhausted) from cache"); } fclose(fp); } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Read error (unable to open disk file) from cache"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, + "Read error (unable to open disk file) from cache"); } } } // Data in cache. else { // lire fichier (d'un coup) - r.adr = (char*) malloct((int) r.size+4); - if (r.adr!=NULL) { - if (unzReadCurrentFile((unzFile) cache->zipInput, r.adr, (int) r.size) != r.size) { // erreur + r.adr = (char *) malloct((int) r.size + 4); + if (r.adr != NULL) { + if (unzReadCurrentFile((unzFile) cache->zipInput, r.adr, (int) r.size) != r.size) { // erreur freet(r.adr); - r.adr=NULL; - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Read Error : Read Data"); + r.adr = NULL; + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Read Error : Read Data"); } else - *(r.adr+r.size)='\0'; + *(r.adr + r.size) = '\0'; //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode); - } else { // erreur - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Memory Error"); + } else { // erreur + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Memory Error"); } } } } } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Read Error : Read Header Data"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Read Error : Read Header Data"); } unzCloseCurrentFile((unzFile) cache->zipInput); } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Read Error : Open File"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Read Error : Open File"); } } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Read Error : Bad Offset"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Read Error : Bad Offset"); } } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"File Cache Entry Not Found"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "File Cache Entry Not Found"); } - if (!location) { /* don't export internal buffer */ + if (!location) { /* don't export internal buffer */ r.location = NULL; } return r; } - // lecture d'un fichier dans le cache // si save==null alors test unqiquement -static htsblk cache_readex_old(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location, - char* return_save, int readonly) { +static htsblk cache_readex_old(httrackp * opt, cache_back * cache, + const char *adr, const char *fil, + const char *save, char *location, + char *return_save, int readonly) { #if HTS_FAST_CACHE intptr_t hash_pos; int hash_pos_return; #else - char* a; + char *a; #endif - char BIGSTK buff[HTS_URLMAXSIZE*2]; - char BIGSTK location_default[HTS_URLMAXSIZE*2]; - char BIGSTK previous_save[HTS_URLMAXSIZE*2]; - char catbuff[CATBUFF_SIZE]; + char BIGSTK buff[HTS_URLMAXSIZE * 2]; + char BIGSTK location_default[HTS_URLMAXSIZE * 2]; + char BIGSTK previous_save[HTS_URLMAXSIZE * 2]; + char catbuff[CATBUFF_SIZE]; htsblk r; - int ok=0; - int header_only=0; + int ok = 0; + int header_only = 0; hts_init_htsblk(&r); //memset(&r, 0, sizeof(htsblk)); r.soc=INVALID_SOCKET; @@ -942,16 +1026,22 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,const char* adr,c } else { r.location = location_default; } - strcpybuff(r.location, ""); + strcpybuff(r.location, ""); #if HTS_FAST_CACHE - strcpybuff(buff,adr); strcatbuff(buff,fil); - hash_pos_return=inthash_read(cache->hashtable,buff,&hash_pos); + strcpybuff(buff, adr); + strcatbuff(buff, fil); + hash_pos_return = inthash_read(cache->hashtable, buff, &hash_pos); #else - buff[0]='\0'; strcatbuff(buff,"\n"); strcatbuff(buff,adr); strcatbuff(buff,"\n"); strcatbuff(buff,fil); strcatbuff(buff,"\n"); + buff[0] = '\0'; + strcatbuff(buff, "\n"); + strcatbuff(buff, adr); + strcatbuff(buff, "\n"); + strcatbuff(buff, fil); + strcatbuff(buff, "\n"); if (cache->use) - a=strstr(cache->use,buff); + a = strstr(cache->use, buff); else - a=NULL; // forcer erreur + a = NULL; // forcer erreur #endif /* avoid errors on data entries */ @@ -962,63 +1052,65 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,const char* adr,c a = NULL; #endif } - // en cas de succès #if HTS_FAST_CACHE if (hash_pos_return != 0) { #else - if (a!=NULL) { // OK existe en cache! + if (a != NULL) { // OK existe en cache! #endif intptr_t pos; + #if DEBUGCA - fprintf(stdout,"..cache: %s%s at ",adr,fil); + fprintf(stdout, "..cache: %s%s at ", adr, fil); #endif - + #if HTS_FAST_CACHE - pos = hash_pos; /* simply */ + pos = hash_pos; /* simply */ #else - a+=strlen(buff); - sscanf(a,"%d",&pos); // lire position + a += strlen(buff); + sscanf(a, "%d", &pos); // lire position #endif #if DEBUGCA - printf("%d\n",pos); + printf("%d\n", pos); #endif - fflush(cache->olddat); - if (fseek(cache->olddat,(long)((pos>0)?pos:(-pos)),SEEK_SET) == 0) { + fflush(cache->olddat); + if (fseek(cache->olddat, (long) ((pos > 0) ? pos : (-pos)), SEEK_SET) == 0) { /* Importer cache1.0 */ - if (cache->version==0) { + if (cache->version == 0) { OLD_htsblk old_r; - if (fread((char*) &old_r,1,sizeof(old_r),cache->olddat)==sizeof(old_r)) { // lire tout (y compris statuscode etc) - r.statuscode=old_r.statuscode; - r.size=old_r.size; // taille fichier - strcpybuff(r.msg,old_r.msg); - strcpybuff(r.contenttype,old_r.contenttype); - ok=1; /* import ok */ + + if (fread((char *) &old_r, 1, sizeof(old_r), cache->olddat) == sizeof(old_r)) { // lire tout (y compris statuscode etc) + r.statuscode = old_r.statuscode; + r.size = old_r.size; // taille fichier + strcpybuff(r.msg, old_r.msg); + strcpybuff(r.contenttype, old_r.contenttype); + ok = 1; /* import ok */ } - /* */ - /* Cache 1.1 */ + /* */ + /* Cache 1.1 */ } else { char check[256]; LLint size_read; - check[0]='\0'; + + check[0] = '\0'; // - cache_rint(cache->olddat,&r.statuscode); - cache_rLLint(cache->olddat,&r.size); - cache_rstr(cache->olddat,r.msg); - cache_rstr(cache->olddat,r.contenttype); + cache_rint(cache->olddat, &r.statuscode); + cache_rLLint(cache->olddat, &r.size); + cache_rstr(cache->olddat, r.msg); + cache_rstr(cache->olddat, r.contenttype); if (cache->version >= 3) - cache_rstr(cache->olddat,r.charset); - cache_rstr(cache->olddat,r.lastmodified); - cache_rstr(cache->olddat,r.etag); - cache_rstr(cache->olddat,r.location); + cache_rstr(cache->olddat, r.charset); + cache_rstr(cache->olddat, r.lastmodified); + cache_rstr(cache->olddat, r.etag); + cache_rstr(cache->olddat, r.location); if (cache->version >= 2) - cache_rstr(cache->olddat,r.cdispo); + cache_rstr(cache->olddat, r.cdispo); if (cache->version >= 4) { - cache_rstr(cache->olddat, previous_save); // adr - cache_rstr(cache->olddat, previous_save); // fil + cache_rstr(cache->olddat, previous_save); // adr + cache_rstr(cache->olddat, previous_save); // fil previous_save[0] = '\0'; - cache_rstr(cache->olddat, previous_save); // save + cache_rstr(cache->olddat, previous_save); // save if (return_save != NULL) { strcpybuff(return_save, previous_save); } @@ -1027,180 +1119,187 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,const char* adr,c r.headers = cache_rstr_addr(cache->olddat); } // - cache_rstr(cache->olddat,check); - if (strcmp(check,"HTS")==0) { /* intégrité OK */ - ok=1; + cache_rstr(cache->olddat, check); + if (strcmp(check, "HTS") == 0) { /* intégrité OK */ + ok = 1; } - cache_rLLint(cache->olddat,&size_read); /* lire size pour être sûr de la taille déclarée (réécrire) */ - if (size_read>0) { /* si inscrite ici */ - r.size=size_read; - } else { /* pas de données directement dans le cache, fichier présent? */ - if (r.statuscode!=HTTP_OK) - header_only=1; /* que l'en tête ici! */ + cache_rLLint(cache->olddat, &size_read); /* lire size pour être sûr de la taille déclarée (réécrire) */ + if (size_read > 0) { /* si inscrite ici */ + r.size = size_read; + } else { /* pas de données directement dans le cache, fichier présent? */ + if (r.statuscode != HTTP_OK) + header_only = 1; /* que l'en tête ici! */ } } /* Remplir certains champs */ - r.totalsize=r.size; + r.totalsize = r.size; // lecture du header (y compris le statuscode) - /*if (fread((char*) &r,1,sizeof(htsblk),cache->olddat)==sizeof(htsblk)) { // lire tout (y compris statuscode etc)*/ + /*if (fread((char*) &r,1,sizeof(htsblk),cache->olddat)==sizeof(htsblk)) { // lire tout (y compris statuscode etc) */ if (ok) { // sécurité - r.adr=NULL; - r.out=NULL; + r.adr = NULL; + r.out = NULL; ////r.location=NULL; non, fixée lors des 301 ou 302 - r.fp=NULL; - - if ( (r.statuscode>=0) && (r.statuscode<=999) - && (r.notmodified>=0) && (r.notmodified<=9) ) { // petite vérif intégrité - if ((save) && (!header_only) ) { /* ne pas lire uniquement header */ + r.fp = NULL; + + if ((r.statuscode >= 0) && (r.statuscode <= 999) + && (r.notmodified >= 0) && (r.notmodified <= 9)) { // petite vérif intégrité + if ((save) && (!header_only)) { /* ne pas lire uniquement header */ //int to_file=0; - - r.adr=NULL; r.soc=INVALID_SOCKET; + + r.adr = NULL; + r.soc = INVALID_SOCKET; // // r.location=NULL; - + #if HTS_DIRECTDISK // Court-circuit: // Peut-on stocker le fichier directement sur disque? - if (!readonly && r.statuscode==HTTP_OK && !is_hypertext_mime(opt,r.contenttype, fil) && strnotempty(save)) { // pas HTML, écrire sur disk directement - int ok=0; - - r.is_write=1; // écrire + if (!readonly && r.statuscode == HTTP_OK && !is_hypertext_mime(opt, r.contenttype, fil) && strnotempty(save)) { // pas HTML, écrire sur disk directement + int ok = 0; + + r.is_write = 1; // écrire if (fexist_utf8(fconv(catbuff, save))) { // un fichier existe déja //if (fsize_utf8(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree) - ok=1; // plus rien à faire - filenote(&opt->state.strc,save,NULL); // noter comme connu - file_notify(opt,adr, fil, save, 0, 0, 0); + ok = 1; // plus rien à faire + filenote(&opt->state.strc, save, NULL); // noter comme connu + file_notify(opt, adr, fil, save, 0, 0, 0); //} } - - if ((pos<0) && (!ok)) { // Pas de donnée en cache et fichier introuvable : erreur! + + if ((pos < 0) && (!ok)) { // Pas de donnée en cache et fichier introuvable : erreur! if (opt->norecatch) { - file_notify(opt,adr, fil, save, 1, 0, 0); + file_notify(opt, adr, fil, save, 1, 0, 0); filecreateempty(&opt->state.strc, save); // - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"File deleted by user not recaught"); - ok=1; // ne pas récupérer (et pas d'erreur) + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "File deleted by user not recaught"); + ok = 1; // ne pas récupérer (et pas d'erreur) } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Previous cache file not found"); - ok=1; // ne pas récupérer + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Previous cache file not found"); + ok = 1; // ne pas récupérer } } - - if (!ok) { - r.out=filecreate(&opt->state.strc, save); + + if (!ok) { + r.out = filecreate(&opt->state.strc, save); #if HDEBUG - printf("direct-disk: %s\n",save); + printf("direct-disk: %s\n", save); #endif - if (r.out!=NULL) { - char BIGSTK buff[32768+4]; + if (r.out != NULL) { + char BIGSTK buff[32768 + 4]; size_t size = (size_t) r.size; + if (size > 0) { size_t nl; + do { - nl=fread(buff,1,minimum(size,32768),cache->olddat); - if (nl>0) { - size-=nl; - if (fwrite(buff,1,nl,r.out)!=nl) { // erreur - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Read Error : Read To Disk"); + nl = fread(buff, 1, minimum(size, 32768), cache->olddat); + if (nl > 0) { + size -= nl; + if (fwrite(buff, 1, nl, r.out) != nl) { // erreur + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Read Error : Read To Disk"); } } - } while((nl>0) && (size>0) && (r.statuscode!=-1)); + } while((nl > 0) && (size > 0) && (r.statuscode != -1)); } - + fclose(r.out); - r.out=NULL; + r.out = NULL; #ifndef _WIN32 - chmod(save,HTS_ACCESS_FILE); -#endif + chmod(save, HTS_ACCESS_FILE); +#endif } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Write Error : Unable to Create File"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, + "Cache Write Error : Unable to Create File"); //printf("%s\n",save); } } - + } else #endif - { // lire en mémoire - - if (pos<0) { - if (strnotempty(save)) { // Pas de donnée en cache, bizarre car html!!! - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Previous cache file not found (2)"); - } else { /* Read in memory from cache */ + { // lire en mémoire + + if (pos < 0) { + if (strnotempty(save)) { // Pas de donnée en cache, bizarre car html!!! + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Previous cache file not found (2)"); + } else { /* Read in memory from cache */ if (strnotempty(return_save) && fexist_utf8(return_save)) { - FILE* fp = FOPEN(fconv(catbuff, return_save), "rb"); + FILE *fp = FOPEN(fconv(catbuff, return_save), "rb"); + if (fp != NULL) { - r.adr = (char*) malloct((size_t)r.size + 4); + r.adr = (char *) malloct((size_t) r.size + 4); if (r.adr != NULL) { - if (r.size > 0 && fread(r.adr, 1, (size_t)r.size, fp) != r.size) { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Read error in cache disk data"); + if (r.size > 0 + && fread(r.adr, 1, (size_t) r.size, fp) != r.size) { + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Read error in cache disk data"); } } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Read error (memory exhausted) from cache"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, + "Read error (memory exhausted) from cache"); } fclose(fp); } } else { - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache file not found on disk"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache file not found on disk"); } } } else { // lire fichier (d'un coup) - r.adr=(char*) malloct((size_t)r.size+4); - if (r.adr!=NULL) { - if (fread(r.adr,1,(size_t)r.size,cache->olddat)!=r.size) { // erreur + r.adr = (char *) malloct((size_t) r.size + 4); + if (r.adr != NULL) { + if (fread(r.adr, 1, (size_t) r.size, cache->olddat) != r.size) { // erreur freet(r.adr); - r.adr=NULL; - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Read Error : Read Data"); + r.adr = NULL; + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Read Error : Read Data"); } else - *(r.adr+r.size)='\0'; + *(r.adr + r.size) = '\0'; //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode); - } else { // erreur - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Memory Error"); + } else { // erreur + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Memory Error"); } } } - } // si save==null, ne rien charger (juste en tête) + } // si save==null, ne rien charger (juste en tête) } else { #if DEBUGCA printf("Cache Read Error : Bad Data"); #endif - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Read Error : Bad Data"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Read Error : Bad Data"); } - } else { // erreur + } else { // erreur #if DEBUGCA printf("Cache Read Error : Read Header"); #endif - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Read Error : Read Header"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Read Error : Read Header"); } } else { #if DEBUGCA printf("Cache Read Error : Seek Failed"); #endif - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"Cache Read Error : Seek Failed"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "Cache Read Error : Seek Failed"); } } else { #if DEBUGCA printf("File Cache Not Found"); #endif - r.statuscode=STATUSCODE_INVALID; - strcpybuff(r.msg,"File Cache Entry Not Found"); + r.statuscode = STATUSCODE_INVALID; + strcpybuff(r.msg, "File Cache Entry Not Found"); } - if (!location) { /* don't export internal buffer */ + if (!location) { /* don't export internal buffer */ r.location = NULL; } return r; @@ -1208,22 +1307,30 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,const char* adr,c /* write (string1-string2)-data in cache */ /* 0 if failed */ -int cache_writedata(FILE* cache_ndx,FILE* cache_dat,const char* str1,const char* str2,char* outbuff,int len) { +int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1, + const char *str2, char *outbuff, int len) { if (cache_dat) { - char BIGSTK buff[HTS_URLMAXSIZE*4]; + char BIGSTK buff[HTS_URLMAXSIZE * 4]; char s[256]; int pos; - fflush(cache_dat); fflush(cache_ndx); - pos=ftell(cache_dat); + + fflush(cache_dat); + fflush(cache_ndx); + pos = ftell(cache_dat); /* first write data */ - if (cache_wint(cache_dat,len)!=-1) { // length - if (fwrite(outbuff,1,len,cache_dat) == len) { // data + if (cache_wint(cache_dat, len) != -1) { // length + if (fwrite(outbuff, 1, len, cache_dat) == len) { // data /* then write index */ - sprintf(s,"%d\n",pos); - buff[0]='\0'; strcatbuff(buff,str1); strcatbuff(buff,"\n"); strcatbuff(buff,str2); strcatbuff(buff,"\n"); - cache_wstr(cache_ndx,buff); - if (fwrite(s,1,strlen(s),cache_ndx) == strlen(s)) { - fflush(cache_dat); fflush(cache_ndx); + sprintf(s, "%d\n", pos); + buff[0] = '\0'; + strcatbuff(buff, str1); + strcatbuff(buff, "\n"); + strcatbuff(buff, str2); + strcatbuff(buff, "\n"); + cache_wstr(cache_ndx, buff); + if (fwrite(s, 1, strlen(s), cache_ndx) == strlen(s)) { + fflush(cache_dat); + fflush(cache_ndx); return 1; } } @@ -1234,22 +1341,28 @@ int cache_writedata(FILE* cache_ndx,FILE* cache_dat,const char* str1,const char* /* read the data corresponding to (string1-string2) in cache */ /* 0 if failed */ -int cache_readdata(cache_back* cache,const char* str1,const char* str2,char** inbuff,int* inlen) { +int cache_readdata(cache_back * cache, const char *str1, const char *str2, + char **inbuff, int *inlen) { #if HTS_FAST_CACHE if (cache->hashtable) { - char BIGSTK buff[HTS_URLMAXSIZE*4]; + char BIGSTK buff[HTS_URLMAXSIZE * 4]; intptr_t pos; - strcpybuff(buff,str1); strcatbuff(buff,str2); - if (inthash_read(cache->hashtable,buff,&pos)) { - if (fseek(cache->olddat,(long)((pos>0)?pos:(-pos)),SEEK_SET) == 0) { + + strcpybuff(buff, str1); + strcatbuff(buff, str2); + if (inthash_read(cache->hashtable, buff, &pos)) { + if (fseek(cache->olddat, (long) ((pos > 0) ? pos : (-pos)), SEEK_SET) == + 0) { INTsys len; - cache_rint(cache->olddat,&len); - if (len>0) { - char* mem_buff=(char*)malloct(len+4); /* Plus byte 0 */ + + cache_rint(cache->olddat, &len); + if (len > 0) { + char *mem_buff = (char *) malloct(len + 4); /* Plus byte 0 */ + if (mem_buff) { - if (fread(mem_buff,1,len,cache->olddat)==len) { // lire tout (y compris statuscode etc)*/ - *inbuff=mem_buff; - *inlen=len; + if (fread(mem_buff, 1, len, cache->olddat) == len) { // lire tout (y compris statuscode etc)*/ + *inbuff = mem_buff; + *inlen = len; return 1; } else freet(mem_buff); @@ -1259,179 +1372,266 @@ int cache_readdata(cache_back* cache,const char* str1,const char* str2,char** in } } #endif - *inbuff=NULL; - *inlen=0; + *inbuff = NULL; + *inlen = 0; return 0; } -static int hts_rename(httrackp* opt, const char *a, const char *b) { +static int hts_rename(httrackp * opt, const char *a, const char *b) { hts_log_print(opt, LOG_DEBUG, "Cache: rename %s -> %s (%p %p)", a, b, a, b); return rename(a, b); } // renvoyer uniquement en tête, ou NULL si erreur // return NULL upon error, and set -1 to r.statuscode -htsblk* cache_header(httrackp* opt,cache_back* cache,const char* adr,const char* fil,htsblk* r) { - *r=cache_read(opt,cache,adr,fil,NULL,NULL); // test uniquement +htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr, + const char *fil, htsblk * r) { + *r = cache_read(opt, cache, adr, fil, NULL, NULL); // test uniquement if (r->statuscode != -1) return r; else return NULL; } - // Initialisation du cache: créer nouveau, renomer ancien, charger.. -void cache_init(cache_back* cache,httrackp* opt) { +void cache_init(cache_back * cache, httrackp * opt) { // --- // utilisation du cache: renommer ancien éventuel et charger index - hts_log_print(opt, LOG_DEBUG, "Cache: enabled=%d, base=%s, ro=%d", (int) opt->cache, fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/"), (int) cache->ro); + hts_log_print(opt, LOG_DEBUG, "Cache: enabled=%d, base=%s, ro=%d", + (int) opt->cache, fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/"), (int) cache->ro); if (opt->cache) { #if DEBUGCA printf("cache init: "); #endif if (!cache->ro) { #ifdef _WIN32 - mkdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache")); + mkdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache")); #else - mkdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache"),HTS_PROTECT_FOLDER); + mkdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache"), + HTS_PROTECT_FOLDER); #endif - if ((fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")))) { // il existe déja un cache précédent.. renommer + if ((fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/new.zip")))) { // il existe déja un cache précédent.. renommer /* Previous cache from the previous cache version */ #if 0 /* No.. reuse with old httrack releases! */ - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx")); #endif /* Previous cache version */ - if ((fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx")); + if ((fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/old.dat")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/old.ndx")); } /* Remove OLD cache */ - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip"))) { - if (remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip")) != 0) { - hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "Cache: error while moving previous cache"); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip"))) { + if (remove + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip")) != 0) { + hts_log_print(opt, LOG_WARNING | LOG_ERRNO, + "Cache: error while moving previous cache"); } } - + /* Rename */ - if (hts_rename(opt, fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip"), fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip")) != 0) { - hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "Cache: error while moving previous cache"); + if (hts_rename + (opt, + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/old.zip")) != 0) { + hts_log_print(opt, LOG_WARNING | LOG_ERRNO, + "Cache: error while moving previous cache"); } else { hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed"); } - } - else if ((fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer + } else if ((fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer #if DEBUGCA printf("work with former cache\n"); #endif - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx")); - - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx")); - } else { // un des deux (ou les deux) fichiers cache absents: effacer l'autre éventuel + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx")); + + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/old.dat")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/old.ndx")); + } else { // un des deux (ou les deux) fichiers cache absents: effacer l'autre éventuel #if DEBUGCA printf("new cache\n"); #endif - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx")); } } else { hts_log_print(opt, LOG_DEBUG, "Cache: no cache found"); } - hts_log_print(opt, LOG_DEBUG, "Cache: size %d", (int)fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip"))); + hts_log_print(opt, LOG_DEBUG, "Cache: size %d", + (int) + fsize(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip"))); // charger index cache précédent - if ( - ( - !cache->ro && - fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip")) > 0 - ) - || - ( - cache->ro && - fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")) > 0 - ) - ) - { + if ((!cache->ro + && + fsize(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip")) > 0) + || (cache->ro + && + fsize(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip")) > 0) + ) { if (!cache->ro) { - cache->zipInput = unzOpen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip")); + cache->zipInput = + unzOpen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip")); } else { - cache->zipInput = unzOpen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")); + cache->zipInput = + unzOpen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip")); } - + // Corrupted ZIP file ? Try to repair! if (cache->zipInput == NULL && !cache->ro) { - char* name; + char *name; uLong repaired = 0; uLong repairedBytes = 0; + if (!cache->ro) { - name = fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip"); + name = + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip"); } else { - name = fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip"); + name = + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip"); + } + hts_log_print(opt, LOG_WARNING, + "Cache: damaged cache, trying to repair"); + if (unzRepair + (name, + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/repair.zip"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/repair.tmp"), + &repaired, &repairedBytes) == Z_OK) { + unlink(name); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/repair.zip"), name); + cache->zipInput = unzOpen(name); + hts_log_print(opt, LOG_WARNING, + "Cache: %d bytes successfully recovered in %d entries", + (int) repairedBytes, (int) repaired); + } else { + hts_log_print(opt, LOG_WARNING, "Cache: could not repair the cache"); } - hts_log_print(opt, LOG_WARNING, "Cache: damaged cache, trying to repair"); - if (unzRepair(name, - fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/repair.zip"), - fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/repair.tmp"), - &repaired, &repairedBytes - ) == Z_OK) { - unlink(name); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/repair.zip"), name); - cache->zipInput = unzOpen(name); - hts_log_print(opt, LOG_WARNING, "Cache: %d bytes successfully recovered in %d entries", - (int) repairedBytes, (int) repaired); - } else { - hts_log_print(opt, LOG_WARNING, "Cache: could not repair the cache"); - } } - // Opened ? - if (cache->zipInput!=NULL) { + if (cache->zipInput != NULL) { int zErr; /* Ready directory entries */ - if ( ( zErr = unzGoToFirstFile((unzFile) cache->zipInput) ) == Z_OK) { + if ((zErr = unzGoToFirstFile((unzFile) cache->zipInput)) == Z_OK) { char comment[128]; char BIGSTK filename[HTS_URLMAXSIZE * 4]; int entries = 0; - memset(comment, 0, sizeof(comment)); // for truncated reads - do { + + memset(comment, 0, sizeof(comment)); // for truncated reads + do { int readSizeHeader = 0; + filename[0] = '\0'; comment[0] = '\0'; if (unzOpenCurrentFile((unzFile) cache->zipInput) == Z_OK) { - if ( - (readSizeHeader = unzGetLocalExtrafield((unzFile) cache->zipInput, comment, sizeof(comment) - 2)) > 0 - && - unzGetCurrentFileInfo((unzFile) cache->zipInput, NULL, filename, sizeof(filename) - 2, NULL, 0, NULL, 0) == Z_OK - ) - { - long int pos = (long int) unzGetOffset((unzFile) cache->zipInput); + if ((readSizeHeader = + unzGetLocalExtrafield((unzFile) cache->zipInput, comment, + sizeof(comment) - 2)) > 0 + && unzGetCurrentFileInfo((unzFile) cache->zipInput, NULL, + filename, sizeof(filename) - 2, NULL, + 0, NULL, 0) == Z_OK) { + long int pos = + (long int) unzGetOffset((unzFile) cache->zipInput); assertf(readSizeHeader < sizeof(comment)); comment[readSizeHeader] = '\0'; entries++; if (pos > 0) { - int dataincache = 0; // data in cache ? - char* filenameIndex = filename; + int dataincache = 0; // data in cache ? + char *filenameIndex = filename; + if (strfield(filenameIndex, "http://")) { filenameIndex += 7; } if (comment[0] != '\0') { int maxLine = 2; - char* a = comment; - while(*a && maxLine-- > 0) { // parse only few first lines + char *a = comment; + + while(*a && maxLine-- > 0) { // parse only few first lines char BIGSTK line[1024]; + line[0] = '\0'; - a+=binput(a, line, sizeof(line) - 2); + a += binput(a, line, sizeof(line) - 2); if (strfield(line, "X-In-Cache:")) { if (strfield2(line, "X-In-Cache: 1")) { dataincache = 1; @@ -1447,273 +1647,401 @@ void cache_init(cache_back* cache,httrackp* opt) { else inthash_add(cache->hashtable, filenameIndex, -pos); } else { - hts_log_print(opt, LOG_WARNING, "Corrupted cache meta entry #%d", (int)entries); + hts_log_print(opt, LOG_WARNING, + "Corrupted cache meta entry #%d", + (int) entries); } } else { - hts_log_print(opt, LOG_WARNING, "Corrupted cache entry #%d", (int)entries); + hts_log_print(opt, LOG_WARNING, "Corrupted cache entry #%d", + (int) entries); } unzCloseCurrentFile((unzFile) cache->zipInput); } else { - hts_log_print(opt, LOG_WARNING, "Corrupted cache entry #%d", (int)entries); + hts_log_print(opt, LOG_WARNING, "Corrupted cache entry #%d", + (int) entries); } - } while( unzGoToNextFile((unzFile) cache->zipInput) == Z_OK ); - hts_log_print(opt, LOG_DEBUG, "Cache index loaded: %d entries loaded", (int)entries); - opt->is_update=1; // signaler comme update - + } while(unzGoToNextFile((unzFile) cache->zipInput) == Z_OK); + hts_log_print(opt, LOG_DEBUG, "Cache index loaded: %d entries loaded", + (int) entries); + opt->is_update = 1; // signaler comme update + } else { - hts_log_print(opt, LOG_WARNING, "Cache: error trying to read the cache: %s", hts_get_zerror(zErr)); - } - + hts_log_print(opt, LOG_WARNING, + "Cache: error trying to read the cache: %s", + hts_get_zerror(zErr)); + } + } else { - hts_log_print(opt, LOG_WARNING, "Cache: error trying to open the cache"); + hts_log_print(opt, LOG_WARNING, + "Cache: error trying to open the cache"); } - - } else if ( - ( - !cache->ro && - fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat")) >=0 && fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx")) >0 - ) - || - ( - cache->ro && - fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")) >=0 && fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")) > 0 - ) + + } else + if ((!cache->ro + && + fsize(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat")) >= 0 + && + fsize(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx")) > 0) + || (cache->ro + && + fsize(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat")) >= 0 + && + fsize(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx")) > 0) ) { - FILE* oldndx=NULL; + FILE *oldndx = NULL; + #if DEBUGCA printf("..load cache\n"); #endif if (!cache->ro) { - cache->olddat=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"),"rb"); - oldndx=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"),"rb"); + cache->olddat = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat"), "rb"); + oldndx = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx"), "rb"); } else { - cache->olddat=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"),"rb"); - oldndx=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx"),"rb"); + cache->olddat = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat"), "rb"); + oldndx = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx"), "rb"); } // les deux doivent être ouvrables - if ((cache->olddat==NULL) && (oldndx!=NULL)) { + if ((cache->olddat == NULL) && (oldndx != NULL)) { fclose(oldndx); - oldndx=NULL; + oldndx = NULL; } - if ((cache->olddat!=NULL) && (oldndx==NULL)) { + if ((cache->olddat != NULL) && (oldndx == NULL)) { fclose(cache->olddat); - cache->olddat=NULL; + cache->olddat = NULL; } // lire index - if (oldndx!=NULL) { + if (oldndx != NULL) { int buffl; - fclose(oldndx); oldndx=NULL; + + fclose(oldndx); + oldndx = NULL; // lire ndx, et lastmodified if (!cache->ro) { - buffl=fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx")); - cache->use=readfile(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx")); + buffl = + fsize(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx")); + cache->use = + readfile(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx")); } else { - buffl=fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); - cache->use=readfile(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); + buffl = + fsize(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx")); + cache->use = + readfile(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx")); } - if (cache->use!=NULL) { + if (cache->use != NULL) { char firstline[256]; - char* a=cache->use; - a+=cache_brstr(a,firstline); - if (strncmp(firstline,"CACHE-",6)==0) { // Nouvelle version du cache - if (strncmp(firstline,"CACHE-1.",8)==0) { // Version 1.1x - cache->version=(int)(firstline[8]-'0'); // cache 1.x + char *a = cache->use; + + a += cache_brstr(a, firstline); + if (strncmp(firstline, "CACHE-", 6) == 0) { // Nouvelle version du cache + if (strncmp(firstline, "CACHE-1.", 8) == 0) { // Version 1.1x + cache->version = (int) (firstline[8] - '0'); // cache 1.x if (cache->version <= 5) { - a+=cache_brstr(a,firstline); - strcpybuff(cache->lastmodified,firstline); + a += cache_brstr(a, firstline); + strcpybuff(cache->lastmodified, firstline); } else { - hts_log_print(opt, LOG_ERROR, "Cache: version 1.%d not supported, ignoring current cache",cache->version); + hts_log_print(opt, LOG_ERROR, + "Cache: version 1.%d not supported, ignoring current cache", + cache->version); fclose(cache->olddat); - cache->olddat=NULL; + cache->olddat = NULL; freet(cache->use); - cache->use=NULL; + cache->use = NULL; } - } else { // non supporté - hts_log_print(opt, LOG_ERROR, "Cache: %s not supported, ignoring current cache",firstline); + } else { // non supporté + hts_log_print(opt, LOG_ERROR, + "Cache: %s not supported, ignoring current cache", + firstline); fclose(cache->olddat); - cache->olddat=NULL; + cache->olddat = NULL; freet(cache->use); - cache->use=NULL; + cache->use = NULL; } /* */ } else { // Vieille version du cache /* */ - hts_log_print(opt, LOG_WARNING, "Cache: importing old cache format"); - cache->version=0; // cache 1.0 - strcpybuff(cache->lastmodified,firstline); + hts_log_print(opt, LOG_WARNING, + "Cache: importing old cache format"); + cache->version = 0; // cache 1.0 + strcpybuff(cache->lastmodified, firstline); } - opt->is_update=1; // signaler comme update - + opt->is_update = 1; // signaler comme update + /* Create hash table for the cache (MUCH FASTER!) */ #if HTS_FAST_CACHE if (cache->use) { - char BIGSTK line[HTS_URLMAXSIZE*2]; + char BIGSTK line[HTS_URLMAXSIZE * 2]; char linepos[256]; - int pos; - while ( (a!=NULL) && (a < (cache->use+buffl) ) ) { - a=strchr(a+1,'\n'); /* start of line */ + int pos; + + while((a != NULL) && (a < (cache->use + buffl))) { + a = strchr(a + 1, '\n'); /* start of line */ if (a) { a++; /* read "host/file" */ - a+=binput(a,line,HTS_URLMAXSIZE); - a+=binput(a,line+strlen(line),HTS_URLMAXSIZE); + a += binput(a, line, HTS_URLMAXSIZE); + a += binput(a, line + strlen(line), HTS_URLMAXSIZE); /* read position */ - a+=binput(a,linepos,200); - sscanf(linepos,"%d",&pos); - inthash_add(cache->hashtable,line,pos); + a += binput(a, linepos, 200); + sscanf(linepos, "%d", &pos); + inthash_add(cache->hashtable, line, pos); } } /* Not needed anymore! */ freet(cache->use); - cache->use=NULL; + cache->use = NULL; } #endif } } - } else { - hts_log_print(opt, LOG_DEBUG, "Cache: no cache found in %s", fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/")); - } + } else { + hts_log_print(opt, LOG_DEBUG, "Cache: no cache found in %s", + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/")); + } #if DEBUGCA - printf("..create cache\n"); + printf("..create cache\n"); #endif - if (!cache->ro) { - // ouvrir caches actuels - structcheck(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/")); - - if (1) { - /* Create ZIP file cache */ - cache->zipOutput = (void*) zipOpen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip"), 0); - - if (cache->zipOutput != NULL) { - // supprimer old.lst - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.lst"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.lst")); - // renommer - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst"))) - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.lst")); - // ouvrir - cache->lst=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst"),"wb"); - strcpybuff(opt->state.strc.path, StringBuff(opt->path_html)); - opt->state.strc.lst = cache->lst; - //{ - //filecreate_params tmp; - //strcpybuff(tmp.path,StringBuff(opt->path_html)); // chemin - //tmp.lst=cache->lst; // fichier lst - //filenote("",&tmp); // initialiser filecreate - //} - - // supprimer old.txt - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.txt"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.txt")); - // renommer - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.txt"))) - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.txt"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.txt")); - // ouvrir - cache->txt=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.txt"),"wb"); - if (cache->txt) { - fprintf(cache->txt,"date\tsize'/'remotesize\tflags(request:Update,Range state:File response:Modified,Chunked,gZipped)\t"); - fprintf(cache->txt,"statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)"LF); - } - } - } else { - cache->dat=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"),"wb"); - cache->ndx=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx"),"wb"); - // les deux doivent être ouvrables - if ((cache->dat==NULL) && (cache->ndx!=NULL)) { - fclose(cache->ndx); - cache->ndx=NULL; - } - if ((cache->dat!=NULL) && (cache->ndx==NULL)) { - fclose(cache->dat); - cache->dat=NULL; + if (!cache->ro) { + // ouvrir caches actuels + structcheck(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/")); + + if (1) { + /* Create ZIP file cache */ + cache->zipOutput = + (void *) + zipOpen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip"), 0); + + if (cache->zipOutput != NULL) { + // supprimer old.lst + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.lst"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.lst")); + // renommer + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst"))) + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/old.lst")); + // ouvrir + cache->lst = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst"), "wb"); + strcpybuff(opt->state.strc.path, StringBuff(opt->path_html)); + opt->state.strc.lst = cache->lst; + //{ + //filecreate_params tmp; + //strcpybuff(tmp.path,StringBuff(opt->path_html)); // chemin + //tmp.lst=cache->lst; // fichier lst + //filenote("",&tmp); // initialiser filecreate + //} + + // supprimer old.txt + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.txt"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.txt")); + // renommer + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.txt"))) + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.txt"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/old.txt")); + // ouvrir + cache->txt = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.txt"), "wb"); + if (cache->txt) { + fprintf(cache->txt, + "date\tsize'/'remotesize\tflags(request:Update,Range state:File response:Modified,Chunked,gZipped)\t"); + fprintf(cache->txt, + "statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)" + LF); } + } + } else { + cache->dat = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat"), "wb"); + cache->ndx = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx"), "wb"); + // les deux doivent être ouvrables + if ((cache->dat == NULL) && (cache->ndx != NULL)) { + fclose(cache->ndx); + cache->ndx = NULL; + } + if ((cache->dat != NULL) && (cache->ndx == NULL)) { + fclose(cache->dat); + cache->dat = NULL; + } - if (cache->ndx!=NULL) { - char s[256]; - - cache_wstr(cache->dat,"CACHE-1.5"); - fflush(cache->dat); - cache_wstr(cache->ndx,"CACHE-1.5"); - fflush(cache->ndx); - // - time_gmt_rfc822(s); // date et heure actuelle GMT pour If-Modified-Since.. - cache_wstr(cache->ndx,s); - fflush(cache->ndx); // un petit fflush au cas où - - // supprimer old.lst - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.lst"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.lst")); - // renommer - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst"))) - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.lst")); - // ouvrir - cache->lst=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst"),"wb"); - strcpybuff(opt->state.strc.path, StringBuff(opt->path_html)); - opt->state.strc.lst = cache->lst; - //{ - // filecreate_params tmp; - // strcpybuff(tmp.path,StringBuff(opt->path_html)); // chemin - // tmp.lst=cache->lst; // fichier lst - // filenote("",&tmp); // initialiser filecreate - //} - - // supprimer old.txt - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.txt"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.txt")); - // renommer - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.txt"))) - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.txt"),fconcat(OPT_GET_BUFF(opt),StringBuff(opt->path_log),"hts-cache/old.txt")); - // ouvrir - cache->txt=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.txt"),"wb"); - if (cache->txt) { - fprintf(cache->txt,"date\tsize'/'remotesize\tflags(request:Update,Range state:File response:Modified,Chunked,gZipped)\t"); - fprintf(cache->txt,"statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)"LF); - } + if (cache->ndx != NULL) { + char s[256]; + + cache_wstr(cache->dat, "CACHE-1.5"); + fflush(cache->dat); + cache_wstr(cache->ndx, "CACHE-1.5"); + fflush(cache->ndx); + // + time_gmt_rfc822(s); // date et heure actuelle GMT pour If-Modified-Since.. + cache_wstr(cache->ndx, s); + fflush(cache->ndx); // un petit fflush au cas où + + // supprimer old.lst + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.lst"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.lst")); + // renommer + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst"))) + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/old.lst")); + // ouvrir + cache->lst = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst"), "wb"); + strcpybuff(opt->state.strc.path, StringBuff(opt->path_html)); + opt->state.strc.lst = cache->lst; + //{ + // filecreate_params tmp; + // strcpybuff(tmp.path,StringBuff(opt->path_html)); // chemin + // tmp.lst=cache->lst; // fichier lst + // filenote("",&tmp); // initialiser filecreate + //} + + // supprimer old.txt + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.txt"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.txt")); + // renommer + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.txt"))) + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.txt"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/old.txt")); + // ouvrir + cache->txt = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.txt"), "wb"); + if (cache->txt) { + fprintf(cache->txt, + "date\tsize'/'remotesize\tflags(request:Update,Range state:File response:Modified,Chunked,gZipped)\t"); + fprintf(cache->txt, + "statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)" + LF); + } + // test + // cache_writedata(cache->ndx,cache->dat,"//[TEST]//","test1","TEST PIPO",9); + } // cache->ndx!=NULL + } //cache->zipOutput != NULL - // test - // cache_writedata(cache->ndx,cache->dat,"//[TEST]//","test1","TEST PIPO",9); - } // cache->ndx!=NULL - } //cache->zipOutput != NULL + } else { + cache->lst = cache->dat = cache->ndx = NULL; + } - } else { - cache->lst = cache->dat = cache->ndx = NULL; - } - } else { hts_log_print(opt, LOG_DEBUG, "Cache: no cache enabled"); } - -} - - +} // lire un fichier.. (compatible \0) /* Note: NOT utf-8 */ -char* readfile(char* fil) { +char *readfile(char *fil) { return readfile2(fil, NULL); } /* Note: NOT utf-8 */ -char* readfile2(char* fil, LLint* size) { - char* adr=NULL; - char catbuff[CATBUFF_SIZE]; - INTsys len=0; - len=fsize(fil); - if (len >= 0) { // exists - FILE* fp; - fp=fopen(fconv(catbuff, fil),"rb"); - if (fp!=NULL) { // n'existe pas (!) - adr=(char*) malloct(len+1); +char *readfile2(char *fil, LLint * size) { + char *adr = NULL; + char catbuff[CATBUFF_SIZE]; + INTsys len = 0; + + len = fsize(fil); + if (len >= 0) { // exists + FILE *fp; + + fp = fopen(fconv(catbuff, fil), "rb"); + if (fp != NULL) { // n'existe pas (!) + adr = (char *) malloct(len + 1); if (size != NULL) *size = len; - if (adr!=NULL) { - if (len > 0 && fread(adr,1,len,fp) != len) { // fichier endommagé ? + if (adr != NULL) { + if (len > 0 && fread(adr, 1, len, fp) != len) { // fichier endommagé ? freet(adr); - adr=NULL; + adr = NULL; } else - *(adr+len)='\0'; + *(adr + len) = '\0'; } fclose(fp); } @@ -1722,18 +2050,20 @@ char* readfile2(char* fil, LLint* size) { } /* Note: utf-8 */ -char* readfile_utf8(char* fil) { - char* adr=NULL; - char catbuff[CATBUFF_SIZE]; +char *readfile_utf8(char *fil) { + char *adr = NULL; + char catbuff[CATBUFF_SIZE]; const off_t len = fsize_utf8(fil); - if (len >= 0) { // exists - FILE*const fp = FOPEN(fconv(catbuff, fil),"rb"); - if (fp!=NULL) { // n'existe pas (!) - adr = (char*) malloct(len+1); - if (adr!=NULL) { - if (len > 0 && fread(adr,1,len,fp) != len) { // fichier endommagé ? + + if (len >= 0) { // exists + FILE *const fp = FOPEN(fconv(catbuff, fil), "rb"); + + if (fp != NULL) { // n'existe pas (!) + adr = (char *) malloct(len + 1); + if (adr != NULL) { + if (len > 0 && fread(adr, 1, len, fp) != len) { // fichier endommagé ? freet(adr); - adr=NULL; + adr = NULL; } else { adr[len] = '\0'; } @@ -1745,19 +2075,21 @@ char* readfile_utf8(char* fil) { } /* Note: NOT utf-8 */ -char* readfile_or(char* fil,char* defaultdata) { - char* realfile=fil; - char* ret; - char catbuff[CATBUFF_SIZE]; +char *readfile_or(char *fil, char *defaultdata) { + char *realfile = fil; + char *ret; + char catbuff[CATBUFF_SIZE]; + if (!fexist(fil)) - realfile=fconcat(catbuff,hts_rootdir(NULL),fil); - ret=readfile(realfile); + realfile = fconcat(catbuff, hts_rootdir(NULL), fil); + ret = readfile(realfile); if (ret) return ret; else { - char *adr=malloct(strlen(defaultdata)+2); + char *adr = malloct(strlen(defaultdata) + 2); + if (adr) { - strcpybuff(adr,defaultdata); + strcpybuff(adr, defaultdata); return adr; } } @@ -1766,104 +2098,118 @@ char* readfile_or(char* fil,char* defaultdata) { // écriture/lecture d'une chaîne sur un fichier // -1 : erreur, sinon 0 -int cache_wstr(FILE* fp,const char* s) { +int cache_wstr(FILE * fp, const char *s) { INTsys i; - char buff[256+4]; - i = (s != NULL) ? ((INTsys)strlen(s)) : 0; - sprintf(buff,INTsysP "\n",i); - if (fwrite(buff,1,strlen(buff),fp) != strlen(buff)) + char buff[256 + 4]; + + i = (s != NULL) ? ((INTsys) strlen(s)) : 0; + sprintf(buff, INTsysP "\n", i); + if (fwrite(buff, 1, strlen(buff), fp) != strlen(buff)) return -1; - if (i > 0 && fwrite(s,1,i,fp) != i) + if (i > 0 && fwrite(s, 1, i, fp) != i) return -1; return 0; } -void cache_rstr(FILE* fp,char* s) { +void cache_rstr(FILE * fp, char *s) { INTsys i; - char buff[256+4]; - linput(fp,buff,256); - sscanf(buff,INTsysP,&i); - if (i < 0 || i > 32768) /* error, something nasty happened */ - i=0; - if (i>0) { - if ((int) fread(s,1,i,fp) != i) { + char buff[256 + 4]; + + linput(fp, buff, 256); + sscanf(buff, INTsysP, &i); + if (i < 0 || i > 32768) /* error, something nasty happened */ + i = 0; + if (i > 0) { + if ((int) fread(s, 1, i, fp) != i) { int fread_cache_failed = 0; + assertf(fread_cache_failed); } } - *(s+i)='\0'; + *(s + i) = '\0'; } -char* cache_rstr_addr(FILE* fp) { +char *cache_rstr_addr(FILE * fp) { INTsys i; - char* addr = NULL; - char buff[256+4]; - linput(fp,buff,256); - sscanf(buff,INTsysP,&i); - if (i < 0 || i > 32768) /* error, something nasty happened */ - i=0; + char *addr = NULL; + char buff[256 + 4]; + + linput(fp, buff, 256); + sscanf(buff, INTsysP, &i); + if (i < 0 || i > 32768) /* error, something nasty happened */ + i = 0; if (i > 0) { addr = malloct(i + 1); if (addr != NULL) { - if ((int) fread(addr,1,i,fp) != i) { + if ((int) fread(addr, 1, i, fp) != i) { int fread_cache_failed = 0; + assertf(fread_cache_failed); } - *(addr+i)='\0'; + *(addr + i) = '\0'; } } return addr; } -int cache_brstr(char* adr,char* s) { +int cache_brstr(char *adr, char *s) { int i; int off; - char buff[256+4]; - off=binput(adr,buff,256); - adr+=off; - sscanf(buff,"%d",&i); - if (i>0) - strncpy(s,adr,i); - *(s+i)='\0'; - off+=i; + char buff[256 + 4]; + + off = binput(adr, buff, 256); + adr += off; + sscanf(buff, "%d", &i); + if (i > 0) + strncpy(s, adr, i); + *(s + i) = '\0'; + off += i; return off; } -int cache_quickbrstr(char* adr,char* s) { +int cache_quickbrstr(char *adr, char *s) { int i; int off; - char buff[256+4]; - off=binput(adr,buff,256); - adr+=off; - sscanf(buff,"%d",&i); - if (i>0) - strncpy(s,adr,i); - *(s+i)='\0'; - off+=i; + char buff[256 + 4]; + + off = binput(adr, buff, 256); + adr += off; + sscanf(buff, "%d", &i); + if (i > 0) + strncpy(s, adr, i); + *(s + i) = '\0'; + off += i; return off; } + /* idem, mais en int */ -int cache_brint(char* adr,int* i) { +int cache_brint(char *adr, int *i) { char s[256]; - int r=cache_brstr(adr,s); - if (r!=-1) - sscanf(s,"%d",i); + int r = cache_brstr(adr, s); + + if (r != -1) + sscanf(s, "%d", i); return r; } -void cache_rint(FILE* fp,int* i) { +void cache_rint(FILE * fp, int *i) { char s[256]; - cache_rstr(fp,s); - sscanf(s,"%d",i); + + cache_rstr(fp, s); + sscanf(s, "%d", i); } -int cache_wint(FILE* fp,int i) { +int cache_wint(FILE * fp, int i) { char s[256]; - sprintf(s,"%d",(int) i); - return cache_wstr(fp,s); + + sprintf(s, "%d", (int) i); + return cache_wstr(fp, s); } -void cache_rLLint(FILE* fp,LLint* i) { +void cache_rLLint(FILE * fp, LLint * i) { char s[256]; - cache_rstr(fp,s); - sscanf(s,LLintP,i); + + cache_rstr(fp, s); + sscanf(s, LLintP, i); } -int cache_wLLint(FILE* fp,LLint i) { +int cache_wLLint(FILE * fp, LLint i) { char s[256]; - sprintf(s,LLintP,(LLint) i); - return cache_wstr(fp,s); + + sprintf(s, LLintP, (LLint) i); + return cache_wstr(fp, s); } + // -- cache -- diff --git a/src/htscache.h b/src/htscache.h index 2ebbab6..5512846 100644 --- a/src/htscache.h +++ b/src/htscache.h @@ -17,28 +17,23 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* cache system (index and stores files in cache) */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - - #ifndef HTSCACHE_DEFH -#define HTSCACHE_DEFH +#define HTSCACHE_DEFH /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE @@ -62,29 +57,41 @@ typedef struct htsblk htsblk; #endif // cache -void cache_mayadd(httrackp* opt,cache_back* cache,htsblk* r,const char* url_adr,const char* url_fil,const char* url_save); -void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,const char* url_adr,const char* url_fil,const char* url_save,int all_in_cache,const char* path_prefix); -htsblk cache_read(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location); -htsblk cache_read_ro(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location); -htsblk cache_read_including_broken(httrackp* opt,cache_back* cache,const char* adr,const char* fil); -htsblk cache_readex(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location,char* return_save,int readonly); -htsblk* cache_header(httrackp* opt,cache_back* cache,const char* adr,const char* fil,htsblk* r); -void cache_init(cache_back* cache,httrackp* opt); - -int cache_writedata(FILE* cache_ndx,FILE* cache_dat,const char* str1,const char* str2,char* outbuff,int len); -int cache_readdata(cache_back* cache,const char* str1,const char* str2,char** inbuff,int* len); - -void cache_rstr(FILE* fp,char* s); -char* cache_rstr_addr(FILE* fp); -int cache_brstr(char* adr,char* s); -int cache_quickbrstr(char* adr,char* s); -int cache_brint(char* adr,int* i); -void cache_rint(FILE* fp,int* i); -void cache_rLLint(FILE* fp,LLint* i); - -int cache_wstr(FILE* fp,const char* s); -int cache_wint(FILE* fp,int i); -int cache_wLLint(FILE* fp,LLint i); +void cache_mayadd(httrackp * opt, cache_back * cache, htsblk * r, + const char *url_adr, const char *url_fil, + const char *url_save); +void cache_add(httrackp * opt, cache_back * cache, const htsblk * r, + const char *url_adr, const char *url_fil, const char *url_save, + int all_in_cache, const char *path_prefix); +htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr, + const char *fil, const char *save, char *location); +htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr, + const char *fil, const char *save, char *location); +htsblk cache_read_including_broken(httrackp * opt, cache_back * cache, + const char *adr, const char *fil); +htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr, + const char *fil, const char *save, char *location, + char *return_save, int readonly); +htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr, + const char *fil, htsblk * r); +void cache_init(cache_back * cache, httrackp * opt); + +int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1, + const char *str2, char *outbuff, int len); +int cache_readdata(cache_back * cache, const char *str1, const char *str2, + char **inbuff, int *len); + +void cache_rstr(FILE * fp, char *s); +char *cache_rstr_addr(FILE * fp); +int cache_brstr(char *adr, char *s); +int cache_quickbrstr(char *adr, char *s); +int cache_brint(char *adr, int *i); +void cache_rint(FILE * fp, int *i); +void cache_rLLint(FILE * fp, LLint * i); + +int cache_wstr(FILE * fp, const char *s); +int cache_wint(FILE * fp, int i); +int cache_wLLint(FILE * fp, LLint i); #endif diff --git a/src/htscatchurl.c b/src/htscatchurl.c index 0d486ac..f015a75 100644 --- a/src/htscatchurl.c +++ b/src/htscatchurl.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: URL catch .h */ /* Author: Xavier Roche */ @@ -62,57 +59,60 @@ Please visit our Website: http://www.httrack.com /* catch url */ #include "htscatchurl.h" - // URL Link catcher // 0- Init the URL catcher with standard port // catch_url_init(&port,&return_host); -HTSEXT_API T_SOC catch_url_init_std(int* port_prox,char* adr_prox) { +HTSEXT_API T_SOC catch_url_init_std(int *port_prox, char *adr_prox) { T_SOC soc; - int try_to_listen_to[]={8080,3128,80,81,82,8081,3129,31337,0,-1}; - int i=0; + int try_to_listen_to[] = { 8080, 3128, 80, 81, 82, 8081, 3129, 31337, 0, -1 }; + int i = 0; + do { - soc=catch_url_init(&try_to_listen_to[i],adr_prox); - *port_prox=try_to_listen_to[i]; + soc = catch_url_init(&try_to_listen_to[i], adr_prox); + *port_prox = try_to_listen_to[i]; i++; - } while( (soc == INVALID_SOCKET) && (try_to_listen_to[i]>=0)); + } while((soc == INVALID_SOCKET) && (try_to_listen_to[i] >= 0)); return soc; } - // 1- Init the URL catcher // catch_url_init(&port,&return_host); -HTSEXT_API T_SOC catch_url_init(int* port,char* adr) { +HTSEXT_API T_SOC catch_url_init(int *port, char *adr) { T_SOC soc = INVALID_SOCKET; - char h_loc[256+2]; + char h_loc[256 + 2]; - if (gethostname(h_loc,256)==0) { // host name + if (gethostname(h_loc, 256) == 0) { // host name SOCaddr server; - int server_size=sizeof(server); - t_hostent* hp_loc; + int server_size = sizeof(server); + t_hostent *hp_loc; t_fullhostent buffer; // effacer structure memset(&server, 0, sizeof(server)); - - if ( (hp_loc=vxgethostbyname(h_loc, &buffer)) ) { // notre host + + if ((hp_loc = vxgethostbyname(h_loc, &buffer))) { // notre host // copie adresse - SOCaddr_copyaddr(server, server_size, hp_loc->h_addr_list[0], hp_loc->h_length); + SOCaddr_copyaddr(server, server_size, hp_loc->h_addr_list[0], + hp_loc->h_length); - if ( (soc = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0)) != INVALID_SOCKET) { + if ((soc = + (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, + 0)) != INVALID_SOCKET) { SOCaddr_initport(server, *port); - if ( bind(soc,(struct sockaddr*) &server,server_size) == 0 ) { + if (bind(soc, (struct sockaddr *) &server, server_size) == 0) { SOCaddr server2; SOClen len; - len=sizeof(server2); + + len = sizeof(server2); // effacer structure memset(&server2, 0, sizeof(server2)); - if (getsockname(soc,(struct sockaddr*) &server2,&len) == 0) { - *port=ntohs(SOCaddr_sinport(server)); // récupérer port - if (listen(soc,10)>=0) { // au pif le 10 + if (getsockname(soc, (struct sockaddr *) &server2, &len) == 0) { + *port = ntohs(SOCaddr_sinport(server)); // récupérer port + if (listen(soc, 10) >= 0) { // au pif le 10 SOCaddr_inetntoa(adr, 128, server2, len); } else { #ifdef _WIN32 @@ -120,27 +120,25 @@ HTSEXT_API T_SOC catch_url_init(int* port,char* adr) { #else close(soc); #endif - soc=INVALID_SOCKET; + soc = INVALID_SOCKET; } - - + } else { #ifdef _WIN32 closesocket(soc); #else close(soc); #endif - soc=INVALID_SOCKET; + soc = INVALID_SOCKET; } - - + } else { #ifdef _WIN32 closesocket(soc); #else close(soc); #endif - soc=INVALID_SOCKET; + soc = INVALID_SOCKET; } } } @@ -154,31 +152,34 @@ HTSEXT_API T_SOC catch_url_init(int* port,char* adr) { // returns 0 if error // url: buffer where URL must be stored - or ip:port in case of failure // data: 32Kb -HTSEXT_API int catch_url(T_SOC soc,char* url,char* method,char* data) { - int retour=0; +HTSEXT_API int catch_url(T_SOC soc, char *url, char *method, char *data) { + int retour = 0; // connexion (accept) if (soc != INVALID_SOCKET) { T_SOC soc2; - while ( (soc2 = (T_SOC) accept(soc, NULL, NULL)) == INVALID_SOCKET); - /* -#ifdef _WIN32 - closesocket(soc); -#else - close(soc); -#endif - */ + + while((soc2 = (T_SOC) accept(soc, NULL, NULL)) == INVALID_SOCKET) ; + /* + #ifdef _WIN32 + closesocket(soc); + #else + close(soc); + #endif + */ soc = soc2; /* INFOS */ { SOCaddr server2; SOClen len = sizeof(server2); + // effacer structure memset(&server2, 0, sizeof(server2)); - if (getpeername(soc,(struct sockaddr*) &server2,&len) == 0) { - char dot[256+2]; + if (getpeername(soc, (struct sockaddr *) &server2, &len) == 0) { + char dot[256 + 2]; + SOCaddr_inetntoa(dot, 256, server2, sizeof(server2)); - sprintf(url,"%s:%d", dot, htons(SOCaddr_sinport(server2))); + sprintf(url, "%s:%d", dot, htons(SOCaddr_sinport(server2))); } } /* INFOS */ @@ -187,65 +188,71 @@ HTSEXT_API int catch_url(T_SOC soc,char* url,char* method,char* data) { if (soc != INVALID_SOCKET) { char line[1000]; char protocol[256]; - line[0]=protocol[0]='\0'; + + line[0] = protocol[0] = '\0'; // - socinput(soc,line,1000); + socinput(soc, line, 1000); if (strnotempty(line)) { - if (sscanf(line,"%s %s %s",method,url,protocol) == 3) { - char BIGSTK url_adr[HTS_URLMAXSIZE*2]; - char BIGSTK url_fil[HTS_URLMAXSIZE*2]; + if (sscanf(line, "%s %s %s", method, url, protocol) == 3) { + char BIGSTK url_adr[HTS_URLMAXSIZE * 2]; + char BIGSTK url_fil[HTS_URLMAXSIZE * 2]; + // méthode en majuscule - int i,r=0; - url_adr[0]=url_fil[0]='\0'; + int i, r = 0; + + url_adr[0] = url_fil[0] = '\0'; // - for(i=0;i<(int) strlen(method);i++) { - if ((method[i]>='a') && (method[i]<='z')) - method[i]-=('a'-'A'); + for(i = 0; i < (int) strlen(method); i++) { + if ((method[i] >= 'a') && (method[i] <= 'z')) + method[i] -= ('a' - 'A'); } // adresse du lien - if (ident_url_absolute(url,url_adr,url_fil)>=0) { + if (ident_url_absolute(url, url_adr, url_fil) >= 0) { // Traitement des en-têtes - char BIGSTK loc[HTS_URLMAXSIZE*2]; + char BIGSTK loc[HTS_URLMAXSIZE * 2]; htsblk blkretour; + hts_init_htsblk(&blkretour); //memset(&blkretour, 0, sizeof(htsblk)); // effacer - blkretour.location=loc; // si non nul, contiendra l'adresse véritable en cas de moved xx + blkretour.location = loc; // si non nul, contiendra l'adresse véritable en cas de moved xx // Lire en têtes restants - sprintf(data,"%s %s %s\r\n",method,url_fil,protocol); + sprintf(data, "%s %s %s\r\n", method, url_fil, protocol); while(strnotempty(line)) { - socinput(soc,line,1000); - treathead(NULL,NULL,NULL,&blkretour,line); // traiter - strcatbuff(data,line); - strcatbuff(data,"\r\n"); + socinput(soc, line, 1000); + treathead(NULL, NULL, NULL, &blkretour, line); // traiter + strcatbuff(data, line); + strcatbuff(data, "\r\n"); } // CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus //strcatbuff(data,"\r\n"); - if (blkretour.totalsize>0) { - int len=(int)min(blkretour.totalsize,32000); + if (blkretour.totalsize > 0) { + int len = (int) min(blkretour.totalsize, 32000); int pos = (int) strlen(data); + // Copier le reste (post éventuel) - while((len>0) && ((r=recv(soc,(char*) data+pos,len,0))>0) ) { - pos+=r; - len-=r; - data[pos]='\0'; // terminer par NULL + while((len > 0) + && ((r = recv(soc, (char *) data + pos, len, 0)) > 0)) { + pos += r; + len -= r; + data[pos] = '\0'; // terminer par NULL } } // Envoyer page - sprintf(line,CATCH_RESPONSE); - send(soc,line,(int)strlen(line),0); + sprintf(line, CATCH_RESPONSE); + send(soc, line, (int) strlen(line), 0); // OK! - retour=1; + retour = 1; } } - } // sinon erreur + } // sinon erreur } } if (soc != INVALID_SOCKET) { #ifdef _WIN32 closesocket(soc); /* - WSACleanup(); - */ + WSACleanup(); + */ #else close(soc); #endif @@ -253,25 +260,31 @@ HTSEXT_API int catch_url(T_SOC soc,char* url,char* method,char* data) { return retour; } - - // Lecture de ligne sur socket -void socinput(T_SOC soc,char* s,int max) { +void socinput(T_SOC soc, char *s, int max) { int c; - int j=0; + int j = 0; + do { unsigned char b; - if (recv(soc,(char*) &b,1,0)==1) { - c=b; - switch(c) { - case 13: break; // sauter CR - case 10: c=-1; break; - case 9: case 12: break; // sauter ces caractères - default: s[j++]=(char) c; break; + + if (recv(soc, (char *) &b, 1, 0) == 1) { + c = b; + switch (c) { + case 13: + break; // sauter CR + case 10: + c = -1; + break; + case 9: + case 12: + break; // sauter ces caractères + default: + s[j++] = (char) c; + break; } } else - c=EOF; - } while((c!=-1) && (c!=EOF) && (j<(max-1))); - s[j++]='\0'; + c = EOF; + } while((c != -1) && (c != EOF) && (j < (max - 1))); + s[j++] = '\0'; } - diff --git a/src/htscharset.c b/src/htscharset.c index 351fa8c..ea96eff 100644 --- a/src/htscharset.c +++ b/src/htscharset.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Charset conversion functions */ /* Author: Xavier Roche */ @@ -39,8 +36,10 @@ Please visit our Website: http://www.httrack.com static int hts_isStringAscii(const char *s, size_t size) { size_t i; - for(i = 0 ; i < size ; i++) { + + for(i = 0; i < size; i++) { const unsigned char c = (const unsigned char) s[i]; + if (c >= 0x80) { return 0; } @@ -205,16 +204,19 @@ static const wincodepage_t codepages[] = { /* Get a Windows codepage, by its name. Return 0 upon error. */ UINT hts_getCodepage(const char *name) { int id; + #define IS_ALNUM(C) ( ((C) >= 'A' && (C) <= 'Z') || ((C) >= 'a' && (C) <= 'z') || ((C) >= '0' && (C) <= '9') ) #define CHAR_LOWER(C) ( ((C) >= 'A' && (C) <= 'Z') ? ((C) + 'a' - 'A') : (C) ) - for(id = 0 ; codepages[id].name != NULL ; id++) { + for(id = 0; codepages[id].name != NULL; id++) { int i, j; + /* Compare the two strings, lowercase and alphanum only (ISO88591 == iso-8859-1) */ const char *a = name, *b = codepages[id].name; - for(i = 0, j = 0 ; ; i++, j++) { + + for(i = 0, j = 0;; i++, j++) { /* Skip non-alnum */ - for( ; a[i] != '\0' && !IS_ALNUM(a[i]) ; i++) ; - for( ; b[j] != '\0' && !IS_ALNUM(b[j]) ; j++) ; + for(; a[i] != '\0' && !IS_ALNUM(a[i]); i++) ; + for(; b[j] != '\0' && !IS_ALNUM(b[j]); j++) ; /* Compare */ if (CHAR_LOWER(a[i]) != CHAR_LOWER(b[j])) { break; @@ -233,6 +235,7 @@ UINT hts_getCodepage(const char *name) { static char *strndup(const char *s, size_t size) { char *dest = malloc(size + 1); + if (dest != NULL) { memcpy(dest, s, size); dest[size] = '\0'; @@ -241,14 +244,19 @@ static char *strndup(const char *s, size_t size) { return NULL; } -LPWSTR hts_convertStringToUCS2(const char *s, int size, UINT cp, int* pwsize) { +LPWSTR hts_convertStringToUCS2(const char *s, int size, UINT cp, int *pwsize) { /* Size in wide chars of the output */ const int wsize = MultiByteToWideChar(cp, 0, (LPCSTR) s, size, NULL, 0); + if (wsize > 0) { LPSTR uoutput = NULL; - LPWSTR woutput = malloc((wsize + 1)*sizeof(WCHAR)); - if (woutput != NULL && MultiByteToWideChar(cp, 0, (LPCSTR) s, size, woutput, wsize) == wsize) { - const int usize = WideCharToMultiByte(CP_UTF8, 0, woutput, wsize, NULL, 0, NULL, FALSE); + LPWSTR woutput = malloc((wsize + 1) * sizeof(WCHAR)); + + if (woutput != NULL + && MultiByteToWideChar(cp, 0, (LPCSTR) s, size, woutput, + wsize) == wsize) { + const int usize = + WideCharToMultiByte(CP_UTF8, 0, woutput, wsize, NULL, 0, NULL, FALSE); if (usize > 0) { woutput[wsize] = 0x0; if (pwsize != NULL) @@ -262,16 +270,19 @@ LPWSTR hts_convertStringToUCS2(const char *s, int size, UINT cp, int* pwsize) { return NULL; } -LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int* pwsize) { +LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int *pwsize) { return hts_convertStringToUCS2(s, size, CP_UTF8, pwsize); } char *hts_convertUCS2StringToCP(LPWSTR woutput, int wsize, UINT cp) { - const int usize = WideCharToMultiByte(cp, 0, woutput, wsize, NULL, 0, NULL, FALSE); + const int usize = + WideCharToMultiByte(cp, 0, woutput, wsize, NULL, 0, NULL, FALSE); if (usize > 0) { - char *const uoutput = malloc((usize + 1)*sizeof(char)); + char *const uoutput = malloc((usize + 1) * sizeof(char)); + if (uoutput != NULL) { - if (WideCharToMultiByte(cp, 0, woutput, wsize, uoutput, usize, NULL, FALSE) == usize) { + if (WideCharToMultiByte + (cp, 0, woutput, wsize, uoutput, usize, NULL, FALSE) == usize) { uoutput[usize] = '\0'; return uoutput; } else { @@ -300,8 +311,10 @@ char *hts_convertStringCPToUTF8(const char *s, size_t size, UINT cp) { /* Size in wide chars of the output */ int wsize; LPWSTR woutput = hts_convertStringToUCS2(s, (int) size, cp, &wsize); + if (woutput != NULL) { char *const uoutput = hts_convertUCS2StringToUTF8(woutput, wsize); + free(woutput); return uoutput; } @@ -325,8 +338,10 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) { /* Size in wide chars of the output */ int wsize; LPWSTR woutput = hts_convertStringToUCS2(s, (int) size, CP_UTF8, &wsize); + if (woutput != NULL) { char *const uoutput = hts_convertUCS2StringToCP(woutput, wsize, cp); + free(woutput); return uoutput; } @@ -338,11 +353,13 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) { char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) { const UINT cp = hts_getCodepage(charset); + return hts_convertStringCPToUTF8(s, size, cp); } char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset) { const UINT cp = hts_getCodepage(charset); + return hts_convertStringCPFromUTF8(s, size, cp); } @@ -355,7 +372,8 @@ char *hts_convertStringSystemToUTF8(const char *s, size_t size) { #include <errno.h> #include <iconv.h> -static char *hts_convertStringToUTF8_(const char *s, size_t size, const char *to, const char *from) { +static char *hts_convertStringToUTF8_(const char *s, size_t size, + const char *to, const char *from) { /* Empty string ? */ if (size == 0) { return strdup(""); @@ -367,8 +385,9 @@ static char *hts_convertStringToUTF8_(const char *s, size_t size, const char *to /* Find codepage */ else { const iconv_t cp = iconv_open(to, from); - if (cp != (iconv_t) -1) { - char *inbuf = (char*) s; + + if (cp != (iconv_t) - 1) { + char *inbuf = (char *) s; size_t inbytesleft = size; size_t outbufCapa = 0; char *outbuf = NULL; @@ -376,7 +395,7 @@ static char *hts_convertStringToUTF8_(const char *s, size_t size, const char *to size_t finalSize; /* Initial size to around the string size */ - for(outbufCapa = 16 ; outbufCapa < size + 1 ; outbufCapa *= 2) ; + for(outbufCapa = 16; outbufCapa < size + 1; outbufCapa *= 2) ; outbuf = malloc(outbufCapa); outbytesleft = outbufCapa; @@ -384,10 +403,12 @@ static char *hts_convertStringToUTF8_(const char *s, size_t size, const char *to while(outbuf != NULL && inbytesleft != 0) { const size_t offset = outbufCapa - outbytesleft; char *outbufCurrent = outbuf + offset; - const size_t ret = iconv(cp, &inbuf, &inbytesleft, &outbufCurrent, &outbytesleft); - if (ret == (size_t) -1) { + const size_t ret = + iconv(cp, &inbuf, &inbytesleft, &outbufCurrent, &outbytesleft); + if (ret == (size_t) - 1) { if (errno == E2BIG) { const size_t used = outbufCapa - outbytesleft; + outbufCapa *= 2; outbuf = realloc(outbuf, outbufCapa); if (outbuf == NULL) { @@ -430,7 +451,8 @@ char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) { return strdup(""); } /* Already UTF-8 ? */ - if (strcasecmp(charset, "utf-8") == 0 || strcasecmp(charset, "utf8") == 0 || hts_isStringAscii(s, size)) { + if (strcasecmp(charset, "utf-8") == 0 || strcasecmp(charset, "utf8") == 0 + || hts_isStringAscii(s, size)) { return strndup(s, size); } /* Find codepage */ @@ -445,7 +467,8 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset) return strdup(""); } /* Already UTF-8 ? */ - if (strcasecmp(charset, "utf-8") == 0 || strcasecmp(charset, "utf8") == 0 || hts_isStringAscii(s, size)) { + if (strcasecmp(charset, "utf-8") == 0 || strcasecmp(charset, "utf8") == 0 + || hts_isStringAscii(s, size)) { return strndup(s, size); } /* Find codepage */ @@ -456,25 +479,32 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset) #endif -HTS_STATIC char* hts_getCharsetFromContentType(const char *mime) { +HTS_STATIC char *hts_getCharsetFromContentType(const char *mime) { /* text/html; charset=utf-8 */ const char *const charset = "charset"; char *pos = strstr(mime, charset); + if (pos != NULL) { /* Skip spaces */ int eq = 0; - for(pos += strlen(charset) ; *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\'' ; pos++) { + + for(pos += strlen(charset); + *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\''; pos++) { if (*pos == '=') { eq = 1; } } if (eq == 1) { int len; - for(len = 0 ; pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'' ; pos++) ; + + for(len = 0; + pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\''; + pos++) ; if (len != 0) { char *const s = malloc(len + 1); int i; - for(i = 0 ; i < len ; i++) { + + for(i = 0; i < len; i++) { s[i] = pos[i]; } s[len] = '\0'; @@ -503,46 +533,60 @@ static int is_space_or_equal_or_quote(char c) { } size_t hts_stringLengthUTF8(const char *s) { - const unsigned char *const bytes = (const unsigned char*) s; + const unsigned char *const bytes = (const unsigned char *) s; size_t i, len; - for(i = 0, len = 0 ; bytes[i] != '\0' ; i++) { + + for(i = 0, len = 0; bytes[i] != '\0'; i++) { const unsigned char c = bytes[i]; - if (HTS_IS_LEADING_UTF8(c)) { // ASCII or leading byte + + if (HTS_IS_LEADING_UTF8(c)) { // ASCII or leading byte len++; } } return len; } -char* hts_getCharsetFromMeta(const char *html, size_t size) { +char *hts_getCharsetFromMeta(const char *html, size_t size) { int i; + // <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8" > - for(i = 0 ; i < size ; i++) { - if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0 && is_space(html[i + 5]) ) { + for(i = 0; i < size; i++) { + if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0 + && is_space(html[i + 5])) { /* Skip spaces */ - for(i += 5 ; is_space(html[i]) ; i++) ; - if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0 && is_space_or_equal(html[i + 10]) ) { - for(i += 10 ; is_space_or_equal_or_quote(html[i]) ; i++) ; + for(i += 5; is_space(html[i]); i++) ; + if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0 + && is_space_or_equal(html[i + 10])) { + for(i += 10; is_space_or_equal_or_quote(html[i]); i++) ; if (strncasecmp(&html[i], "CONTENT-TYPE", 12) == 0) { - for(i += 12 ; is_space_or_equal_or_quote(html[i]) ; i++) ; - if (strncasecmp(&html[i], "CONTENT", 7) == 0 && is_space_or_equal(html[i + 7]) ) { - for(i += 7 ; is_space_or_equal_or_quote(html[i]) ; i++) ; + for(i += 12; is_space_or_equal_or_quote(html[i]); i++) ; + if (strncasecmp(&html[i], "CONTENT", 7) == 0 + && is_space_or_equal(html[i + 7])) { + for(i += 7; is_space_or_equal_or_quote(html[i]); i++) ; /* Skip content-type */ - for( ; i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'' ; i++) ; + for(; + i < size && html[i] != ';' && html[i] != '"' && html[i] != '\''; + i++) ; /* Expect charset attribute here */ if (html[i] == ';') { - for(i++ ; is_space(html[i]) ; i++) ; + for(i++; is_space(html[i]); i++) ; /* Look for charset */ - if (strncasecmp(&html[i], "charset", 7) == 0 && is_space_or_equal(html[i + 7])) { + if (strncasecmp(&html[i], "charset", 7) == 0 + && is_space_or_equal(html[i + 7])) { int len; - for(i += 7 ; is_space_or_equal(html[i]) || html[i] == '\'' ; i++) ; + + for(i += 7; is_space_or_equal(html[i]) || html[i] == '\''; + i++) ; /* Charset */ - for(len = 0 ; i + len < size && html[i + len] != '"' && html[i + len] != '\'' && html[i + len] != ' ' ; len++) ; + for(len = 0; + i + len < size && html[i + len] != '"' + && html[i + len] != '\'' && html[i + len] != ' '; len++) ; /* No error ? */ if (len != 0 && i < size) { char *const s = malloc(len + 1); int j; - for(j = 0 ; j < len ; j++) { + + for(j = 0; j < len; j++) { s[j] = html[i + j]; } s[len] = '\0'; @@ -557,4 +601,3 @@ char* hts_getCharsetFromMeta(const char *html, size_t size) { } return NULL; } - diff --git a/src/htscharset.h b/src/htscharset.h index 53751bd..2b9238a 100644 --- a/src/htscharset.h +++ b/src/htscharset.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Charset conversion functions */ /* Author: Xavier Roche */ @@ -51,18 +48,20 @@ Please visit our Website: http://www.httrack.com * Convert the string "s" from charset "charset" to UTF-8. * Return NULL upon error. **/ -extern char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset); +extern char *hts_convertStringToUTF8(const char *s, size_t size, + const char *charset); /** * Convert the string "s" from UTF-8 to charset "charset". * Return NULL upon error. **/ -extern char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset); +extern char *hts_convertStringFromUTF8(const char *s, size_t size, + const char *charset); /** * Extract the charset from the HTML buffer "html" **/ -extern char* hts_getCharsetFromMeta(const char *html, size_t size); +extern char *hts_getCharsetFromMeta(const char *html, size_t size); /** * Get an UTF-8 string length in characters. @@ -74,7 +73,7 @@ extern size_t hts_stringLengthUTF8(const char *s); /** * Convert UTF-8 to WCHAR. **/ -extern LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int* pwsize); +extern LPWSTR hts_convertUTF8StringToUCS2(const char *s, int size, int *pwsize); /** * Convert from WCHAR. diff --git a/src/htsconfig.h b/src/htsconfig.h index 25f3e0e..ced2679 100644 --- a/src/htsconfig.h +++ b/src/htsconfig.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Global engine definition file */ /* Author: Xavier Roche */ diff --git a/src/htscore.c b/src/htscore.c index 3f623b7..abf390a 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Main source */ /* Author: Xavier Roche */ @@ -67,19 +64,18 @@ Please visit our Website: http://www.httrack.com /* Charset handling */ #include "htscharset.h" - /* END specific definitions */ /* external modules */ -extern int hts_parse_externals(htsmoduleStruct* str); +extern int hts_parse_externals(htsmoduleStruct * str); extern void htspe_init(void); /* debug */ #if DEBUG_SHOWTYPES -char REG[32768]="\n"; +char REG[32768] = "\n"; #endif #if NSDEBUG -int nsocDEBUG=0; +int nsocDEBUG = 0; #endif // @@ -87,8 +83,8 @@ int nsocDEBUG=0; #define _GOTOXY(X,Y) printf("\33[" X ";" Y "f"); #if DEBUG_CHECKINT - #define _CHECKINT_FAIL(a) printf("\n%s\n",a); fflush(stdout); abort(); - #define _CHECKINT(obj_ptr,message) \ +#define _CHECKINT_FAIL(a) printf("\n%s\n",a); fflush(stdout); abort(); +#define _CHECKINT(obj_ptr,message) \ if (obj_ptr) {\ if (( * ((char*) (obj_ptr)) != 0) || ( * ((char*) (((char*) (obj_ptr)) + sizeof(*(obj_ptr))-1)) != 0)) {\ char msg[1100];\ @@ -109,7 +105,7 @@ int nsocDEBUG=0; #if DEBUG_HASH // longest hash chain? - int longest_hash[3]={0,0,0},hashnumber=0; +int longest_hash[3] = { 0, 0, 0 }, hashnumber = 0; #endif // Début de httpmirror, routines annexes @@ -187,7 +183,7 @@ RUN_CALLBACK0(opt, end); \ // enfin on écrit à l'adresse courante du buffer, qu'on incrémente. on décrémente la taille dispo d'autant ensuite // codebase: si non nul et si .class stockee on le note pour chemin primaire pour classes // FA,FS: former_adr et former_fil, lien original -#define liens_record_sav_len(A) +#define liens_record_sav_len(A) #define liens_record(A,F,S,FA,FF,NORM) { \ int notecode=0; \ @@ -236,7 +232,6 @@ hash_write(hashptr,lien_tot,NORM); \ } \ } - #define HT_INDEX_END do { \ if (!makeindex_done) { \ if (makeindex_fp) { \ @@ -261,113 +256,132 @@ if (makeindex_fp) { \ makeindex_done=1; /* ok c'est fait */ \ } while(0) - - - // Début de httpmirror, robot // url1 peut être multiple -int httpmirror(char* url1, httrackp* opt) { - char* primary=NULL; // première page, contenant les liens à scanner - int lien_tot=0; // nombre de liens pour le moment - lien_url** liens=NULL; // les pointeurs sur les liens - hash_struct hash; // système de hachage, accélère la recherche dans les liens - hash_struct* hashptr = &hash; - t_cookie BIGSTK cookie; // gestion des cookies - int lien_max=0; - size_t lien_size=0; // octets restants dans buffer liens dispo - char* lien_buffer=NULL; // buffer liens actuel - int add_tab_alloc=256000; // +256K de liens à chaque fois +int httpmirror(char *url1, httrackp * opt) { + char *primary = NULL; // première page, contenant les liens à scanner + int lien_tot = 0; // nombre de liens pour le moment + lien_url **liens = NULL; // les pointeurs sur les liens + hash_struct hash; // système de hachage, accélère la recherche dans les liens + hash_struct *hashptr = &hash; + t_cookie BIGSTK cookie; // gestion des cookies + int lien_max = 0; + size_t lien_size = 0; // octets restants dans buffer liens dispo + char *lien_buffer = NULL; // buffer liens actuel + int add_tab_alloc = 256000; // +256K de liens à chaque fois + //char* tab_alloc=NULL; - int ptr; // pointeur actuel sur les liens + int ptr; // pointeur actuel sur les liens + // - int numero_passe=0; // deux passes pour html puis images - struct_back* sback=NULL; - htsblk BIGSTK r; // retour de certaines fonctions + int numero_passe = 0; // deux passes pour html puis images + struct_back *sback = NULL; + htsblk BIGSTK r; // retour de certaines fonctions + // pour les stats, nombre de fichiers & octets écrits - LLint stat_fragment=0; // pour la fragmentation + LLint stat_fragment = 0; // pour la fragmentation + //TStamp istat_timestart; // départ pour calcul instantanné // - TStamp last_info_shell=0; - int info_shell=0; + TStamp last_info_shell = 0; + int info_shell = 0; + // filtres - char** filters = NULL; + char **filters = NULL; + //int filter_max=0; - int filptr=0; + int filptr = 0; + // - int makeindex_done=0; // lorsque l'index sera fait - FILE* makeindex_fp=NULL; - int makeindex_links=0; - char BIGSTK makeindex_firstlink[HTS_URLMAXSIZE*2]; + int makeindex_done = 0; // lorsque l'index sera fait + FILE *makeindex_fp = NULL; + int makeindex_links = 0; + char BIGSTK makeindex_firstlink[HTS_URLMAXSIZE * 2]; + // statistiques (mode #Z) - FILE* makestat_fp=NULL; // fichier de stats taux transfert - FILE* maketrack_fp=NULL; // idem pour le tracking - TStamp makestat_time=0; // attente (secondes) - LLint makestat_total=0; // repère du nombre d'octets transférés depuis denrière stat - int makestat_lnk=0; // idem, pour le nombre de liens + FILE *makestat_fp = NULL; // fichier de stats taux transfert + FILE *maketrack_fp = NULL; // idem pour le tracking + TStamp makestat_time = 0; // attente (secondes) + LLint makestat_total = 0; // repère du nombre d'octets transférés depuis denrière stat + int makestat_lnk = 0; // idem, pour le nombre de liens + // - char BIGSTK codebase[HTS_URLMAXSIZE*2]; // base pour applet java - char BIGSTK base[HTS_URLMAXSIZE*2]; // base pour les autres fichiers + char BIGSTK codebase[HTS_URLMAXSIZE * 2]; // base pour applet java + char BIGSTK base[HTS_URLMAXSIZE * 2]; // base pour les autres fichiers + // cache_back BIGSTK cache; - robots_wizard BIGSTK robots; // gestion robots.txt - inthash cache_hashtable=NULL; - inthash cache_tests=NULL; - int cache_hash_size=0; + robots_wizard BIGSTK robots; // gestion robots.txt + inthash cache_hashtable = NULL; + inthash cache_tests = NULL; + int cache_hash_size = 0; + // - char *template_header=NULL,*template_body=NULL,*template_footer=NULL; + char *template_header = NULL, *template_body = NULL, *template_footer = NULL; + // - codebase[0]='\0'; base[0]='\0'; + codebase[0] = '\0'; + base[0] = '\0'; // - cookie.auth.next=NULL; - cookie.auth.auth[0]=cookie.auth.prefix[0]='\0'; + cookie.auth.next = NULL; + cookie.auth.auth[0] = cookie.auth.prefix[0] = '\0'; // // noter heure actuelle de départ en secondes memset(&HTS_STAT, 0, sizeof(HTS_STAT)); - HTS_STAT.stat_timestart=time_local(); + HTS_STAT.stat_timestart = time_local(); //istat_timestart=stat_timestart; - HTS_STAT.istat_timestart[0]=HTS_STAT.istat_timestart[1]=mtime_local(); + HTS_STAT.istat_timestart[0] = HTS_STAT.istat_timestart[1] = mtime_local(); /* reset stats */ - HTS_STAT.HTS_TOTAL_RECV=0; - HTS_STAT.istat_bytes[0]=HTS_STAT.istat_bytes[1]=0; + HTS_STAT.HTS_TOTAL_RECV = 0; + HTS_STAT.istat_bytes[0] = HTS_STAT.istat_bytes[1] = 0; if (opt->shell) { - last_info_shell=HTS_STAT.stat_timestart; + last_info_shell = HTS_STAT.stat_timestart; } - if ((opt->makestat) || (opt->maketrack)){ - makestat_time=HTS_STAT.stat_timestart; + if ((opt->makestat) || (opt->maketrack)) { + makestat_time = HTS_STAT.stat_timestart; } // init external modules htspe_init(); // initialiser cookie if (opt->accept_cookie) { - opt->cookie=&cookie; - cookie.max_len=30000; // max len - strcpybuff(cookie.data,""); + opt->cookie = &cookie; + cookie.max_len = 30000; // max len + strcpybuff(cookie.data, ""); // Charger cookies.txt par défaut ou cookies.txt du miroir - cookie_load(opt->cookie,StringBuff(opt->path_log),"cookies.txt"); - cookie_load(opt->cookie,"","cookies.txt"); + cookie_load(opt->cookie, StringBuff(opt->path_log), "cookies.txt"); + cookie_load(opt->cookie, "", "cookies.txt"); } else - opt->cookie=NULL; + opt->cookie = NULL; // initialiser exit_xh - opt->state.exit_xh=0; // sortir prématurément (var globale) + opt->state.exit_xh = 0; // sortir prématurément (var globale) // initialiser usercommand - usercommand(opt,opt->sys_com_exec,StringBuff(opt->sys_com),"","",""); + usercommand(opt, opt->sys_com_exec, StringBuff(opt->sys_com), "", "", ""); // initialiser structcheck // structcheck_init(1); // initialiser verif_backblue - verif_backblue(opt,NULL); - verif_external(opt,0,0); - verif_external(opt,1,0); + verif_backblue(opt, NULL); + verif_external(opt, 0, 0); + verif_external(opt, 1, 0); // et templates html - template_header=readfile_or(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_bin),"templates/index-header.html"),HTS_INDEX_HEADER); - template_body=readfile_or(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_bin),"templates/index-body.html"),HTS_INDEX_BODY); - template_footer=readfile_or(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_bin),"templates/index-footer.html"),HTS_INDEX_FOOTER); + template_header = + readfile_or(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_bin), + "templates/index-header.html"), HTS_INDEX_HEADER); + template_body = + readfile_or(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_bin), + "templates/index-body.html"), HTS_INDEX_BODY); + template_footer = + readfile_or(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_bin), + "templates/index-footer.html"), HTS_INDEX_FOOTER); // initialiser mimedefs //get_userhttptype(opt,1,StringBuff(opt->mimedefs),NULL); @@ -378,143 +392,158 @@ int httpmirror(char* url1, httrackp* opt) { // effacer bloc cache memset(&cache, 0, sizeof(cache_back)); - cache.type=opt->cache; // cache? - cache.errlog=cache.log=opt->log; // err log? - cache.ptr_ant=cache.ptr_last=0; // pointeur pour anticiper + cache.type = opt->cache; // cache? + cache.errlog = cache.log = opt->log; // err log? + cache.ptr_ant = cache.ptr_last = 0; // pointeur pour anticiper // initialiser hash cache - if (!cache_hash_size) - cache_hash_size=HTS_HASH_SIZE; - cache_hashtable=inthash_new(cache_hash_size); - cache_tests=inthash_new(cache_hash_size); - if (cache_hashtable==NULL || cache_tests==NULL) { - printf("PANIC! : Not enough memory [%d]\n",__LINE__); - filters[0]=NULL; // uniquement a cause du warning de XH_extuninit + if (!cache_hash_size) + cache_hash_size = HTS_HASH_SIZE; + cache_hashtable = inthash_new(cache_hash_size); + cache_tests = inthash_new(cache_hash_size); + if (cache_hashtable == NULL || cache_tests == NULL) { + printf("PANIC! : Not enough memory [%d]\n", __LINE__); + filters[0] = NULL; // uniquement a cause du warning de XH_extuninit XH_extuninit; return 0; } - inthash_value_is_malloc(cache_tests, 1); /* malloc */ - cache.hashtable=(void*)cache_hashtable; /* copy backcache hash */ - cache.cached_tests=(void*)cache_tests; /* copy of cache_tests */ + inthash_value_is_malloc(cache_tests, 1); /* malloc */ + cache.hashtable = (void *) cache_hashtable; /* copy backcache hash */ + cache.cached_tests = (void *) cache_tests; /* copy of cache_tests */ // robots.txt - strcpybuff(robots.adr,"!"); // dummy - robots.token[0]='\0'; - robots.next=NULL; // suivant + strcpybuff(robots.adr, "!"); // dummy + robots.token[0] = '\0'; + robots.next = NULL; // suivant opt->robotsptr = &robots; - + // effacer filters opt->maxfilter = maximum(opt->maxfilter, 128); if (filters_init(&filters, opt->maxfilter, 0) == 0) { - printf("PANIC! : Not enough memory [%d]\n",__LINE__); + printf("PANIC! : Not enough memory [%d]\n", __LINE__); XH_extuninit; return 0; } - opt->filters.filters=&filters; + opt->filters.filters = &filters; // - opt->filters.filptr=&filptr; + opt->filters.filptr = &filptr; //opt->filters.filter_max=&filter_max; - + // hash table opt->hash = &hash; // tableau de pointeurs sur les liens - lien_max=maximum(opt->maxlink,32); - liens=(lien_url**) malloct(lien_max*sizeof(lien_url*)); // tableau de pointeurs sur les liens - if (liens==NULL) { - printf("PANIC! : Not enough memory [%d]\n",__LINE__); + lien_max = maximum(opt->maxlink, 32); + liens = (lien_url **) malloct(lien_max * sizeof(lien_url *)); // tableau de pointeurs sur les liens + if (liens == NULL) { + printf("PANIC! : Not enough memory [%d]\n", __LINE__); //XH_uninit; return 0; } else { int i; - for(i=0;i<lien_max;i++) { - liens[i]=NULL; + + for(i = 0; i < lien_max; i++) { + liens[i] = NULL; } } // initialiser ptr et lien_tot - ptr=0; - lien_tot=0; + ptr = 0; + lien_tot = 0; // initialiser hachage { int i; - for(i=0;i<HTS_HASH_SIZE;i++) - hash.hash[0][i]=hash.hash[1][i]=hash.hash[2][i] = -1; // pas d'entrées + + for(i = 0; i < HTS_HASH_SIZE; i++) + hash.hash[0][i] = hash.hash[1][i] = hash.hash[2][i] = -1; // pas d'entrées hash.liens = liens; - hash.max_lien=0; + hash.max_lien = 0; } // copier adresse(s) dans liste des adresses { - char *a=url1; - int primary_len=8192; + char *a = url1; + int primary_len = 8192; + if (StringNotEmpty(opt->filelist)) { - primary_len += max(0, fsize(StringBuff(opt->filelist))*2); + primary_len += max(0, fsize(StringBuff(opt->filelist)) * 2); } - primary_len += (int) strlen(url1)*2; + primary_len += (int) strlen(url1) * 2; // création de la première page, qui contient les liens de base à scanner // c'est plus propre et plus logique que d'entrer à la main les liens dans la pile // on bénéficie ainsi des vérifications et des tests du robot pour les liens "primaires" - primary=(char*) malloct(primary_len); + primary = (char *) malloct(primary_len); if (primary) { - primary[0]='\0'; + primary[0] = '\0'; } else { - printf("PANIC! : Not enough memory [%d]\n",__LINE__); + printf("PANIC! : Not enough memory [%d]\n", __LINE__); XH_extuninit; return 0; } - + while(*a) { int i; - int joker=0; + int joker = 0; // vérifier qu'il n'y a pas de * dans l'url - if (*a=='+') - joker=1; - else if (*a=='-') - joker=1; - - if (joker) { // joker ou filters + if (*a == '+') + joker = 1; + else if (*a == '-') + joker = 1; + + if (joker) { // joker ou filters //char* p; - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - int type; int plus=0; + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + int type; + int plus = 0; // noter joker (dans b) - if (*a=='+') { // champ + - type=1; plus=1; a++; - } else if (*a=='-') { // champ forbidden[] - type=0; a++; - } else { // champ + avec joker sans doute - type=1; + if (*a == '+') { // champ + + type = 1; + plus = 1; + a++; + } else if (*a == '-') { // champ forbidden[] + type = 0; + a++; + } else { // champ + avec joker sans doute + type = 1; } // recopier prochaine chaine (+ ou -) - i=0; - while((*a!=0) && (!isspace((unsigned char)*a))) { tempo[i++]=*a; a++; } - tempo[i++]='\0'; - while(isspace((unsigned char)*a)) { a++; } + i = 0; + while((*a != 0) && (!isspace((unsigned char) *a))) { + tempo[i++] = *a; + a++; + } + tempo[i++] = '\0'; + while(isspace((unsigned char) *a)) { + a++; + } // sauter les + sans rien après.. if (strnotempty(tempo)) { - if ((plus==0) && (type==1)) { // implicite: *www.edf.fr par exemple - if (tempo[strlen(tempo)-1]!='*') { - strcatbuff(tempo,"*"); // ajouter un * + if ((plus == 0) && (type == 1)) { // implicite: *www.edf.fr par exemple + if (tempo[strlen(tempo) - 1] != '*') { + strcatbuff(tempo, "*"); // ajouter un * } } if (type) - strcpybuff(filters[filptr],"+"); + strcpybuff(filters[filptr], "+"); else - strcpybuff(filters[filptr],"-"); - strcatbuff(filters[filptr],tempo); + strcpybuff(filters[filptr], "-"); + strcatbuff(filters[filptr], tempo); filptr++; - + /* sanity check */ if (filptr + 1 >= opt->maxfilter) { opt->maxfilter += HTS_FILTERSINC; if (filters_init(&filters, opt->maxfilter, HTS_FILTERSINC) == 0) { - printf("PANIC! : Too many filters : >%d [%d]\n",filptr,__LINE__); - hts_log_print(opt, LOG_PANIC, "Too many filters, giving up..(>%d)",filptr); - hts_log_print(opt, LOG_INFO, "To avoid that: use #F option for more filters (example: -#F5000)"); + printf("PANIC! : Too many filters : >%d [%d]\n", filptr, + __LINE__); + hts_log_print(opt, LOG_PANIC, + "Too many filters, giving up..(>%d)", filptr); + hts_log_print(opt, LOG_INFO, + "To avoid that: use #F option for more filters (example: -#F5000)"); XH_extuninit; return 0; } @@ -522,37 +551,45 @@ int httpmirror(char* url1, httrackp* opt) { } } - - } else { // adresse normale - char BIGSTK url[HTS_URLMAXSIZE*2]; + + } else { // adresse normale + char BIGSTK url[HTS_URLMAXSIZE * 2]; + // prochaine adresse - i=0; - while((*a!=0) && (!isspace((unsigned char)*a))) { url[i++]=*a; a++; } - while(isspace((unsigned char)*a)) { a++; } - url[i++]='\0'; + i = 0; + while((*a != 0) && (!isspace((unsigned char) *a))) { + url[i++] = *a; + a++; + } + while(isspace((unsigned char) *a)) { + a++; + } + url[i++] = '\0'; //strcatbuff(primary,"<PRIMARY=\""); - if (strstr(url,":/")==NULL) - strcatbuff(primary,"http://"); - strcatbuff(primary,url); + if (strstr(url, ":/") == NULL) + strcatbuff(primary, "http://"); + strcatbuff(primary, url); //strcatbuff(primary,"\">"); - strcatbuff(primary,"\n"); + strcatbuff(primary, "\n"); } - } // while + } // while /* load URL file list */ /* OPTIMIZED for fast load */ if (StringNotEmpty(opt->filelist)) { - char* filelist_buff=NULL; + char *filelist_buff = NULL; off_t filelist_sz = fsize(StringBuff(opt->filelist)); - if (filelist_sz>0) { - FILE* fp=fopen(StringBuff(opt->filelist),"rb"); + + if (filelist_sz > 0) { + FILE *fp = fopen(StringBuff(opt->filelist), "rb"); + if (fp) { filelist_buff = malloct(filelist_sz + 2); if (filelist_buff) { - if (fread(filelist_buff,1,filelist_sz,fp) != filelist_sz) { + if (fread(filelist_buff, 1, filelist_sz, fp) != filelist_sz) { freet(filelist_buff); - filelist_buff=NULL; + filelist_buff = NULL; } else { *(filelist_buff + filelist_sz) = '\0'; } @@ -560,18 +597,20 @@ int httpmirror(char* url1, httrackp* opt) { fclose(fp); } } - + if (filelist_buff) { - int filelist_ptr=0; - int n=0; - char BIGSTK line[HTS_URLMAXSIZE*2]; - char* primary_ptr = primary + strlen(primary); - while( filelist_ptr < filelist_sz ) { - int count=binput(filelist_buff+filelist_ptr,line,HTS_URLMAXSIZE); - filelist_ptr+=count; + int filelist_ptr = 0; + int n = 0; + char BIGSTK line[HTS_URLMAXSIZE * 2]; + char *primary_ptr = primary + strlen(primary); + + while(filelist_ptr < filelist_sz) { + int count = + binput(filelist_buff + filelist_ptr, line, HTS_URLMAXSIZE); + filelist_ptr += count; if (count && line[0]) { n++; - if (strstr(line,":/")==NULL) { + if (strstr(line, ":/") == NULL) { strcpybuff(primary_ptr, "http://"); primary_ptr += strlen(primary_ptr); } @@ -582,63 +621,72 @@ int httpmirror(char* url1, httrackp* opt) { } } // fclose(fp); - hts_log_print(opt, LOG_INFO, "%d links added from %s",n,StringBuff(opt->filelist)); + hts_log_print(opt, LOG_INFO, "%d links added from %s", n, + StringBuff(opt->filelist)); // Free buffer freet(filelist_buff); } else { - hts_log_print(opt, LOG_ERROR, "Could not include URL list: %s",StringBuff(opt->filelist)); + hts_log_print(opt, LOG_ERROR, "Could not include URL list: %s", + StringBuff(opt->filelist)); } } - // lien primaire - liens_record("primary","/primary",fslash(OPT_GET_BUFF(opt),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html_utf8),"index.html")),"","",opt->urlhack); - if (liens[lien_tot]==NULL) { // erreur, pas de place réservée - printf("PANIC! : Not enough memory [%d]\n",__LINE__); - hts_log_print(opt, LOG_PANIC, "Not enough memory, can not re-allocate %d bytes",(int)((add_tab_alloc+1)*sizeof(lien_url))); - XH_extuninit; // désallocation mémoire & buffers + liens_record("primary", "/primary", + fslash(OPT_GET_BUFF(opt), + fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_html_utf8), "index.html")), + "", "", opt->urlhack); + if (liens[lien_tot] == NULL) { // erreur, pas de place réservée + printf("PANIC! : Not enough memory [%d]\n", __LINE__); + hts_log_print(opt, LOG_PANIC, + "Not enough memory, can not re-allocate %d bytes", + (int) ((add_tab_alloc + 1) * sizeof(lien_url))); + XH_extuninit; // désallocation mémoire & buffers return 0; - } - liens[lien_tot]->testmode=0; // pas mode test - liens[lien_tot]->link_import=0; // pas mode import - liens[lien_tot]->depth=opt->depth+1; // lien de priorité maximale - liens[lien_tot]->pass2=0; // 1ère passe - liens[lien_tot]->retry=opt->retry; // lien de priorité maximale - liens[lien_tot]->premier=lien_tot; // premier lien, objet-père=objet - liens[lien_tot]->precedent=lien_tot; // lien précédent - lien_tot++; + } + liens[lien_tot]->testmode = 0; // pas mode test + liens[lien_tot]->link_import = 0; // pas mode import + liens[lien_tot]->depth = opt->depth + 1; // lien de priorité maximale + liens[lien_tot]->pass2 = 0; // 1ère passe + liens[lien_tot]->retry = opt->retry; // lien de priorité maximale + liens[lien_tot]->premier = lien_tot; // premier lien, objet-père=objet + liens[lien_tot]->precedent = lien_tot; // lien précédent + lien_tot++; // Initialiser cache { int backupXFR = htsMemoryFastXfr; - opt->state._hts_in_html_parsing=4; - if (!RUN_CALLBACK7(opt, loop, NULL,0,0,0,lien_tot,0,NULL)) { - opt->state.exit_xh=1; // exit requested + + opt->state._hts_in_html_parsing = 4; + if (!RUN_CALLBACK7(opt, loop, NULL, 0, 0, 0, lien_tot, 0, NULL)) { + opt->state.exit_xh = 1; // exit requested } - htsMemoryFastXfr = 1; /* fast load */ - cache_init(&cache,opt); + htsMemoryFastXfr = 1; /* fast load */ + cache_init(&cache, opt); htsMemoryFastXfr = backupXFR; - opt->state._hts_in_html_parsing=0; + opt->state._hts_in_html_parsing = 0; } } - + #if BDEBUG==3 { int i; - for(i=0;i<lien_tot;i++) { - printf("%d>%s%s as %s\n",i,liens[i]->adr,liens[i]->fil,liens[i]->sav); + + for(i = 0; i < lien_tot; i++) { + printf("%d>%s%s as %s\n", i, liens[i]->adr, liens[i]->fil, liens[i]->sav); } - for(i=0;i<filptr;i++) { - printf("%d>filters=%s\n",i,filters[i]); + for(i = 0; i < filptr; i++) { + printf("%d>filters=%s\n", i, filters[i]); } } #endif - + // backing //soc_max=opt->maxsoc; - if (opt->maxsoc>0) { + if (opt->maxsoc > 0) { #if BDEBUG==2 _CLRSCR; #endif @@ -646,122 +694,134 @@ int httpmirror(char* url1, httrackp* opt) { // On prévoit large: les fichiers HTML ne prennent que peu de place en mémoire, et les // fichiers non html sont sauvés en direct sur disque. // --> 1024 entrées + 32 entrées par socket en supplément - sback = back_new(opt->maxsoc*32+1024); + sback = back_new(opt->maxsoc * 32 + 1024); if (sback == NULL) { - hts_log_print(opt, LOG_PANIC, "Not enough memory, can not allocate %d bytes",(int)((opt->maxsoc+1)*sizeof(lien_back))); + hts_log_print(opt, LOG_PANIC, + "Not enough memory, can not allocate %d bytes", + (int) ((opt->maxsoc + 1) * sizeof(lien_back))); return 0; } } - // statistiques if (opt->makestat) { - makestat_fp=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-stats.txt"),"wb"); + makestat_fp = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-stats.txt"), + "wb"); if (makestat_fp != NULL) { - fprintf(makestat_fp,"HTTrack statistics report, every minutes"LF LF); - fflush(makestat_fp); + fprintf(makestat_fp, "HTTrack statistics report, every minutes" LF LF); + fflush(makestat_fp); } } - // tracking -- débuggage if (opt->maketrack) { - maketrack_fp=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-track.txt"),"wb"); + maketrack_fp = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-track.txt"), + "wb"); if (maketrack_fp != NULL) { - fprintf(maketrack_fp,"HTTrack tracking report, every minutes"LF LF); - fflush(maketrack_fp); + fprintf(maketrack_fp, "HTTrack tracking report, every minutes" LF LF); + fflush(maketrack_fp); } } - // on n'a pas de liens!! (exemple: httrack www.* impossible sans départ..) - if (lien_tot<=0) { - hts_log_print(opt, LOG_ERROR, "You MUST specify at least one complete URL, and not only wildcards!"); + if (lien_tot <= 0) { + hts_log_print(opt, LOG_ERROR, + "You MUST specify at least one complete URL, and not only wildcards!"); } /* Send options to callback functions */ RUN_CALLBACK0(opt, chopt); // attendre une certaine heure.. - if (opt->waittime>0) { - int rollover=0; - int ok=0; + if (opt->waittime > 0) { + int rollover = 0; + int ok = 0; + { - TStamp tl=0; + TStamp tl = 0; time_t tt; - struct tm* A; - tt=time(NULL); - A=localtime(&tt); - tl+=A->tm_sec; - tl+=A->tm_min*60; - tl+=A->tm_hour*60*60; - if (tl>opt->waittime) // attendre minuit - rollover=1; + struct tm *A; + + tt = time(NULL); + A = localtime(&tt); + tl += A->tm_sec; + tl += A->tm_min * 60; + tl += A->tm_hour * 60 * 60; + if (tl > opt->waittime) // attendre minuit + rollover = 1; } // attendre.. - opt->state._hts_in_html_parsing=5; + opt->state._hts_in_html_parsing = 5; do { - TStamp tl=0; + TStamp tl = 0; time_t tt; - struct tm* A; - tt=time(NULL); - A=localtime(&tt); - tl+=A->tm_sec; - tl+=A->tm_min*60; - tl+=A->tm_hour*60*60; + struct tm *A; + + tt = time(NULL); + A = localtime(&tt); + tl += A->tm_sec; + tl += A->tm_min * 60; + tl += A->tm_hour * 60 * 60; if (rollover) { - if (tl<=opt->waittime) - rollover=0; // attendre heure + if (tl <= opt->waittime) + rollover = 0; // attendre heure } else { - if (tl>opt->waittime) - ok=1; // ok! + if (tl > opt->waittime) + ok = 1; // ok! } - - { + + { int r; + if (rollover) - r = RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,0,0,lien_tot,(int) (opt->waittime-tl+24*3600),NULL); + r = + RUN_CALLBACK7(opt, loop, sback->lnk, sback->count, 0, 0, lien_tot, + (int) (opt->waittime - tl + 24 * 3600), NULL); else - r = RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,0,0,lien_tot,(int) (opt->waittime-tl),NULL); + r = + RUN_CALLBACK7(opt, loop, sback->lnk, sback->count, 0, 0, lien_tot, + (int) (opt->waittime - tl), NULL); if (!r) { - opt->state.exit_xh=1; // exit requested - ok=1; + opt->state.exit_xh = 1; // exit requested + ok = 1; } else Sleep(100); } - } while(!ok); - opt->state._hts_in_html_parsing=0; - + } while(!ok); + opt->state._hts_in_html_parsing = 0; + // note: recopie de plus haut // noter heure actuelle de départ en secondes - HTS_STAT.stat_timestart=time_local(); + HTS_STAT.stat_timestart = time_local(); if (opt->shell) { - last_info_shell=HTS_STAT.stat_timestart; + last_info_shell = HTS_STAT.stat_timestart; } - if ((opt->makestat) || (opt->maketrack)){ - makestat_time=HTS_STAT.stat_timestart; + if ((opt->makestat) || (opt->maketrack)) { + makestat_time = HTS_STAT.stat_timestart; } - - } + } /* Info for wrappers */ hts_log_print(opt, LOG_INFO, "engine: start"); if (!RUN_CALLBACK0(opt, start)) { XH_extuninit; return 1; } - // ------------------------------------------------------------ // ------------------------------------------------------------ // Boucle générale de parcours des liens // ------------------------------------------------------------ do { - int error=0; // si error alors sauter - int store_errpage=0; // c'est une erreur mais on enregistre le html - int is_binary=0; // is a binary file - int is_loaded_from_file=0; // has been loaded from a file (implies is_write=1) - char BIGSTK loc[HTS_URLMAXSIZE*2]; // adresse de relocation + int error = 0; // si error alors sauter + int store_errpage = 0; // c'est une erreur mais on enregistre le html + int is_binary = 0; // is a binary file + int is_loaded_from_file = 0; // has been loaded from a file (implies is_write=1) + char BIGSTK loc[HTS_URLMAXSIZE * 2]; // adresse de relocation // Ici on charge le fichier (html, gif..) en mémoire // Les HTMLs sont traités (si leur priorité est suffisante) @@ -769,7 +829,7 @@ int httpmirror(char* url1, httrackp* opt) { // effacer r hts_init_htsblk(&r); //memset(&r, 0, sizeof(htsblk)); r.soc=INVALID_SOCKET; - r.location=loc; // en cas d'erreur 3xx (moved) + r.location = loc; // en cas d'erreur 3xx (moved) // recopier proxy if ((r.req.proxy.active = opt->proxy.active)) { if (StringBuff(opt->proxy.bindhost) != NULL) @@ -779,27 +839,32 @@ int httpmirror(char* url1, httrackp* opt) { r.req.proxy.port = opt->proxy.port; } // et user-agent - strcpy(r.req.user_agent,StringBuff(opt->user_agent)); - strcpy(r.req.referer,StringBuff(opt->referer)); - strcpy(r.req.from,StringBuff(opt->from)); - strcpy(r.req.lang_iso,StringBuff(opt->lang_iso)); - r.req.user_agent_send=opt->user_agent_send; + strcpy(r.req.user_agent, StringBuff(opt->user_agent)); + strcpy(r.req.referer, StringBuff(opt->referer)); + strcpy(r.req.from, StringBuff(opt->from)); + strcpy(r.req.lang_iso, StringBuff(opt->lang_iso)); + r.req.user_agent_send = opt->user_agent_send; if (!error) { - + // Skip empty/invalid/done in background if (liens[ptr]) { - while ( (liens[ptr]) && ( - ( ((urladr != NULL)?(urladr):(" "))[0]=='!') || - ( ((urlfil != NULL)?(urlfil):(" "))[0]=='\0') || - ( (liens[ptr]->pass2 == -1) ) - ) - ) { // sauter si lien annulé (ou fil vide) - if (liens[ptr] != NULL && liens[ptr]->pass2 == -1) { - hts_log_print(opt, LOG_DEBUG, "link #%d is ready, skipping: %s%s..",ptr,((urladr != NULL)?(urladr):(" ")),((urlfil != NULL)?(urlfil):(" "))); - } else { - hts_log_print(opt, LOG_DEBUG, "link #%d seems ready, skipping: %s%s..",ptr,((urladr != NULL)?(urladr):(" ")),((urlfil != NULL)?(urlfil):(" "))); - } + while((liens[ptr]) + && ((((urladr != NULL) ? (urladr) : (" "))[0] == '!') + || (((urlfil != NULL) ? (urlfil) : (" "))[0] == '\0') + || ((liens[ptr]->pass2 == -1)) + ) + ) { // sauter si lien annulé (ou fil vide) + if (liens[ptr] != NULL && liens[ptr]->pass2 == -1) { + hts_log_print(opt, LOG_DEBUG, "link #%d is ready, skipping: %s%s..", + ptr, ((urladr != NULL) ? (urladr) : (" ")), + ((urlfil != NULL) ? (urlfil) : (" "))); + } else { + hts_log_print(opt, LOG_DEBUG, + "link #%d seems ready, skipping: %s%s..", ptr, + ((urladr != NULL) ? (urladr) : (" ")), + ((urlfil != NULL) ? (urlfil) : (" "))); + } // remove from stats if (liens[ptr]->pass2 == -1) { HTS_STAT.stat_background--; @@ -807,36 +872,38 @@ int httpmirror(char* url1, httrackp* opt) { ptr++; } } - if (liens[ptr] != NULL) { // on a qq chose à récupérer? + if (liens[ptr] != NULL) { // on a qq chose à récupérer? - hts_log_print(opt, LOG_DEBUG, "Wait get: %s%s",urladr,urlfil); + hts_log_print(opt, LOG_DEBUG, "Wait get: %s%s", urladr, urlfil); #if DEBUG_ROBOTS - if (strcmp(urlfil,"/robots.txt") == 0) { + if (strcmp(urlfil, "/robots.txt") == 0) { printf("robots.txt detected\n"); } #endif // ------------------------------------------------------------ // DEBUT --RECUPERATION LIEN--- - if (ptr==0) { // premier lien à parcourir: lien primaire construit avant - r.adr=primary; primary=NULL; - r.statuscode=HTTP_OK; - r.size=strlen(r.adr); - r.soc=INVALID_SOCKET; - strcpybuff(r.contenttype,"text/html"); - /*} else if (opt->maxsoc<=0) { // fichiers 1 à 1 en attente (pas de backing) - // charger le fichier en mémoire tout bêtement - r=xhttpget(urladr,urlfil); - // - */ - } else { // backing, multiples sockets + if (ptr == 0) { // premier lien à parcourir: lien primaire construit avant + r.adr = primary; + primary = NULL; + r.statuscode = HTTP_OK; + r.size = strlen(r.adr); + r.soc = INVALID_SOCKET; + strcpybuff(r.contenttype, "text/html"); + /*} else if (opt->maxsoc<=0) { // fichiers 1 à 1 en attente (pas de backing) + // charger le fichier en mémoire tout bêtement + r=xhttpget(urladr,urlfil); + // + */ + } else { // backing, multiples sockets /* - ************************************** - Get the next link, waiting for other files, handling external callbacks - */ + ************************************** + Get the next link, waiting for other files, handling external callbacks + */ { char BIGSTK buff_err_msg[1024]; htsmoduleStruct BIGSTK str; htsmoduleStructExtended BIGSTK stre; + buff_err_msg[0] = '\0'; memset(&str, 0, sizeof(str)); memset(&stre, 0, sizeof(stre)); @@ -902,7 +969,7 @@ int httpmirror(char* url1, httrackp* opt) { stre.info_shell_ = &info_shell; /* Parse */ - switch(hts_mirror_wait_for_next_file(&str, &stre)) { + switch (hts_mirror_wait_for_next_file(&str, &stre)) { case -1: XH_uninit; return -1; @@ -914,50 +981,51 @@ int httpmirror(char* url1, httrackp* opt) { // A supplemental flag and if( ) { } would be really messy goto jump_if_done; } - + } - - + } // FIN --RECUPERATION LIEN--- // ------------------------------------------------------------ - - - - } else { // lien vide.. - hts_log_print(opt, LOG_WARNING, "Warning, link #%d empty",ptr); - error=1; - goto jump_if_done; - } // test si url existe (non vide!) - + } else { // lien vide.. + hts_log_print(opt, LOG_WARNING, "Warning, link #%d empty", ptr); + error = 1; + goto jump_if_done; + } // test si url existe (non vide!) // ---tester taille a posteriori--- // tester r.adr if (!error) { // erreur, pas de fichier chargé: - if ((!r.adr) && (r.is_write==0) - && (r.statuscode!=301) - && (r.statuscode!=302) - && (r.statuscode!=303) - && (r.statuscode!=307) - && (r.statuscode!=412) - && (r.statuscode!=416) - ) { + if ((!r.adr) && (r.is_write == 0) + && (r.statuscode != 301) + && (r.statuscode != 302) + && (r.statuscode != 303) + && (r.statuscode != 307) + && (r.statuscode != 412) + && (r.statuscode != 416) + ) { // error=1; - + // peut être que le fichier était trop gros? - if ((istoobig(opt,r.totalsize,opt->maxfile_html,opt->maxfile_nonhtml,r.contenttype)) - || (istoobig(opt,r.totalsize,opt->maxfile_html,opt->maxfile_nonhtml,r.contenttype))) { - error=0; - hts_log_print(opt, LOG_WARNING, "Big file cancelled according to user's preferences: %s%s",urladr,urlfil); + if ((istoobig + (opt, r.totalsize, opt->maxfile_html, opt->maxfile_nonhtml, + r.contenttype)) + || + (istoobig + (opt, r.totalsize, opt->maxfile_html, opt->maxfile_nonhtml, + r.contenttype))) { + error = 0; + hts_log_print(opt, LOG_WARNING, + "Big file cancelled according to user's preferences: %s%s", + urladr, urlfil); } // // // error=1; // ne pas traiter la suite -- euhh si finalement.. } } // ---fin tester taille a posteriori--- - // -------------------- // BOGUS MIME TYPE HACK // Check if we have a bogus MIME type @@ -967,23 +1035,26 @@ int httpmirror(char* url1, httrackp* opt) { // Content-disposition="foo.jpg" // -------------------- if (!error) { - if (r.statuscode == HTTP_OK) { // OK (ou 304 en backing) - if (r.adr) { // Written file - if ( (is_hypertext_mime(opt,r.contenttype, urlfil)) /* Is HTML or Js, .. */ - /* NO - real media is real media, and mms is mms, not HTML */ - /*|| (may_be_hypertext_mime(r.contenttype, urlfil) && (r.adr) )*/ /* Is real media, .. */ + if (r.statuscode == HTTP_OK) { // OK (ou 304 en backing) + if (r.adr) { // Written file + if ((is_hypertext_mime(opt, r.contenttype, urlfil)) + + /* Is HTML or Js, .. */ + /* NO - real media is real media, and mms is mms, not HTML */ + /*|| (may_be_hypertext_mime(r.contenttype, urlfil) && (r.adr) ) */ + /* Is real media, .. */ ) { if (strnotempty(r.cdispo)) { // Content-disposition set! - if (ishtml(opt, savename) == 0) { // Non HTML!! + if (ishtml(opt, savename) == 0) { // Non HTML!! // patch it! - strcpybuff(r.contenttype,"application/octet-stream"); + strcpybuff(r.contenttype, "application/octet-stream"); } } } } } - /* Load file if necessary and decode. */ \ + /* Load file if necessary and decode. */ #define LOAD_IN_MEMORY_IF_NECESSARY() do { \ if ( \ may_be_hypertext_mime(opt,r.contenttype, urlfil) /* Is HTML or Js, .. */ \ @@ -1008,9 +1079,8 @@ int httpmirror(char* url1, httrackp* opt) { // ------------------------------------ // BOGUS MIME TYPE HACK II (the revenge) // Check if we have a bogus MIME type - if (HTTP_IS_OK(r.statuscode) && - (is_hypertext_mime(opt,r.contenttype, urlfil) /* Is HTML or Js, .. */ - || may_be_hypertext_mime(opt,r.contenttype, urlfil)) /* Is real media, .. */ + if (HTTP_IS_OK(r.statuscode) && (is_hypertext_mime(opt, r.contenttype, urlfil) /* Is HTML or Js, .. */ + ||may_be_hypertext_mime(opt, r.contenttype, urlfil)) /* Is real media, .. */ ) { /* Convert charset to UTF-8 - NOT! (what about links ? remote server side will have troubles with converted names) */ @@ -1043,27 +1113,25 @@ int httpmirror(char* url1, httrackp* opt) { unsigned int map[256]; int i; unsigned int nspec = 0; - map_characters((unsigned char*)r.adr, (unsigned int)r.size, &map[0]); - for(i = 1 ; i < 32 ; i++) { // null chars ignored.. - if (!is_realspace(i) - && i != 27 /* Damn you ISO2022-xx! */ + + map_characters((unsigned char *) r.adr, (unsigned int) r.size, + &map[0]); + for(i = 1; i < 32; i++) { // null chars ignored.. + if (!is_realspace(i) + && i != 27 /* Damn you ISO2022-xx! */ ) { nspec += map[i]; } } /* On-the-fly UCS2 to UTF-8 conversion (note: UCS2 should never be used on the net) */ - if ( - map[0] > r.size/10 - && - r.size % 2 == 0 - && - ( - ( ((unsigned char) r.adr[0]) == 0xff && ((unsigned char) r.adr[1]) == 0xfe) - || - ( ((unsigned char) r.adr[0]) == 0xfe && ((unsigned char) r.adr[1]) == 0xff) - ) - ) - { + if (map[0] > r.size / 10 && r.size % 2 == 0 + && + ((((unsigned char) r.adr[0]) == 0xff + && ((unsigned char) r.adr[1]) == 0xfe) + || (((unsigned char) r.adr[0]) == 0xfe + && ((unsigned char) r.adr[1]) == 0xff) + ) + ) { #define CH_ADD(c) do { \ if (new_offs + 1 > new_capa) { \ new_capa *= 2; \ @@ -1083,98 +1151,104 @@ int httpmirror(char* url1, httrackp* opt) { #define CH_ADD_RNG2(c, r, r2, o) do { \ CH_ADD_RNG1(c, (r) * (r2), o); \ } while(0) - int new_capa = (int) ( r.size / 2 + 1 ); - int new_offs = 0; - unsigned char* prev_adr = (unsigned char*) r.adr; - unsigned char* new_adr = (unsigned char*) malloct(new_capa); - int i; - int swap = (((unsigned char)r.adr[0]) == 0xff); - assertf(new_adr != NULL); - /* - See http://www.unicode.org/reports/tr28/tr28-3.html#conformance - U+0000..U+007F 00..7F - U+0080..U+07FF C2..DF 80..BF - U+0800..U+0FFF E0 A0..BF 80..BF - U+1000..U+CFFF E1..EC 80..BF 80..BF - U+D000..U+D7FF ED 80..9F 80..BF - U+D800..U+DFFF - U+E000..U+FFFF EE..EF 80..BF 80..BF - */ - for(i = 0 ; i < r.size / 2 ; i++) { - unsigned short int unic = 0; - if (swap) - unic = prev_adr[i*2] + (prev_adr[i*2 + 1] << 8); - else - unic = (prev_adr[i*2] << 8) + prev_adr[i*2 + 1]; - if (unic <= 0x7F) { - /* U+0000..U+007F 00..7F */ - CH_ADD_RNG0( unic, 0x00 ); - } else if (unic <= 0x07FF) { - /* U+0080..U+07FF C2..DF 80..BF */ - unic -= 0x0080; - CH_ADD_RNG1( unic, 0xbf - 0x80 + 1, 0xc2 ); - CH_ADD_RNG0( unic, 0x80 ); - } else if (unic <= 0x0FFF) { - /* U+0800..U+0FFF E0 A0..BF 80..BF */ - unic -= 0x0800; - CH_ADD_RNG2( unic, 0xbf - 0x80 + 1, 0xbf - 0xa0 + 1, 0xe0 ); - CH_ADD_RNG1( unic, 0xbf - 0x80 + 1, 0xa0 ); - CH_ADD_RNG0( unic, 0x80 ); - } else if (unic <= 0xCFFF) { - /* U+1000..U+CFFF E1..EC 80..BF 80..BF */ - unic -= 0x1000; - CH_ADD_RNG2( unic, 0xbf - 0x80 + 1, 0xbf - 0x80 + 1, 0xe1 ); - CH_ADD_RNG1( unic, 0xbf - 0x80 + 1, 0x80 ); - CH_ADD_RNG0( unic, 0x80 ); - } else if (unic <= 0xD7FF) { - /* U+D000..U+D7FF ED 80..9F 80..BF */ - unic -= 0xD000; - CH_ADD_RNG2( unic, 0xbf - 0x80 + 1, 0x9f - 0x80 + 1, 0xed ); - CH_ADD_RNG1( unic, 0xbf - 0x80 + 1, 0x80 ); - CH_ADD_RNG0( unic, 0x80 ); - } else if (unic <= 0xDFFF) { - /* U+D800..U+DFFF */ - CH_ADD('?'); - /* ill-formed */ - } else /* if (unic <= 0xFFFF) */ { - /* U+E000..U+FFFF EE..EF 80..BF 80..BF */ - unic -= 0xE000; - CH_ADD_RNG2( unic, 0xbf - 0x80 + 1, 0xbf - 0x80 + 1, 0xee ); - CH_ADD_RNG1( unic, 0xbf - 0x80 + 1, 0x80 ); - CH_ADD_RNG0( unic, 0x80 ); - } - } - hts_log_print(opt, LOG_WARNING, "File %s%s converted from UCS2 to UTF-8 (old size: %d bytes, new size: %d bytes)", urladr, urlfil, (int)r.size, new_offs); - freet(r.adr); - r.adr = NULL; - r.size = new_offs; - CH_ADD(0); - r.adr = (char*) new_adr; + int new_capa = (int) (r.size / 2 + 1); + int new_offs = 0; + unsigned char *prev_adr = (unsigned char *) r.adr; + unsigned char *new_adr = (unsigned char *) malloct(new_capa); + int i; + int swap = (((unsigned char) r.adr[0]) == 0xff); + + assertf(new_adr != NULL); + /* + See http://www.unicode.org/reports/tr28/tr28-3.html#conformance + U+0000..U+007F 00..7F + U+0080..U+07FF C2..DF 80..BF + U+0800..U+0FFF E0 A0..BF 80..BF + U+1000..U+CFFF E1..EC 80..BF 80..BF + U+D000..U+D7FF ED 80..9F 80..BF + U+D800..U+DFFF + U+E000..U+FFFF EE..EF 80..BF 80..BF + */ + for(i = 0; i < r.size / 2; i++) { + unsigned short int unic = 0; + + if (swap) + unic = prev_adr[i * 2] + (prev_adr[i * 2 + 1] << 8); + else + unic = (prev_adr[i * 2] << 8) + prev_adr[i * 2 + 1]; + if (unic <= 0x7F) { + /* U+0000..U+007F 00..7F */ + CH_ADD_RNG0(unic, 0x00); + } else if (unic <= 0x07FF) { + /* U+0080..U+07FF C2..DF 80..BF */ + unic -= 0x0080; + CH_ADD_RNG1(unic, 0xbf - 0x80 + 1, 0xc2); + CH_ADD_RNG0(unic, 0x80); + } else if (unic <= 0x0FFF) { + /* U+0800..U+0FFF E0 A0..BF 80..BF */ + unic -= 0x0800; + CH_ADD_RNG2(unic, 0xbf - 0x80 + 1, 0xbf - 0xa0 + 1, 0xe0); + CH_ADD_RNG1(unic, 0xbf - 0x80 + 1, 0xa0); + CH_ADD_RNG0(unic, 0x80); + } else if (unic <= 0xCFFF) { + /* U+1000..U+CFFF E1..EC 80..BF 80..BF */ + unic -= 0x1000; + CH_ADD_RNG2(unic, 0xbf - 0x80 + 1, 0xbf - 0x80 + 1, 0xe1); + CH_ADD_RNG1(unic, 0xbf - 0x80 + 1, 0x80); + CH_ADD_RNG0(unic, 0x80); + } else if (unic <= 0xD7FF) { + /* U+D000..U+D7FF ED 80..9F 80..BF */ + unic -= 0xD000; + CH_ADD_RNG2(unic, 0xbf - 0x80 + 1, 0x9f - 0x80 + 1, 0xed); + CH_ADD_RNG1(unic, 0xbf - 0x80 + 1, 0x80); + CH_ADD_RNG0(unic, 0x80); + } else if (unic <= 0xDFFF) { + /* U+D800..U+DFFF */ + CH_ADD('?'); + /* ill-formed */ + } else { /* if (unic <= 0xFFFF) */ + + /* U+E000..U+FFFF EE..EF 80..BF 80..BF */ + unic -= 0xE000; + CH_ADD_RNG2(unic, 0xbf - 0x80 + 1, 0xbf - 0x80 + 1, 0xee); + CH_ADD_RNG1(unic, 0xbf - 0x80 + 1, 0x80); + CH_ADD_RNG0(unic, 0x80); + } + } + hts_log_print(opt, LOG_WARNING, + "File %s%s converted from UCS2 to UTF-8 (old size: %d bytes, new size: %d bytes)", + urladr, urlfil, (int) r.size, new_offs); + freet(r.adr); + r.adr = NULL; + r.size = new_offs; + CH_ADD(0); + r.adr = (char *) new_adr; #undef CH_ADD #undef CH_ADD_RNG0 #undef CH_ADD_RNG1 #undef CH_ADD_RNG2 - } else if ((nspec > r.size / 100) && (nspec > 10)) { // too many special characters + } else if ((nspec > r.size / 100) && (nspec > 10)) { // too many special characters is_binary = 1; - strcpybuff(r.contenttype,"application/octet-stream"); - hts_log_print(opt, LOG_WARNING, "File not parsed, looks like binary: %s%s",urladr,urlfil); - } + strcpybuff(r.contenttype, "application/octet-stream"); + hts_log_print(opt, LOG_WARNING, + "File not parsed, looks like binary: %s%s", urladr, + urlfil); + } - /* This hack allows to avoid problems with parsing '\0' characters */ + /* This hack allows to avoid problems with parsing '\0' characters */ if (!is_binary) { - for(i = 0 ; i < r.size ; i++) { - if (r.adr[i] == '\0') r.adr[i] = ' '; + for(i = 0; i < r.size; i++) { + if (r.adr[i] == '\0') + r.adr[i] = ' '; } } } - } } - - // MOVED IN back_finalize() - // + // MOVED IN back_finalize() + // // -------------------- // REAL MEDIA HACK // Check if we have to load locally the file @@ -1193,8 +1267,8 @@ int httpmirror(char* url1, httrackp* opt) { // if (r.adr) { // if (fread(r.adr,1,sz,fp) == sz) { // r.size=sz; - // r.adr[sz] = '\0'; - // r.is_write = 0; + // r.adr[sz] = '\0'; + // r.is_write = 0; // } else { // freet(r.adr); // r.size=0; @@ -1217,33 +1291,31 @@ int httpmirror(char* url1, httrackp* opt) { // } //} // EN OF REAL MEDIA HACK - // ---stockage en cache--- // stocker dans le cache? /* - if (!error) { - if (ptr>0) { - if (liens[ptr]) { - xxcache_mayadd(opt,&cache,&r,urladr,urlfil,savename); - } else - error=1; - } - } - */ + if (!error) { + if (ptr>0) { + if (liens[ptr]) { + xxcache_mayadd(opt,&cache,&r,urladr,urlfil,savename); + } else + error=1; + } + } + */ // ---fin stockage en cache--- - - - + /* - ************************************** - Check "Moved permanently" and other similar errors, retrying URLs if necessary and handling - redirect pages. - */ + ************************************** + Check "Moved permanently" and other similar errors, retrying URLs if necessary and handling + redirect pages. + */ if (!error) { char BIGSTK buff_err_msg[1024]; htsmoduleStruct BIGSTK str; htsmoduleStructExtended BIGSTK stre; + buff_err_msg[0] = '\0'; memset(&str, 0, sizeof(str)); memset(&stre, 0, sizeof(stre)); @@ -1303,82 +1375,80 @@ int httpmirror(char* url1, httrackp* opt) { stre.makestat_total_ = &makestat_total; stre.makestat_lnk_ = &makestat_lnk; stre.maketrack_fp = maketrack_fp; - + /* Parse */ if (hts_mirror_check_moved(&str, &stre) != 0) { XH_uninit; return -1; } - + } - } // if !error - + } // if !error + if (!error) { #if DEBUG_SHOWTYPES - if (strstr(REG,r.contenttype)==NULL) { - strcatbuff(REG,r.contenttype); - strcatbuff(REG,"\n"); - printf("%s\n",r.contenttype); + if (strstr(REG, r.contenttype) == NULL) { + strcatbuff(REG, r.contenttype); + strcatbuff(REG, "\n"); + printf("%s\n", r.contenttype); io_flush; } #endif /* Load file and decode if necessary, after redirect check. */ LOAD_IN_MEMORY_IF_NECESSARY(); - + // ------------------------------------------------------ // ok, fichier chargé localement // ------------------------------------------------------ - + // Vérificateur d'intégrité - #if DEBUG_CHECKINT +#if DEBUG_CHECKINT { int i; - for(i = 0 ; i < sback->count ; i++) { + + for(i = 0; i < sback->count; i++) { char si[256]; - sprintf(si,"Test global après back_wait, index %d",i); - _CHECKINT(&back[i],si) + + sprintf(si, "Test global après back_wait, index %d", i); + _CHECKINT(&back[i], si) } } - #endif - +#endif /* info: updated */ /* - if (ptr>0) { - // "mis à jour" - if ((!r.notmodified) && (opt->is_update) && (!store_errpage)) { // page modifiée - if (strnotempty(savename)) { - HTS_STAT.stat_updated_files++; - //if ((opt->debug>0) && (opt->log!=NULL)) { - hts_log_print(opt, LOG_INFO, "File updated: %s%s",urladr,urlfil); - } - } else { - if (!store_errpage) { - hts_log_print(opt, LOG_INFO, "File recorded: %s%s",urladr,urlfil); - } - } - } - */ - + if (ptr>0) { + // "mis à jour" + if ((!r.notmodified) && (opt->is_update) && (!store_errpage)) { // page modifiée + if (strnotempty(savename)) { + HTS_STAT.stat_updated_files++; + //if ((opt->debug>0) && (opt->log!=NULL)) { + hts_log_print(opt, LOG_INFO, "File updated: %s%s",urladr,urlfil); + } + } else { + if (!store_errpage) { + hts_log_print(opt, LOG_INFO, "File recorded: %s%s",urladr,urlfil); + } + } + } + */ + // ------------------------------------------------------ // traitement (parsing) // ------------------------------------------------------ // traiter - if ( - ! is_binary - && - ( (is_hypertext_mime(opt,r.contenttype, urlfil)) /* Is HTML or Js, .. */ - || (may_be_hypertext_mime(opt,r.contenttype, urlfil) && r.adr != NULL ) /* Is real media, .. */ - ) - && (liens[ptr]->depth>0) /* Depth > 0 (recurse depth) */ - && (r.adr!=NULL) /* HTML Data exists */ - && (r.size>0) /* And not empty */ - && (!store_errpage) /* Not an html error page */ - && (savename[0]!='\0') /* Output filename exists */ - ) { // ne traiter que le html si autorisé + if (!is_binary && ((is_hypertext_mime(opt, r.contenttype, urlfil)) /* Is HTML or Js, .. */ + ||(may_be_hypertext_mime(opt, r.contenttype, urlfil) && r.adr != NULL) /* Is real media, .. */ + ) + && (liens[ptr]->depth > 0) /* Depth > 0 (recurse depth) */ + &&(r.adr != NULL) /* HTML Data exists */ + &&(r.size > 0) /* And not empty */ + &&(!store_errpage) /* Not an html error page */ + &&(savename[0] != '\0') /* Output filename exists */ + ) { // ne traiter que le html si autorisé // -- -- -- -- // Parsing HTML if (!error) { @@ -1386,7 +1456,7 @@ int httpmirror(char* url1, httrackp* opt) { /* Remove file if being processed */ if (is_loaded_from_file) { - (void) unlink(fconv(OPT_GET_BUFF(opt),savename)); + (void) unlink(fconv(OPT_GET_BUFF(opt), savename)); is_loaded_from_file = 0; } @@ -1402,6 +1472,7 @@ int httpmirror(char* url1, httrackp* opt) { /* Attempt to find a meta charset */ else if (is_html_mime_type(r.contenttype)) { char *const charset = hts_getCharsetFromMeta(r.adr, r.size); + if (charset != NULL && strlen(charset) < sizeof(page_charset)) { strcpy(page_charset, charset); } @@ -1416,18 +1487,22 @@ int httpmirror(char* url1, httrackp* opt) { } /* Could not detect charset */ if (page_charset[0] == '\0') { - hts_log_print(opt, LOG_INFO, "Warning: could not detect encoding for: %s%s",urladr,urlfil); + hts_log_print(opt, LOG_INFO, + "Warning: could not detect encoding for: %s%s", + urladr, urlfil); /* Fallback to ISO-8859-1 (~== identity) ; accents will look weird */ strcpy(page_charset, "iso-8859-1"); } } /* Info for wrappers */ - hts_log_print(opt, LOG_INFO, "engine: check-html: %s%s",urladr,urlfil); + hts_log_print(opt, LOG_INFO, "engine: check-html: %s%s", urladr, + urlfil); { char BIGSTK buff_err_msg[1024]; htsmoduleStruct BIGSTK str; htsmoduleStructExtended BIGSTK stre; + buff_err_msg[0] = '\0'; memset(&str, 0, sizeof(str)); memset(&stre, 0, sizeof(stre)); @@ -1487,98 +1562,104 @@ int httpmirror(char* url1, httrackp* opt) { stre.makestat_total_ = &makestat_total; stre.makestat_lnk_ = &makestat_lnk; stre.maketrack_fp = maketrack_fp; - + /* Parse */ if (htsparse(&str, &stre) != 0) { XH_uninit; return -1; } - - // I'll have to segment this part + // I'll have to segment this part // #include "htsparse.c" - } } // Fin parsing HTML // -- -- -- -- - - } // si text/html + } // si text/html // -- -- -- - else { // sauver fichier quelconque + else { // sauver fichier quelconque // -- -- -- // sauver fichier - /* En cas d'erreur, vérifier que fichier d'erreur existe */ - if (strnotempty(savename) == 0) { // chemin de sauvegarde existant - if (strcmp(urlfil,"/robots.txt")==0) { // pas robots.txt - if (store_errpage) { // c'est une page d'erreur - int create_html_warning=0; - int create_gif_warning=0; - switch (ishtml(opt,urlfil)) { /* pas fichier html */ - case 0: /* non html */ + if (strnotempty(savename) == 0) { // chemin de sauvegarde existant + if (strcmp(urlfil, "/robots.txt") == 0) { // pas robots.txt + if (store_errpage) { // c'est une page d'erreur + int create_html_warning = 0; + int create_gif_warning = 0; + + switch (ishtml(opt, urlfil)) { /* pas fichier html */ + case 0: /* non html */ { char buff[256]; - guess_httptype(opt,buff,urlfil); - if (strcmp(buff,"image/gif")==0) - create_gif_warning=1; + + guess_httptype(opt, buff, urlfil); + if (strcmp(buff, "image/gif") == 0) + create_gif_warning = 1; } break; - case 1: /* html */ + case 1: /* html */ if (!r.adr) { } break; - default: /* don't know.. */ - break; + default: /* don't know.. */ + break; } /* Créer message d'erreur ? */ if (create_html_warning) { - char* adr=(char*)malloct(strlen(HTS_DATA_ERROR_HTML)+1100); - hts_log_print(opt, LOG_INFO, "Creating HTML warning file (%s)",r.msg); + char *adr = + (char *) malloct(strlen(HTS_DATA_ERROR_HTML) + 1100); + hts_log_print(opt, LOG_INFO, "Creating HTML warning file (%s)", + r.msg); if (adr) { if (r.adr) { freet(r.adr); - r.adr=NULL; + r.adr = NULL; } - sprintf(adr,HTS_DATA_ERROR_HTML,r.msg); - r.adr=adr; + sprintf(adr, HTS_DATA_ERROR_HTML, r.msg); + r.adr = adr; } } else if (create_gif_warning) { - char* adr=(char*)malloct(HTS_DATA_UNKNOWN_GIF_LEN); - hts_log_print(opt, LOG_INFO, "Creating GIF dummy file (%s)",r.msg); + char *adr = (char *) malloct(HTS_DATA_UNKNOWN_GIF_LEN); + + hts_log_print(opt, LOG_INFO, "Creating GIF dummy file (%s)", + r.msg); if (r.adr) { freet(r.adr); - r.adr=NULL; + r.adr = NULL; } memcpy(adr, HTS_DATA_UNKNOWN_GIF, HTS_DATA_UNKNOWN_GIF_LEN); - r.adr=adr; + r.adr = adr; } } } } - if (strnotempty(savename) == 0) { // pas de chemin de sauvegarde - if (strcmp(urlfil,"/robots.txt")==0) { // robots.txt + if (strnotempty(savename) == 0) { // pas de chemin de sauvegarde + if (strcmp(urlfil, "/robots.txt") == 0) { // robots.txt if (r.adr) { - int bptr=0; + int bptr = 0; char BIGSTK line[1024]; char BIGSTK buff[8192]; char BIGSTK infobuff[8192]; - int record=0; - line[0]='\0'; buff[0]='\0'; infobuff[0]='\0'; + int record = 0; + + line[0] = '\0'; + buff[0] = '\0'; + infobuff[0] = '\0'; // #if DEBUG_ROBOTS - printf("robots.txt dump:\n%s\n",r.adr); + printf("robots.txt dump:\n%s\n", r.adr); #endif do { - char* comm; + char *comm; int llen; - bptr+=binput(r.adr+bptr, line, sizeof(line) - 2); + + bptr += binput(r.adr + bptr, line, sizeof(line) - 2); /* strip comment */ - comm=strchr(line, '#'); + comm = strchr(line, '#'); if (comm != NULL) { *comm = '\0'; } @@ -1588,63 +1669,75 @@ int httpmirror(char* url1, httrackp* opt) { line[llen - 1] = '\0'; llen--; } - if (strfield(line,"user-agent:")) { - char* a; - a=line+11; - while(is_realspace(*a)) a++; // sauter espace(s) - if ( *a == '*') { + if (strfield(line, "user-agent:")) { + char *a; + + a = line + 11; + while(is_realspace(*a)) + a++; // sauter espace(s) + if (*a == '*') { if (record != 2) - record=1; // c pour nous - } else if (strfield(a,"httrack") || strfield(a,"winhttrack") || strfield(a,"webhttrack")) { - buff[0]='\0'; // re-enregistrer - infobuff[0]='\0'; - record=2; // locked + record = 1; // c pour nous + } else if (strfield(a, "httrack") || strfield(a, "winhttrack") + || strfield(a, "webhttrack")) { + buff[0] = '\0'; // re-enregistrer + infobuff[0] = '\0'; + record = 2; // locked #if DEBUG_ROBOTS printf("explicit disallow for httrack\n"); #endif - } - else record=0; + } else + record = 0; } else if (record) { - if (strfield(line,"disallow:")) { - char* a=line+9; + if (strfield(line, "disallow:")) { + char *a = line + 9; + while(is_realspace(*a)) - a++; // sauter espace(s) + a++; // sauter espace(s) if (strnotempty(a)) { -#ifdef IGNORE_RESTRICTIVE_ROBOTS - if (strcmp(a,"/") != 0 || opt->robots >= 3) +#ifdef IGNORE_RESTRICTIVE_ROBOTS + if (strcmp(a, "/") != 0 || opt->robots >= 3) #endif - { /* ignoring disallow: / */ - if ( (strlen(buff) + strlen(a) + 8) < sizeof(buff)) { - strcatbuff(buff,a); - strcatbuff(buff,"\n"); - if ( (strlen(infobuff) + strlen(a) + 8) < sizeof(infobuff)) { - if (strnotempty(infobuff)) strcatbuff(infobuff,", "); - strcatbuff(infobuff,a); + { /* ignoring disallow: / */ + if ((strlen(buff) + strlen(a) + 8) < sizeof(buff)) { + strcatbuff(buff, a); + strcatbuff(buff, "\n"); + if ((strlen(infobuff) + strlen(a) + 8) < + sizeof(infobuff)) { + if (strnotempty(infobuff)) + strcatbuff(infobuff, ", "); + strcatbuff(infobuff, a); } } } -#ifdef IGNORE_RESTRICTIVE_ROBOTS +#ifdef IGNORE_RESTRICTIVE_ROBOTS else { - hts_log_print(opt, LOG_INFO, "Note: %s robots.txt rules are too restrictive, ignoring /",urladr); + hts_log_print(opt, LOG_INFO, + "Note: %s robots.txt rules are too restrictive, ignoring /", + urladr); } #endif } } } - } while( (bptr<r.size) && (strlen(buff) < (sizeof(buff) - 32) ) ); + } while((bptr < r.size) && (strlen(buff) < (sizeof(buff) - 32))); if (strnotempty(buff)) { - checkrobots_set(&robots,urladr,buff); - hts_log_print(opt, LOG_INFO, "Note: robots.txt forbidden links for %s are: %s",urladr,infobuff); - hts_log_print(opt, LOG_INFO, "Note: due to %s remote robots.txt rules, links beginning with these path will be forbidden: %s (see in the options to disable this)",urladr,infobuff); + checkrobots_set(&robots, urladr, buff); + hts_log_print(opt, LOG_INFO, + "Note: robots.txt forbidden links for %s are: %s", + urladr, infobuff); + hts_log_print(opt, LOG_INFO, + "Note: due to %s remote robots.txt rules, links beginning with these path will be forbidden: %s (see in the options to disable this)", + urladr, infobuff); } } } - } else if (r.is_write) { // déja sauvé sur disque + } else if (r.is_write) { // déja sauvé sur disque /* - if (!ishttperror(r.statuscode)) - HTS_STAT.stat_files++; - HTS_STAT.stat_bytes+=r.size; - */ + if (!ishttperror(r.statuscode)) + HTS_STAT.stat_files++; + HTS_STAT.stat_bytes+=r.size; + */ //printf("ok......\n"); } else { // Si on doit sauver une page HTML sans la scanner, cela signifie que le niveau de @@ -1653,100 +1746,119 @@ int httpmirror(char* url1, httrackp* opt) { // Si par la suite on doit retraiter ce fichier avec un niveau de récursion plus // fort, on supprimera le readme, et on scannera le fichier html! // note: sauté si store_errpage (càd si page d'erreur, non à scanner!) - if ( (is_hypertext_mime(opt,r.contenttype, urlfil)) && (!store_errpage) && (r.size>0)) { // c'est du html!! - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - FILE* fp; - tempo[0]='\0'; - strcpybuff(tempo,savename); - strcatbuff(tempo,".readme"); - + if ((is_hypertext_mime(opt, r.contenttype, urlfil)) && (!store_errpage) && (r.size > 0)) { // c'est du html!! + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + FILE *fp; + + tempo[0] = '\0'; + strcpybuff(tempo, savename); + strcatbuff(tempo, ".readme"); + #if HTS_DOSNAME // remplacer / par des slash arrière { - int i=0; + int i = 0; + while(tempo[i]) { - if (tempo[i]=='/') - tempo[i]='\\'; + if (tempo[i] == '/') + tempo[i] = '\\'; i++; - } - } + } + } // a partir d'ici le slash devient antislash #endif - - if ((fp=FOPEN(tempo,"wb"))!=NULL) { - fprintf(fp,"Info-file generated by HTTrack Website Copier "HTTRACK_VERSION"%s"CRLF""CRLF, hts_get_version_info(opt)); - fprintf(fp,"The file %s has not been scanned by HTS"CRLF,savename); - fprintf(fp,"Some links contained in it may be unreachable locally."CRLF); - fprintf(fp,"If you want to get these files, you have to set an upper recurse level, "); - fprintf(fp,"and to rescan the URL."CRLF); + + if ((fp = FOPEN(tempo, "wb")) != NULL) { + fprintf(fp, + "Info-file generated by HTTrack Website Copier " + HTTRACK_VERSION "%s" CRLF "" CRLF, + hts_get_version_info(opt)); + fprintf(fp, "The file %s has not been scanned by HTS" CRLF, + savename); + fprintf(fp, + "Some links contained in it may be unreachable locally." + CRLF); + fprintf(fp, + "If you want to get these files, you have to set an upper recurse level, "); + fprintf(fp, "and to rescan the URL." CRLF); fclose(fp); #ifndef _WIN32 - chmod(tempo,HTS_ACCESS_FILE); + chmod(tempo, HTS_ACCESS_FILE); #endif - usercommand(opt,0,NULL,fconv(OPT_GET_BUFF(opt),tempo),"",""); + usercommand(opt, 0, NULL, fconv(OPT_GET_BUFF(opt), tempo), "", + ""); } - - - hts_log_print(opt, LOG_WARNING, "Warning: store %s without scan: %s",r.contenttype,savename); + + hts_log_print(opt, LOG_WARNING, + "Warning: store %s without scan: %s", r.contenttype, + savename); } else { - if ((opt->getmode & 2)!=0) { // ok autorisé - hts_log_print(opt, LOG_DEBUG, "Store %s: %s",r.contenttype,savename); - } else { // lien non autorisé! (ex: cgi-bin en html) - hts_log_print(opt, LOG_DEBUG, "non-html file ignored after upload at %s : %s",urladr,urlfil); + if ((opt->getmode & 2) != 0) { // ok autorisé + hts_log_print(opt, LOG_DEBUG, "Store %s: %s", r.contenttype, + savename); + } else { // lien non autorisé! (ex: cgi-bin en html) + hts_log_print(opt, LOG_DEBUG, + "non-html file ignored after upload at %s : %s", + urladr, urlfil); if (r.adr) { - freet(r.adr); r.adr=NULL; + freet(r.adr); + r.adr = NULL; } } } - + //printf("extern=%s\n",r.contenttype); // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!! if (r.adr != NULL || r.size == 0) { - file_notify(opt, urladr,urlfil, savename, 1, 1, r.notmodified); - if (filesave(opt,r.adr,(int)r.size,savename,urladr,urlfil)!=0) { + file_notify(opt, urladr, urlfil, savename, 1, 1, r.notmodified); + if (filesave(opt, r.adr, (int) r.size, savename, urladr, urlfil) != + 0) { int fcheck; - if ((fcheck=check_fatal_io_errno())) { - hts_log_print(opt, LOG_ERROR, "Mirror aborted: disk full or filesystem problems"); - opt->state.exit_xh=-1; /* fatal error */ + + if ((fcheck = check_fatal_io_errno())) { + hts_log_print(opt, LOG_ERROR, + "Mirror aborted: disk full or filesystem problems"); + opt->state.exit_xh = -1; /* fatal error */ } - hts_log_print(opt, LOG_ERROR | LOG_ERRNO, "Unable to save file %s", savename); + hts_log_print(opt, LOG_ERROR | LOG_ERRNO, + "Unable to save file %s", savename); } else { /* - if (!ishttperror(r.statuscode)) - HTS_STAT.stat_files++; - HTS_STAT.stat_bytes+=r.size; - */ + if (!ishttperror(r.statuscode)) + HTS_STAT.stat_files++; + HTS_STAT.stat_bytes+=r.size; + */ } } - + } - /* Parsing of other media types (java, ram..) */ /* - if (strfield2(r.contenttype,"audio/x-pn-realaudio")) { - hts_log_print(opt, LOG_DEBUG, "(Real Media): parsing %s",savename); - if (fexist(savename)) { // ok, existe bien! - FILE* fp=FOPEN(savename,"r+b"); - if (fp) { - if (!fseek(fp,0,SEEK_SET)) { - char BIGSTK line[HTS_URLMAXSIZE*2]; - linput(fp,line,HTS_URLMAXSIZE); - if (strnotempty(line)) { - hts_log_print(opt, LOG_DEBUG, "(Real Media): detected %s",line); - } - } - fclose(fp); - } - } - } else */ - + if (strfield2(r.contenttype,"audio/x-pn-realaudio")) { + hts_log_print(opt, LOG_DEBUG, "(Real Media): parsing %s",savename); + if (fexist(savename)) { // ok, existe bien! + FILE* fp=FOPEN(savename,"r+b"); + if (fp) { + if (!fseek(fp,0,SEEK_SET)) { + char BIGSTK line[HTS_URLMAXSIZE*2]; + linput(fp,line,HTS_URLMAXSIZE); + if (strnotempty(line)) { + hts_log_print(opt, LOG_DEBUG, "(Real Media): detected %s",line); + } + } + fclose(fp); + } + } + } else */ /* External modules */ - if ( opt->parsejava && ( opt->parsejava & HTSPARSE_NO_CLASS ) == 0 && fexist(savename)) { + if (opt->parsejava && (opt->parsejava & HTSPARSE_NO_CLASS) == 0 + && fexist(savename)) { char BIGSTK buff_err_msg[1024]; htsmoduleStruct BIGSTK str; + buff_err_msg[0] = '\0'; memset(&str, 0, sizeof(str)); /* */ @@ -1772,171 +1884,224 @@ int httpmirror(char* url1, httrackp* opt) { str.lien_size_ = &lien_size; str.lien_buffer_ = &lien_buffer; /* Parse if recognized */ - switch(hts_parse_externals(&str)) { + switch (hts_parse_externals(&str)) { case 1: - hts_log_print(opt, LOG_DEBUG, "(External module): parsed successfully %s",savename); + hts_log_print(opt, LOG_DEBUG, + "(External module): parsed successfully %s", + savename); break; case 0: - hts_log_print(opt, LOG_DEBUG, "(External module): couldn't parse successfully %s : %s",savename, str.err_msg); + hts_log_print(opt, LOG_DEBUG, + "(External module): couldn't parse successfully %s : %s", + savename, str.err_msg); break; } } - - - } // text/html ou autre - + + } // text/html ou autre /* Post-processing */ if (fexist(savename)) { usercommand(opt, 0, NULL, savename, urladr, urlfil); } - } // if !error + } // if !error -jump_if_done: + jump_if_done: // libérer les liens - if (r.adr) { - freet(r.adr); - r.adr=NULL; - } // libérer la mémoire! - + if (r.adr) { + freet(r.adr); + r.adr = NULL; + } // libérer la mémoire! + // prochain lien ptr++; - + // faut-il sauter le(s) lien(s) suivant(s)? (fichiers images à passer après les html) - if (opt->getmode & 4) { // sauver les non html après + if (opt->getmode & 4) { // sauver les non html après // sauter les fichiers selon la passe if (!numero_passe) { - while((ptr<lien_tot)?( liens[ptr]->pass2):0) ptr++; + while((ptr < lien_tot) ? (liens[ptr]->pass2) : 0) + ptr++; } else { - while((ptr<lien_tot)?( ! liens[ptr]->pass2):0) ptr++; + while((ptr < lien_tot) ? (!liens[ptr]->pass2) : 0) + ptr++; } - if (ptr>=lien_tot) { // fin de boucle - if (!numero_passe) { // première boucle + if (ptr >= lien_tot) { // fin de boucle + if (!numero_passe) { // première boucle hts_log_print(opt, LOG_DEBUG, "Now getting non-html files..."); - numero_passe=1; // seconde boucle - ptr=0; + numero_passe = 1; // seconde boucle + ptr = 0; // prochain pass2 - while((ptr<lien_tot)?(!liens[ptr]->pass2):0) ptr++; - + while((ptr < lien_tot) ? (!liens[ptr]->pass2) : 0) + ptr++; + //printf("first link==%d\n"); - + } - } + } } - // copy abort state if necessary from outside //if (!exit_xh && opt->state.exit_xh) { // exit_xh=opt->state.exit_xh; //} // a-t-on dépassé le quota? if (!back_checkmirror(opt)) { - ptr=lien_tot; - } else if (opt->state.exit_xh) { // sortir - if (opt->state.exit_xh==1) { + ptr = lien_tot; + } else if (opt->state.exit_xh) { // sortir + if (opt->state.exit_xh == 1) { hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user"); } else { hts_log_print(opt, LOG_ERROR, "Exit requested by engine"); } - ptr=lien_tot; + ptr = lien_tot; } - } while(ptr<lien_tot); + } while(ptr < lien_tot); // // // - + /* - Ensure the index is being closed - */ + Ensure the index is being closed + */ HT_INDEX_END; - + /* - updating-a-remotely-deteted-website hack - no much data transfered, no data saved - <no files successfulyl saved> - we assume that something was bad (no connection) - just backup old cache and restore everything - */ - if ( - (HTS_STAT.stat_files <= 0) - && - (HTS_STAT.HTS_TOTAL_RECV < 32768) /* should be fine */ + updating-a-remotely-deteted-website hack + no much data transfered, no data saved + <no files successfulyl saved> + we assume that something was bad (no connection) + just backup old cache and restore everything + */ + if ((HTS_STAT.stat_files <= 0) + && (HTS_STAT.HTS_TOTAL_RECV < 32768) /* should be fine */ ) { - hts_log_print(opt, LOG_INFO, "No data seems to have been transfered during this session! : restoring previous one!"); + hts_log_print(opt, LOG_INFO, + "No data seems to have been transfered during this session! : restoring previous one!"); XH_uninit; - if ( (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"))) ) { - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")); - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst")); - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.txt")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.lst"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.txt"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.txt")); - } - opt->state.exit_xh=2; /* interrupted (no connection detected) */ + if ((fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/old.dat"))) + && + (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx")))) { + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat")); + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx")); + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst")); + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.txt")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.dat")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.ndx")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.lst"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.lst")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.txt"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.txt")); + } + opt->state.exit_xh = 2; /* interrupted (no connection detected) */ return 1; } - // info text if (cache.txt) { - fclose(cache.txt); cache.txt=NULL; + fclose(cache.txt); + cache.txt = NULL; } - // purger! if (cache.lst) { - fclose(cache.lst); cache.lst=opt->state.strc.lst=NULL; + fclose(cache.lst); + cache.lst = opt->state.strc.lst = NULL; if (opt->delete_old) { - FILE *old_lst,*new_lst; + FILE *old_lst, *new_lst; + // - opt->state._hts_in_html_parsing=3; + opt->state._hts_in_html_parsing = 3; // - old_lst=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.lst"),"rb"); + old_lst = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.lst"), "rb"); if (old_lst) { - off_t sz=fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst")); - new_lst=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst"),"rb"); - if ((new_lst) && (sz>0)) { - char* adr=(char*) malloct(sz); + off_t sz = + fsize(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst")); + new_lst = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst"), "rb"); + if ((new_lst) && (sz > 0)) { + char *adr = (char *) malloct(sz); + if (adr) { - if (fread(adr,1,sz,new_lst) == sz) { + if (fread(adr, 1, sz, new_lst) == sz) { char line[1100]; - int purge=0; + int purge = 0; + while(!feof(old_lst)) { - linput(old_lst,line,1000); - if (!strstr(adr,line)) { // fichier non trouvé dans le nouveau? - char BIGSTK file[HTS_URLMAXSIZE*2]; - strcpybuff(file,StringBuff(opt->path_html)); - strcatbuff(file,line+1); - file[strlen(file)-1]='\0'; - if (fexist(file)) { // toujours sur disque: virer - hts_log_print(opt, LOG_INFO, "Purging %s",file); - remove(file); purge=1; + linput(old_lst, line, 1000); + if (!strstr(adr, line)) { // fichier non trouvé dans le nouveau? + char BIGSTK file[HTS_URLMAXSIZE * 2]; + + strcpybuff(file, StringBuff(opt->path_html)); + strcatbuff(file, line + 1); + file[strlen(file) - 1] = '\0'; + if (fexist(file)) { // toujours sur disque: virer + hts_log_print(opt, LOG_INFO, "Purging %s", file); + remove(file); + purge = 1; } } } { - fseek(old_lst,0,SEEK_SET); + fseek(old_lst, 0, SEEK_SET); while(!feof(old_lst)) { - linput(old_lst,line,1000); - while(strnotempty(line) && (line[strlen(line)-1]!='/') && (line[strlen(line)-1]!='\\')) { - line[strlen(line)-1]='\0'; + linput(old_lst, line, 1000); + while(strnotempty(line) && (line[strlen(line) - 1] != '/') + && (line[strlen(line) - 1] != '\\')) { + line[strlen(line) - 1] = '\0'; } if (strnotempty(line)) - line[strlen(line)-1]='\0'; + line[strlen(line) - 1] = '\0'; if (strnotempty(line)) - if (!strstr(adr,line)) { // non trouvé? - char BIGSTK file[HTS_URLMAXSIZE*2]; - strcpybuff(file,StringBuff(opt->path_html)); - strcatbuff(file,line+1); - while ((strnotempty(file)) && (rmdir(file)==0)) { // ok, éliminé (existait) - purge=1; + if (!strstr(adr, line)) { // non trouvé? + char BIGSTK file[HTS_URLMAXSIZE * 2]; + + strcpybuff(file, StringBuff(opt->path_html)); + strcatbuff(file, line + 1); + while((strnotempty(file)) && (rmdir(file) == 0)) { // ok, éliminé (existait) + purge = 1; if (opt->log) { - hts_log_print(opt, LOG_INFO, "Purging directory %s/",file); - while(strnotempty(file) && (file[strlen(file)-1]!='/') && (file[strlen(file)-1]!='\\')) { - file[strlen(file)-1]='\0'; + hts_log_print(opt, LOG_INFO, "Purging directory %s/", + file); + while(strnotempty(file) + && (file[strlen(file) - 1] != '/') + && (file[strlen(file) - 1] != '\\')) { + file[strlen(file) - 1] = '\0'; } if (strnotempty(file)) - file[strlen(file)-1]='\0'; + file[strlen(file) - 1] = '\0'; } } } @@ -1954,66 +2119,75 @@ jump_if_done: fclose(old_lst); } // - opt->state._hts_in_html_parsing=0; + opt->state._hts_in_html_parsing = 0; } } // fin purge! // Indexation if (opt->kindex) - index_finish(StringBuff(opt->path_html),opt->kindex); + index_finish(StringBuff(opt->path_html), opt->kindex); // afficher résumé dans log - if (opt->log!=NULL) { + if (opt->log != NULL) { char BIGSTK finalInfo[8192]; - int error = fspc(opt,NULL,"error"); - int warning = fspc(opt,NULL,"warning"); - int info = fspc(opt,NULL,"info"); + int error = fspc(opt, NULL, "error"); + int warning = fspc(opt, NULL, "warning"); + int info = fspc(opt, NULL, "info"); char BIGSTK htstime[256]; char BIGSTK infoupdated[256]; + // int n=(int) (stat_loaded/(time_local()-HTS_STAT.stat_timestart)); - LLint n=(LLint) (HTS_STAT.HTS_TOTAL_RECV/(max(1,time_local()-HTS_STAT.stat_timestart))); - - sec2str(htstime,time_local()-HTS_STAT.stat_timestart); + LLint n = + (LLint) (HTS_STAT.HTS_TOTAL_RECV / + (max(1, time_local() - HTS_STAT.stat_timestart))); + + sec2str(htstime, time_local() - HTS_STAT.stat_timestart); //sprintf(finalInfo + strlen(finalInfo),LF"HTS-mirror complete in %s : %d links scanned, %d files written (%d bytes overall) [%d bytes received at %d bytes/sec]"LF,htstime,lien_tot-1,HTS_STAT.stat_files,stat_bytes,stat_loaded,n); infoupdated[0] = '\0'; if (opt->is_update) { if (HTS_STAT.stat_updated_files > 0) { - sprintf(infoupdated, ", %d files updated", (int)HTS_STAT.stat_updated_files); + sprintf(infoupdated, ", %d files updated", + (int) HTS_STAT.stat_updated_files); } else { sprintf(infoupdated, ", no files updated"); } } finalInfo[0] = '\0'; sprintf(finalInfo + strlen(finalInfo), - "HTTrack Website Copier/"HTTRACK_VERSION" mirror complete in %s : " - "%d links scanned, %d files written ("LLintP" bytes overall)%s " - "["LLintP" bytes received at "LLintP" bytes/sec]", - htstime, - (int)lien_tot-1, - (int)HTS_STAT.stat_files, - (LLint)HTS_STAT.stat_bytes, - infoupdated, - (LLint)HTS_STAT.HTS_TOTAL_RECV, - (LLint)n - ); + "HTTrack Website Copier/" HTTRACK_VERSION + " mirror complete in %s : " "%d links scanned, %d files written (" + LLintP " bytes overall)%s " "[" LLintP " bytes received at " LLintP + " bytes/sec]", htstime, (int) lien_tot - 1, + (int) HTS_STAT.stat_files, (LLint) HTS_STAT.stat_bytes, infoupdated, + (LLint) HTS_STAT.HTS_TOTAL_RECV, (LLint) n); if (HTS_STAT.total_packed > 0 && HTS_STAT.total_unpacked > 0) { - int packed_ratio=(int)((LLint)(HTS_STAT.total_packed*100)/HTS_STAT.total_unpacked); - sprintf(finalInfo + strlen(finalInfo),", "LLintP" bytes transfered using HTTP compression in %d files, ratio %d%%",(LLint)HTS_STAT.total_unpacked,HTS_STAT.total_packedfiles,(int)packed_ratio); - } - if (!opt->nokeepalive && HTS_STAT.stat_sockid > 0 && HTS_STAT.stat_nrequests > HTS_STAT.stat_sockid) { + int packed_ratio = + (int) ((LLint) (HTS_STAT.total_packed * 100) / HTS_STAT.total_unpacked); + sprintf(finalInfo + strlen(finalInfo), + ", " LLintP + " bytes transfered using HTTP compression in %d files, ratio %d%%", + (LLint) HTS_STAT.total_unpacked, HTS_STAT.total_packedfiles, + (int) packed_ratio); + } + if (!opt->nokeepalive && HTS_STAT.stat_sockid > 0 + && HTS_STAT.stat_nrequests > HTS_STAT.stat_sockid) { int rq = (HTS_STAT.stat_nrequests * 10) / HTS_STAT.stat_sockid; - sprintf(finalInfo + strlen(finalInfo),", %d.%d requests per connection", rq/10, rq%10); + + sprintf(finalInfo + strlen(finalInfo), ", %d.%d requests per connection", + rq / 10, rq % 10); } - sprintf(finalInfo + strlen(finalInfo),LF); + sprintf(finalInfo + strlen(finalInfo), LF); if (error) - sprintf(finalInfo + strlen(finalInfo),"(%d errors, %d warnings, %d messages)"LF,error,warning,info); + sprintf(finalInfo + strlen(finalInfo), + "(%d errors, %d warnings, %d messages)" LF, error, warning, info); else - sprintf(finalInfo + strlen(finalInfo),"(No errors, %d warnings, %d messages)"LF,warning,info); + sprintf(finalInfo + strlen(finalInfo), + "(No errors, %d warnings, %d messages)" LF, warning, info); // Log - fprintf(opt->log,LF"%s", finalInfo); + fprintf(opt->log, LF "%s", finalInfo); // Close ZIP if (cache.zipOutput) { @@ -2025,8 +2199,9 @@ jump_if_done: // noter les collisions { int i; - int empty1=0,empty2=0,empty3=0; - for(i=0;i<HTS_HASH_SIZE;i++) { + int empty1 = 0, empty2 = 0, empty3 = 0; + + for(i = 0; i < HTS_HASH_SIZE; i++) { if (hash.hash[0][i] == -1) empty1++; if (hash.hash[1][i] == -1) @@ -2036,25 +2211,29 @@ jump_if_done: } printf("\n"); printf("Debug info: Hash-table report\n"); - printf("Number of files entered: %d\n",hashnumber); - printf("Table size: %d\n",HTS_HASH_SIZE); + printf("Number of files entered: %d\n", hashnumber); + printf("Table size: %d\n", HTS_HASH_SIZE); printf("\n"); - printf("Longest chain sav: %d, empty: %d\n",longest_hash[0],empty1); - printf("Longest chain adr,fil: %d, empty: %d\n",longest_hash[1],empty2); - printf("Longest chain former_adr/fil: %d, empty: %d\n",longest_hash[2],empty3); + printf("Longest chain sav: %d, empty: %d\n", longest_hash[0], + empty1); + printf("Longest chain adr,fil: %d, empty: %d\n", longest_hash[1], + empty2); + printf("Longest chain former_adr/fil: %d, empty: %d\n", longest_hash[2], + empty3); printf("\n"); } -#endif +#endif // fin afficher résumé dans log // ending - usercommand(opt,0,NULL,NULL,NULL,NULL); + usercommand(opt, 0, NULL, NULL, NULL, NULL); // désallocation mémoire & buffers XH_uninit; - return 1; // OK + return 1; // OK } + // version 2 pour le reste // Estimate transfer rate @@ -2076,45 +2255,50 @@ jump_if_done: */ int engine_stats(void) { #if 0 - static FILE* debug_fp=NULL; /* ok */ + static FILE *debug_fp = NULL; /* ok */ + if (!debug_fp) - debug_fp=fopen("esstat.txt","wb"); + debug_fp = fopen("esstat.txt", "wb"); #endif - HTS_STAT.stat_nsocket=HTS_STAT.stat_errors=HTS_STAT.nbk=0; - HTS_STAT.nb=0; - if (HTS_STAT.HTS_TOTAL_RECV>2048) { - TStamp cdif=mtime_local(); + HTS_STAT.stat_nsocket = HTS_STAT.stat_errors = HTS_STAT.nbk = 0; + HTS_STAT.nb = 0; + if (HTS_STAT.HTS_TOTAL_RECV > 2048) { + TStamp cdif = mtime_local(); int i; - for(i=0;i<2;i++) { - if ( (cdif - HTS_STAT.istat_timestart[i]) >= 2000) { + for(i = 0; i < 2; i++) { + if ((cdif - HTS_STAT.istat_timestart[i]) >= 2000) { TStamp dif; + #if 0 -fprintf(debug_fp,"set timer %d\n",i); fflush(debug_fp); + fprintf(debug_fp, "set timer %d\n", i); + fflush(debug_fp); #endif - dif=cdif - HTS_STAT.istat_timestart[i]; - if ((TStamp)(dif/1000)>0) { - LLint byt=(HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[i]); - HTS_STAT.rate=(LLint)((TStamp) ((TStamp)byt/(dif/1000))); - HTS_STAT.istat_idlasttimer=i; // this timer recently sets the stats + dif = cdif - HTS_STAT.istat_timestart[i]; + if ((TStamp) (dif / 1000) > 0) { + LLint byt = (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[i]); + + HTS_STAT.rate = (LLint) ((TStamp) ((TStamp) byt / (dif / 1000))); + HTS_STAT.istat_idlasttimer = i; // this timer recently sets the stats // - HTS_STAT.istat_bytes[i]=HTS_STAT.HTS_TOTAL_RECV; - HTS_STAT.istat_timestart[i]=cdif; + HTS_STAT.istat_bytes[i] = HTS_STAT.HTS_TOTAL_RECV; + HTS_STAT.istat_timestart[i] = cdif; } - return 1; /* refreshed */ + return 1; /* refreshed */ } } // resynchronization between timer 0 (master) and 1 (slave) // timer #0 resync timer #1 when reaching 1 second limit if (HTS_STAT.istat_reference01 != HTS_STAT.istat_timestart[0]) { - if ( (cdif - HTS_STAT.istat_timestart[0]) >= 1000) { + if ((cdif - HTS_STAT.istat_timestart[0]) >= 1000) { #if 0 -fprintf(debug_fp,"resync timer 1\n"); fflush(debug_fp); + fprintf(debug_fp, "resync timer 1\n"); + fflush(debug_fp); #endif - HTS_STAT.istat_bytes[1]=HTS_STAT.HTS_TOTAL_RECV; - HTS_STAT.istat_timestart[1]=cdif; - HTS_STAT.istat_reference01=HTS_STAT.istat_timestart[0]; + HTS_STAT.istat_bytes[1] = HTS_STAT.HTS_TOTAL_RECV; + HTS_STAT.istat_timestart[1] = cdif; + HTS_STAT.istat_reference01 = HTS_STAT.istat_timestart[0]; } } @@ -2122,116 +2306,139 @@ fprintf(debug_fp,"resync timer 1\n"); fflush(debug_fp); return 0; } - #define _FILTERS (*opt->filters.filters) #define _FILTERS_PTR (opt->filters.filptr) #define _ROBOTS ((robots_wizard*)opt->robotsptr) // bannir host (trop lent etc) -void host_ban(httrackp* opt,lien_url** liens,int ptr,int lien_tot,struct_back* sback,char* host) { - lien_back* const back = sback->lnk; +void host_ban(httrackp * opt, lien_url ** liens, int ptr, int lien_tot, + struct_back * sback, char *host) { + lien_back *const back = sback->lnk; const int back_max = sback->count; + //int l; int i; - if (host[0]=='!') - return; // erreur.. déja cancellé.. bizarre.. devrait pas arriver + if (host[0] == '!') + return; // erreur.. déja cancellé.. bizarre.. devrait pas arriver /* sanity check */ if (*_FILTERS_PTR + 1 >= opt->maxfilter) { opt->maxfilter += HTS_FILTERSINC; if (filters_init(&_FILTERS, opt->maxfilter, HTS_FILTERSINC) == 0) { - printf("PANIC! : Too many filters : >%d [%d]\n",*_FILTERS_PTR,__LINE__); - hts_log_print(opt, LOG_PANIC, "Too many filters, giving up..(>%d)",*_FILTERS_PTR); - hts_log_print(opt, LOG_INFO, "To avoid that: use #F option for more filters (example: -#F5000)"); + printf("PANIC! : Too many filters : >%d [%d]\n", *_FILTERS_PTR, __LINE__); + hts_log_print(opt, LOG_PANIC, "Too many filters, giving up..(>%d)", + *_FILTERS_PTR); + hts_log_print(opt, LOG_INFO, + "To avoid that: use #F option for more filters (example: -#F5000)"); assertf("too many filters - giving up" == NULL); } } - // interdire host assertf((*_FILTERS_PTR) < opt->maxfilter); if (*_FILTERS_PTR < opt->maxfilter) { - strcpybuff(_FILTERS[*_FILTERS_PTR],"-"); - strcatbuff(_FILTERS[*_FILTERS_PTR],host); - strcatbuff(_FILTERS[*_FILTERS_PTR],"/*"); // host/ * interdit - (*_FILTERS_PTR)++; + strcpybuff(_FILTERS[*_FILTERS_PTR], "-"); + strcatbuff(_FILTERS[*_FILTERS_PTR], host); + strcatbuff(_FILTERS[*_FILTERS_PTR], "/*"); // host/ * interdit + (*_FILTERS_PTR)++; } - // oups - if (strlen(host)<=1) { // euhh?? longueur <= 1 - if (strcmp(host,"file://")) { + if (strlen(host) <= 1) { // euhh?? longueur <= 1 + if (strcmp(host, "file://")) { //## if (host[0]!=lOCAL_CHAR) { // pas local - hts_log_print(opt, LOG_PANIC, "PANIC! HostCancel detected memory leaks [char %d]",host[0]); - return; // purée + hts_log_print(opt, LOG_PANIC, + "PANIC! HostCancel detected memory leaks [char %d]", + host[0]); + return; // purée } } - // couper connexion - for(i=0;i<back_max;i++) { - if (back[i].status>=0) // réception OU prêt - if (strfield2(back[i].url_adr,host)) { + for(i = 0; i < back_max; i++) { + if (back[i].status >= 0) // réception OU prêt + if (strfield2(back[i].url_adr, host)) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("host control: deletehttp\n"); #endif - back[i].status=0; // terminé + back[i].status = 0; // terminé back_set_finished(sback, i); - if (back[i].r.soc!=INVALID_SOCKET) deletehttp(&back[i].r); - back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=STATUSCODE_TIMEOUT; // timeout (peu importe si c'est un traffic jam) - strcpybuff(back[i].r.msg,"Link Cancelled by host control"); - hts_log_print(opt, LOG_DEBUG, "Shutdown: %s%s",back[i].url_adr,back[i].url_fil); + if (back[i].r.soc != INVALID_SOCKET) + deletehttp(&back[i].r); + back[i].r.soc = INVALID_SOCKET; + back[i].r.statuscode = STATUSCODE_TIMEOUT; // timeout (peu importe si c'est un traffic jam) + strcpybuff(back[i].r.msg, "Link Cancelled by host control"); + hts_log_print(opt, LOG_DEBUG, "Shutdown: %s%s", back[i].url_adr, + back[i].url_fil); } } - + // effacer liens //l=strlen(host); - for(i=0;i<lien_tot;i++) { + for(i = 0; i < lien_tot; i++) { //if (liens[i]->adr_len==l) { // même taille de chaîne // Calcul de taille sécurisée if (liens[i]) { if (liens[i]->adr) { int l = 0; - while((liens[i]->adr[l]) && (l<1020)) l++; - if ((l > 0) && (l<1020)) { // sécurité - if (strfield2(jump_identification(liens[i]->adr),host)) { // host - hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s",liens[i]->adr,liens[i]->fil); - strcpybuff(liens[i]->adr,"!"); // cancel (invalide hash) + + while((liens[i]->adr[l]) && (l < 1020)) + l++; + if ((l > 0) && (l < 1020)) { // sécurité + if (strfield2(jump_identification(liens[i]->adr), host)) { // host + hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", liens[i]->adr, + liens[i]->fil); + strcpybuff(liens[i]->adr, "!"); // cancel (invalide hash) // on efface pas le hash, because si on rencontre le lien, reverif sav.. } } else { - if (opt->log!=NULL) { + if (opt->log != NULL) { char dmp[1040]; - dmp[0]='\0'; - strncatbuff(dmp,liens[i]->adr,1024); - hts_log_print(opt, LOG_WARNING, "WARNING! HostCancel detected memory leaks [len %d at %d]",l,i); - hts_log_print(opt, LOG_WARNING, "dump 1024 bytes (address %p): "LF"%s",liens[i]->adr,dmp); - } + + dmp[0] = '\0'; + strncatbuff(dmp, liens[i]->adr, 1024); + hts_log_print(opt, LOG_WARNING, + "WARNING! HostCancel detected memory leaks [len %d at %d]", + l, i); + hts_log_print(opt, LOG_WARNING, + "dump 1024 bytes (address %p): " LF "%s", + liens[i]->adr, dmp); + } } } else { - hts_log_print(opt, LOG_WARNING, "WARNING! HostCancel detected memory leaks [adr at %d]",i); + hts_log_print(opt, LOG_WARNING, + "WARNING! HostCancel detected memory leaks [adr at %d]", + i); } } else { - hts_log_print(opt, LOG_WARNING, "WARNING! HostCancel detected memory leaks [null at %d]",i); + hts_log_print(opt, LOG_WARNING, + "WARNING! HostCancel detected memory leaks [null at %d]", + i); } //} } } -int filters_init(char*** ptrfilters, int maxfilter, int filterinc) { - char** filters = *ptrfilters; - int filter_max=maximum(maxfilter, 128); +int filters_init(char ***ptrfilters, int maxfilter, int filterinc) { + char **filters = *ptrfilters; + int filter_max = maximum(maxfilter, 128); + if (filters == NULL) { - filters=(char**) malloct( sizeof(char*) * (filter_max+2) ); - memset(filters, 0, sizeof(char*) * (filter_max+2)); // filters[0] == 0 + filters = (char **) malloct(sizeof(char *) * (filter_max + 2)); + memset(filters, 0, sizeof(char *) * (filter_max + 2)); // filters[0] == 0 } else { - filters=(char**) realloct(filters, sizeof(char*) * (filter_max+2) ); + filters = (char **) realloct(filters, sizeof(char *) * (filter_max + 2)); } if (filters) { if (filters[0] == NULL) { - filters[0]=(char*) malloct( sizeof(char) * (filter_max+2) * (HTS_URLMAXSIZE*2) ); - memset(filters[0], 0, sizeof(char) * (filter_max+2) * (HTS_URLMAXSIZE*2) ); + filters[0] = + (char *) malloct(sizeof(char) * (filter_max + 2) * + (HTS_URLMAXSIZE * 2)); + memset(filters[0], 0, + sizeof(char) * (filter_max + 2) * (HTS_URLMAXSIZE * 2)); } else { - filters[0]=(char*) realloct(filters[0], sizeof(char) * (filter_max+2) * (HTS_URLMAXSIZE*2) ); + filters[0] = + (char *) realloct(filters[0], + sizeof(char) * (filter_max + + 2) * (HTS_URLMAXSIZE * 2)); } if (filters[0] == NULL) { freet(filters); @@ -2241,15 +2448,16 @@ int filters_init(char*** ptrfilters, int maxfilter, int filterinc) { if (filters != NULL) { int i; int from; + if (filterinc == 0) from = 0; else from = filter_max - filterinc; - for(i=0 ; i<=filter_max ; i++) { // PLUS UN (sécurité) - filters[i]=filters[0]+i*(HTS_URLMAXSIZE*2); + for(i = 0; i <= filter_max; i++) { // PLUS UN (sécurité) + filters[i] = filters[0] + i * (HTS_URLMAXSIZE * 2); } - for(i=from ; i<=filter_max ; i++) { // PLUS UN (sécurité) - filters[i][0]='\0'; // clear + for(i = from; i <= filter_max; i++) { // PLUS UN (sécurité) + filters[i][0] = '\0'; // clear } } *ptrfilters = filters; @@ -2259,81 +2467,83 @@ int filters_init(char*** ptrfilters, int maxfilter, int filterinc) { static int mkdir_compat(const char *pathname) { #ifdef _WIN32 return mkdir(pathname); -#else +#else return mkdir(pathname, HTS_ACCESS_FOLDER); #endif } /* path must end with "/" or with the finename (/tmp/bar/ or /tmp/bar/foo.zip) */ /* Note: preserve errno */ -HTSEXT_API int dir_exists(const char* path) { +HTSEXT_API int dir_exists(const char *path) { const int err = errno; STRUCT_STAT st; - char BIGSTK file[HTS_URLMAXSIZE*2]; + char BIGSTK file[HTS_URLMAXSIZE * 2]; int i = 0; + if (strnotempty(path) == 0) { - return 0; + return 0; } if (strlen(path) > HTS_URLMAXSIZE) { - return 0; + return 0; } /* Get a copy */ strcpybuff(file, path); #ifdef _WIN32 /* To system name */ - for(i = 0 ; file[i] != 0 ; i++) { + for(i = 0; file[i] != 0; i++) { if (file[i] == '/') { file[i] = PATH_SEPARATOR; } } #endif /* Get prefix (note: file can not be empty here) */ - for(i = (int) strlen(file) - 1 ; i > 0 && file[i] != PATH_SEPARATOR ; i--); - for( ; i > 0 && file[i] == PATH_SEPARATOR ; i--); + for(i = (int) strlen(file) - 1; i > 0 && file[i] != PATH_SEPARATOR; i--) ; + for(; i > 0 && file[i] == PATH_SEPARATOR; i--) ; file[i + 1] = '\0'; /* Check the final dir */ if (STAT(file, &st) == 0 && S_ISDIR(st.st_mode)) { - return 1; /* EXISTS */ + return 1; /* EXISTS */ } errno = err; - return 0; /* DOES NOT EXIST */ + return 0; /* DOES NOT EXIST */ } /* path must end with "/" or with the finename (/tmp/bar/ or /tmp/bar/foo.zip) */ /* Note: *not* UTF-8 */ -HTSEXT_API int structcheck(const char* path) { +HTSEXT_API int structcheck(const char *path) { struct stat st; - char BIGSTK tmpbuf[HTS_URLMAXSIZE*2]; - char BIGSTK file[HTS_URLMAXSIZE*2]; + char BIGSTK tmpbuf[HTS_URLMAXSIZE * 2]; + char BIGSTK file[HTS_URLMAXSIZE * 2]; int i = 0; int npaths; + if (strnotempty(path) == 0) - return 0; + return 0; if (strlen(path) > HTS_URLMAXSIZE) { errno = EINVAL; - return -1; + return -1; } /* Get a copy */ strcpybuff(file, path); #ifdef _WIN32 /* To system name */ - for(i = 0 ; file[i] != 0 ; i++) { + for(i = 0; file[i] != 0; i++) { if (file[i] == '/') { file[i] = PATH_SEPARATOR; } } #endif /* Get prefix (note: file can not be empty here) */ - for(i = (int) strlen(file) - 1 ; i > 0 && file[i] != PATH_SEPARATOR ; i--); - for( ; i > 0 && file[i] == PATH_SEPARATOR ; i--); + for(i = (int) strlen(file) - 1; i > 0 && file[i] != PATH_SEPARATOR; i--) ; + for(; i > 0 && file[i] == PATH_SEPARATOR; i--) ; file[i + 1] = '\0'; /* First check the final dir */ if (stat(file, &st) == 0 && S_ISDIR(st.st_mode)) { - return 0; /* OK */ + return 0; /* OK */ } /* Start from the beginning */ @@ -2341,18 +2551,19 @@ HTSEXT_API int structcheck(const char* path) { /* Skip irrelevant part (the root slash, or the drive path) */ #ifdef _WIN32 - if (file[0] != 0 && file[1] == ':') { /* f:\ */ - i+= 2; - if (file[i] == PATH_SEPARATOR) { /* f:\ */ + if (file[0] != 0 && file[1] == ':') { /* f:\ */ + i += 2; + if (file[i] == PATH_SEPARATOR) { /* f:\ */ i++; } - } else if (file[0] == PATH_SEPARATOR && file[1] == PATH_SEPARATOR) { /* \\mch */ + } else if (file[0] == PATH_SEPARATOR && file[1] == PATH_SEPARATOR) { /* \\mch */ int j; - i+= 2; + + i += 2; // skip machine and first dir (share) or we'll have troubles (TomZ) - for(j = 0 ; j < 2 ; j++) { + for(j = 0; j < 2; j++) { // skip segment - for( ; file[i] != '\0' && file[i] != PATH_SEPARATOR ; i++) ; + for(; file[i] != '\0' && file[i] != PATH_SEPARATOR; i++) ; // skip final separator if (file[i] == PATH_SEPARATOR) { /* f:\ */ i++; @@ -2362,30 +2573,30 @@ HTSEXT_API int structcheck(const char* path) { #endif /* Check paths */ - for(npaths = 1 ; ; npaths++) { + for(npaths = 1;; npaths++) { char end_char; /* Go to next path */ /* Skip separator(s) */ - for( ; file[i] == PATH_SEPARATOR ; i++); + for(; file[i] == PATH_SEPARATOR; i++) ; /* Next separator */ - for( ; file[i] != 0 && file[i] != PATH_SEPARATOR ; i++); + for(; file[i] != 0 && file[i] != PATH_SEPARATOR; i++) ; /* Check */ end_char = file[i]; if (end_char != 0) { file[i] = '\0'; } - if (stat(file, &st) == 0) { /* Something exists */ + if (stat(file, &st) == 0) { /* Something exists */ if (!S_ISDIR(st.st_mode)) { #if HTS_REMOVE_ANNOYING_INDEX - if (S_ISREG(st.st_mode)) { /* Regular file in place ; move it and create directory */ + if (S_ISREG(st.st_mode)) { /* Regular file in place ; move it and create directory */ sprintf(tmpbuf, "%s.txt", file); - if (rename(file, tmpbuf) != 0) { /* Can't rename regular file */ + if (rename(file, tmpbuf) != 0) { /* Can't rename regular file */ return -1; } - if (mkdir_compat(file) != 0) { /* Can't create directory */ + if (mkdir_compat(file) != 0) { /* Can't create directory */ return -1; } } @@ -2393,15 +2604,15 @@ HTSEXT_API int structcheck(const char* path) { #error Not implemented #endif } - } else { /* Nothing exists ; create directory */ - if (mkdir_compat(file) != 0) { /* Can't create directory */ + } else { /* Nothing exists ; create directory */ + if (mkdir_compat(file) != 0) { /* Can't create directory */ return -1; } } - if (end_char == 0) { /* End */ + if (end_char == 0) { /* End */ break; } else { - file[i] = end_char; /* Restore / */ + file[i] = end_char; /* Restore / */ } } return 0; @@ -2409,37 +2620,38 @@ HTSEXT_API int structcheck(const char* path) { /* path must end with "/" or with the finename (/tmp/bar/ or /tmp/bar/foo.zip) */ /* Note: UTF-8 */ -HTSEXT_API int structcheck_utf8(const char* path) { +HTSEXT_API int structcheck_utf8(const char *path) { STRUCT_STAT st; - char BIGSTK tmpbuf[HTS_URLMAXSIZE*2]; - char BIGSTK file[HTS_URLMAXSIZE*2]; + char BIGSTK tmpbuf[HTS_URLMAXSIZE * 2]; + char BIGSTK file[HTS_URLMAXSIZE * 2]; int i = 0; int npaths; + if (strnotempty(path) == 0) - return 0; + return 0; if (strlen(path) > HTS_URLMAXSIZE) { errno = EINVAL; - return -1; + return -1; } /* Get a copy */ strcpybuff(file, path); #ifdef _WIN32 /* To system name */ - for(i = 0 ; file[i] != 0 ; i++) { + for(i = 0; file[i] != 0; i++) { if (file[i] == '/') { file[i] = PATH_SEPARATOR; } } #endif /* Get prefix (note: file can not be empty here) */ - for(i = (int) strlen(file) - 1 ; i > 0 && file[i] != PATH_SEPARATOR ; i--); - for( ; i > 0 && file[i] == PATH_SEPARATOR ; i--); + for(i = (int) strlen(file) - 1; i > 0 && file[i] != PATH_SEPARATOR; i--) ; + for(; i > 0 && file[i] == PATH_SEPARATOR; i--) ; file[i + 1] = '\0'; /* First check the final dir */ if (STAT(file, &st) == 0 && S_ISDIR(st.st_mode)) { - return 0; /* OK */ + return 0; /* OK */ } /* Start from the beginning */ @@ -2447,18 +2659,19 @@ HTSEXT_API int structcheck_utf8(const char* path) { /* Skip irrelevant part (the root slash, or the drive path) */ #ifdef _WIN32 - if (file[0] != 0 && file[1] == ':') { /* f:\ */ - i+= 2; - if (file[i] == PATH_SEPARATOR) { /* f:\ */ + if (file[0] != 0 && file[1] == ':') { /* f:\ */ + i += 2; + if (file[i] == PATH_SEPARATOR) { /* f:\ */ i++; } - } else if (file[0] == PATH_SEPARATOR && file[1] == PATH_SEPARATOR) { /* \\mch */ + } else if (file[0] == PATH_SEPARATOR && file[1] == PATH_SEPARATOR) { /* \\mch */ int j; - i+= 2; + + i += 2; // skip machine and first dir (share) or we'll have troubles (TomZ) - for(j = 0 ; j < 2 ; j++) { + for(j = 0; j < 2; j++) { // skip segment - for( ; file[i] != '\0' && file[i] != PATH_SEPARATOR ; i++) ; + for(; file[i] != '\0' && file[i] != PATH_SEPARATOR; i++) ; // skip final separator if (file[i] == PATH_SEPARATOR) { /* f:\ */ i++; @@ -2468,30 +2681,30 @@ HTSEXT_API int structcheck_utf8(const char* path) { #endif /* Check paths */ - for(npaths = 1 ; ; npaths++) { + for(npaths = 1;; npaths++) { char end_char; /* Go to next path */ /* Skip separator(s) */ - for( ; file[i] == PATH_SEPARATOR ; i++); + for(; file[i] == PATH_SEPARATOR; i++) ; /* Next separator */ - for( ; file[i] != 0 && file[i] != PATH_SEPARATOR ; i++); + for(; file[i] != 0 && file[i] != PATH_SEPARATOR; i++) ; /* Check */ end_char = file[i]; if (end_char != 0) { file[i] = '\0'; } - if (STAT(file, &st) == 0) { /* Something exists */ + if (STAT(file, &st) == 0) { /* Something exists */ if (!S_ISDIR(st.st_mode)) { #if HTS_REMOVE_ANNOYING_INDEX - if (S_ISREG(st.st_mode)) { /* Regular file in place ; move it and create directory */ + if (S_ISREG(st.st_mode)) { /* Regular file in place ; move it and create directory */ sprintf(tmpbuf, "%s.txt", file); - if (RENAME(file, tmpbuf) != 0) { /* Can't rename regular file */ + if (RENAME(file, tmpbuf) != 0) { /* Can't rename regular file */ return -1; } - if (MKDIR(file) != 0) { /* Can't create directory */ + if (MKDIR(file) != 0) { /* Can't create directory */ return -1; } } @@ -2499,49 +2712,52 @@ HTSEXT_API int structcheck_utf8(const char* path) { #error Not implemented #endif } - } else { /* Nothing exists ; create directory */ - if (MKDIR(file) != 0) { /* Can't create directory */ + } else { /* Nothing exists ; create directory */ + if (MKDIR(file) != 0) { /* Can't create directory */ return -1; } } - if (end_char == 0) { /* End */ + if (end_char == 0) { /* End */ break; } else { - file[i] = end_char; /* Restore / */ + file[i] = end_char; /* Restore / */ } } return 0; } // sauver un fichier -int filesave(httrackp* opt,const char* adr,int len,const char* s,const char* url_adr,const char* url_fil) { - FILE* fp; +int filesave(httrackp * opt, const char *adr, int len, const char *s, + const char *url_adr, const char *url_fil) { + FILE *fp; + // écrire le fichier - if ((fp = filecreate(&opt->state.strc, s))!=NULL) { - int nl=0; - if (len>0) { - nl=(int) fwrite(adr,1,len,fp); + if ((fp = filecreate(&opt->state.strc, s)) != NULL) { + int nl = 0; + + if (len > 0) { + nl = (int) fwrite(adr, 1, len, fp); } fclose(fp); - if (nl!=len) // erreur + if (nl != len) // erreur return -1; } else return -1; - + return 0; } /* We should stop */ int check_fatal_io_errno(void) { - switch(errno) { + switch (errno) { #ifdef EMFILE - case EMFILE: /* Too many open files */ + case EMFILE: /* Too many open files */ #endif #ifdef ENOSPC - case ENOSPC: /* No space left on device */ + case ENOSPC: /* No space left on device */ #endif #ifdef EROFS - case EROFS: /* Read-only file system */ + case EROFS: /* Read-only file system */ #endif return 1; break; @@ -2549,34 +2765,35 @@ int check_fatal_io_errno(void) { return 0; } - // ouvrir un fichier (avec chemin Un*x) /* Note: utf-8 */ -FILE* filecreate(filenote_strc *strc, const char* s) { - char BIGSTK fname[HTS_URLMAXSIZE*2]; - FILE* fp; +FILE *filecreate(filenote_strc * strc, const char *s) { + char BIGSTK fname[HTS_URLMAXSIZE * 2]; + FILE *fp; int last_errno = 0; - fname[0]='\0'; + + fname[0] = '\0'; // noter lst - if (strc != NULL) { - filenote(strc, s, NULL); - } - + if (strc != NULL) { + filenote(strc, s, NULL); + } + strcpybuff(fname, s); #if HTS_DOSNAME // remplacer / par des slash arrière { - int i=0; + int i = 0; + while(fname[i]) { - if (fname[i]=='/') - fname[i]='\\'; + if (fname[i] == '/') + fname[i] = '\\'; i++; - } - } + } + } // a partir d'ici le slash devient antislash #endif - + /* Try to open the file */ fp = FOPEN(fname, "wb"); @@ -2595,176 +2812,198 @@ FILE* filecreate(filenote_strc *strc, const char* s) { } #ifndef _WIN32 if (fp != NULL) - chmod(fname, HTS_ACCESS_FILE); + chmod(fname, HTS_ACCESS_FILE); #endif return fp; } // ouvrir un fichier (avec chemin Un*x) -FILE* fileappend(filenote_strc *strc,const char* s) { - char BIGSTK fname[HTS_URLMAXSIZE*2]; - FILE* fp; - fname[0]='\0'; +FILE *fileappend(filenote_strc * strc, const char *s) { + char BIGSTK fname[HTS_URLMAXSIZE * 2]; + FILE *fp; + + fname[0] = '\0'; // noter lst - filenote(strc,s,NULL); - + filenote(strc, s, NULL); + // if (*s=='/') strcpybuff(fname,s+1); else strcpybuff(fname,s); // pas de / (root!!) // ** SIIIIIII!!! à cause de -O <path> - strcpybuff(fname,s); + strcpybuff(fname, s); #if HTS_DOSNAME // remplacer / par des slash arrière { - int i=0; + int i = 0; + while(fname[i]) { - if (fname[i]=='/') - fname[i]='\\'; + if (fname[i] == '/') + fname[i] = '\\'; i++; - } - } + } + } // a partir d'ici le slash devient antislash #endif - + // ouvrir - fp=FOPEN(fname,"ab"); - + fp = FOPEN(fname, "ab"); + #ifndef _WIN32 - if (fp!=NULL) chmod(fname,HTS_ACCESS_FILE); + if (fp != NULL) + chmod(fname, HTS_ACCESS_FILE); #endif return fp; } - // create an empty file -int filecreateempty(filenote_strc *strc, const char* filename) { - FILE* fp; - fp=filecreate(strc, filename); // filenote & co +int filecreateempty(filenote_strc * strc, const char *filename) { + FILE *fp; + + fp = filecreate(strc, filename); // filenote & co if (fp) { fclose(fp); return 1; } else - return 0; + return 0; } // noter fichier -int filenote(filenote_strc *strc, const char* s, filecreate_params* params) { +int filenote(filenote_strc * strc, const char *s, filecreate_params * params) { // gestion du fichier liste liste if (params) { //filecreate_params* p = (filecreate_params*) params; - strcpybuff(strc->path,params->path); - strc->lst=params->lst; + strcpybuff(strc->path, params->path); + strc->lst = params->lst; return 0; } else if (strc->lst) { - char BIGSTK savelst[HTS_URLMAXSIZE*2]; - char catbuff[CATBUFF_SIZE]; - strcpybuff(savelst,fslash(catbuff,s)); + char BIGSTK savelst[HTS_URLMAXSIZE * 2]; + char catbuff[CATBUFF_SIZE]; + + strcpybuff(savelst, fslash(catbuff, s)); // couper chemin? if (strnotempty(strc->path)) { - if (strncmp(fslash(catbuff,strc->path),savelst,strlen(strc->path))==0) { // couper - strcpybuff(savelst,s+strlen(strc->path)); + if (strncmp(fslash(catbuff, strc->path), savelst, strlen(strc->path)) == 0) { // couper + strcpybuff(savelst, s + strlen(strc->path)); } } - fprintf(strc->lst,"[%s]"LF,savelst); + fprintf(strc->lst, "[%s]" LF, savelst); fflush(strc->lst); } return 1; } /* Note: utf-8 */ -void file_notify(httrackp* opt,const char* adr,const char* fil,const char* save,int create,int modify,int not_updated) { +void file_notify(httrackp * opt, const char *adr, const char *fil, + const char *save, int create, int modify, int not_updated) { RUN_CALLBACK6(opt, filesave2, adr, fil, save, create, modify, not_updated); } // executer commande utilisateur -static void postprocess_file(httrackp* opt, const char* save, const char* adr, const char* fil); -void usercommand(httrackp* opt,int _exe,const char* _cmd,const char* file,const char* adr,const char* fil) { - usercommand_strc* strc = &opt->state.usercmd; - +static void postprocess_file(httrackp * opt, const char *save, const char *adr, + const char *fil); +void usercommand(httrackp * opt, int _exe, const char *_cmd, const char *file, + const char *adr, const char *fil) { + usercommand_strc *strc = &opt->state.usercmd; + /* Callback */ if (_exe) { - strcpybuff(strc->cmd,_cmd); + strcpybuff(strc->cmd, _cmd); if (strnotempty(strc->cmd)) - strc->exe=_exe; + strc->exe = _exe; else - strc->exe=0; + strc->exe = 0; } /* post-processing */ postprocess_file(opt, file, adr, fil); - if (file != NULL && strnotempty(file)) { + if (file != NULL && strnotempty(file)) { RUN_CALLBACK1(opt, filesave, file); - } + } if (strc->exe) { if (file != NULL && strnotempty(file)) { if (strnotempty(strc->cmd)) { - usercommand_exe(strc->cmd,file); + usercommand_exe(strc->cmd, file); } } } } -void usercommand_exe(const char* cmd,const char* file) { +void usercommand_exe(const char *cmd, const char *file) { char BIGSTK temp[8192]; - char c[2]=""; + char c[2] = ""; int i; - temp[0]='\0'; + + temp[0] = '\0'; // - for(i=0;i<(int) strlen(cmd);i++) { - if ((cmd[i]=='$') && (cmd[i+1]=='0')) { - strcatbuff(temp,file); + for(i = 0; i < (int) strlen(cmd); i++) { + if ((cmd[i] == '$') && (cmd[i + 1] == '0')) { + strcatbuff(temp, file); i++; } else { - c[0]=cmd[i]; c[1]='\0'; - strcatbuff(temp,c); + c[0] = cmd[i]; + c[1] = '\0'; + strcatbuff(temp, c); } } if (system(temp) == -1) { - assertf(! "can not spawn process"); \ + assertf(!"can not spawn process"); } } - -static void postprocess_file(httrackp* opt,const char* save, const char* adr, const char* fil) { +static void postprocess_file(httrackp * opt, const char *save, const char *adr, + const char *fil) { //int first = 0; /* MIME-html archive to build */ if (opt != NULL && opt->mimehtml) { if (adr != NULL && strcmp(adr, "primary") == 0) { adr = NULL; } - if (save != NULL && opt != NULL && adr != NULL && adr[0] && strnotempty(save) && fexist(save)) { - const char* rsc_save = save; - const char* rsc_fil = strrchr(fil, '/'); + if (save != NULL && opt != NULL && adr != NULL && adr[0] + && strnotempty(save) && fexist(save)) { + const char *rsc_save = save; + const char *rsc_fil = strrchr(fil, '/'); int n; + if (rsc_fil == NULL) rsc_fil = fil; - if (strncmp(fslash(OPT_GET_BUFF(opt),save), fslash(OPT_GET_BUFF(opt),StringBuff(opt->path_html_utf8)), (n = (int)strlen(StringBuff(opt->path_html_utf8)))) == 0) { + if (strncmp + (fslash(OPT_GET_BUFF(opt), save), + fslash(OPT_GET_BUFF(opt), StringBuff(opt->path_html_utf8)), (n = + (int) + strlen + (StringBuff + (opt-> + path_html_utf8)))) + == 0) { rsc_save += n; } if (!opt->state.mimehtml_created) { //first = 1; - opt->state.mimefp = fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),"index.mht"), "wb"); + opt->state.mimefp = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_html), "index.mht"), + "wb"); if (opt->state.mimefp != NULL) { char BIGSTK rndtmp[1024], currtime[256]; - srand((unsigned int)time(NULL)); + + srand((unsigned int) time(NULL)); time_gmt_rfc822(currtime); - sprintf(rndtmp, "%d_%d", (int)time(NULL), (int) rand()); + sprintf(rndtmp, "%d_%d", (int) time(NULL), (int) rand()); StringRoom(opt->state.mimemid, 256); - sprintf(StringBuffRW(opt->state.mimemid), "----=_MIMEPart_%s_=----", rndtmp); + sprintf(StringBuffRW(opt->state.mimemid), "----=_MIMEPart_%s_=----", + rndtmp); StringSetLength(opt->state.mimemid, -1); - fprintf(opt->state.mimefp, "From: HTTrack Website Copier <nobody@localhost>\r\n" - "Subject: Local mirror\r\n" - "Date: %s\r\n" - "Message-ID: <httrack_%s@localhost>\r\n" - "Content-Type: multipart/related;\r\n" - "\tboundary=\"%s\";\r\n" - "\ttype=\"text/html\"\r\n" - "MIME-Version: 1.0\r\n" - "\r\nThis message is a RFC MIME-compliant multipart message.\r\n" - "\r\n" - , currtime, rndtmp, StringBuff(opt->state.mimemid)); + fprintf(opt->state.mimefp, + "From: HTTrack Website Copier <nobody@localhost>\r\n" + "Subject: Local mirror\r\n" "Date: %s\r\n" + "Message-ID: <httrack_%s@localhost>\r\n" + "Content-Type: multipart/related;\r\n" + "\tboundary=\"%s\";\r\n" "\ttype=\"text/html\"\r\n" + "MIME-Version: 1.0\r\n" + "\r\nThis message is a RFC MIME-compliant multipart message.\r\n" + "\r\n", currtime, rndtmp, StringBuff(opt->state.mimemid)); opt->state.mimehtml_created = 1; } else { opt->state.mimehtml_created = -1; @@ -2772,42 +3011,53 @@ static void postprocess_file(httrackp* opt,const char* save, const char* adr, co } } if (opt->state.mimehtml_created == 1 && opt->state.mimefp != NULL) { - FILE* fp = FOPEN(save, "rb"); + FILE *fp = FOPEN(save, "rb"); + if (fp != NULL) { - char buff[60*100 + 2]; + char buff[60 * 100 + 2]; char mimebuff[256]; - char BIGSTK cid[HTS_URLMAXSIZE*3]; + char BIGSTK cid[HTS_URLMAXSIZE * 3]; size_t len; - int isHtml = ( ishtml(opt,save) == 1 ); + int isHtml = (ishtml(opt, save) == 1); + mimebuff[0] = '\0'; /* CID */ strcpybuff(cid, adr); strcatbuff(cid, fil); escape_in_url(cid); - { char* a = cid; while((a = strchr(a, '%'))) { *a = 'X'; a++; } } - - guess_httptype(opt,mimebuff, save); - fprintf(opt->state.mimefp, "--%s\r\n", StringBuff(opt->state.mimemid)); + { + char *a = cid; + + while((a = strchr(a, '%'))) { + *a = 'X'; + a++; + } + } + + guess_httptype(opt, mimebuff, save); + fprintf(opt->state.mimefp, "--%s\r\n", + StringBuff(opt->state.mimemid)); /*if (first) - fprintf(opt->state.mimefp, "Content-disposition: inline\r\n"); - else*/ - fprintf(opt->state.mimefp, "Content-disposition: attachment; filename=\"%s\"\r\n", rsc_save); - fprintf(opt->state.mimefp, - "Content-Type: %s\r\n" - "Content-Transfer-Encoding: %s\r\n" - /*"Content-Location: http://localhost/%s\r\n"*/ - "Content-ID: <%s>\r\n" - "\r\n" - , mimebuff - , isHtml ? "8bit" : "base64" - /*, rsc_save*/ - , cid); + fprintf(opt->state.mimefp, "Content-disposition: inline\r\n"); + else */ + fprintf(opt->state.mimefp, + "Content-disposition: attachment; filename=\"%s\"\r\n", + rsc_save); + fprintf(opt->state.mimefp, + "Content-Type: %s\r\n" "Content-Transfer-Encoding: %s\r\n" + /*"Content-Location: http://localhost/%s\r\n" */ + "Content-ID: <%s>\r\n" "\r\n", mimebuff, + isHtml ? "8bit" : "base64" + /*, rsc_save */ + , cid); while((len = fread(buff, 1, sizeof(buff) - 2, fp)) > 0) { buff[len] = '\0'; if (!isHtml) { - char base64buff[60*100*2]; - code64((unsigned char*)buff, (int)len, (unsigned char*)base64buff, 1); + char base64buff[60 * 100 * 2]; + + code64((unsigned char *) buff, (int) len, + (unsigned char *) base64buff, 1); fprintf(opt->state.mimefp, "%s", base64buff); } else { fprintf(opt->state.mimefp, "%s", buff); @@ -2819,8 +3069,8 @@ static void postprocess_file(httrackp* opt,const char* save, const char* adr, co } } else if (save == NULL) { if (opt->state.mimehtml_created == 1 && opt->state.mimefp != NULL) { - fprintf(opt->state.mimefp, - "--%s--\r\n", StringBuff(opt->state.mimemid)); + fprintf(opt->state.mimefp, "--%s--\r\n", + StringBuff(opt->state.mimemid)); fclose(opt->state.mimefp); opt->state.mimefp = NULL; } @@ -2829,67 +3079,66 @@ static void postprocess_file(httrackp* opt,const char* save, const char* adr, co } // écrire n espaces dans fp -int fspc(httrackp *opt,FILE* fp,const char* type) { - fspc_strc* const strc = ( opt != NULL ) ? &opt->state.fspc : NULL; +int fspc(httrackp * opt, FILE * fp, const char *type) { + fspc_strc *const strc = (opt != NULL) ? &opt->state.fspc : NULL; + if (fp != NULL) { char s[256]; time_t tt; - struct tm* A; - tt=time(NULL); - A=localtime(&tt); + struct tm *A; + + tt = time(NULL); + A = localtime(&tt); if (A == NULL) { - int localtime_returned_null=0; + int localtime_returned_null = 0; + assert(localtime_returned_null); } - strftime(s,250,"%H:%M:%S",A); + strftime(s, 250, "%H:%M:%S", A); if (strnotempty(type)) - fprintf(fp,"%s\t%c%s: \t",s,hichar(*type),type+1); - else - fprintf(fp,"%s\t \t",s); - if (strc != NULL) { - if (strcmp(type,"warning")==0) - strc->warning++; - else if (strcmp(type,"error")==0) - strc->error++; - else if (strcmp(type,"info")==0) - strc->info++; - } - } - else if (strc == NULL) { - return 0; - } - else if (!type) { - strc->error=strc->warning=strc->info=0; // reset - } - else if (strcmp(type,"warning")==0) + fprintf(fp, "%s\t%c%s: \t", s, hichar(*type), type + 1); + else + fprintf(fp, "%s\t \t", s); + if (strc != NULL) { + if (strcmp(type, "warning") == 0) + strc->warning++; + else if (strcmp(type, "error") == 0) + strc->error++; + else if (strcmp(type, "info") == 0) + strc->info++; + } + } else if (strc == NULL) { + return 0; + } else if (!type) { + strc->error = strc->warning = strc->info = 0; // reset + } else if (strcmp(type, "warning") == 0) return strc->warning; - else if (strcmp(type,"error")==0) + else if (strcmp(type, "error") == 0) return strc->error; - else if (strcmp(type,"info")==0) + else if (strcmp(type, "info") == 0) return strc->info; return 0; } - // vérifier taux de transfert #if 0 -void check_rate(TStamp stat_timestart,int maxrate) { +void check_rate(TStamp stat_timestart, int maxrate) { // vérifier taux de transfert (pas trop grand?) /* - if (maxrate>0) { - int r = (int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-stat_timestart)); // taux actuel de transfert - HTS_STAT.HTS_TOTAL_RECV_STATE=0; - if (r>maxrate) { // taux>taux autorisé - int taux = (int) (((TStamp) (r - maxrate) * 100) / (TStamp) maxrate); - if (taux<15) - HTS_STAT.HTS_TOTAL_RECV_STATE=1; // ralentir un peu (<15% dépassement) - else if (taux<50) - HTS_STAT.HTS_TOTAL_RECV_STATE=2; // beaucoup (<50% dépassement) - else - HTS_STAT.HTS_TOTAL_RECV_STATE=3; // énormément (>50% dépassement) - } - } - */ + if (maxrate>0) { + int r = (int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-stat_timestart)); // taux actuel de transfert + HTS_STAT.HTS_TOTAL_RECV_STATE=0; + if (r>maxrate) { // taux>taux autorisé + int taux = (int) (((TStamp) (r - maxrate) * 100) / (TStamp) maxrate); + if (taux<15) + HTS_STAT.HTS_TOTAL_RECV_STATE=1; // ralentir un peu (<15% dépassement) + else if (taux<50) + HTS_STAT.HTS_TOTAL_RECV_STATE=2; // beaucoup (<50% dépassement) + else + HTS_STAT.HTS_TOTAL_RECV_STATE=3; // énormément (>50% dépassement) + } + } + */ } #endif @@ -2897,12 +3146,15 @@ void check_rate(TStamp stat_timestart,int maxrate) { // sous routines liées au moteur et au backing // supplemental links ready (done) after ptr or ready in background -int backlinks_done(struct_back* sback,lien_url** liens,int lien_tot,int ptr) { - int n=0; +int backlinks_done(struct_back * sback, lien_url ** liens, int lien_tot, + int ptr) { + int n = 0; + #if 0 int i; + //Links done and stored in cache - for(i=ptr+1;i<lien_tot;i++) { + for(i = ptr + 1; i < lien_tot; i++) { if (liens[i]) { if (liens[i]->pass2 == -1) { n++; @@ -2911,35 +3163,40 @@ int backlinks_done(struct_back* sback,lien_url** liens,int lien_tot,int ptr) { } #else // finalized in background - n+=HTS_STAT.stat_background; + n += HTS_STAT.stat_background; #endif - n+=back_done_incache(sback); + n += back_done_incache(sback); return n; } // remplir backing si moins de max_bytes en mémoire -HTS_INLINE int back_fillmax(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot) { +HTS_INLINE int back_fillmax(struct_back * sback, httrackp * opt, + cache_back * cache, lien_url ** liens, int ptr, + int numero_passe, int lien_tot) { if (!opt->state.stop) { - if (back_incache(sback)<opt->maxcache) { // pas trop en mémoire? - return back_fill(sback,opt,cache,liens,ptr,numero_passe,lien_tot); + if (back_incache(sback) < opt->maxcache) { // pas trop en mémoire? + return back_fill(sback, opt, cache, liens, ptr, numero_passe, lien_tot); } } - return -1; /* plus de place */ + return -1; /* plus de place */ } -int back_pluggable_sockets_strict(struct_back* sback, httrackp* opt) { +int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt) { int n = opt->maxsoc - back_nsoc(sback); // connect limiter if (n > 0 && opt->maxconn > 0 && HTS_STAT.last_connect > 0) { - TStamp opTime = HTS_STAT.last_request ? HTS_STAT.last_request : HTS_STAT.last_connect; + TStamp opTime = + HTS_STAT.last_request ? HTS_STAT.last_request : HTS_STAT.last_connect; TStamp cTime = mtime_local(); - TStamp lap = ( cTime - opTime ); - TStamp minLap = (TStamp) ( 1000.0 / opt->maxconn ); + TStamp lap = (cTime - opTime); + TStamp minLap = (TStamp) (1000.0 / opt->maxconn); + if (lap < minLap) { n = 0; } else { - int nMax = (int) ( lap / minLap ); + int nMax = (int) (lap / minLap); + n = min(n, nMax); } } @@ -2947,49 +3204,51 @@ int back_pluggable_sockets_strict(struct_back* sback, httrackp* opt) { return n; } -int back_pluggable_sockets(struct_back* sback, httrackp* opt) { +int back_pluggable_sockets(struct_back * sback, httrackp * opt) { int n; // ajouter autant de socket qu'on peut ajouter - n=back_pluggable_sockets_strict(sback, opt); + n = back_pluggable_sockets_strict(sback, opt); // vérifier qu'il restera assez de place pour les tests ensuite (en théorie, 1 entrée libre restante suffirait) - n=min( n, back_available(sback) - 8 ); + n = min(n, back_available(sback) - 8); // no space left on backing stack - do not back anymore if (back_stack_available(sback) <= 2) - n=0; + n = 0; return n; } // remplir backing -int back_fill(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot) { +int back_fill(struct_back * sback, httrackp * opt, cache_back * cache, + lien_url ** liens, int ptr, int numero_passe, int lien_tot) { int n = back_pluggable_sockets(sback, opt); - if (opt->savename_delayed == 2 && !opt->delayed_cached) /* cancel (always delayed) */ + + if (opt->savename_delayed == 2 && !opt->delayed_cached) /* cancel (always delayed) */ return 0; - if (n>0) { + if (n > 0) { int p; - if (ptr<cache->ptr_last) { /* restart (2 scans: first html, then non html) */ - cache->ptr_ant=0; + if (ptr < cache->ptr_last) { /* restart (2 scans: first html, then non html) */ + cache->ptr_ant = 0; } - p=ptr+1; + p = ptr + 1; /* on a déja parcouru */ - if (p<cache->ptr_ant) - p=cache->ptr_ant; - while( (p<lien_tot) && (n>0) && back_checkmirror(opt)) { - //while((p<lien_tot) && (n>0) && (p < ptr+opt->maxcache_anticipate)) { - int ok=1; - + if (p < cache->ptr_ant) + p = cache->ptr_ant; + while((p < lien_tot) && (n > 0) && back_checkmirror(opt)) { + //while((p<lien_tot) && (n>0) && (p < ptr+opt->maxcache_anticipate)) { + int ok = 1; + // on ne met pas le fichier en backing si il doit être traité après ou s'il a déja été traité - if (liens[p]->pass2) { // 2è passe - if (numero_passe!=1) - ok=0; + if (liens[p]->pass2) { // 2è passe + if (numero_passe != 1) + ok = 0; } else { - if (numero_passe!=0) - ok=0; + if (numero_passe != 0) + ok = 0; } // Why in hell did I do that ? @@ -2999,65 +3258,55 @@ int back_fill(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** lien // ok = 0; //} if (liens[p]->sav == NULL || liens[p]->sav[0] == '\0' - || hash_read(opt->hash,liens[p]->sav,"",0,opt->urlhack) < 0) { + || hash_read(opt->hash, liens[p]->sav, "", 0, opt->urlhack) < 0) { ok = 0; } - // note: si un backing est fini, il reste en mémoire jusqu'à ce que // le ptr l'atteigne if (ok) { - if (!back_exist(sback, opt, liens[p]->adr,liens[p]->fil,liens[p]->sav)) { - if (back_add(sback,opt,cache,liens[p]->adr,liens[p]->fil,liens[p]->sav,liens[liens[p]->precedent]->adr,liens[liens[p]->precedent]->fil,liens[p]->testmode)==-1) { - hts_log_print(opt, LOG_DEBUG, "error: unable to add more links through back_add for back_fill"); + if (!back_exist + (sback, opt, liens[p]->adr, liens[p]->fil, liens[p]->sav)) { + if (back_add + (sback, opt, cache, liens[p]->adr, liens[p]->fil, liens[p]->sav, + liens[liens[p]->precedent]->adr, liens[liens[p]->precedent]->fil, + liens[p]->testmode) == -1) { + hts_log_print(opt, LOG_DEBUG, + "error: unable to add more links through back_add for back_fill"); #if BDEBUG==1 printf("error while adding\n"); -#endif - n=0; // sortir +#endif + n = 0; // sortir } else { n--; #if BDEBUG==1 - printf("backing: %s%s\n",liens[p]->adr,liens[p]->fil); + printf("backing: %s%s\n", liens[p]->adr, liens[p]->fil); #endif - } - } + } + } } p++; - } // while + } // while /* sauver position dernière anticipation */ - cache->ptr_ant=p; - cache->ptr_last=ptr; + cache->ptr_ant = p; + cache->ptr_last = ptr; } return 0; } -// --- - - - - - - - - - - - - - - - - +// --- // Poll stdin.. si besoin #if HTS_POLL // lecture stdin des caractères disponibles -int read_stdin(char* s,int max) { - int i=0; - while((check_stdin()) && (i<(max-1)) ) - s[i++]=fgetc(stdin); - s[i]='\0'; +int read_stdin(char *s, int max) { + int i = 0; + + while((check_stdin()) && (i < (max - 1))) + s[i++] = fgetc(stdin); + s[i] = '\0'; return i; } + #ifdef _WIN32 int check_stdin(void) { #ifndef _WIN32_WCE @@ -3070,15 +3319,17 @@ int check_stdin(void) { int check_flot(T_SOC s) { fd_set fds; struct timeval tv; + FD_ZERO(&fds); - FD_SET((T_SOC) s,&fds); - tv.tv_sec=0; - tv.tv_usec=0; - select(s+1,&fds,NULL,NULL,&tv); - return FD_ISSET(s,&fds); + FD_SET((T_SOC) s, &fds); + tv.tv_sec = 0; + tv.tv_usec = 0; + select(s + 1, &fds, NULL, NULL, &tv); + return FD_ISSET(s, &fds); } int check_stdin(void) { - fflush(stdout); fflush(stdin); + fflush(stdout); + fflush(stdin); if (check_flot(0)) return 1; return 0; @@ -3089,33 +3340,36 @@ int check_stdin(void) { int check_sockerror(T_SOC s) { fd_set fds; struct timeval tv; + FD_ZERO(&fds); - FD_SET((T_SOC) s,&fds); - tv.tv_sec=0; - tv.tv_usec=0; - select(s+1,NULL,NULL,&fds,&tv); - return FD_ISSET(s,&fds); + FD_SET((T_SOC) s, &fds); + tv.tv_sec = 0; + tv.tv_usec = 0; + select(s + 1, NULL, NULL, &fds, &tv); + return FD_ISSET(s, &fds); } /* check incoming data */ int check_sockdata(T_SOC s) { fd_set fds; struct timeval tv; + FD_ZERO(&fds); - FD_SET((T_SOC) s,&fds); - tv.tv_sec=0; - tv.tv_usec=0; - select(s+1,&fds,NULL,NULL,&tv); - return FD_ISSET(s,&fds); + FD_SET((T_SOC) s, &fds); + tv.tv_sec = 0; + tv.tv_usec = 0; + select(s + 1, &fds, NULL, NULL, &tv); + return FD_ISSET(s, &fds); } // Attente de touche -int ask_continue(httrackp *opt) { - const char* s; +int ask_continue(httrackp * opt) { + const char *s; + s = RUN_CALLBACK1(opt, query2, opt->state.HTbuff); if (s) { if (strnotempty(s)) { - if ((strfield2(s,"N")) || (strfield2(s,"NO")) || (strfield2(s,"NON"))) + if ((strfield2(s, "N")) || (strfield2(s, "NO")) || (strfield2(s, "NON"))) return 0; } return 1; @@ -3125,66 +3379,73 @@ int ask_continue(httrackp *opt) { // nombre de digits dans un nombre int nombre_digit(int n) { - int i=1; - while(n >= 10) { n/=10; i++; } + int i = 1; + + while(n >= 10) { + n /= 10; + i++; + } return i; } - // renvoi adresse de la fin du token dans p // renvoi NULL si la chaine est un token unique // (PATCHE également la chaine) // ex: "test" "test2" renvoi adresse sur espace // flag==1 si chaine comporte des echappements comme \" -char* next_token(char* p,int flag) { - int detect=0; - int quote=0; +char *next_token(char *p, int flag) { + int detect = 0; + int quote = 0; + p--; do { p++; - if (flag && (*p=='\\')) { // sauter \x ou \" + if (flag && (*p == '\\')) { // sauter \x ou \" if (quote) { - char c='\0'; - if (*(p+1)=='\\') - c='\\'; - else if (*(p+1)=='"') - c='"'; + char c = '\0'; + + if (*(p + 1) == '\\') + c = '\\'; + else if (*(p + 1) == '"') + c = '"'; if (c) { char BIGSTK tempo[8192]; - tempo[0]=c; tempo[1]='\0'; - strcatbuff(tempo,p+2); - strcpybuff(p,tempo); + + tempo[0] = c; + tempo[1] = '\0'; + strcatbuff(tempo, p + 2); + strcpybuff(p, tempo); } } - } - else if (*p==34) { // guillemets (de fin) + } else if (*p == 34) { // guillemets (de fin) char BIGSTK tempo[8192]; - tempo[0]='\0'; - strcatbuff(tempo,p+1); - strcpybuff(p,tempo); /* wipe "" */ + + tempo[0] = '\0'; + strcatbuff(tempo, p + 1); + strcpybuff(p, tempo); /* wipe "" */ p--; /* */ - quote=!quote; - } - else if (*p==32) { + quote = !quote; + } else if (*p == 32) { if (!quote) - detect=1; - } - else if (*p=='\0') { - p=NULL; - detect=1; + detect = 1; + } else if (*p == '\0') { + p = NULL; + detect = 1; } } while(!detect); return p; } -static int hts_cancel_file_push_(httrackp *opt, const char *url) { +static int hts_cancel_file_push_(httrackp * opt, const char *url) { if (url != NULL && url[0] != '\0') { htsoptstatecancel **cancel; + /* search for available place to store a new htsoptstatecancel* */ - for( cancel = &opt->state.cancel ; *cancel != NULL ; cancel = & ( (*cancel)->next ) ) { + for(cancel = &opt->state.cancel; *cancel != NULL; + cancel = &((*cancel)->next)) { if (strcmp((*cancel)->url, url) == 0) { - return 1; /* already there */ + return 1; /* already there */ } } *cancel = malloct(sizeof(htsoptstatecancel)); @@ -3196,88 +3457,96 @@ static int hts_cancel_file_push_(httrackp *opt, const char *url) { } /* cancel a file (locked) */ -HTSEXT_API int hts_cancel_file_push(httrackp *opt, const char *url) { +HTSEXT_API int hts_cancel_file_push(httrackp * opt, const char *url) { int ret; + hts_mutexlock(&opt->state.lock); ret = hts_cancel_file_push_(opt, url); hts_mutexrelease(&opt->state.lock); return ret; } -static char* hts_cancel_file_pop_(httrackp *opt) { +static char *hts_cancel_file_pop_(httrackp * opt) { if (opt->state.cancel != NULL) { htsoptstatecancel **cancel; htsoptstatecancel *ret; - for( cancel = &opt->state.cancel ; (*cancel)->next != NULL ; cancel = & ( (*cancel)->next ) ); + + for(cancel = &opt->state.cancel; (*cancel)->next != NULL; + cancel = &((*cancel)->next)) ; ret = *cancel; *cancel = NULL; return ret->url; } - return NULL; /* no entry */ + return NULL; /* no entry */ } -char* hts_cancel_file_pop(httrackp *opt) { - char* ret; +char *hts_cancel_file_pop(httrackp * opt) { + char *ret; + hts_mutexlock(&opt->state.lock); ret = hts_cancel_file_pop_(opt); hts_mutexrelease(&opt->state.lock); return ret; } -HTSEXT_API void hts_cancel_test(httrackp *opt) { - if (opt->state._hts_in_html_parsing==2) - opt->state._hts_cancel=2; +HTSEXT_API void hts_cancel_test(httrackp * opt) { + if (opt->state._hts_in_html_parsing == 2) + opt->state._hts_cancel = 2; } -HTSEXT_API void hts_cancel_parsing(httrackp *opt) { +HTSEXT_API void hts_cancel_parsing(httrackp * opt) { if (opt->state._hts_in_html_parsing) - opt->state._hts_cancel=1; + opt->state._hts_cancel = 1; } // en train de parser un fichier html? réponse: % effectués // flag>0 : refresh demandé -HTSEXT_API int hts_is_parsing(httrackp *opt, int flag) { - if (opt->state._hts_in_html_parsing) { // parsing? +HTSEXT_API int hts_is_parsing(httrackp * opt, int flag) { + if (opt->state._hts_in_html_parsing) { // parsing? if (flag >= 0) - opt->state._hts_in_html_poll = 1; // faudrait un tit refresh - return max(opt->state._hts_in_html_done, 1); // % effectués + opt->state._hts_in_html_poll = 1; // faudrait un tit refresh + return max(opt->state._hts_in_html_done, 1); // % effectués } else { - return 0; // non + return 0; // non } } -HTSEXT_API int hts_is_testing(httrackp *opt) { // 0 non 1 test 2 purge - if (opt->state._hts_in_html_parsing==2) +HTSEXT_API int hts_is_testing(httrackp * opt) { // 0 non 1 test 2 purge + if (opt->state._hts_in_html_parsing == 2) return 1; - else if (opt->state._hts_in_html_parsing==3) + else if (opt->state._hts_in_html_parsing == 3) return 2; - else if (opt->state._hts_in_html_parsing==4) + else if (opt->state._hts_in_html_parsing == 4) return 3; - else if (opt->state._hts_in_html_parsing==5) // scheduling + else if (opt->state._hts_in_html_parsing == 5) // scheduling return 4; - else if (opt->state._hts_in_html_parsing==6) // wait for slot + else if (opt->state._hts_in_html_parsing == 6) // wait for slot return 5; return 0; } -HTSEXT_API int hts_is_exiting(httrackp *opt) { +HTSEXT_API int hts_is_exiting(httrackp * opt) { return opt->state.exit_xh; } + // message d'erreur? -char* hts_errmsg(httrackp *opt) { +char *hts_errmsg(httrackp * opt) { return opt->state._hts_errmsg; } + // mode pause transfer -HTSEXT_API int hts_setpause(httrackp *opt, int p) { +HTSEXT_API int hts_setpause(httrackp * opt, int p) { if (p >= 0) - opt->state._hts_setpause = p; + opt->state._hts_setpause = p; return opt->state._hts_setpause; } + // ask for termination -HTSEXT_API int hts_request_stop(httrackp* opt, int force) { +HTSEXT_API int hts_request_stop(httrackp * opt, int force) { if (opt != NULL) { hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user"); opt->state.stop = 1; } return 0; } + // régler en cours de route les paramètres réglables.. // -1 : erreur //HTSEXT_API int hts_setopt(httrackp* set_opt) { @@ -3292,192 +3561,207 @@ HTSEXT_API int hts_request_stop(httrackp* opt, int force) { //} // ajout d'URL // -1 : erreur -HTSEXT_API int hts_addurl(httrackp *opt, char** url) { +HTSEXT_API int hts_addurl(httrackp * opt, char **url) { if (url) - opt->state._hts_addurl = url; + opt->state._hts_addurl = url; return (opt->state._hts_addurl != NULL); } -HTSEXT_API int hts_resetaddurl(httrackp *opt) { +HTSEXT_API int hts_resetaddurl(httrackp * opt) { opt->state._hts_addurl = NULL; return (opt->state._hts_addurl != NULL); } + // copier nouveaux paramètres si besoin -HTSEXT_API int copy_htsopt(const httrackp* from,httrackp* to) { - if (from->maxsite > -1) +HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) { + if (from->maxsite > -1) to->maxsite = from->maxsite; - - if (from->maxfile_nonhtml > -1) + + if (from->maxfile_nonhtml > -1) to->maxfile_nonhtml = from->maxfile_nonhtml; - - if (from->maxfile_html > -1) + + if (from->maxfile_html > -1) to->maxfile_html = from->maxfile_html; - - if (from->maxsoc > 0) + + if (from->maxsoc > 0) to->maxsoc = from->maxsoc; - - if (from->nearlink > -1) + + if (from->nearlink > -1) to->nearlink = from->nearlink; - - if (from->timeout > -1) + + if (from->timeout > -1) to->timeout = from->timeout; - + if (from->rateout > -1) to->rateout = from->rateout; - - if (from->maxtime > -1) + + if (from->maxtime > -1) to->maxtime = from->maxtime; #if HTS_USEMMS - if (from->mms_maxtime > -1) + if (from->mms_maxtime > -1) to->mms_maxtime = from->mms_maxtime; #endif if (from->maxrate > -1) to->maxrate = from->maxrate; - + if (from->maxconn > 0) to->maxconn = from->maxconn; - if (StringNotEmpty(from->user_agent)) + if (StringNotEmpty(from->user_agent)) StringCopyS(to->user_agent, from->user_agent); - - if (from->retry > -1) + + if (from->retry > -1) to->retry = from->retry; - - if (from->hostcontrol > -1) + + if (from->hostcontrol > -1) to->hostcontrol = from->hostcontrol; - - if (from->errpage > -1) + + if (from->errpage > -1) to->errpage = from->errpage; - if (from->parseall > -1) + if (from->parseall > -1) to->parseall = from->parseall; - // test all: bit 8 de travel - if (from->travel > -1) { + if (from->travel > -1) { if (from->travel & 256) - to->travel|=256; + to->travel |= 256; else - to->travel&=255; + to->travel &= 255; } - return 0; } // /* External modules callback */ -int htsAddLink(htsmoduleStruct* str, char* link) { +int htsAddLink(htsmoduleStruct * str, char *link) { if (link != NULL && str != NULL && link[0] != '\0') { ENGINE_LOAD_CONTEXT_BASE(); /* */ - char BIGSTK adr[HTS_URLMAXSIZE*2], - fil[HTS_URLMAXSIZE*2], - save[HTS_URLMAXSIZE*2]; - char BIGSTK codebase[HTS_URLMAXSIZE*2]; + char BIGSTK adr[HTS_URLMAXSIZE * 2], fil[HTS_URLMAXSIZE * 2], + save[HTS_URLMAXSIZE * 2]; + char BIGSTK codebase[HTS_URLMAXSIZE * 2]; + /* */ int pass_fix, prio_fix; + /* */ int forbidden_url = 1; - - codebase[0]='\0'; - + + codebase[0] = '\0'; + hts_log_print(opt, LOG_DEBUG, "(module): adding link : '%s'", link); // recopie de "creer le lien" // if (!RUN_CALLBACK1(opt, linkdetected, link)) { - hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper", link); + hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper", + link); return 0; } if (!RUN_CALLBACK2(opt, linkdetected2, link, NULL)) { - hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper(2)", link); + hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper(2)", + link); return 0; } - // adr = c'est la même // fil et save: save2 et fil2 - prio_fix=maximum(liens[ptr]->depth-1,0); - pass_fix=max(liens[ptr]->pass2,numero_passe); - if (liens[ptr]->cod) strcpybuff(codebase,liens[ptr]->cod); // codebase valable pour tt les classes descendantes - if (strnotempty(codebase)==0) { // pas de codebase, construire - char* a; + prio_fix = maximum(liens[ptr]->depth - 1, 0); + pass_fix = max(liens[ptr]->pass2, numero_passe); + if (liens[ptr]->cod) + strcpybuff(codebase, liens[ptr]->cod); // codebase valable pour tt les classes descendantes + if (strnotempty(codebase) == 0) { // pas de codebase, construire + char *a; + if (str->relativeToHtmlLink == 0) - strcpybuff(codebase,liens[ptr]->fil); + strcpybuff(codebase, liens[ptr]->fil); else - strcpybuff(codebase,liens[liens[ptr]->precedent]->fil); - a=codebase+strlen(codebase)-1; - while((*a) && (*a!='/') && ( a > codebase)) a--; - if (*a=='/') - *(a+1)='\0'; // couper - } else { // couper http:// éventuel - if (strfield(codebase,"http://")) { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - char* a=codebase+7; - a=strchr(a,'/'); // après host - if (a) { // ** msg erreur et vérifier? - strcpybuff(tempo,a); - strcpybuff(codebase,tempo); // couper host + strcpybuff(codebase, liens[liens[ptr]->precedent]->fil); + a = codebase + strlen(codebase) - 1; + while((*a) && (*a != '/') && (a > codebase)) + a--; + if (*a == '/') + *(a + 1) = '\0'; // couper + } else { // couper http:// éventuel + if (strfield(codebase, "http://")) { + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + char *a = codebase + 7; + + a = strchr(a, '/'); // après host + if (a) { // ** msg erreur et vérifier? + strcpybuff(tempo, a); + strcpybuff(codebase, tempo); // couper host } else { - hts_log_print(opt, LOG_ERROR, "Unexpected strstr error in base %s",codebase); + hts_log_print(opt, LOG_ERROR, "Unexpected strstr error in base %s", + codebase); } } } - - if (!((int) strlen(codebase)<HTS_URLMAXSIZE)) { // trop long - hts_log_print(opt, LOG_ERROR, "Codebase too long, parsing skipped (%s)",codebase); + + if (!((int) strlen(codebase) < HTS_URLMAXSIZE)) { // trop long + hts_log_print(opt, LOG_ERROR, "Codebase too long, parsing skipped (%s)", + codebase); } - + { - char* lien = link; - int dejafait=0; - + char *lien = link; + int dejafait = 0; + if (strnotempty(lien) && strlen(lien) < HTS_URLMAXSIZE) { - + // calculer les chemins et noms de sauvegarde - if (ident_url_relatif(lien,urladr,codebase,adr,fil)>=0) { // reformage selon chemin + if (ident_url_relatif(lien, urladr, codebase, adr, fil) >= 0) { // reformage selon chemin int r; int set_prio_to = 0; int just_test_it = 0; - forbidden_url = hts_acceptlink(opt, ptr, lien_tot, liens, - adr,fil, - NULL, NULL, - &set_prio_to, - &just_test_it); - hts_log_print(opt, LOG_DEBUG, "result for wizard external module link: %d",forbidden_url); + + forbidden_url = + hts_acceptlink(opt, ptr, lien_tot, liens, adr, fil, NULL, NULL, + &set_prio_to, &just_test_it); + hts_log_print(opt, LOG_DEBUG, + "result for wizard external module link: %d", + forbidden_url); /* Link accepted */ if (!forbidden_url) { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - int a,b; - tempo[0]='\0'; - a=opt->savename_type; - b=opt->savename_83; - opt->savename_type=0; - opt->savename_83=0; + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + int a, b; + + tempo[0] = '\0'; + a = opt->savename_type; + b = opt->savename_83; + opt->savename_type = 0; + opt->savename_83 = 0; // note: adr,fil peuvent être patchés - r=url_savename(adr,fil,save,NULL,NULL,NULL,NULL,opt,liens,lien_tot,sback,cache,hashptr,ptr,numero_passe,NULL); + r = + url_savename(adr, fil, save, NULL, NULL, NULL, NULL, opt, liens, + lien_tot, sback, cache, hashptr, ptr, numero_passe, + NULL); // resolve unresolved type - if (r != -1 - && forbidden_url == 0 - && IS_DELAYED_EXT(save) - ) - { // pas d'erreur, on continue - char BIGSTK former_adr[HTS_URLMAXSIZE*2]; - char BIGSTK former_fil[HTS_URLMAXSIZE*2]; + if (r != -1 && forbidden_url == 0 && IS_DELAYED_EXT(save) + ) { // pas d'erreur, on continue + char BIGSTK former_adr[HTS_URLMAXSIZE * 2]; + char BIGSTK former_fil[HTS_URLMAXSIZE * 2]; + former_adr[0] = former_fil[0] = '\0'; - r = hts_wait_delayed(str, adr, fil, save, NULL, NULL, former_adr, former_fil, &forbidden_url); + r = + hts_wait_delayed(str, adr, fil, save, NULL, NULL, former_adr, + former_fil, &forbidden_url); } // end resolve unresolved type - opt->savename_type=a; - opt->savename_83=b; + opt->savename_type = a; + opt->savename_83 = b; if (r != -1 && !forbidden_url) { if (savename) { - if (lienrelatif(tempo,save,savename)==0) { - hts_log_print(opt, LOG_DEBUG, "(module): relative link at %s build with %s and %s: %s",adr,save,savename,tempo); - if (str->localLink && str->localLinkSize > (int) strlen(tempo) + 1) { + if (lienrelatif(tempo, save, savename) == 0) { + hts_log_print(opt, LOG_DEBUG, + "(module): relative link at %s build with %s and %s: %s", + adr, save, savename, tempo); + if (str->localLink + && str->localLinkSize > (int) strlen(tempo) + 1) { strcpybuff(str->localLink, tempo); } } @@ -3486,81 +3770,87 @@ int htsAddLink(htsmoduleStruct* str, char* link) { } if (forbidden_url) { - hts_log_print(opt, LOG_DEBUG, "(module): file not caught: %s",lien); - if (str->localLink && str->localLinkSize > (int) ( strlen(adr) + strlen(fil) + 8 ) ) { + hts_log_print(opt, LOG_DEBUG, "(module): file not caught: %s", + lien); + if (str->localLink + && str->localLinkSize > (int) (strlen(adr) + strlen(fil) + 8)) { str->localLink[0] = '\0'; if (!link_has_authority(adr)) - strcpybuff(str->localLink,"http://"); + strcpybuff(str->localLink, "http://"); strcatbuff(str->localLink, adr); strcatbuff(str->localLink, fil); } - r=-1; + r = -1; } - // if (r != -1) { - hts_log_print(opt, LOG_DEBUG, "(module): %s%s -> %s (base %s)",adr,fil,save,codebase); - + hts_log_print(opt, LOG_DEBUG, "(module): %s%s -> %s (base %s)", adr, + fil, save, codebase); + // modifié par rapport à l'autre version (cf prio_fix notamment et save2) - + // vérifier que le lien n'a pas déja été noté // si c'est le cas, alors il faut s'assurer que la priorité associée // au fichier est la plus grande des deux priorités // // On part de la fin et on essaye de se presser (économise temps machine) { - int i=hash_read(hashptr,save,"",0,opt->urlhack); // lecture type 0 (sav) - if (i>=0) { - liens[i]->depth=maximum(liens[i]->depth,prio_fix); - dejafait=1; + int i = hash_read(hashptr, save, "", 0, opt->urlhack); // lecture type 0 (sav) + + if (i >= 0) { + liens[i]->depth = maximum(liens[i]->depth, prio_fix); + dejafait = 1; } - } - + } + if (!dejafait) { // // >>>> CREER LE LIEN JAVA <<<< - + // enregistrer fichier (MACRO) - liens_record(adr,fil,save,"","",opt->urlhack); - if (liens[lien_tot]==NULL) { // erreur, pas de place réservée - printf("PANIC! : Not enough memory [%d]\n",__LINE__); - hts_log_print(opt, LOG_PANIC, "Not enough memory, can not re-allocate %d bytes",(int)((add_tab_alloc+1)*sizeof(lien_url))); - opt->state.exit_xh=-1; /* fatal error -> exit */ + liens_record(adr, fil, save, "", "", opt->urlhack); + if (liens[lien_tot] == NULL) { // erreur, pas de place réservée + printf("PANIC! : Not enough memory [%d]\n", __LINE__); + hts_log_print(opt, LOG_PANIC, + "Not enough memory, can not re-allocate %d bytes", + (int) ((add_tab_alloc + 1) * sizeof(lien_url))); + opt->state.exit_xh = -1; /* fatal error -> exit */ return 0; - } - + } // mode test? - liens[lien_tot]->testmode=0; // pas mode test - - liens[lien_tot]->link_import=0; // pas mode import - + liens[lien_tot]->testmode = 0; // pas mode test + + liens[lien_tot]->link_import = 0; // pas mode import + // écrire autres paramètres de la structure-lien //if (meme_adresse) - liens[lien_tot]->premier=liens[ptr]->premier; + liens[lien_tot]->premier = liens[ptr]->premier; //else // sinon l'objet père est le précédent lui même // liens[lien_tot]->premier=ptr; - - liens[lien_tot]->precedent=ptr; + + liens[lien_tot]->precedent = ptr; // noter la priorité if (!set_prio_to) - liens[lien_tot]->depth=prio_fix; + liens[lien_tot]->depth = prio_fix; else - liens[lien_tot]->depth=max(0,min(liens[ptr]->depth-1,set_prio_to-1)); // PRIORITE NULLE (catch page) - liens[lien_tot]->pass2=max(pass_fix,numero_passe); - liens[lien_tot]->retry=opt->retry; - + liens[lien_tot]->depth = max(0, min(liens[ptr]->depth - 1, set_prio_to - 1)); // PRIORITE NULLE (catch page) + liens[lien_tot]->pass2 = max(pass_fix, numero_passe); + liens[lien_tot]->retry = opt->retry; + //strcpybuff(liens[lien_tot]->adr,adr); //strcpybuff(liens[lien_tot]->fil,fil); //strcpybuff(liens[lien_tot]->sav,save); - hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s",liens[lien_tot]->adr,liens[lien_tot]->fil,liens[lien_tot]->sav); - - lien_tot++; // UN LIEN DE PLUS + hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s", + liens[lien_tot]->adr, liens[lien_tot]->fil, + liens[lien_tot]->sav); + + lien_tot++; // UN LIEN DE PLUS } } } } } - + /* Apply changes */ ENGINE_SAVE_CONTEXT_BASE(); @@ -3569,36 +3859,31 @@ int htsAddLink(htsmoduleStruct* str, char* link) { return 0; } - - - - // message copyright interne void voidf(void) { - static const char* a; - a=""CRLF""CRLF; - a="+-----------------------------------------------+"CRLF; - a="|HyperTextTRACKer, Offline Browser Utility |"CRLF; - a="| HTTrack Website Copier |"CRLF; - a="|Code: Windows Interface Xavier Roche |"CRLF; - a="| HTS/HTTrack Xavier Roche |"CRLF; - a="| .class Parser Yann Philippot |"CRLF; - a="| |"CRLF; - a="|Tested on: Windows95,98,NT,2K |"CRLF; - a="| Linux PC |"CRLF; - a="| Sun-Solaris 5.6 |"CRLF; - a="| AIX 4 |"CRLF; - a="| |"CRLF; - a="|Copyright (C) Xavier Roche and other |"CRLF; - a="|contributors |"CRLF; - a="| |"CRLF; - a="|Use this program at your own risks! |"CRLF; - a="+-----------------------------------------------+"CRLF; - a=""CRLF; + static const char *a; + + a = "" CRLF "" CRLF; + a = "+-----------------------------------------------+" CRLF; + a = "|HyperTextTRACKer, Offline Browser Utility |" CRLF; + a = "| HTTrack Website Copier |" CRLF; + a = "|Code: Windows Interface Xavier Roche |" CRLF; + a = "| HTS/HTTrack Xavier Roche |" CRLF; + a = "| .class Parser Yann Philippot |" CRLF; + a = "| |" CRLF; + a = "|Tested on: Windows95,98,NT,2K |" CRLF; + a = "| Linux PC |" CRLF; + a = "| Sun-Solaris 5.6 |" CRLF; + a = "| AIX 4 |" CRLF; + a = "| |" CRLF; + a = "|Copyright (C) Xavier Roche and other |" CRLF; + a = "|contributors |" CRLF; + a = "| |" CRLF; + a = "|Use this program at your own risks! |" CRLF; + a = "+-----------------------------------------------+" CRLF; + a = "" CRLF; (void) a; } - // HTTrack Website Copier Copyright (C) Xavier Roche and other contributors // - diff --git a/src/htscore.h b/src/htscore.h index 4d707e9..d718dfe 100644 --- a/src/htscore.h +++ b/src/htscore.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Main file .h */ /* Author: Xavier Roche */ @@ -143,23 +140,23 @@ typedef struct filecreate_params filecreate_params; typedef struct lien_url lien_url; #endif struct lien_url { - char firstblock; // flag 1=premier malloc - char link_import; // lien importé à la suite d'un moved - ne pas appliquer les règles classiques up/down - int depth; // profondeur autorisée lien ; >0 forte 0=faible - int pass2; // traiter après les autres, seconde passe. si == -1, lien traité en background - int premier; // pointeur sur le premier lien qui a donné lieu aux autres liens du domaine - int precedent; // pointeur sur le lien qui a donné lieu à ce lien précis + char firstblock; // flag 1=premier malloc + char link_import; // lien importé à la suite d'un moved - ne pas appliquer les règles classiques up/down + int depth; // profondeur autorisée lien ; >0 forte 0=faible + int pass2; // traiter après les autres, seconde passe. si == -1, lien traité en background + int premier; // pointeur sur le premier lien qui a donné lieu aux autres liens du domaine + int precedent; // pointeur sur le lien qui a donné lieu à ce lien précis //int moved; // pointeur sur moved - int retry; // nombre de retry restants - int testmode; // mode test uniquement, envoyer juste un head! - char* adr; // adresse - char* fil; // nom du fichier distant - char* sav; // nom à sauver sur disque (avec chemin éventuel) - char* cod; // chemin codebase éventuel si classe java - char* former_adr; // adresse initiale (avant éventuel moved), peut être nulle - char* former_fil; // nom du fichier distant initial (avant éventuel moved), peut être nul + int retry; // nombre de retry restants + int testmode; // mode test uniquement, envoyer juste un head! + char *adr; // adresse + char *fil; // nom du fichier distant + char *sav; // nom à sauver sur disque (avec chemin éventuel) + char *cod; // chemin codebase éventuel si classe java + char *former_adr; // adresse initiale (avant éventuel moved), peut être nulle + char *former_fil; // nom du fichier distant initial (avant éventuel moved), peut être nul // pour optimisation: - int hash_next[3]; // prochain lien avec même valeur hash + int hash_next[3]; // prochain lien avec même valeur hash }; // chargement de fichiers en 'arrière plan' @@ -171,43 +168,43 @@ struct lien_back { #if DEBUG_CHECKINT char magic; #endif - char url_adr[HTS_URLMAXSIZE*2]; // adresse - char url_fil[HTS_URLMAXSIZE*2]; // nom du fichier distant - char url_sav[HTS_URLMAXSIZE*2]; // nom à sauver sur disque (avec chemin éventuel) - char referer_adr[HTS_URLMAXSIZE*2]; // adresse host page referer - char referer_fil[HTS_URLMAXSIZE*2]; // fichier page referer - char location_buffer[HTS_URLMAXSIZE*2]; // "location" en cas de "moved" (302,..) - char* tmpfile; // nom à sauver temporairement (compressé) - char tmpfile_buffer[HTS_URLMAXSIZE*2]; // buffer pour le nom à sauver temporairement - char send_too[1024]; // données à envoyer en même temps que le header - int status; // status (-1=non utilisé, 0: prêt, >0: opération en cours) - int locked; // locked (to be used soon) - int testmode; // mode de test - int timeout; // gérer des timeouts? (!=0 : nombre de secondes) - TStamp timeout_refresh; // si oui, time refresh - int rateout; // timeout refresh? (!=0 : taux minimum toléré en octets/s) - TStamp rateout_time; // si oui, date de départ - LLint maxfile_nonhtml; // taille max d'un fichier non html - LLint maxfile_html; // idem pour un ficheir html - htsblk r; // structure htsblk de chaque objet en background - int is_update; // mode update - int head_request; // requète HEAD? - LLint range_req_size; // range utilisé - TStamp ka_time_start; // refresh time for KA + char url_adr[HTS_URLMAXSIZE * 2]; // adresse + char url_fil[HTS_URLMAXSIZE * 2]; // nom du fichier distant + char url_sav[HTS_URLMAXSIZE * 2]; // nom à sauver sur disque (avec chemin éventuel) + char referer_adr[HTS_URLMAXSIZE * 2]; // adresse host page referer + char referer_fil[HTS_URLMAXSIZE * 2]; // fichier page referer + char location_buffer[HTS_URLMAXSIZE * 2]; // "location" en cas de "moved" (302,..) + char *tmpfile; // nom à sauver temporairement (compressé) + char tmpfile_buffer[HTS_URLMAXSIZE * 2]; // buffer pour le nom à sauver temporairement + char send_too[1024]; // données à envoyer en même temps que le header + int status; // status (-1=non utilisé, 0: prêt, >0: opération en cours) + int locked; // locked (to be used soon) + int testmode; // mode de test + int timeout; // gérer des timeouts? (!=0 : nombre de secondes) + TStamp timeout_refresh; // si oui, time refresh + int rateout; // timeout refresh? (!=0 : taux minimum toléré en octets/s) + TStamp rateout_time; // si oui, date de départ + LLint maxfile_nonhtml; // taille max d'un fichier non html + LLint maxfile_html; // idem pour un ficheir html + htsblk r; // structure htsblk de chaque objet en background + int is_update; // mode update + int head_request; // requète HEAD? + LLint range_req_size; // range utilisé + TStamp ka_time_start; // refresh time for KA // - int http11; // L'en tête doit être signé HTTP/1.1 et non HTTP/1.0 - int is_chunk; // chunk? - char* chunk_adr; // adresse chunk en cours de chargement - LLint chunk_size; // taille chunk en cours de chargement - LLint chunk_blocksize; // taille data declaree par le chunk - LLint compressed_size; // taille compressés (stats uniquement) + int http11; // L'en tête doit être signé HTTP/1.1 et non HTTP/1.0 + int is_chunk; // chunk? + char *chunk_adr; // adresse chunk en cours de chargement + LLint chunk_size; // taille chunk en cours de chargement + LLint chunk_blocksize; // taille data declaree par le chunk + LLint compressed_size; // taille compressés (stats uniquement) // //int links_index; // to access liens[links_index] // - char info[256]; // éventuel status pour le ftp - int stop_ftp; // flag stop pour ftp - int finalized; // finalized (optim memory) - int early_add; // was added before link heap saw it + char info[256]; // éventuel status pour le ftp + int stop_ftp; // flag stop pour ftp + int finalized; // finalized (optim memory) + int early_add; // was added before link heap saw it #if DEBUG_CHECKINT char magic2; #endif @@ -218,10 +215,10 @@ struct lien_back { typedef struct struct_back struct_back; #endif struct struct_back { - lien_back* lnk; + lien_back *lnk; int count; inthash ready; - LLint ready_size_bytes; + LLint ready_size_bytes; }; typedef struct cache_back_zip_entry cache_back_zip_entry; @@ -232,29 +229,29 @@ typedef struct cache_back_zip_entry cache_back_zip_entry; typedef struct cache_back cache_back; #endif struct cache_back { - int version; // 0 ou 1 + int version; // 0 ou 1 /* */ int type; int ro; - FILE *dat,*ndx,*olddat; - char *use; // liste des adr+fil - FILE *lst; // liste des fichiers pour la "purge" - FILE *txt; // liste des fichiers (info) + FILE *dat, *ndx, *olddat; + char *use; // liste des adr+fil + FILE *lst; // liste des fichiers pour la "purge" + FILE *txt; // liste des fichiers (info) char lastmodified[256]; // HASH inthash hashtable; // HASH for tests (naming subsystem) inthash cached_tests; // fichiers log optionnels - FILE* log; - FILE* errlog; + FILE *log; + FILE *errlog; // variables - int ptr_ant; // pointeur pour anticiper - int ptr_last; // pointeur pour anticiper + int ptr_ant; // pointeur pour anticiper + int ptr_last; // pointeur pour anticiper // void *zipInput; void *zipOutput; - cache_back_zip_entry* zipEntries; + cache_back_zip_entry *zipEntries; int zipEntriesOffs; int zipEntriesCapa; }; @@ -264,9 +261,9 @@ struct cache_back { typedef struct hash_struct hash_struct; #endif struct hash_struct { - lien_url** liens; // pointeur sur liens - int max_lien; // indice le plus grand rencontré - int hash[3][HTS_HASH_SIZE]; // tables pour sav/adr-fil/former_adr-former_fil + lien_url **liens; // pointeur sur liens + int max_lien; // indice le plus grand rencontré + int hash[3][HTS_HASH_SIZE]; // tables pour sav/adr-fil/former_adr-former_fil }; #ifndef HTS_DEF_FWSTRUCT_filecreate_params @@ -274,19 +271,19 @@ struct hash_struct { typedef struct filecreate_params filecreate_params; #endif struct filecreate_params { - FILE* lst; - char path[HTS_URLMAXSIZE*2]; + FILE *lst; + char path[HTS_URLMAXSIZE * 2]; }; /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -HTS_STATIC int cache_writable(cache_back* cache) { - return (cache != NULL && ( cache->dat != NULL || cache->zipOutput != NULL ) ); +HTS_STATIC int cache_writable(cache_back * cache) { + return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL)); } -HTS_STATIC int cache_readable(cache_back* cache) { - return (cache != NULL && ( cache->olddat != NULL || cache->zipInput != NULL ) ); +HTS_STATIC int cache_readable(cache_back * cache) { + return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL)); } #endif @@ -300,97 +297,109 @@ HTS_STATIC int cache_readable(cache_back* cache) { // #ifndef HTTRACK_DEFLIB -HTSEXT_API int hts_is_parsing(httrackp *opt, int flag); -HTSEXT_API int hts_is_testing(httrackp *opt); -HTSEXT_API int hts_addurl(httrackp *opt, char** url); -HTSEXT_API int hts_resetaddurl(httrackp *opt); -HTSEXT_API int copy_htsopt(const httrackp* from,httrackp* to); -HTSEXT_API char* hts_errmsg(httrackp *opt); -HTSEXT_API int hts_setpause(httrackp *opt, int); +HTSEXT_API int hts_is_parsing(httrackp * opt, int flag); +HTSEXT_API int hts_is_testing(httrackp * opt); +HTSEXT_API int hts_addurl(httrackp * opt, char **url); +HTSEXT_API int hts_resetaddurl(httrackp * opt); +HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to); +HTSEXT_API char *hts_errmsg(httrackp * opt); +HTSEXT_API int hts_setpause(httrackp * opt, int); + // -HTSEXT_API int hts_is_exiting(httrackp *opt); -HTSEXT_API int hts_request_stop(httrackp* opt, int force); +HTSEXT_API int hts_is_exiting(httrackp * opt); +HTSEXT_API int hts_request_stop(httrackp * opt, int force); + // -HTSEXT_API int hts_cancel_file_push(httrackp *opt, const char *url); -HTSEXT_API void hts_cancel_test(httrackp *opt); -HTSEXT_API void hts_cancel_parsing(httrackp *opt); +HTSEXT_API int hts_cancel_file_push(httrackp * opt, const char *url); +HTSEXT_API void hts_cancel_test(httrackp * opt); +HTSEXT_API void hts_cancel_parsing(httrackp * opt); #endif -char* hts_cancel_file_pop(httrackp *opt); +char *hts_cancel_file_pop(httrackp * opt); #endif // - //int httpmirror(char* url,int level,httrackp opt); -int httpmirror(char* url1,httrackp* opt); -int filesave(httrackp* opt,const char* adr,int len,const char* s,const char* url_adr /* = NULL */,const char* url_fil /* = NULL */); -char* hts_cancel_file_pop(httrackp *opt); +int httpmirror(char *url1, httrackp * opt); +int filesave(httrackp * opt, const char *adr, int len, const char *s, + const char *url_adr /* = NULL */ , + const char *url_fil /* = NULL */ ); +char *hts_cancel_file_pop(httrackp * opt); int check_fatal_io_errno(void); int engine_stats(void); -void host_ban(httrackp* opt,lien_url** liens,int ptr,int lien_tot,struct_back* sback,char* host); -FILE* filecreate(filenote_strc *strct,const char* s); -FILE* fileappend(filenote_strc *strct,const char* s); -int filecreateempty(filenote_strc *strct, const char* filename); -int filenote(filenote_strc *strct,const char* s,filecreate_params* params); -void file_notify(httrackp* opt,const char* adr,const char* fil,const char* save,int create,int modify,int wasupdated); -void usercommand(httrackp* opt,int exe,const char* cmd,const char* file,const char* adr,const char* fil); -void usercommand_exe(const char* cmd,const char* file); -int filters_init(char*** ptrfilters, int maxfilter, int filterinc); +void host_ban(httrackp * opt, lien_url ** liens, int ptr, int lien_tot, + struct_back * sback, char *host); +FILE *filecreate(filenote_strc * strct, const char *s); +FILE *fileappend(filenote_strc * strct, const char *s); +int filecreateempty(filenote_strc * strct, const char *filename); +int filenote(filenote_strc * strct, const char *s, filecreate_params * params); +void file_notify(httrackp * opt, const char *adr, const char *fil, + const char *save, int create, int modify, int wasupdated); +void usercommand(httrackp * opt, int exe, const char *cmd, const char *file, + const char *adr, const char *fil); +void usercommand_exe(const char *cmd, const char *file); +int filters_init(char ***ptrfilters, int maxfilter, int filterinc); + #ifndef HTTRACK_DEFLIB -HTSEXT_API int structcheck(const char* path); -HTSEXT_API int structcheck_utf8(const char* path); -HTSEXT_API int dir_exists(const char* path); +HTSEXT_API int structcheck(const char *path); +HTSEXT_API int structcheck_utf8(const char *path); +HTSEXT_API int dir_exists(const char *path); #endif -int fspc(httrackp *opt,FILE* fp,const char* type); -char* next_token(char* p,int flag); +int fspc(httrackp * opt, FILE * fp, const char *type); +char *next_token(char *p, int flag); + // -char* readfile(char* fil); -char* readfile2(char* fil, LLint* size); -char* readfile_utf8(char* fil); -char* readfile_or(char* fil,char* defaultdata); +char *readfile(char *fil); +char *readfile2(char *fil, LLint * size); +char *readfile_utf8(char *fil); +char *readfile_or(char *fil, char *defaultdata); + #if 0 -void check_rate(TStamp stat_timestart,int maxrate); +void check_rate(TStamp stat_timestart, int maxrate); #endif // liens -int liens_record(char* adr,char* fil,char* save,char* former_adr,char* former_fil,char* codebase); - +int liens_record(char *adr, char *fil, char *save, char *former_adr, + char *former_fil, char *codebase); // backing, routines externes -int back_pluggable_sockets(struct_back* sback, httrackp* opt); -int back_pluggable_sockets_strict(struct_back* sback, httrackp* opt); -int back_fill(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot); -int backlinks_done(struct_back* sback,lien_url** liens,int lien_tot,int ptr); -int back_fillmax(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot); +int back_pluggable_sockets(struct_back * sback, httrackp * opt); +int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt); +int back_fill(struct_back * sback, httrackp * opt, cache_back * cache, + lien_url ** liens, int ptr, int numero_passe, int lien_tot); +int backlinks_done(struct_back * sback, lien_url ** liens, int lien_tot, + int ptr); +int back_fillmax(struct_back * sback, httrackp * opt, cache_back * cache, + lien_url ** liens, int ptr, int numero_passe, int lien_tot); // cancel file #ifndef HTTRACK_DEFLIB -HTSEXT_API int hts_cancel_file_push(httrackp *opt, const char *url); -HTSEXT_API void hts_cancel_test(httrackp *opt); -HTSEXT_API void hts_cancel_parsing(httrackp *opt); +HTSEXT_API int hts_cancel_file_push(httrackp * opt, const char *url); +HTSEXT_API void hts_cancel_test(httrackp * opt); +HTSEXT_API void hts_cancel_parsing(httrackp * opt); #endif -int ask_continue(httrackp *opt); +int ask_continue(httrackp * opt); int nombre_digit(int n); // Java #if 0 -int hts_add_file(char* file,int file_position); +int hts_add_file(char *file, int file_position); #endif // Polling #if HTS_POLL int check_flot(T_SOC s); int check_stdin(void); -int read_stdin(char* s,int max); +int read_stdin(char *s, int max); #endif int check_sockerror(T_SOC s); int check_sockdata(T_SOC s); /* external modules */ -int htsAddLink(htsmoduleStruct* str, char* link); +int htsAddLink(htsmoduleStruct * str, char *link); // Void void voidf(void); diff --git a/src/htscoremain.c b/src/htscoremain.c index 4775432..d815952 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: opt->c subroutines: */ /* main routine (first called) */ @@ -69,7 +66,6 @@ Please visit our Website: http://www.httrack.com /* Resolver */ extern int IPV6_resolver; - // Add a command in the argc/argv #define cmdl_add(token,argc,argv,buff,ptr) \ argv[argc]=(buff+ptr); \ @@ -108,43 +104,47 @@ extern int IPV6_resolver; } \ } while(0) -HTSEXT_API int hts_main(int argc, char **argv) -{ +HTSEXT_API int hts_main(int argc, char **argv) { httrackp *opt = hts_create_opt(); - int ret = hts_main2(argc, argv, opt); - hts_free_opt(opt); - return ret; + int ret = hts_main2(argc, argv, opt); + + hts_free_opt(opt); + return ret; } // Main, récupère les paramètres et appelle le robot -HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { - char** x_argv=NULL; // Patch pour argv et argc: en cas de récupération de ligne de commande - char* x_argvblk=NULL; // (reprise ou update) - int x_ptr=0; // offset +HTSEXT_API int hts_main2(int argc, char **argv, httrackp * opt) { + char **x_argv = NULL; // Patch pour argv et argc: en cas de récupération de ligne de commande + char *x_argvblk = NULL; // (reprise ou update) + int x_ptr = 0; // offset + // - int argv_url=-1; // ==0 : utiliser cache et doit.log - char* argv_firsturl=NULL; // utilisé pour nommage par défaut - char* url = NULL; // URLS séparées par un espace - int url_sz = 65535; + int argv_url = -1; // ==0 : utiliser cache et doit.log + char *argv_firsturl = NULL; // utilisé pour nommage par défaut + char *url = NULL; // URLS séparées par un espace + int url_sz = 65535; + //char url[65536]; // URLS séparées par un espace // the parametres - int httrack_logmode=3; // ONE log file + int httrack_logmode = 3; // ONE log file + #ifndef _WIN32 #ifndef HTS_DO_NOT_USE_UID - int switch_uid=-1,switch_gid=-1; /* setuid/setgid */ + int switch_uid = -1, switch_gid = -1; /* setuid/setgid */ #endif - int switch_chroot=0; /* chroot ? */ + int switch_chroot = 0; /* chroot ? */ #endif // ensureUrlCapacity(url, url_sz, 65536); - // Create options - _DEBUG_HEAD=0; // pas de debuggage en têtes + // Create options + _DEBUG_HEAD = 0; // pas de debuggage en têtes /* command max-size check (3.43 ; 3.42-4) */ { int i; - for(i = 0 ; i < argc ; i++) { + + for(i = 0; i < argc; i++) { if (strlen(argv[i]) >= HTS_CDLMAXSIZE) { HTS_PANIC_PRINTF("argument too long"); htsmain_free(); @@ -160,24 +160,25 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { #else /* Terminal is a tty, may ask questions and display funny information */ if (isatty(1)) { - opt->quiet=0; - opt->verbosedisplay=1; + opt->quiet = 0; + opt->verbosedisplay = 1; } /* Not a tty, no stdin input or funny output! */ else { - opt->quiet=1; - opt->verbosedisplay=0; + opt->quiet = 1; + opt->verbosedisplay = 0; } #endif // Binary program path? #ifndef HTS_HTTRACKDIR { - char catbuff[CATBUFF_SIZE]; - char* path=fslash(catbuff,argv[0]); - char* a; - if ((a=strrchr(path,'/'))) { - StringCopyN(opt->path_bin,argv[0],a - path); + char catbuff[CATBUFF_SIZE]; + char *path = fslash(catbuff, argv[0]); + char *a; + + if ((a = strrchr(path, '/'))) { + StringCopyN(opt->path_bin, argv[0], a - path); } } #else @@ -187,276 +188,316 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { /* filter CR, LF, TAB.. */ { int na; - for(na=1;na<argc;na++) { - char* a; - while( (a=strchr(argv[na],'\x0d')) ) *a=' '; - while( (a=strchr(argv[na],'\x0a')) ) *a=' '; - while( (a=strchr(argv[na],9)) ) *a=' '; + + for(na = 1; na < argc; na++) { + char *a; + + while((a = strchr(argv[na], '\x0d'))) + *a = ' '; + while((a = strchr(argv[na], '\x0a'))) + *a = ' '; + while((a = strchr(argv[na], 9))) + *a = ' '; /* equivalent to "empty parameter" */ - if ((strcmp(argv[na],HTS_NOPARAM)==0) || (strcmp(argv[na],HTS_NOPARAM2)==0)) // (none) - strcpybuff(argv[na],"\"\""); - if (strncmp(argv[na],"-&",2)==0) - argv[na][1]='%'; + if ((strcmp(argv[na], HTS_NOPARAM) == 0) || (strcmp(argv[na], HTS_NOPARAM2) == 0)) // (none) + strcpybuff(argv[na], "\"\""); + if (strncmp(argv[na], "-&", 2) == 0) + argv[na][1] = '%'; } } /* create x_argvblk buffer for transformed command line */ { - int current_size=0; + int current_size = 0; int size; int na; - for(na=0;na<argc;na++) + + for(na = 0; na < argc; na++) current_size += (int) (strlen(argv[na]) + 1); - if ((size=fsize("config"))>0) + if ((size = fsize("config")) > 0) current_size += size; - x_argvblk=(char*) malloct(current_size+32768); + x_argvblk = (char *) malloct(current_size + 32768); if (x_argvblk == NULL) { HTS_PANIC_PRINTF("Error, not enough memory"); htsmain_free(); return -1; } - x_argvblk[0]='\0'; - x_ptr=0; + x_argvblk[0] = '\0'; + x_ptr = 0; /* Create argv */ - x_argv = (char**) malloct(sizeof(char*) * ( argc + 1024 )); + x_argv = (char **) malloct(sizeof(char *) * (argc + 1024)); } /* Create new argc/argv, replace alias, count URLs, treat -h, -q, -i */ { char BIGSTK _tmp_argv[2][HTS_CDLMAXSIZE]; char BIGSTK tmp_error[HTS_CDLMAXSIZE]; - char* tmp_argv[2]; + char *tmp_argv[2]; int tmp_argc; - int x_argc=0; + int x_argc = 0; int na; - tmp_argv[0]=_tmp_argv[0]; - tmp_argv[1]=_tmp_argv[1]; + + tmp_argv[0] = _tmp_argv[0]; + tmp_argv[1] = _tmp_argv[1]; // - argv_url=0; /* pour comptage */ + argv_url = 0; /* pour comptage */ // - cmdl_add(argv[0],x_argc,x_argv,x_argvblk,x_ptr); - na=1; /* commencer après nom_prg */ - while(na<argc) { - int result=1; - tmp_argv[0][0]=tmp_argv[1][0]='\0'; + cmdl_add(argv[0], x_argc, x_argv, x_argvblk, x_ptr); + na = 1; /* commencer après nom_prg */ + while(na < argc) { + int result = 1; + + tmp_argv[0][0] = tmp_argv[1][0] = '\0'; /* Vérifier argv[] non vide */ if (strnotempty(argv[na])) { - + /* Vérifier Commande (alias) */ - result=optalias_check(argc,(const char * const *)argv,na, - &tmp_argc,(char**)tmp_argv,tmp_error); + result = + optalias_check(argc, (const char *const *) argv, na, &tmp_argc, + (char **) tmp_argv, tmp_error); if (!result) { HTS_PANIC_PRINTF(tmp_error); htsmain_free(); return -1; } - + /* Copier */ - cmdl_add(tmp_argv[0],x_argc,x_argv,x_argvblk,x_ptr); + cmdl_add(tmp_argv[0], x_argc, x_argv, x_argvblk, x_ptr); if (tmp_argc > 1) { - cmdl_add(tmp_argv[1],x_argc,x_argv,x_argvblk,x_ptr); + cmdl_add(tmp_argv[1], x_argc, x_argv, x_argvblk, x_ptr); } - + /* Compter URLs et détecter -i,-q.. */ - if (tmp_argc == 1) { /* pas -P & co */ - if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */ - if (argv_url<0) argv_url=0; // -1==force -> 1=one url already detected, wipe all previous options + if (tmp_argc == 1) { /* pas -P & co */ + if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */ + if (argv_url < 0) + argv_url = 0; // -1==force -> 1=one url already detected, wipe all previous options //if (argv_url>=0) { argv_url++; if (!argv_firsturl) - argv_firsturl=x_argv[x_argc-1]; + argv_firsturl = x_argv[x_argc - 1]; //} } else { - if (strcmp(tmp_argv[0],"-h")==0) { - help(argv[0],!opt->quiet); + if (strcmp(tmp_argv[0], "-h") == 0) { + help(argv[0], !opt->quiet); htsmain_free(); return 0; - } else if (strcmp(tmp_argv[0],"-#h")==0) { - printf("HTTrack version "HTTRACK_VERSION"%s\n", hts_get_version_info(opt)); + } else if (strcmp(tmp_argv[0], "-#h") == 0) { + printf("HTTrack version " HTTRACK_VERSION "%s\n", + hts_get_version_info(opt)); return 0; } else { - if (strncmp(tmp_argv[0],"--",2)) { /* pas */ - if ((strchr(tmp_argv[0],'q')!=NULL)) - opt->quiet=1; // ne pas poser de questions! (nohup par exemple) - if ((strchr(tmp_argv[0],'i')!=NULL)) { // doit.log! - argv_url=-1; /* forcer */ - opt->quiet=1; + if (strncmp(tmp_argv[0], "--", 2)) { /* pas */ + if ((strchr(tmp_argv[0], 'q') != NULL)) + opt->quiet = 1; // ne pas poser de questions! (nohup par exemple) + if ((strchr(tmp_argv[0], 'i') != NULL)) { // doit.log! + argv_url = -1; /* forcer */ + opt->quiet = 1; } - } else if (strcmp(tmp_argv[0] + 2,"quiet") == 0) { - opt->quiet=1; // ne pas poser de questions! (nohup par exemple) - } else if (strcmp(tmp_argv[0] + 2,"continue") == 0) { - argv_url=-1; /* forcer */ - opt->quiet=1; + } else if (strcmp(tmp_argv[0] + 2, "quiet") == 0) { + opt->quiet = 1; // ne pas poser de questions! (nohup par exemple) + } else if (strcmp(tmp_argv[0] + 2, "continue") == 0) { + argv_url = -1; /* forcer */ + opt->quiet = 1; } } } } else if (tmp_argc == 2) { - if ((strcmp(tmp_argv[0],"-%L")==0)) { // liste d'URLs - if (argv_url<0) argv_url=0; // -1==force -> 1=one url already detected, wipe all previous options + if ((strcmp(tmp_argv[0], "-%L") == 0)) { // liste d'URLs + if (argv_url < 0) + argv_url = 0; // -1==force -> 1=one url already detected, wipe all previous options //if (argv_url>=0) - argv_url++; /* forcer */ + argv_url++; /* forcer */ } } } - na+=result; + na += result; } - if (argv_url<0) - argv_url=0; + if (argv_url < 0) + argv_url = 0; /* Nouveaux argc et argv */ - argv=x_argv; - argc=x_argc; + argv = x_argv; + argc = x_argc; } // Option O and includerc - { - int loops=0; - while (loops<2) { - char* com; + { + int loops = 0; + + while(loops < 2) { + char *com; int na; - - for(na=1;na<argc;na++) { - - if (argv[na][0]=='"') { + + for(na = 1; na < argc; na++) { + + if (argv[na][0] == '"') { char BIGSTK tempo[HTS_CDLMAXSIZE]; - strcpybuff(tempo,argv[na]+1); - if (tempo[strlen(tempo)-1]!='"') { + + strcpybuff(tempo, argv[na] + 1); + if (tempo[strlen(tempo) - 1] != '"') { char BIGSTK s[HTS_CDLMAXSIZE]; - sprintf(s,"Missing quote in %s",argv[na]); + + sprintf(s, "Missing quote in %s", argv[na]); HTS_PANIC_PRINTF(s); htsmain_free(); return -1; } - tempo[strlen(tempo)-1]='\0'; - strcpybuff(argv[na],tempo); + tempo[strlen(tempo) - 1] = '\0'; + strcpybuff(argv[na], tempo); } - - if (cmdl_opt(argv[na])) { // option - com=argv[na]+1; - + + if (cmdl_opt(argv[na])) { // option + com = argv[na] + 1; + while(*com) { - switch(*com) { - case 'O': // output path - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option O needs to be followed by a blank space, and a path (or path,path)"); + switch (*com) { + case 'O': // output path + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option O needs to be followed by a blank space, and a path (or path,path)"); printf("Example: -O /binary/\n"); printf("Example: -O /binary/,/log/\n"); htsmain_free(); return -1; } else { - int i /*, j*/; - int inQuote; - String *path; - int noDbl = 0; - if (com[1] == '1') { /* only 1 arg */ - com++; - noDbl = 1; - } + int i /*, j */ ; + int inQuote; + String *path; + int noDbl = 0; + + if (com[1] == '1') { /* only 1 arg */ + com++; + noDbl = 1; + } na++; - StringClear(opt->path_html); + StringClear(opt->path_html); StringClear(opt->path_log); - for(i = 0/*, j = 0*/, inQuote = 0, path = &opt->path_html ; argv[na][i] != 0 ; i++) { - if (argv[na][i] == '"') { - if (inQuote) - inQuote = 0; - else - inQuote = 1; - } else if (!inQuote && !noDbl && argv[na][i] == ',') { + for(i = 0 /*, j = 0 */ , inQuote = 0, path = &opt->path_html; + argv[na][i] != 0; i++) { + if (argv[na][i] == '"') { + if (inQuote) + inQuote = 0; + else + inQuote = 1; + } else if (!inQuote && !noDbl && argv[na][i] == ',') { //StringAddchar(path, '\0'); - //j = 0; - path = &opt->path_log; - } else { + //j = 0; + path = &opt->path_log; + } else { StringAddchar(*path, argv[na][i]); - //path[j++] = argv[na][i]; - } - } - //path[j++] = '\0'; - if (StringLength(opt->path_log) == 0) { - StringCopyS(opt->path_log, opt->path_html); - } + //path[j++] = argv[na][i]; + } + } + //path[j++] = '\0'; + if (StringLength(opt->path_log) == 0) { + StringCopyS(opt->path_log, opt->path_html); + } check_path(&opt->path_log, argv_firsturl); if (check_path(&opt->path_html, argv_firsturl)) { - opt->dir_topindex=1; // rebuilt top index + opt->dir_topindex = 1; // rebuilt top index } - //printf("-->%s\n%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log)); } break; - } // switch - com++; - } // while - - } // arg - - } // for - + } // switch + com++; + } // while + + } // arg + + } // for + // Convert path to UTF-8 #ifdef _WIN32 { - char *const path = hts_convertStringSystemToUTF8(StringBuff(opt->path_html), (int) StringLength(opt->path_html)); + char *const path = + hts_convertStringSystemToUTF8(StringBuff(opt->path_html), + (int) StringLength(opt->path_html)); if (path != NULL) { StringCopy(opt->path_html_utf8, path); free(path); } else { - StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html), StringLength(opt->path_html)); + StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html), + StringLength(opt->path_html)); } } #else // Assume UTF-8 filesystem. - StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html), StringLength(opt->path_html)); + StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html), + StringLength(opt->path_html)); #endif - /* if doit.log exists, or if new URL(s) defined, - then DO NOT load standard config files */ + /* if doit.log exists, or if new URL(s) defined, + then DO NOT load standard config files */ /* (config files are added in doit.log) */ #if DEBUG_STEPS printf("Loading httrackrc/doit.log\n"); #endif /* recreate a doit.log (no old doit.log or new URLs (and parameters)) */ - if ((strnotempty(StringBuff(opt->path_log))) || (strnotempty(StringBuff(opt->path_html)))) - loops++; // do not loop once again and do not include rc file (O option exists) + if ((strnotempty(StringBuff(opt->path_log))) + || (strnotempty(StringBuff(opt->path_html)))) + loops++; // do not loop once again and do not include rc file (O option exists) else { - if ( (!fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/doit.log"))) || (argv_url>0) ) { - if (!optinclude_file(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),HTS_HTTRACKRC),&argc,argv,x_argvblk,&x_ptr)) - if (!optinclude_file(HTS_HTTRACKRC,&argc,argv,x_argvblk,&x_ptr)) { - if (!optinclude_file(fconcat(OPT_GET_BUFF(opt), hts_gethome(),"/"HTS_HTTRACKRC),&argc,argv,x_argvblk,&x_ptr)) { + if ((!fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/doit.log"))) || (argv_url > 0)) { + if (!optinclude_file + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), HTS_HTTRACKRC), + &argc, argv, x_argvblk, &x_ptr)) + if (!optinclude_file(HTS_HTTRACKRC, &argc, argv, x_argvblk, &x_ptr)) { + if (!optinclude_file + (fconcat(OPT_GET_BUFF(opt), hts_gethome(), "/" HTS_HTTRACKRC), + &argc, argv, x_argvblk, &x_ptr)) { #ifdef HTS_HTTRACKCNF - optinclude_file(HTS_HTTRACKCNF,&argc,argv,x_argvblk,&x_ptr); + optinclude_file(HTS_HTTRACKCNF, &argc, argv, x_argvblk, &x_ptr); #endif } } } else - loops++; // do not loop once again + loops++; // do not loop once again } loops++; - } // while + } // while - } // traiter -O + } // traiter -O /* load doit.log and insert in current command line */ - if ( fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/doit.log")) && (argv_url<=0) ) { - FILE* fp=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/doit.log"),"rb"); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/doit.log")) + && (argv_url <= 0)) { + FILE *fp = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/doit.log"), "rb"); if (fp) { - int insert_after=1; /* insérer après nom au début */ + int insert_after = 1; /* insérer après nom au début */ + // char BIGSTK buff[8192]; - char *p,*lastp; - linput(fp,buff,8000); - fclose(fp); fp=NULL; - p=buff; + char *p, *lastp; + + linput(fp, buff, 8000); + fclose(fp); + fp = NULL; + p = buff; do { int insert_after_argc; + // read next - lastp=p; + lastp = p; if (p) { - p=next_token(p,1); + p = next_token(p, 1); if (p) { - *p=0; // null + *p = 0; // null p++; } } @@ -464,52 +505,99 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { /* Insert parameters BUT so that they can be in the same order */ if (lastp) { if (strnotempty(lastp)) { - insert_after_argc=argc-insert_after; - cmdl_ins(lastp,insert_after_argc,(argv+insert_after),x_argvblk,x_ptr); - argc=insert_after_argc+insert_after; + insert_after_argc = argc - insert_after; + cmdl_ins(lastp, insert_after_argc, (argv + insert_after), x_argvblk, + x_ptr); + argc = insert_after_argc + insert_after; insert_after++; } } - } while(lastp!=NULL); + } while(lastp != NULL); //fclose(fp); } } - // Existence d'un cache - pas de new mais un old.. renommer #if DEBUG_STEPS printf("Checking cache\n"); #endif - if (!fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip"))) { - if ( fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip")) ) { - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")); + if (!fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/new.zip"))) { + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/old.zip"))) { + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.zip")); } - } else if ( (!fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"))) || (!fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx"))) ) { - if ( (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"))) ) { - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")); - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); + } else + if ((!fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) + || + (!fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx")))) { + if ((fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/old.dat"))) + && + (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx")))) { + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat")); + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx")); //remove(fconcat(StringBuff(opt->path_log),"hts-cache/new.lst")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.dat")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.ndx")); //rename(fconcat(StringBuff(opt->path_log),"hts-cache/old.lst"),fconcat(StringBuff(opt->path_log),"hts-cache/new.lst")); } } /* Interrupted mirror detected */ if (!opt->quiet) { - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"))) { + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-in_progress.lock"))) { /* Old cache */ - if ( (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"))) ) { + if ((fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat"))) + && + (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx")))) { if (opt->log != NULL) { - fprintf(opt->log,"Warning!\n"); - fprintf(opt->log,"An aborted mirror has been detected!\nThe current temporary cache is required for any update operation and only contains data downloaded during the last aborted session.\nThe former cache might contain more complete information; if you do not want to lose that information, you have to restore it and delete the current cache.\nThis can easily be done here by erasing the hts-cache/new.* files\n"); - fprintf(opt->log,"Please restart HTTrack with --continue (-iC1) option to override this message!\n"); + fprintf(opt->log, "Warning!\n"); + fprintf(opt->log, + "An aborted mirror has been detected!\nThe current temporary cache is required for any update operation and only contains data downloaded during the last aborted session.\nThe former cache might contain more complete information; if you do not want to lose that information, you have to restore it and delete the current cache.\nThis can easily be done here by erasing the hts-cache/new.* files\n"); + fprintf(opt->log, + "Please restart HTTrack with --continue (-iC1) option to override this message!\n"); } return 0; } } } - // remplacer "macros" comme --spider // permet de lancer httrack sans a avoir à se rappeler de syntaxes comme p0C0I0Qc32 .. #if DEBUG_STEPS @@ -517,56 +605,131 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { #endif { int i; - for(i=0;i<argc;i++) { + + for(i = 0; i < argc; i++) { #if DEBUG_STEPS - printf("Checking #%d:\n",argv[i]); - printf("%s\n",argv[i]); + printf("Checking #%d:\n", argv[i]); + printf("%s\n", argv[i]); #endif - if (argv[i][0]=='-') { - if (argv[i][1]=='-') { // --xxx - if ((strfield2(argv[i]+2,"clean")) || (strfield2(argv[i]+2,"tide"))) { // nettoyer - strcpybuff(argv[i]+1,""); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-log.txt"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-log.txt")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-err.txt"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-err.txt")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),"index.html"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),"index.html")); + if (argv[i][0] == '-') { + if (argv[i][1] == '-') { // --xxx + if ((strfield2(argv[i] + 2, "clean")) || (strfield2(argv[i] + 2, "tide"))) { // nettoyer + strcpybuff(argv[i] + 1, ""); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-log.txt"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-log.txt")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-err.txt"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-err.txt")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_html), "index.html"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_html), + "index.html")); /* */ - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.lst")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.lst"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.lst")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.txt"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.txt")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.txt"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.txt")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/doit.log"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/doit.log")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock")); - rmdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.lst")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.lst"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.lst")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.txt"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.txt")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.txt"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.txt")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/doit.log"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/doit.log")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-in_progress.lock"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-in_progress.lock")); + rmdir(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache")); // - } else if (strfield2(argv[i]+2,"catchurl")) { // capture d'URL via proxy temporaire! - argv_url=1; // forcer a passer les parametres - strcpybuff(argv[i]+1,"#P"); + } else if (strfield2(argv[i] + 2, "catchurl")) { // capture d'URL via proxy temporaire! + argv_url = 1; // forcer a passer les parametres + strcpybuff(argv[i] + 1, "#P"); // - } else if (strfield2(argv[i]+2,"updatehttrack")) { + } else if (strfield2(argv[i] + 2, "updatehttrack")) { #ifdef _WIN32 char s[HTS_CDLMAXSIZE + 256]; - sprintf(s,"%s not available in this version",argv[i]); + + sprintf(s, "%s not available in this version", argv[i]); HTS_PANIC_PRINTF(s); htsmain_free(); return -1; @@ -574,18 +737,18 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { #if 0 char _args[8][256]; char *args[8]; - + printf("Cheking for updates...\n"); - strcpybuff(_args[0],argv[0]); - strcpybuff(_args[1],"--get"); - sprintf(_args[2],HTS_UPDATE_WEBSITE,0,""); - strcpybuff(_args[3],"--quickinfo"); - args[0]=_args[0]; - args[1]=_args[1]; - args[2]=_args[2]; - args[3]=_args[3]; - args[4]=NULL; - if (execvp(args[0],args)==-1) { + strcpybuff(_args[0], argv[0]); + strcpybuff(_args[1], "--get"); + sprintf(_args[2], HTS_UPDATE_WEBSITE, 0, ""); + strcpybuff(_args[3], "--quickinfo"); + args[0] = _args[0]; + args[1] = _args[1]; + args[2] = _args[2]; + args[3] = _args[3]; + args[4] = NULL; + if (execvp(args[0], args) == -1) { } #endif #endif @@ -593,172 +756,184 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { // else { char s[HTS_CDLMAXSIZE + 256]; - sprintf(s,"%s not recognized",argv[i]); + + sprintf(s, "%s not recognized", argv[i]); HTS_PANIC_PRINTF(s); htsmain_free(); return -1; } - } + } } } } // Compter urls/jokers /* - if (argv_url<=0) { - int na; - argv_url=0; - for(na=1;na<argc;na++) { - if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) { - na++; // sauter nom de proxy - } else if (!cmdl_opt(argv[na])) { - argv_url++; // un de plus - } else if (strcmp(argv[na],"-h")==0) { - help(argv[0],!opt->quiet); - htsmain_free(); - return 0; - } else { - if ((strchr(argv[na],'q')!=NULL)) - opt->quiet=1; // ne pas poser de questions! (nohup par exemple) - if ((strchr(argv[na],'i')!=NULL)) { // doit.log! - argv_url=0; - na=argc; - } - } - } - } - */ + if (argv_url<=0) { + int na; + argv_url=0; + for(na=1;na<argc;na++) { + if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) { + na++; // sauter nom de proxy + } else if (!cmdl_opt(argv[na])) { + argv_url++; // un de plus + } else if (strcmp(argv[na],"-h")==0) { + help(argv[0],!opt->quiet); + htsmain_free(); + return 0; + } else { + if ((strchr(argv[na],'q')!=NULL)) + opt->quiet=1; // ne pas poser de questions! (nohup par exemple) + if ((strchr(argv[na],'i')!=NULL)) { // doit.log! + argv_url=0; + na=argc; + } + } + } + } + */ // Ici on ajoute les arguments qui ont été appelés avant au cas où on récupère une session // Exemple: httrack www.truc.fr -L0 puis ^C puis httrack sans URL : ajouter URL précédente /* - if (argv_url==0) { - //if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer - if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) { // un cache est présent - - x_argvblk=(char*) calloct(32768,1); - - if (x_argvblk!=NULL) { - FILE* fp; - int x_argc; - - //strcpybuff(x_argvblk,"httrack "); - fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb"); - if (fp) { - linput(fp,x_argvblk+strlen(x_argvblk),8192); - fclose(fp); fp=NULL; - } - - // calculer arguments selon derniers arguments - x_argv[0]=argv[0]; - x_argc=1; - { - char* p=x_argvblk; - do { - x_argv[x_argc++]=p; - //p=strstr(p," "); - // exemple de chaine: "echo \"test\"" c:\a "\$0" - p=next_token(p,1); // prochain token - if (p) { - *p=0; // octet nul (tableau) - p++; - } - } while(p!=NULL); - } - // recopier arguments actuels (pointeurs uniquement) - { - int na; - for(na=1;na<argc;na++) { - if (strcmp(argv[na],"-O") != 0) // SAUF le path! - x_argv[x_argc++]=argv[na]; - else - na++; - } - } - argc=x_argc; // nouvel argc - argv=x_argv; // nouvel argv - } - - - } - //} - } - */ - + if (argv_url==0) { + //if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer + if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) { // un cache est présent + + x_argvblk=(char*) calloct(32768,1); + + if (x_argvblk!=NULL) { + FILE* fp; + int x_argc; + + //strcpybuff(x_argvblk,"httrack "); + fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb"); + if (fp) { + linput(fp,x_argvblk+strlen(x_argvblk),8192); + fclose(fp); fp=NULL; + } + + // calculer arguments selon derniers arguments + x_argv[0]=argv[0]; + x_argc=1; + { + char* p=x_argvblk; + do { + x_argv[x_argc++]=p; + //p=strstr(p," "); + // exemple de chaine: "echo \"test\"" c:\a "\$0" + p=next_token(p,1); // prochain token + if (p) { + *p=0; // octet nul (tableau) + p++; + } + } while(p!=NULL); + } + // recopier arguments actuels (pointeurs uniquement) + { + int na; + for(na=1;na<argc;na++) { + if (strcmp(argv[na],"-O") != 0) // SAUF le path! + x_argv[x_argc++]=argv[na]; + else + na++; + } + } + argc=x_argc; // nouvel argc + argv=x_argv; // nouvel argv + } + + } + //} + } + */ + // Vérifier quiet /* - { - int na; - for(na=1;na<argc;na++) { - if (!cmdl_opt(argv[na])) { - if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0)) - na++; // sauter nom de proxy - } else { - if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL)) - opt->quiet=1; // ne pas poser de questions! (nohup par exemple) - } - } - } - */ + { + int na; + for(na=1;na<argc;na++) { + if (!cmdl_opt(argv[na])) { + if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0)) + na++; // sauter nom de proxy + } else { + if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL)) + opt->quiet=1; // ne pas poser de questions! (nohup par exemple) + } + } + } + */ // Pas d'URL #if DEBUG_STEPS printf("Checking URLs\n"); #endif - if (argv_url==0) { + if (argv_url == 0) { // Présence d'un cache, que faire?.. - if ( - ( fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")) ) - || - ( fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")) && fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")) ) - ) { // il existe déja un cache précédent.. renommer - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/doit.log"))) { // un cache est présent - if (x_argvblk!=NULL) { - int m; + if ((fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/new.zip"))) + || + (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/new.dat")) + && + fexist(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx"))) + ) { // il existe déja un cache précédent.. renommer + if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache/doit.log"))) { // un cache est présent + if (x_argvblk != NULL) { + int m; + // établir mode - mode cache: 1 (cache valide) 2 (cache à vérifier) - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"))) { // cache prioritaire - m=1; + if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-in_progress.lock"))) { // cache prioritaire + m = 1; } else { - m=2; + m = 2; } - opt->cache=m; - - if (opt->quiet==0) { // sinon on continue automatiquement + opt->cache = m; + + if (opt->quiet == 0) { // sinon on continue automatiquement HT_REQUEST_START; HT_PRINT("A cache (hts-cache/) has been found in the directory "); HT_PRINT(StringBuff(opt->path_log)); HT_PRINT(LF); - if (m==1) { - HT_PRINT("That means that a transfer has been aborted"LF); + if (m == 1) { + HT_PRINT("That means that a transfer has been aborted" LF); HT_PRINT("OK to Continue "); } else { - HT_PRINT("That means you can update faster the remote site(s)"LF); + HT_PRINT("That means you can update faster the remote site(s)" + LF); HT_PRINT("OK to Update "); } - HT_PRINT("httrack "); HT_PRINT(x_argvblk); HT_PRINT("?"LF); + HT_PRINT("httrack "); + HT_PRINT(x_argvblk); + HT_PRINT("?" LF); HT_REQUEST_END; - if (!ask_continue(opt)) { + if (!ask_continue(opt)) { htsmain_free(); return 0; } } - + } else { HTS_PANIC_PRINTF("Error, not enough memory"); htsmain_free(); return -1; } - } else { // log existe pas + } else { // log existe pas HTS_PANIC_PRINTF("A cache has been found, but no command line"); - printf("Please launch httrack with proper parameters to reuse the cache\n"); + printf + ("Please launch httrack with proper parameters to reuse the cache\n"); htsmain_free(); return -1; } - - } else { // aucune URL définie et pas de cache + + } else { // aucune URL définie et pas de cache if (opt->quiet) { - help(argv[0],!opt->quiet); + help(argv[0], !opt->quiet); htsmain_free(); return -1; } else { @@ -769,24 +944,33 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { htsmain_free(); return 0; } - } else { // plus de 2 paramètres + } else { // plus de 2 paramètres // un fichier log existe? - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"))) { // fichier lock? + if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-in_progress.lock"))) { // fichier lock? //char s[32]; - - opt->cache=1; // cache prioritaire - if (opt->quiet==0) { - if ( - ( fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")) ) - || - ( fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")) && fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")) ) + + opt->cache = 1; // cache prioritaire + if (opt->quiet == 0) { + if ((fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip"))) + || + (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat")) + && + fexist(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx"))) ) { HT_REQUEST_START; HT_PRINT("There is a lock-file in the directory "); HT_PRINT(StringBuff(opt->path_log)); - HT_PRINT(LF"That means that a mirror has not been terminated"LF); - HT_PRINT("Be sure you call httrack with proper parameters"LF); - HT_PRINT("(The cache allows you to restart faster the transfer)"LF); + HT_PRINT(LF "That means that a mirror has not been terminated" LF); + HT_PRINT("Be sure you call httrack with proper parameters" LF); + HT_PRINT("(The cache allows you to restart faster the transfer)" LF); HT_REQUEST_END; if (!ask_continue(opt)) { htsmain_free(); @@ -794,20 +978,35 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { } } } - } else if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html),"index.html"))) { + } else + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_html), "index.html"))) { //char s[32]; - opt->cache=2; // cache vient après test de validité - if (opt->quiet==0) { - if ( - ( fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")) ) - || - ( fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")) && fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")) ) + opt->cache = 2; // cache vient après test de validité + if (opt->quiet == 0) { + if ((fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip"))) + || + (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat")) + && + fexist(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx"))) ) { HT_REQUEST_START; - HT_PRINT("There is an index.html and a hts-cache folder in the directory "); + HT_PRINT + ("There is an index.html and a hts-cache folder in the directory "); HT_PRINT(StringBuff(opt->path_log)); - HT_PRINT(LF"A site may have been mirrored here, that could mean that you want to update it"LF); - HT_PRINT("Be sure parameters are ok"LF); + HT_PRINT(LF + "A site may have been mirrored here, that could mean that you want to update it" + LF); + HT_PRINT("Be sure parameters are ok" LF); HT_REQUEST_END; if (!ask_continue(opt)) { htsmain_free(); @@ -817,10 +1016,10 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { HT_REQUEST_START; HT_PRINT("There is an index.html in the directory "); HT_PRINT(StringBuff(opt->path_log)); - HT_PRINT(" but no cache"LF); - HT_PRINT("There is an index.html in the directory, but no cache"LF); - HT_PRINT("A site may have been mirrored here, and erased.."LF); - HT_PRINT("Be sure parameters are ok"LF); + HT_PRINT(" but no cache" LF); + HT_PRINT("There is an index.html in the directory, but no cache" LF); + HT_PRINT("A site may have been mirrored here, and erased.." LF); + HT_PRINT("Be sure parameters are ok" LF); HT_REQUEST_END; if (!ask_continue(opt)) { htsmain_free(); @@ -830,69 +1029,78 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { } } } - - + // Treat parameters // Traiter les paramètres #if DEBUG_STEPS printf("Analyze parameters\n"); #endif - { - char* com; + { + char *com; int na; - - for(na=1;na<argc;na++) { - if (argv[na][0]=='"') { + for(na = 1; na < argc; na++) { + + if (argv[na][0] == '"') { char BIGSTK tempo[HTS_CDLMAXSIZE + 256]; - strcpybuff(tempo,argv[na]+1); - if (tempo[strlen(tempo)-1]!='"') { + + strcpybuff(tempo, argv[na] + 1); + if (tempo[strlen(tempo) - 1] != '"') { char s[HTS_CDLMAXSIZE + 256]; - sprintf(s,"Missing quote in %s",argv[na]); + + sprintf(s, "Missing quote in %s", argv[na]); HTS_PANIC_PRINTF(s); htsmain_free(); return -1; } - tempo[strlen(tempo)-1]='\0'; - strcpybuff(argv[na],tempo); + tempo[strlen(tempo) - 1] = '\0'; + strcpybuff(argv[na], tempo); } if (cmdl_opt(argv[na])) { // option - com=argv[na]+1; - + com = argv[na] + 1; + while(*com) { - switch(*com) { - case ' ': case 9: case '-': case '\0': break; + switch (*com) { + case ' ': + case 9: + case '-': + case '\0': + break; // - case 'h': - help(argv[0],0); + case 'h': + help(argv[0], 0); htsmain_free(); - return 0; // déja fait normalement + return 0; // déja fait normalement // - case 'g': // récupérer un (ou plusieurs) fichiers isolés - opt->wizard=2; // le wizard on peut plus s'en passer.. + case 'g': // récupérer un (ou plusieurs) fichiers isolés + opt->wizard = 2; // le wizard on peut plus s'en passer.. //opt->wizard=0; // pas de wizard - opt->cache=0; // ni de cache - opt->makeindex=0; // ni d'index - httrack_logmode=1; // erreurs à l'écran - opt->savename_type=1003; // mettre dans le répertoire courant - opt->depth=0; // ne pas explorer la page - opt->accept_cookie=0; // pas de cookies - opt->robots=0; // pas de robots + opt->cache = 0; // ni de cache + opt->makeindex = 0; // ni d'index + httrack_logmode = 1; // erreurs à l'écran + opt->savename_type = 1003; // mettre dans le répertoire courant + opt->depth = 0; // ne pas explorer la page + opt->accept_cookie = 0; // pas de cookies + opt->robots = 0; // pas de robots break; - case 'w': opt->wizard=2; // wizard 'soft' (ne pose pas de questions) - opt->travel=0; - opt->seeker=1; + case 'w': + opt->wizard = 2; // wizard 'soft' (ne pose pas de questions) + opt->travel = 0; + opt->seeker = 1; break; - case 'W': opt->wizard=1; // Wizard-Help (pose des questions) - opt->travel=0; - opt->seeker=1; + case 'W': + opt->wizard = 1; // Wizard-Help (pose des questions) + opt->travel = 0; + opt->seeker = 1; break; - case 'r': // n'est plus le recurse get bestial mais wizard itou! - if (isdigit((unsigned char)*(com+1))) { - sscanf(com+1,"%d",&opt->depth); - while(isdigit((unsigned char)*(com+1))) com++; - } else opt->depth=3; + case 'r': // n'est plus le recurse get bestial mais wizard itou! + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &opt->depth); + while(isdigit((unsigned char) *(com + 1))) + com++; + } else + opt->depth = 3; break; /* case 'r': opt->wizard=0; @@ -905,1006 +1113,1399 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { // // note: les tests opt->depth sont pour éviter de faire // un miroir du web (:-O) accidentelement ;-) - case 'a': /*if (opt->depth==9999) opt->depth=3;*/ - opt->travel=0+(opt->travel&256); break; - case 'd': /*if (opt->depth==9999) opt->depth=3;*/ - opt->travel=1+(opt->travel&256); break; - case 'l': /*if (opt->depth==9999) opt->depth=3;*/ - opt->travel=2+(opt->travel&256); break; - case 'e': /*if (opt->depth==9999) opt->depth=3;*/ - opt->travel=7+(opt->travel&256); break; - case 't': opt->travel|=256; break; - case 'n': opt->nearlink=1; break; - case 'x': opt->external=1; break; + case 'a': /*if (opt->depth==9999) opt->depth=3; */ + opt->travel = 0 + (opt->travel & 256); + break; + case 'd': /*if (opt->depth==9999) opt->depth=3; */ + opt->travel = 1 + (opt->travel & 256); + break; + case 'l': /*if (opt->depth==9999) opt->depth=3; */ + opt->travel = 2 + (opt->travel & 256); + break; + case 'e': /*if (opt->depth==9999) opt->depth=3; */ + opt->travel = 7 + (opt->travel & 256); + break; + case 't': + opt->travel |= 256; + break; + case 'n': + opt->nearlink = 1; + break; + case 'x': + opt->external = 1; + break; // - case 'U': opt->seeker=2; break; - case 'D': opt->seeker=1; break; - case 'S': opt->seeker=0; break; - case 'B': opt->seeker=3; break; + case 'U': + opt->seeker = 2; + break; + case 'D': + opt->seeker = 1; + break; + case 'S': + opt->seeker = 0; + break; + case 'B': + opt->seeker = 3; + break; // - case 'Y': opt->mirror_first_page=1; break; + case 'Y': + opt->mirror_first_page = 1; + break; // - case 'q': case 'i': opt->quiet=1; break; + case 'q': + case 'i': + opt->quiet = 1; + break; // - case 'Q': httrack_logmode=0; break; - case 'v': httrack_logmode=1; break; - case 'f': httrack_logmode=2; if (*(com+1)=='2') httrack_logmode=3; while(isdigit((unsigned char)*(com+1))) com++; break; + case 'Q': + httrack_logmode = 0; + break; + case 'v': + httrack_logmode = 1; + break; + case 'f': + httrack_logmode = 2; + if (*(com + 1) == '2') + httrack_logmode = 3; + while(isdigit((unsigned char) *(com + 1))) + com++; + break; // - //case 'A': opt->urlmode=1; break; - //case 'R': opt->urlmode=2; break; - case 'K': opt->urlmode=0; - if (isdigit((unsigned char)*(com+1))) { - sscanf(com+1,"%d",&opt->urlmode); + //case 'A': opt->urlmode=1; break; + //case 'R': opt->urlmode=2; break; + case 'K': + opt->urlmode = 0; + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &opt->urlmode); if (opt->urlmode == 0) { // in fact K0 ==> K2 - // and K ==> K0 - opt->urlmode=2; + // and K ==> K0 + opt->urlmode = 2; } - while(isdigit((unsigned char)*(com+1))) com++; + while(isdigit((unsigned char) *(com + 1))) + com++; } //if (*(com+1)=='0') { opt->urlmode=2; com++; } break; // case 'c': - if (isdigit((unsigned char)*(com+1))) { - sscanf(com+1,"%d",&opt->maxsoc); - while(isdigit((unsigned char)*(com+1))) com++; - opt->maxsoc=max(opt->maxsoc,1); // FORCER A 1 - } else opt->maxsoc=4; - + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &opt->maxsoc); + while(isdigit((unsigned char) *(com + 1))) + com++; + opt->maxsoc = max(opt->maxsoc, 1); // FORCER A 1 + } else + opt->maxsoc = 4; + break; // - case 'p': sscanf(com+1,"%d",&opt->getmode); while(isdigit((unsigned char)*(com+1))) com++; break; + case 'p': + sscanf(com + 1, "%d", &opt->getmode); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; // - case 'G': sscanf(com+1,LLintP,&opt->fragment); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'M': sscanf(com+1,LLintP,&opt->maxsite); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'm': sscanf(com+1,LLintP,&opt->maxfile_nonhtml); while(isdigit((unsigned char)*(com+1))) com++; - if (*(com+1)==',') { + case 'G': + sscanf(com + 1, LLintP, &opt->fragment); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'M': + sscanf(com + 1, LLintP, &opt->maxsite); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'm': + sscanf(com + 1, LLintP, &opt->maxfile_nonhtml); + while(isdigit((unsigned char) *(com + 1))) + com++; + if (*(com + 1) == ',') { com++; - sscanf(com+1,LLintP,&opt->maxfile_html); while(isdigit((unsigned char)*(com+1))) com++; - } else opt->maxfile_html=-1; + sscanf(com + 1, LLintP, &opt->maxfile_html); + while(isdigit((unsigned char) *(com + 1))) + com++; + } else + opt->maxfile_html = -1; break; // - case 'T': sscanf(com+1,"%d",&opt->timeout); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'J': sscanf(com+1,"%d",&opt->rateout); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'R': sscanf(com+1,"%d",&opt->retry); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'E': sscanf(com+1,"%d",&opt->maxtime); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'H': sscanf(com+1,"%d",&opt->hostcontrol); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'A': sscanf(com+1,"%d",&opt->maxrate); while(isdigit((unsigned char)*(com+1))) com++; break; + case 'T': + sscanf(com + 1, "%d", &opt->timeout); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'J': + sscanf(com + 1, "%d", &opt->rateout); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'R': + sscanf(com + 1, "%d", &opt->retry); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'E': + sscanf(com + 1, "%d", &opt->maxtime); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'H': + sscanf(com + 1, "%d", &opt->hostcontrol); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'A': + sscanf(com + 1, "%d", &opt->maxrate); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; case 'j': opt->parsejava = HTSPARSE_DEFAULT; - if (isdigit((unsigned char)*(com+1))) { - sscanf(com+1,"%d",&opt->parsejava); - while(isdigit((unsigned char)*(com+1))) com++; + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &opt->parsejava); + while(isdigit((unsigned char) *(com + 1))) + com++; } break; // - case 'I': opt->makeindex=1; if (*(com+1)=='0') { opt->makeindex=0; com++; } break; + case 'I': + opt->makeindex = 1; + if (*(com + 1) == '0') { + opt->makeindex = 0; + com++; + } + break; // - case 'X': opt->delete_old=1; if (*(com+1)=='0') { opt->delete_old=0; com++; } break; + case 'X': + opt->delete_old = 1; + if (*(com + 1) == '0') { + opt->delete_old = 0; + com++; + } + break; // - case 'y': opt->background_on_suspend=1; if (*(com+1)=='0') { opt->background_on_suspend=0; com++; } break; + case 'y': + opt->background_on_suspend = 1; + if (*(com + 1) == '0') { + opt->background_on_suspend = 0; + com++; + } + break; // - case 'b': sscanf(com+1,"%d",&opt->accept_cookie); while(isdigit((unsigned char)*(com+1))) com++; break; + case 'b': + sscanf(com + 1, "%d", &opt->accept_cookie); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; // case 'N': - if (strcmp(argv[na],"-N")==0) { // Tout seul - if ((na+1>=argc) || (argv[na+1][0]=='-')) { // erreur - HTS_PANIC_PRINTF("Option N needs a number, or needs to be followed by a blank space, and a string"); + if (strcmp(argv[na], "-N") == 0) { // Tout seul + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { // erreur + HTS_PANIC_PRINTF + ("Option N needs a number, or needs to be followed by a blank space, and a string"); printf("Example: -N4\n"); htsmain_free(); return -1; } else { na++; - if (strlen(argv[na])>=127) { + if (strlen(argv[na]) >= 127) { HTS_PANIC_PRINTF("Userdef structure string too long"); htsmain_free(); return -1; } StringCopy(opt->savename_userdef, argv[na]); if (StringLength(opt->savename_userdef) > 0) - opt->savename_type = -1; // userdef! + opt->savename_type = -1; // userdef! else - opt->savename_type = 0; // -N "" : par défaut + opt->savename_type = 0; // -N "" : par défaut } } else { - sscanf(com+1,"%d",&opt->savename_type); while(isdigit((unsigned char)*(com+1))) com++; + sscanf(com + 1, "%d", &opt->savename_type); + while(isdigit((unsigned char) *(com + 1))) + com++; } break; - case 'L': + case 'L': { - sscanf(com+1,"%d",&opt->savename_83); - switch(opt->savename_83) { - case 0: // 8-3 (ISO9660 L1) - opt->savename_83=1; + sscanf(com + 1, "%d", &opt->savename_83); + switch (opt->savename_83) { + case 0: // 8-3 (ISO9660 L1) + opt->savename_83 = 1; break; case 1: - opt->savename_83=0; + opt->savename_83 = 0; break; - default: // 2 == ISO9660 (ISO9660 L2) - opt->savename_83=2; + default: // 2 == ISO9660 (ISO9660 L2) + opt->savename_83 = 2; break; } - while(isdigit((unsigned char)*(com+1))) com++; + while(isdigit((unsigned char) *(com + 1))) + com++; } break; - case 's': - if (isdigit((unsigned char)*(com+1))) { - sscanf(com+1,"%d",&opt->robots); - while(isdigit((unsigned char)*(com+1))) com++; - } else opt->robots=1; + case 's': + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &opt->robots); + while(isdigit((unsigned char) *(com + 1))) + com++; + } else + opt->robots = 1; #if DEBUG_ROBOTS - printf("robots.txt mode set to %d\n",opt->robots); + printf("robots.txt mode set to %d\n", opt->robots); #endif break; - case 'o': sscanf(com+1,"%d",&opt->errpage); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'u': sscanf(com+1,"%d",&opt->check_type); while(isdigit((unsigned char)*(com+1))) com++; break; + case 'o': + sscanf(com + 1, "%d", &opt->errpage); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'u': + sscanf(com + 1, "%d", &opt->check_type); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; // - case 'C': - if (isdigit((unsigned char)*(com+1))) { - sscanf(com+1,"%d",&opt->cache); - while(isdigit((unsigned char)*(com+1))) com++; - } else opt->cache=1; + case 'C': + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &opt->cache); + while(isdigit((unsigned char) *(com + 1))) + com++; + } else + opt->cache = 1; + break; + case 'k': + opt->all_in_cache = 1; break; - case 'k': opt->all_in_cache=1; break; // - case 'z': opt->debug=1; break; // petit debug - case 'Z': opt->debug=2; break; // GROS debug + case 'z': + opt->debug = 1; + break; // petit debug + case 'Z': + opt->debug = 2; + break; // GROS debug // - case '&': case '%': { // deuxième jeu d'options - com++; - switch(*com) { - case 'M': opt->mimehtml = 1; if (*(com+1)=='0') { opt->mimehtml=0; com++; } break; - case 'k': opt->nokeepalive = 0; if (*(com+1)=='0') { opt->nokeepalive = 1; com++; } break; - case 'x': opt->passprivacy=1; if (*(com+1)=='0') { opt->passprivacy=0; com++; } break; // No passwords in html files - case 'q': opt->includequery=1; if (*(com+1)=='0') { opt->includequery=0; com++; } break; // No passwords in html files - case 'I': opt->kindex=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->kindex); while(isdigit((unsigned char)*(com+1))) com++; } - break; // Keyword Index - case 'c': sscanf(com+1,"%f",&opt->maxconn); while(isdigit((unsigned char)*(com+1)) || *(com+1) == '.') com++; break; - case 'e': sscanf(com+1,"%d",&opt->extdepth); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'B': opt->tolerant=1; if (*(com+1)=='0') { opt->tolerant=0; com++; } break; // HTTP/1.0 notamment - case 'h': opt->http10=1; if (*(com+1)=='0') { opt->http10=0; com++; } break; // HTTP/1.0 - case 'z': opt->nocompression=1; if (*(com+1)=='0') { opt->nocompression=0; com++; } break; // pas de compression - case 'f': opt->ftp_proxy=1; if (*(com+1)=='0') { opt->ftp_proxy=0; com++; } break; // proxy http pour ftp - case 'P': opt->parseall=1; if (*(com+1)=='0') { opt->parseall=0; com++; } break; // tout parser - case 'n': opt->norecatch=1; if (*(com+1)=='0') { opt->norecatch=0; com++; } break; // ne pas reprendre fichiers effacés localement - case 's': opt->sizehack=1; if (*(com+1)=='0') { opt->sizehack=0; com++; } break; // hack sur content-length - case 'u': opt->urlhack=1; if (*(com+1)=='0') { opt->urlhack=0; com++; } break; // url hack - case 'v': opt->verbosedisplay=2; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->verbosedisplay); while(isdigit((unsigned char)*(com+1))) com++; } break; - case 'i': opt->dir_topindex = 1; if (*(com+1)=='0') { opt->dir_topindex=0; com++; } break; - case 'N': opt->savename_delayed = 2; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->savename_delayed); while(isdigit((unsigned char)*(com+1))) com++; } break; - case 'D': opt->delayed_cached=1; if (*(com+1)=='0') { opt->delayed_cached=0; com++; } break; // url hack - case 'T': opt->convert_utf8=1; if (*(com+1)=='0') { opt->convert_utf8=0; com++; } break; // convert to utf-8 - case '!': opt->bypass_limits = 1; if (*(com+1)=='0') { opt->bypass_limits=0; com++; } break; + case '&': + case '%':{ // deuxième jeu d'options + com++; + switch (*com) { + case 'M': + opt->mimehtml = 1; + if (*(com + 1) == '0') { + opt->mimehtml = 0; + com++; + } + break; + case 'k': + opt->nokeepalive = 0; + if (*(com + 1) == '0') { + opt->nokeepalive = 1; + com++; + } + break; + case 'x': + opt->passprivacy = 1; + if (*(com + 1) == '0') { + opt->passprivacy = 0; + com++; + } + break; // No passwords in html files + case 'q': + opt->includequery = 1; + if (*(com + 1) == '0') { + opt->includequery = 0; + com++; + } + break; // No passwords in html files + case 'I': + opt->kindex = 1; + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &opt->kindex); + while(isdigit((unsigned char) *(com + 1))) + com++; + } + break; // Keyword Index + case 'c': + sscanf(com + 1, "%f", &opt->maxconn); + while(isdigit((unsigned char) *(com + 1)) || *(com + 1) == '.') + com++; + break; + case 'e': + sscanf(com + 1, "%d", &opt->extdepth); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'B': + opt->tolerant = 1; + if (*(com + 1) == '0') { + opt->tolerant = 0; + com++; + } + break; // HTTP/1.0 notamment + case 'h': + opt->http10 = 1; + if (*(com + 1) == '0') { + opt->http10 = 0; + com++; + } + break; // HTTP/1.0 + case 'z': + opt->nocompression = 1; + if (*(com + 1) == '0') { + opt->nocompression = 0; + com++; + } + break; // pas de compression + case 'f': + opt->ftp_proxy = 1; + if (*(com + 1) == '0') { + opt->ftp_proxy = 0; + com++; + } + break; // proxy http pour ftp + case 'P': + opt->parseall = 1; + if (*(com + 1) == '0') { + opt->parseall = 0; + com++; + } + break; // tout parser + case 'n': + opt->norecatch = 1; + if (*(com + 1) == '0') { + opt->norecatch = 0; + com++; + } + break; // ne pas reprendre fichiers effacés localement + case 's': + opt->sizehack = 1; + if (*(com + 1) == '0') { + opt->sizehack = 0; + com++; + } + break; // hack sur content-length + case 'u': + opt->urlhack = 1; + if (*(com + 1) == '0') { + opt->urlhack = 0; + com++; + } + break; // url hack + case 'v': + opt->verbosedisplay = 2; + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &opt->verbosedisplay); + while(isdigit((unsigned char) *(com + 1))) + com++; + } + break; + case 'i': + opt->dir_topindex = 1; + if (*(com + 1) == '0') { + opt->dir_topindex = 0; + com++; + } + break; + case 'N': + opt->savename_delayed = 2; + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &opt->savename_delayed); + while(isdigit((unsigned char) *(com + 1))) + com++; + } + break; + case 'D': + opt->delayed_cached = 1; + if (*(com + 1) == '0') { + opt->delayed_cached = 0; + com++; + } + break; // url hack + case 'T': + opt->convert_utf8 = 1; + if (*(com + 1) == '0') { + opt->convert_utf8 = 0; + com++; + } + break; // convert to utf-8 + case '!': + opt->bypass_limits = 1; + if (*(com + 1) == '0') { + opt->bypass_limits = 0; + com++; + } + break; #if HTS_USEMMS - case 'm': sscanf(com+1,"%d",&opt->mms_maxtime); while(isdigit((unsigned char)*(com+1))) com++; break; + case 'm': + sscanf(com + 1, "%d", &opt->mms_maxtime); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; #endif - case 'w': // disable specific plugin - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %w needs to be followed by a blank space, and a module name"); - printf("Example: -%%w htsswf\n"); - htsmain_free(); - return -1; - } else{ - na++; - StringCat(opt->mod_blacklist, argv[na]); - StringCat(opt->mod_blacklist, "\n"); - } - break; + case 'w': // disable specific plugin + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %w needs to be followed by a blank space, and a module name"); + printf("Example: -%%w htsswf\n"); + htsmain_free(); + return -1; + } else { + na++; + StringCat(opt->mod_blacklist, argv[na]); + StringCat(opt->mod_blacklist, "\n"); + } + break; - // preserve: no footer, original links - case 'p': - StringClear(opt->footer); - opt->urlmode=4; - break; - case 'L': // URL list - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %L needs to be followed by a blank space, and a text filename"); - printf("Example: -%%L \"mylist.txt\"\n"); - htsmain_free(); - return -1; - } else{ - na++; - if (strlen(argv[na])>=254) { - HTS_PANIC_PRINTF("File list string too long"); + // preserve: no footer, original links + case 'p': + StringClear(opt->footer); + opt->urlmode = 4; + break; + case 'L': // URL list + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %L needs to be followed by a blank space, and a text filename"); + printf("Example: -%%L \"mylist.txt\"\n"); htsmain_free(); return -1; + } else { + na++; + if (strlen(argv[na]) >= 254) { + HTS_PANIC_PRINTF("File list string too long"); + htsmain_free(); + return -1; + } + StringCopy(opt->filelist, argv[na]); } - StringCopy(opt->filelist,argv[na]); - } - break; - case 'b': // bind - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %b needs to be followed by a blank space, and a local hostname"); - printf("Example: -%%b \"ip4.localhost\"\n"); - htsmain_free(); - return -1; - } else{ - na++; - if (strlen(argv[na])>=254) { - HTS_PANIC_PRINTF("Hostname string too long"); + break; + case 'b': // bind + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %b needs to be followed by a blank space, and a local hostname"); + printf("Example: -%%b \"ip4.localhost\"\n"); htsmain_free(); return -1; + } else { + na++; + if (strlen(argv[na]) >= 254) { + HTS_PANIC_PRINTF("Hostname string too long"); + htsmain_free(); + return -1; + } + StringCopy(opt->proxy.bindhost, argv[na]); } - StringCopy(opt->proxy.bindhost, argv[na]); - } - break; - case 'S': // Scan Rules list - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %S needs to be followed by a blank space, and a text filename"); - printf("Example: -%%S \"myfilterlist.txt\"\n"); - htsmain_free(); - return -1; - } else{ - off_t fz; - na++; - fz = fsize(argv[na]); - if (fz < 0) { - HTS_PANIC_PRINTF("File url list could not be opened"); + break; + case 'S': // Scan Rules list + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %S needs to be followed by a blank space, and a text filename"); + printf("Example: -%%S \"myfilterlist.txt\"\n"); htsmain_free(); return -1; } else { - FILE* fp = fopen(argv[na], "rb"); - if (fp != NULL) { - int cl = (int) strlen(url); - ensureUrlCapacity(url, url_sz, cl + fz + 8192); - if (cl > 0) { /* don't stick! (3.43) */ - url[cl] = ' '; - cl++; - } - if (fread(url + cl, 1, fz, fp) != fz) { - HTS_PANIC_PRINTF("File url list could not be read"); - htsmain_free(); - return -1; + off_t fz; + + na++; + fz = fsize(argv[na]); + if (fz < 0) { + HTS_PANIC_PRINTF("File url list could not be opened"); + htsmain_free(); + return -1; + } else { + FILE *fp = fopen(argv[na], "rb"); + + if (fp != NULL) { + int cl = (int) strlen(url); + + ensureUrlCapacity(url, url_sz, cl + fz + 8192); + if (cl > 0) { /* don't stick! (3.43) */ + url[cl] = ' '; + cl++; + } + if (fread(url + cl, 1, fz, fp) != fz) { + HTS_PANIC_PRINTF("File url list could not be read"); + htsmain_free(); + return -1; + } + fclose(fp); + *(url + cl + fz) = '\0'; } - fclose(fp); - *(url + cl + fz) = '\0'; } } - } - break; - case 'A': // assume - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %A needs to be followed by a blank space, and a filesystemtype=mimetype/mimesubtype parameters"); - printf("Example: -%%A php3=text/html,asp=text/html\n"); - htsmain_free(); - return -1; - } else{ - na++; - // --assume standard - if (strcmp(argv[na], "standard") == 0) { - StringCopy(opt->mimedefs,"\n"); - StringCat(opt->mimedefs,HTS_ASSUME_STANDARD); - StringCat(opt->mimedefs,"\n"); + break; + case 'A': // assume + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %A needs to be followed by a blank space, and a filesystemtype=mimetype/mimesubtype parameters"); + printf("Example: -%%A php3=text/html,asp=text/html\n"); + htsmain_free(); + return -1; } else { - char* a; - //char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs); - for(a = argv[na] ; *a != '\0' ; a++) { - if (*a == ';') { /* next one */ - StringAddchar(opt->mimedefs, '\n'); - //*b++ = '\n'; - } else if (*a == ',' || *a == '\n' || *a == '\r' || *a == '\t') { - StringAddchar(opt->mimedefs, ' '); - //*b++ = ' '; - } else { - StringAddchar(opt->mimedefs, *a); - //*b++ = *a; + na++; + // --assume standard + if (strcmp(argv[na], "standard") == 0) { + StringCopy(opt->mimedefs, "\n"); + StringCat(opt->mimedefs, HTS_ASSUME_STANDARD); + StringCat(opt->mimedefs, "\n"); + } else { + char *a; + + //char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs); + for(a = argv[na]; *a != '\0'; a++) { + if (*a == ';') { /* next one */ + StringAddchar(opt->mimedefs, '\n'); + //*b++ = '\n'; + } else if (*a == ',' || *a == '\n' || *a == '\r' + || *a == '\t') { + StringAddchar(opt->mimedefs, ' '); + //*b++ = ' '; + } else { + StringAddchar(opt->mimedefs, *a); + //*b++ = *a; + } } + StringAddchar(opt->mimedefs, '\n'); + //*b++ = '\n'; /* next def */ + //*b++ = '\0'; } - StringAddchar(opt->mimedefs, '\n'); - //*b++ = '\n'; /* next def */ - //*b++ = '\0'; } - } - break; - // - case 'l': - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %l needs to be followed by a blank space, and an ISO language code"); - printf("Example: -%%l \"en\"\n"); - htsmain_free(); - return -1; - } else{ - na++; - if (strlen(argv[na])>=62) { - HTS_PANIC_PRINTF("Lang list string too long"); + break; + // + case 'l': + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %l needs to be followed by a blank space, and an ISO language code"); + printf("Example: -%%l \"en\"\n"); htsmain_free(); return -1; + } else { + na++; + if (strlen(argv[na]) >= 62) { + HTS_PANIC_PRINTF("Lang list string too long"); + htsmain_free(); + return -1; + } + StringCopy(opt->lang_iso, argv[na]); } - StringCopy(opt->lang_iso,argv[na]); - } - break; - // - case 'F': // footer id - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %F needs to be followed by a blank space, and a footer string"); - printf("Example: -%%F \"<!-- Mirrored from %%s by HTTrack Website Copier/"HTTRACK_AFF_VERSION" "HTTRACK_AFF_AUTHORS", %%s -->\"\n"); - htsmain_free(); - return -1; - } else{ - na++; - if (strlen(argv[na])>=254) { - HTS_PANIC_PRINTF("Footer string too long"); + break; + // + case 'F': // footer id + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %F needs to be followed by a blank space, and a footer string"); + printf + ("Example: -%%F \"<!-- Mirrored from %%s by HTTrack Website Copier/" + HTTRACK_AFF_VERSION " " HTTRACK_AFF_AUTHORS + ", %%s -->\"\n"); htsmain_free(); return -1; + } else { + na++; + if (strlen(argv[na]) >= 254) { + HTS_PANIC_PRINTF("Footer string too long"); + htsmain_free(); + return -1; + } + StringCopy(opt->footer, argv[na]); } - StringCopy(opt->footer,argv[na]); - } - break; - case 'H': // debug headers - _DEBUG_HEAD=1; - break; - case 'O': + break; + case 'H': // debug headers + _DEBUG_HEAD = 1; + break; + case 'O': #ifdef _WIN32 - printf("Warning option -%%O has no effect in this system (chroot)\n"); + printf + ("Warning option -%%O has no effect in this system (chroot)\n"); #else - switch_chroot=1; + switch_chroot = 1; #endif - break; - case 'U': // setuid - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %U needs to be followed by a blank space, and a username"); - printf("Example: -%%U smith\n"); - htsmain_free(); - return -1; - } else { - na++; + break; + case 'U': // setuid + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %U needs to be followed by a blank space, and a username"); + printf("Example: -%%U smith\n"); + htsmain_free(); + return -1; + } else { + na++; #ifdef _WIN32 - printf("Warning option -%%U has no effect on this system (setuid)\n"); + printf + ("Warning option -%%U has no effect on this system (setuid)\n"); #else #ifndef HTS_DO_NOT_USE_UID - /* Change the user id and gid */ - { - struct passwd* userdef=getpwnam((const char*)argv[na]); - if (userdef) { /* we'll have to switch the user id */ - switch_gid=userdef->pw_gid; - switch_uid=userdef->pw_uid; + /* Change the user id and gid */ + { + struct passwd *userdef = getpwnam((const char *) argv[na]); + + if (userdef) { /* we'll have to switch the user id */ + switch_gid = userdef->pw_gid; + switch_uid = userdef->pw_uid; + } } - } #else - printf("Warning option -%%U has no effect with this compiled version (setuid)\n"); + printf + ("Warning option -%%U has no effect with this compiled version (setuid)\n"); #endif #endif - } - break; - - case 'W': // Wrapper callback - // --wrapper check-link=obj.so:check_link - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %W needs to be followed by a blank space, and a <callback-name>=<myfile.so>:<function-name> field"); - printf("Example: -%%W check-link=checklink.so:check\n"); - htsmain_free(); - return -1; - } else { - char* pos; - na++; - for(pos = argv[na] ; *pos != '\0' && *pos != '=' && *pos != ',' && *pos != ':' ; pos++); - /* httrack --wrapper callback[,foo] */ - if (*pos == 0 || *pos == ',' || *pos == ':') { - int ret; - char *moduleName; - if (*pos == ',' || *pos == ':') { - *pos = '\0'; - moduleName = strdupt(argv[na]); - *pos = ','; /* foce seperator to ',' */ - } else { - moduleName = strdupt(argv[na]); + } + break; + + case 'W': // Wrapper callback + // --wrapper check-link=obj.so:check_link + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %W needs to be followed by a blank space, and a <callback-name>=<myfile.so>:<function-name> field"); + printf("Example: -%%W check-link=checklink.so:check\n"); + htsmain_free(); + return -1; + } else { + char *pos; + + na++; + for(pos = argv[na]; + *pos != '\0' && *pos != '=' && *pos != ',' && *pos != ':'; + pos++) ; + /* httrack --wrapper callback[,foo] */ + if (*pos == 0 || *pos == ',' || *pos == ':') { + int ret; + char *moduleName; + + if (*pos == ',' || *pos == ':') { + *pos = '\0'; + moduleName = strdupt(argv[na]); + *pos = ','; /* foce seperator to ',' */ + } else { + moduleName = strdupt(argv[na]); + } + ret = plug_wrapper(opt, moduleName, argv[na]); + freet(moduleName); + if (ret == 0) { + char BIGSTK tmp[1024 * 2]; + + sprintf(tmp, + "option %%W : unable to plug the module %s (returncode != 1)", + argv[na]); + HTS_PANIC_PRINTF(tmp); + htsmain_free(); + return -1; + } else if (ret == -1) { + char BIGSTK tmp[1024 * 2]; + int last_errno = errno; + + sprintf(tmp, + "option %%W : unable to load the module %s: %s (check the library path ?)", + argv[na], strerror(last_errno)); + HTS_PANIC_PRINTF(tmp); + htsmain_free(); + return -1; + } } - ret = plug_wrapper(opt, moduleName, argv[na]); - freet(moduleName); - if (ret == 0) { - char BIGSTK tmp[1024 * 2]; - sprintf(tmp, "option %%W : unable to plug the module %s (returncode != 1)", argv[na]); - HTS_PANIC_PRINTF(tmp); + /* Old style */ + /* httrack --wrapper save-name=callback:process,string */ + else if (*pos == '=') { + fprintf(stderr, + "Syntax error in option %%W : the old (<3.41) API is no more supported!\n"); + HTS_PANIC_PRINTF + ("Syntax error in option %W : the old (<3.41) API is no more supported!"); + printf("Example: -%%W check-link=checklink.so:check\n"); htsmain_free(); return -1; - } else if (ret == -1) { - char BIGSTK tmp[1024 * 2]; - int last_errno = errno; - sprintf(tmp, "option %%W : unable to load the module %s: %s (check the library path ?)", argv[na], strerror(last_errno)); - HTS_PANIC_PRINTF(tmp); + } else { + HTS_PANIC_PRINTF + ("Syntax error in option %W : this function needs to be followed by a blank space, and a module name"); + printf("Example: -%%W check-link=checklink.so:check\n"); htsmain_free(); return -1; } } - /* Old style */ - /* httrack --wrapper save-name=callback:process,string */ - else if (*pos == '=') { - fprintf(stderr, "Syntax error in option %%W : the old (<3.41) API is no more supported!\n"); - HTS_PANIC_PRINTF("Syntax error in option %W : the old (<3.41) API is no more supported!"); - printf("Example: -%%W check-link=checklink.so:check\n"); + break; + + case 'R': // Referer + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %R needs to be followed by a blank space, and a referer URL"); + printf("Example: -%%R \"http://www.example.com/\"\n"); htsmain_free(); return -1; } else { - HTS_PANIC_PRINTF("Syntax error in option %W : this function needs to be followed by a blank space, and a module name"); - printf("Example: -%%W check-link=checklink.so:check\n"); - htsmain_free(); - return -1; + na++; + if (strlen(argv[na]) >= 254) { + HTS_PANIC_PRINTF("Referer URL too long"); + htsmain_free(); + return -1; + } + StringCopy(opt->referer, argv[na]); } - } - break; - - case 'R': // Referer - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %R needs to be followed by a blank space, and a referer URL"); - printf("Example: -%%R \"http://www.example.com/\"\n"); - htsmain_free(); - return -1; - } else{ - na++; - if (strlen(argv[na])>=254) { - HTS_PANIC_PRINTF("Referer URL too long"); + break; + case 'E': // From Email address + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option %E needs to be followed by a blank space, and an email"); + printf("Example: -%%E \"postmaster@example.com\"\n"); htsmain_free(); return -1; + } else { + na++; + if (strlen(argv[na]) >= 254) { + HTS_PANIC_PRINTF("From email too long"); + htsmain_free(); + return -1; + } + StringCopy(opt->from, argv[na]); } - StringCopy(opt->referer, argv[na]); - } - break; - case 'E': // From Email address - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option %E needs to be followed by a blank space, and an email"); - printf("Example: -%%E \"postmaster@example.com\"\n"); - htsmain_free(); - return -1; - } else{ - na++; - if (strlen(argv[na])>=254) { - HTS_PANIC_PRINTF("From email too long"); + break; + + default:{ + char s[HTS_CDLMAXSIZE + 256]; + + sprintf(s, "invalid option %%%c\n", *com); + HTS_PANIC_PRINTF(s); htsmain_free(); return -1; } - StringCopy(opt->from, argv[na]); - } - break; + break; - default: { - char s[HTS_CDLMAXSIZE + 256]; - sprintf(s,"invalid option %%%c\n",*com); - HTS_PANIC_PRINTF(s); - htsmain_free(); - return -1; - } - break; - + } } - } break; // - case '@': { // troisième jeu d'options - com++; - switch(*com) { - case 'i': + case '@':{ // troisième jeu d'options + com++; + switch (*com) { + case 'i': #if HTS_INET6==0 - printf("Warning, option @i has no effect (v6 routines not compiled)\n"); -#else - { - int res=0; - if (isdigit((unsigned char)*(com+1))) { - sscanf(com+1,"%d",&res); while(isdigit((unsigned char)*(com+1))) com++; - } - switch(res) { - case 1: - case 4: - IPV6_resolver=1; - break; - case 2: - case 6: - IPV6_resolver=2; - break; - case 0: - IPV6_resolver=0; - break; - default: - printf("Unknown flag @i%d\n", res); - htsmain_free(); - return -1; - break; + printf + ("Warning, option @i has no effect (v6 routines not compiled)\n"); +#else + { + int res = 0; + + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &res); + while(isdigit((unsigned char) *(com + 1))) + com++; + } + switch (res) { + case 1: + case 4: + IPV6_resolver = 1; + break; + case 2: + case 6: + IPV6_resolver = 2; + break; + case 0: + IPV6_resolver = 0; + break; + default: + printf("Unknown flag @i%d\n", res); + htsmain_free(); + return -1; + break; + } } - } #endif - break; - - default: { + break; + + default:{ char s[HTS_CDLMAXSIZE + 256]; - sprintf(s,"invalid option %%%c\n",*com); + + sprintf(s, "invalid option %%%c\n", *com); HTS_PANIC_PRINTF(s); htsmain_free(); return -1; - } - break; - - //case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break; + } + break; + + //case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break; + } } - } break; - + // - case '#': { // non documenté - com++; - switch(*com) { - case 'C': // list cache files : httrack -#C '*spid*.gif' will attempt to find the matching file - { - int hasFilter = 0; - int found = 0; - char* filter=NULL; - cache_back cache; - inthash cache_hashtable=inthash_new(HTS_HASH_SIZE); - int backupXFR = htsMemoryFastXfr; - int sendb = 0; - if (isdigit((unsigned char)*(com+1))) { - sscanf(com+1,"%d",&sendb); - while(isdigit((unsigned char)*(com+1))) com++; - } else sendb=0; - if (!((na+1>=argc) || (argv[na+1][0]=='-'))) { - na++; - hasFilter = 1; - filter=argv[na]; - } - htsMemoryFastXfr = 1; /* fast load */ - - memset(&cache, 0, sizeof(cache_back)); - cache.type=1; // cache? - cache.log=stdout; // log? - cache.errlog=stderr; // err log? - cache.ptr_ant=cache.ptr_last=0; // pointeur pour anticiper - cache.hashtable=(void*)cache_hashtable; /* copy backcache hash */ - cache.ro = 1; /* read only */ - if (cache.hashtable) { - char BIGSTK adr[HTS_URLMAXSIZE*2]; - char BIGSTK fil[HTS_URLMAXSIZE*2]; - char BIGSTK url[HTS_URLMAXSIZE*2]; - char linepos[256]; - int pos; - char* cacheNdx = readfile(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); - cache_init(&cache,opt); /* load cache */ - if (cacheNdx != NULL) { - char firstline[256]; - char* a = cacheNdx; - a+=cache_brstr(a, firstline); - a+=cache_brstr(a, firstline); - while ( a != NULL ) { - a=strchr(a+1,'\n'); /* start of line */ - if (a) { - htsblk r; - /* */ - a++; - /* read "host/file" */ - a+=binput(a,adr,HTS_URLMAXSIZE); - a+=binput(a,fil,HTS_URLMAXSIZE); - url[0]='\0'; - if (!link_has_authority(adr)) - strcatbuff(url, "http://"); - strcatbuff(url, adr); - strcatbuff(url, fil); - /* read position */ - a+=binput(a,linepos,200); - sscanf(linepos,"%d",&pos); - if (!hasFilter - || - (strjoker(url, filter, NULL, NULL) != NULL) - ) { - r = cache_read_ro(opt, &cache, adr, fil, "", NULL); // lire entrée cache + data - if (r.statuscode != -1) { // No errors - found++; - if (!hasFilter) { - fprintf(stdout, "%s%s%s\r\n", - (link_has_authority(adr)) ? "" : "http://", - adr, fil); - } else { - char msg[256], cdate[256]; - char BIGSTK sav[HTS_URLMAXSIZE*2]; - infostatuscode(msg, r.statuscode); - time_gmt_rfc822(cdate); - - fprintf(stdout, "HTTP/1.1 %d %s\r\n", - r.statuscode, - r.msg[0] ? r.msg : msg - ); - fprintf(stdout, "X-Host: %s\r\n", adr); - fprintf(stdout, "X-File: %s\r\n", fil); - fprintf(stdout, "X-URL: %s%s%s\r\n", - (link_has_authority(adr)) ? "" : "http://", - adr, fil); - if (url_savename(adr, fil, sav, /*former_adr*/NULL, /*former_fil*/NULL, /*referer_adr*/NULL, /*referer_fil*/NULL, - /*opt*/opt, /*liens*/NULL, /*lien_tot*/0, /*sback*/NULL, /*cache*/&cache, /*hash*/NULL, /*ptr*/0, /*numero_passe*/0, /*mime_type*/NULL)!=-1) { - if (fexist(sav)) { - fprintf(stdout, "Content-location: %s\r\n", sav); + case '#':{ // non documenté + com++; + switch (*com) { + case 'C': // list cache files : httrack -#C '*spid*.gif' will attempt to find the matching file + { + int hasFilter = 0; + int found = 0; + char *filter = NULL; + cache_back cache; + inthash cache_hashtable = inthash_new(HTS_HASH_SIZE); + int backupXFR = htsMemoryFastXfr; + int sendb = 0; + + if (isdigit((unsigned char) *(com + 1))) { + sscanf(com + 1, "%d", &sendb); + while(isdigit((unsigned char) *(com + 1))) + com++; + } else + sendb = 0; + if (!((na + 1 >= argc) || (argv[na + 1][0] == '-'))) { + na++; + hasFilter = 1; + filter = argv[na]; + } + htsMemoryFastXfr = 1; /* fast load */ + + memset(&cache, 0, sizeof(cache_back)); + cache.type = 1; // cache? + cache.log = stdout; // log? + cache.errlog = stderr; // err log? + cache.ptr_ant = cache.ptr_last = 0; // pointeur pour anticiper + cache.hashtable = (void *) cache_hashtable; /* copy backcache hash */ + cache.ro = 1; /* read only */ + if (cache.hashtable) { + char BIGSTK adr[HTS_URLMAXSIZE * 2]; + char BIGSTK fil[HTS_URLMAXSIZE * 2]; + char BIGSTK url[HTS_URLMAXSIZE * 2]; + char linepos[256]; + int pos; + char *cacheNdx = + readfile(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx")); + cache_init(&cache, opt); /* load cache */ + if (cacheNdx != NULL) { + char firstline[256]; + char *a = cacheNdx; + + a += cache_brstr(a, firstline); + a += cache_brstr(a, firstline); + while(a != NULL) { + a = strchr(a + 1, '\n'); /* start of line */ + if (a) { + htsblk r; + + /* */ + a++; + /* read "host/file" */ + a += binput(a, adr, HTS_URLMAXSIZE); + a += binput(a, fil, HTS_URLMAXSIZE); + url[0] = '\0'; + if (!link_has_authority(adr)) + strcatbuff(url, "http://"); + strcatbuff(url, adr); + strcatbuff(url, fil); + /* read position */ + a += binput(a, linepos, 200); + sscanf(linepos, "%d", &pos); + if (!hasFilter + || (strjoker(url, filter, NULL, NULL) != NULL) + ) { + r = cache_read_ro(opt, &cache, adr, fil, "", NULL); // lire entrée cache + data + if (r.statuscode != -1) { // No errors + found++; + if (!hasFilter) { + fprintf(stdout, "%s%s%s\r\n", + (link_has_authority(adr)) ? "" : + "http://", adr, fil); + } else { + char msg[256], cdate[256]; + char BIGSTK sav[HTS_URLMAXSIZE * 2]; + + infostatuscode(msg, r.statuscode); + time_gmt_rfc822(cdate); + + fprintf(stdout, "HTTP/1.1 %d %s\r\n", + r.statuscode, r.msg[0] ? r.msg : msg); + fprintf(stdout, "X-Host: %s\r\n", adr); + fprintf(stdout, "X-File: %s\r\n", fil); + fprintf(stdout, "X-URL: %s%s%s\r\n", + (link_has_authority(adr)) ? "" : + "http://", adr, fil); + if (url_savename + (adr, fil, sav, /*former_adr */ NULL, + /*former_fil */ NULL, /*referer_adr */ + NULL, /*referer_fil */ NULL, + /*opt */ opt, /*liens */ NULL, + /*lien_tot */ 0, /*sback */ NULL, + /*cache */ &cache, /*hash */ NULL, /*ptr */ + 0, /*numero_passe */ 0, /*mime_type */ + NULL) != -1) { + if (fexist(sav)) { + fprintf(stdout, "Content-location: %s\r\n", + sav); + } + } + fprintf(stdout, "Date: %s\r\n", cdate); + fprintf(stdout, + "Server: HTTrack Website Copier/" + HTTRACK_VERSION "\r\n"); + if (r.lastmodified[0]) { + fprintf(stdout, "Last-Modified: %s\r\n", + r.lastmodified); + } + if (r.etag[0]) { + fprintf(stdout, "Etag: %s\r\n", r.etag); + } + if (r.totalsize >= 0) { + fprintf(stdout, + "Content-Length: " LLintP "\r\n", + r.totalsize); + } + fprintf(stdout, + "X-Content-Length: " LLintP "\r\n", + (r.size >= 0) ? r.size : (-r.size)); + if (r.contenttype >= 0) { + fprintf(stdout, "Content-Type: %s\r\n", + r.contenttype); + } + if (r.cdispo[0]) { + fprintf(stdout, "Content-Disposition: %s\r\n", + r.cdispo); + } + if (r.contentencoding[0]) { + fprintf(stdout, "Content-Encoding: %s\r\n", + r.contentencoding); + } + if (r.is_chunk) { + fprintf(stdout, + "Transfer-Encoding: chunked\r\n"); } - } - fprintf(stdout, "Date: %s\r\n", cdate); - fprintf(stdout, "Server: HTTrack Website Copier/"HTTRACK_VERSION"\r\n"); - if (r.lastmodified[0]) { - fprintf(stdout, "Last-Modified: %s\r\n", r.lastmodified); - } - if (r.etag[0]) { - fprintf(stdout, "Etag: %s\r\n", r.etag); - } - if (r.totalsize >= 0) { - fprintf(stdout, "Content-Length: "LLintP"\r\n", r.totalsize); - } - fprintf(stdout, "X-Content-Length: "LLintP"\r\n", (r.size >= 0) ? r.size : (-r.size) ); - if (r.contenttype >= 0) { - fprintf(stdout, "Content-Type: %s\r\n", r.contenttype); - } - if (r.cdispo[0]) { - fprintf(stdout, "Content-Disposition: %s\r\n", r.cdispo); - } - if (r.contentencoding[0]) { - fprintf(stdout, "Content-Encoding: %s\r\n", r.contentencoding); - } - if (r.is_chunk) { - fprintf(stdout, "Transfer-Encoding: chunked\r\n"); - } #if HTS_USEOPENSSL - if (r.ssl) { - fprintf(stdout, "X-SSL: yes\r\n"); - } + if (r.ssl) { + fprintf(stdout, "X-SSL: yes\r\n"); + } #endif - if (r.is_write) { - fprintf(stdout, "X-Direct-To-Disk: yes\r\n"); - } - if (r.compressed) { - fprintf(stdout, "X-Compressed: yes\r\n"); - } - if (r.notmodified) { - fprintf(stdout, "X-Not-Modified: yes\r\n"); - } - if (r.is_chunk) { - fprintf(stdout, "X-Chunked: yes\r\n"); - } - fprintf(stdout, "\r\n"); - /* Send the body */ - if (sendb && r.adr) { - fprintf(stdout, "%s\r\n", r.adr); + if (r.is_write) { + fprintf(stdout, "X-Direct-To-Disk: yes\r\n"); + } + if (r.compressed) { + fprintf(stdout, "X-Compressed: yes\r\n"); + } + if (r.notmodified) { + fprintf(stdout, "X-Not-Modified: yes\r\n"); + } + if (r.is_chunk) { + fprintf(stdout, "X-Chunked: yes\r\n"); + } + fprintf(stdout, "\r\n"); + /* Send the body */ + if (sendb && r.adr) { + fprintf(stdout, "%s\r\n", r.adr); + } } } } } } + freet(cacheNdx); } - freet(cacheNdx); } + if (!found) { + fprintf(stderr, "No cache entry found%s%s%s\r\n", + (hasFilter) ? " for '" : "", + (hasFilter) ? filter : "", (hasFilter) ? "'" : ""); + } + htsMemoryFastXfr = backupXFR; + return 0; } - if (!found) { - fprintf(stderr, "No cache entry found%s%s%s\r\n", - (hasFilter)?" for '":"", - (hasFilter)?filter:"", - (hasFilter)?"'":"" - ); + break; + case 'E': // extract cache + if (!hts_extract_meta(StringBuff(opt->path_log))) { + fprintf(stderr, "* error extracting meta-data\n"); + return 1; } - htsMemoryFastXfr = backupXFR; + fprintf(stderr, "* successfully extracted meta-data\n"); return 0; - } - break; - case 'E': // extract cache - if (!hts_extract_meta(StringBuff(opt->path_log))) { - fprintf(stderr, "* error extracting meta-data\n"); - return 1; - } - fprintf(stderr, "* successfully extracted meta-data\n"); - return 0; - break; - case 'X': + break; + case 'X': #ifndef STRDEBUG - fprintf(stderr, "warning: no string debugging support built, option has no effect\n"); + fprintf(stderr, + "warning: no string debugging support built, option has no effect\n"); #endif - htsMemoryFastXfr=1; - if (*(com+1)=='0') { htsMemoryFastXfr=0; com++; } - break; - case 'R': - { - char* name; - uLong repaired = 0; - uLong repairedBytes = 0; - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip"))) { - name = fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip"); - } else if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip"))) { - name = fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip"); - } else { - fprintf(stderr, "* error: no cache found in %s\n", fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")); - return 1; + htsMemoryFastXfr = 1; + if (*(com + 1) == '0') { + htsMemoryFastXfr = 0; + com++; } - fprintf(stderr, "Cache: trying to repair %s\n", name); - if (unzRepair(name, - fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/repair.zip"), - fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/repair.tmp"), - &repaired, &repairedBytes - ) == Z_OK) { - unlink(name); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/repair.zip"), name); - fprintf(stderr,"Cache: %d bytes successfully recovered in %d entries\n", (int) repairedBytes, (int) repaired); - } else { - fprintf(stderr, "Cache: could not repair the cache\n"); + break; + case 'R': + { + char *name; + uLong repaired = 0; + uLong repairedBytes = 0; + + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip"))) { + name = + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip"); + } else + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip"))) { + name = + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip"); + } else { + fprintf(stderr, "* error: no cache found in %s\n", + fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.zip")); + return 1; + } + fprintf(stderr, "Cache: trying to repair %s\n", name); + if (unzRepair + (name, + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/repair.zip"), + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/repair.tmp"), &repaired, + &repairedBytes) == Z_OK) { + unlink(name); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/repair.zip"), name); + fprintf(stderr, + "Cache: %d bytes successfully recovered in %d entries\n", + (int) repairedBytes, (int) repaired); + } else { + fprintf(stderr, "Cache: could not repair the cache\n"); + } } - } - return 0; - break; - case '~': /* internal lib test */ - //Disabled because choke on GCC 4.3 (toni from links2linux.de) - //{ - // char thisIsATestYouShouldSeeAnError[12]; - // const char *const bufferOverflowTest = "0123456789012345678901234567890123456789"; - // strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest); - // return 0; - //} - break; - case 'f': opt->flush=1; break; - case 'h': - printf("HTTrack version "HTTRACK_VERSION"%s\n", hts_get_version_info(opt)); - return 0; - break; - case 'p': /* opt->aff_progress=1; deprecated */ break; - case 'S': opt->shell=1; break; // stdin sur un shell - case 'K': opt->keyboard=1; break; // vérifier stdin - // - case 'L': sscanf(com+1,"%d",&opt->maxlink); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'F': sscanf(com+1,"%d",&opt->maxfilter); while(isdigit((unsigned char)*(com+1))) com++; break; - case 'Z': opt->makestat=1; break; - case 'T': opt->maketrack=1; break; - case 'u': sscanf(com+1,"%d",&opt->waittime); while(isdigit((unsigned char)*(com+1))) com++; break; - - /*case 'R': // ohh ftp, catch->ftpget - HTS_PANIC_PRINTF("Unexpected internal error with -#R command"); - htsmain_free(); - return -1; - break; - */ - case 'P': { // catchurl - help_catchurl(StringBuff(opt->path_log)); - htsmain_free(); - return 0; - } - break; - - case '0': /* test #0 : filters */ - if (na+2>=argc) { - HTS_PANIC_PRINTF("Option #0 needs to be followed by a filter string and a string"); - printf("Example: '-#0' '*.gif' 'foo.gif'\n"); - htsmain_free(); - return -1; - } else { - if (strjoker(argv[na+2],argv[na+1],NULL,NULL)) - printf("%s does match %s\n",argv[na+2],argv[na+1]); - else - printf("%s does NOT match %s\n",argv[na+2],argv[na+1]); - htsmain_free(); return 0; - } - break; - case '1': /* test #1 : fil_simplifie */ - if (na+1>=argc) { - HTS_PANIC_PRINTF("Option #1 needs to be followed by an URL"); - printf("Example: '-#1' ./foo/bar/../foobar\n"); - htsmain_free(); - return -1; - } else { - fil_simplifie(argv[na+1]); - printf("simplified=%s\n", argv[na+1]); - htsmain_free(); + break; + case '~': /* internal lib test */ + //Disabled because choke on GCC 4.3 (toni from links2linux.de) + //{ + // char thisIsATestYouShouldSeeAnError[12]; + // const char *const bufferOverflowTest = "0123456789012345678901234567890123456789"; + // strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest); + // return 0; + //} + break; + case 'f': + opt->flush = 1; + break; + case 'h': + printf("HTTrack version " HTTRACK_VERSION "%s\n", + hts_get_version_info(opt)); return 0; - } - break; - case '2': // mimedefs - if (na+1>=argc) { - HTS_PANIC_PRINTF("Option #2 needs to be followed by an URL"); - printf("Example: '-#2' /foo/bar.php\n"); - htsmain_free(); - return -1; - } else { - char mime[256]; - // initialiser mimedefs - //get_userhttptype(opt,1,opt->mimedefs,NULL); - // check - mime[0] = '\0'; - get_httptype(opt, mime, argv[na+1], 0); - if (mime[0] != '\0') { - char ext[256]; - printf("%s is '%s'\n", argv[na+1], mime); - ext[0] = '\0'; - give_mimext(ext, mime); - if (ext[0]) { - printf("and its local type is '.%s'\n", ext); + break; + case 'p': /* opt->aff_progress=1; deprecated */ + break; + case 'S': + opt->shell = 1; + break; // stdin sur un shell + case 'K': + opt->keyboard = 1; + break; // vérifier stdin + // + case 'L': + sscanf(com + 1, "%d", &opt->maxlink); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'F': + sscanf(com + 1, "%d", &opt->maxfilter); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + case 'Z': + opt->makestat = 1; + break; + case 'T': + opt->maketrack = 1; + break; + case 'u': + sscanf(com + 1, "%d", &opt->waittime); + while(isdigit((unsigned char) *(com + 1))) + com++; + break; + + /*case 'R': // ohh ftp, catch->ftpget + HTS_PANIC_PRINTF("Unexpected internal error with -#R command"); + htsmain_free(); + return -1; + break; + */ + case 'P':{ // catchurl + help_catchurl(StringBuff(opt->path_log)); + htsmain_free(); + return 0; + } + break; + + case '0': /* test #0 : filters */ + if (na + 2 >= argc) { + HTS_PANIC_PRINTF + ("Option #0 needs to be followed by a filter string and a string"); + printf("Example: '-#0' '*.gif' 'foo.gif'\n"); + htsmain_free(); + return -1; + } else { + if (strjoker(argv[na + 2], argv[na + 1], NULL, NULL)) + printf("%s does match %s\n", argv[na + 2], argv[na + 1]); + else + printf("%s does NOT match %s\n", argv[na + 2], + argv[na + 1]); + htsmain_free(); + return 0; + } + break; + case '1': /* test #1 : fil_simplifie */ + if (na + 1 >= argc) { + HTS_PANIC_PRINTF("Option #1 needs to be followed by an URL"); + printf("Example: '-#1' ./foo/bar/../foobar\n"); + htsmain_free(); + return -1; + } else { + fil_simplifie(argv[na + 1]); + printf("simplified=%s\n", argv[na + 1]); + htsmain_free(); + return 0; + } + break; + case '2': // mimedefs + if (na + 1 >= argc) { + HTS_PANIC_PRINTF("Option #2 needs to be followed by an URL"); + printf("Example: '-#2' /foo/bar.php\n"); + htsmain_free(); + return -1; + } else { + char mime[256]; + + // initialiser mimedefs + //get_userhttptype(opt,1,opt->mimedefs,NULL); + // check + mime[0] = '\0'; + get_httptype(opt, mime, argv[na + 1], 0); + if (mime[0] != '\0') { + char ext[256]; + + printf("%s is '%s'\n", argv[na + 1], mime); + ext[0] = '\0'; + give_mimext(ext, mime); + if (ext[0]) { + printf("and its local type is '.%s'\n", ext); + } + } else { + printf("%s is of an unknown MIME type\n", argv[na + 1]); + } + htsmain_free(); + return 0; + } + break; + case '3': // charset tests: httrack #3 "iso-8859-1" "café" + if (argc == 3) { + char *s = + hts_convertStringToUTF8(argv[2], strlen(argv[2]), argv[1]); + if (s != NULL) { + printf(">> %s\n", s); + free(s); + } else { + fprintf(stderr, "invalid string for charset %s\n", argv[1]); } } else { - printf("%s is of an unknown MIME type\n", argv[na+1]); + HTS_PANIC_PRINTF + ("Option #3 needs to be followed by a charset and a string"); } htsmain_free(); return 0; - } - break; - case '3': // charset tests: httrack #3 "iso-8859-1" "café" - if (argc == 3) { - char *s = hts_convertStringToUTF8(argv[2], strlen(argv[2]), argv[1]); - if (s != NULL) { - printf(">> %s\n", s); - free(s); + break; + case '!': + if (na + 1 >= argc) { + HTS_PANIC_PRINTF + ("Option #! needs to be followed by a commandline"); + printf("Example: '-#!' 'echo hello'\n"); + htsmain_free(); + return -1; } else { - fprintf(stderr, "invalid string for charset %s\n", argv[1]); - } - } else { - HTS_PANIC_PRINTF("Option #3 needs to be followed by a charset and a string"); - } - htsmain_free(); - return 0; - break; - case '!': - if (na+1>=argc) { - HTS_PANIC_PRINTF("Option #! needs to be followed by a commandline"); - printf("Example: '-#!' 'echo hello'\n"); - htsmain_free(); - return -1; - } else { - int code; - if ((code = system(argv[na+1])) != 0) { - fprintf(stderr, "process returned error code %d\n", code); + int code; + + if ((code = system(argv[na + 1])) != 0) { + fprintf(stderr, "process returned error code %d\n", code); + } } - } - break; - case 'd': - opt->parsedebug = 1; - break; + break; + case 'd': + opt->parsedebug = 1; + break; - /* autotest */ - case 't': /* not yet implemented */ - fprintf(stderr, "** AUTOCHECK OK\n"); - return 0; - break; + /* autotest */ + case 't': /* not yet implemented */ + fprintf(stderr, "** AUTOCHECK OK\n"); + return 0; + break; - default: printf("Internal option %c not recognized\n",*com); break; + default: + printf("Internal option %c not recognized\n", *com); + break; + } + } + break; + case 'O': // output path + while(isdigit(com[1])) { + com++; } - } - break; - case 'O': // output path - while(isdigit(com[1])) { - com++; - } - na++; // sauter, déja traité + na++; // sauter, déja traité break; - case 'P': // proxy - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option P needs to be followed by a blank space, and a proxy proxy:port or user:id@proxy:port"); + case 'P': // proxy + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option P needs to be followed by a blank space, and a proxy proxy:port or user:id@proxy:port"); printf("Example: -P proxy.myhost.com:8080\n"); htsmain_free(); return -1; } else { - char* a; + char *a; + na++; - opt->proxy.active=1; + opt->proxy.active = 1; // Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port - a = argv[na] + strlen(argv[na]) -1; + a = argv[na] + strlen(argv[na]) - 1; // a=strstr(argv[na],":"); // port - while( (a > argv[na]) && (*a != ':') && (*a != '@') ) a--; + while((a > argv[na]) && (*a != ':') && (*a != '@')) + a--; if (*a == ':') { // un port est présent, <proxy>:port - sscanf(a+1,"%d",&opt->proxy.port); - StringCopyN(opt->proxy.name,argv[na],(int) (a - argv[na])); - } else { // <proxy> - opt->proxy.port=8080; - StringCopy(opt->proxy.name,argv[na]); + sscanf(a + 1, "%d", &opt->proxy.port); + StringCopyN(opt->proxy.name, argv[na], (int) (a - argv[na])); + } else { // <proxy> + opt->proxy.port = 8080; + StringCopy(opt->proxy.name, argv[na]); } } break; - case 'F': // user-agent field - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option F needs to be followed by a blank space, and a user-agent name"); + case 'F': // user-agent field + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option F needs to be followed by a blank space, and a user-agent name"); printf("Example: -F \"my_user_agent/1.0\"\n"); htsmain_free(); return -1; - } else{ + } else { na++; - if (strlen(argv[na])>=126) { + if (strlen(argv[na]) >= 126) { HTS_PANIC_PRINTF("User-agent length too long"); htsmain_free(); return -1; } - StringCopy(opt->user_agent,argv[na]); + StringCopy(opt->user_agent, argv[na]); if (StringNotEmpty(opt->user_agent)) - opt->user_agent_send=1; + opt->user_agent_send = 1; else - opt->user_agent_send=0; // -F "" désactive l'option + opt->user_agent_send = 0; // -F "" désactive l'option } break; // - case 'V': // execute command - if ((na+1>=argc) || (argv[na+1][0]=='-')) { - HTS_PANIC_PRINTF("Option V needs to be followed by a system-command string"); + case 'V': // execute command + if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) { + HTS_PANIC_PRINTF + ("Option V needs to be followed by a system-command string"); printf("Example: -V \"tar uvf some.tar \\$0\"\n"); htsmain_free(); return -1; - } else{ + } else { na++; - if (strlen(argv[na])>=2048) { + if (strlen(argv[na]) >= 2048) { HTS_PANIC_PRINTF("System-command length too long"); htsmain_free(); return -1; } - StringCopy(opt->sys_com,argv[na]); + StringCopy(opt->sys_com, argv[na]); if (StringNotEmpty(opt->sys_com)) - opt->sys_com_exec=1; + opt->sys_com_exec = 1; else - opt->sys_com_exec=0; // -V "" désactive l'option + opt->sys_com_exec = 0; // -V "" désactive l'option } break; // - default: { - char s[HTS_CDLMAXSIZE + 256]; - sprintf(s,"invalid option %c\n",*com); - HTS_PANIC_PRINTF(s); - htsmain_free(); - return -1; - } + default:{ + char s[HTS_CDLMAXSIZE + 256]; + + sprintf(s, "invalid option %c\n", *com); + HTS_PANIC_PRINTF(s); + htsmain_free(); + return -1; + } break; - } // switch - com++; - } // while - - } else { // URL/filters - char catbuff[CATBUFF_SIZE]; + } // switch + com++; + } // while + + } else { // URL/filters + char catbuff[CATBUFF_SIZE]; char BIGSTK tempo[CATBUFF_SIZE]; - const int urlSize = (int) strlen(argv[na]); - const int capa = (int) ( strlen(url) + urlSize + 32 ); - assertf(urlSize < HTS_URLMAXSIZE); - if (urlSize < HTS_URLMAXSIZE) { - ensureUrlCapacity(url, url_sz, capa); - if (strnotempty(url)) strcatbuff(url," "); // espace de séparation - strcpybuff(tempo,unescape_http_unharm(catbuff,argv[na],1)); - escape_spc_url(tempo); - strcatbuff(url,tempo); - } - } // if argv=- etc. - - } // for + const int urlSize = (int) strlen(argv[na]); + const int capa = (int) (strlen(url) + urlSize + 32); + + assertf(urlSize < HTS_URLMAXSIZE); + if (urlSize < HTS_URLMAXSIZE) { + ensureUrlCapacity(url, url_sz, capa); + if (strnotempty(url)) + strcatbuff(url, " "); // espace de séparation + strcpybuff(tempo, unescape_http_unharm(catbuff, argv[na], 1)); + escape_spc_url(tempo); + strcatbuff(url, tempo); + } + } // if argv=- etc. + + } // for } - -#if BDEBUG==3 - printf("URLs/filters=%s\n",url); + +#if BDEBUG==3 + printf("URLs/filters=%s\n", url); #endif #if DEBUG_STEPS printf("Analyzing parameters done\n"); #endif - #ifdef _WIN32 #else #ifndef HTS_DO_NOT_USE_UID /* Chroot - xxc */ if (switch_chroot) { - uid_t userid=getuid(); + uid_t userid = getuid(); + //struct passwd* userdef=getpwuid(userid); //if (userdef) { if (!userid) { //if (strcmp(userdef->pw_name,"root")==0) { char BIGSTK rpath[1024]; + //printf("html=%s log=%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log)); // xxc if ((StringBuff(opt->path_html)[0]) && (StringBuff(opt->path_log)[0])) { - const char *a=StringBuff(opt->path_html),*b=StringBuff(opt->path_log),*c=NULL,*d=NULL; - c=a; d=b; - while ((*a) && (*a == *b)) { - if (*a=='/') { c=a; d=b; } + const char *a = StringBuff(opt->path_html), *b = + StringBuff(opt->path_log), *c = NULL, *d = NULL; + c = a; + d = b; + while((*a) && (*a == *b)) { + if (*a == '/') { + c = a; + d = b; + } a++; b++; } - rpath[0]='\0'; + rpath[0] = '\0'; if (c != StringBuff(opt->path_html)) { - if (StringBuff(opt->path_html)[0]!='/') - strcatbuff(rpath,"./"); - strncatbuff(rpath,StringBuff(opt->path_html),(int) (c - StringBuff(opt->path_html))); + if (StringBuff(opt->path_html)[0] != '/') + strcatbuff(rpath, "./"); + strncatbuff(rpath, StringBuff(opt->path_html), + (int) (c - StringBuff(opt->path_html))); } StringCopyOverlapped(opt->path_html, c); StringCopyOverlapped(opt->path_log, d); } else { - strcpybuff(rpath,"./"); - StringCopy(opt->path_html,"/"); - StringCopy(opt->path_log,"/"); + strcpybuff(rpath, "./"); + StringCopy(opt->path_html, "/"); + StringCopy(opt->path_log, "/"); } if (rpath[0]) { - printf("[changing root path to %s (path_data=%s,path_log=%s)]\n",rpath,StringBuff(opt->path_html),StringBuff(opt->path_log)); + printf("[changing root path to %s (path_data=%s,path_log=%s)]\n", rpath, + StringBuff(opt->path_html), StringBuff(opt->path_log)); if (chroot(rpath)) { - printf("ERROR! Can not chroot to %s!\n",rpath); + printf("ERROR! Can not chroot to %s!\n", rpath); return -1; } - if (chdir("/")) { /* new root */ - printf("ERROR! Can not chdir to %s!\n",rpath); + if (chdir("/")) { /* new root */ + printf("ERROR! Can not chdir to %s!\n", rpath); return -1; } } else @@ -1914,62 +2515,128 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { } /* Setuid */ - if (switch_uid>=0) { - printf("[setting user/group to %d/%d]\n",switch_uid,switch_gid); + if (switch_uid >= 0) { + printf("[setting user/group to %d/%d]\n", switch_uid, switch_gid); if (setgid(switch_gid)) - printf("WARNING! Can not setgid to %d!\n",switch_gid); + printf("WARNING! Can not setgid to %d!\n", switch_gid); if (setuid(switch_uid)) - printf("WARNING! Can not setuid to %d!\n",switch_uid); + printf("WARNING! Can not setuid to %d!\n", switch_uid); } /* Final check */ { - uid_t userid=getuid(); + uid_t userid = getuid(); + if (!userid) { /* running as r00t */ printf("WARNING! You are running this program as root!\n"); - printf("It might be a good idea to use the -%%U option to change the userid:\n"); + printf + ("It might be a good idea to use the -%%U option to change the userid:\n"); printf("Example: -%%U smith\n\n"); } } #endif #endif - + //printf("WARNING! This is *only* a beta-release of HTTrack\n"); io_flush; - + #if DEBUG_STEPS printf("Cache & log settings\n"); #endif - + // on utilise le cache.. // en cas de présence des deux versions, garder la version la plus avancée, // cad la version contenant le plus de fichiers if (opt->cache) { - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"))) { // problemes.. - if ( fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")) ) { - if ( fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip")) ) { - if (fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip"))<32768) { - if (fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip"))>65536) { - if (fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip")) > fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip"))) { - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.zip")); + if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-in_progress.lock"))) { // problemes.. + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat"))) { + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip"))) { + if (fsize + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip")) < 32768) { + if (fsize + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip")) > 65536) { + if (fsize + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip")) > fsize(fconcat(OPT_GET_BUFF(opt), + StringBuff(opt-> + path_log), + "hts-cache/new.zip"))) + { + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.zip")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.zip")); } } } } - } - else if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")) && fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx"))) { - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat")) && fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"))) { + } else + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat")) + && + fexist(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx"))) { + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat")) + && + fexist(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx"))) { // switcher si new<32Ko et old>65Ko (tailles arbitraires) ? // ce cas est peut être une erreur ou un crash d'un miroir ancien, prendre // alors l'ancien cache - if (fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"))<32768) { - if (fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"))>65536) { - if (fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat")) > fsize(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat"))) { - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")); - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.dat"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.dat")); - rename(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/old.ndx"),fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/new.ndx")); + if (fsize + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat")) < 32768) { + if (fsize + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat")) > 65536) { + if (fsize + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat")) > fsize(fconcat(OPT_GET_BUFF(opt), + StringBuff(opt-> + path_log), + "hts-cache/new.dat"))) + { + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.dat")); + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/new.ndx")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.dat")); + rename(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), + StringBuff(opt->path_log), + "hts-cache/new.ndx")); //} else { // ne rien faire // remove("hts-cache/old.dat"); // remove("hts-cache/old.ndx"); @@ -1980,248 +2647,319 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { } } } - // Débuggage des en têtes if (_DEBUG_HEAD) { - ioinfo=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-ioinfo.txt"),"wb"); + ioinfo = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-ioinfo.txt"), + "wb"); } - + { char n_lock[256]; + // on peut pas avoir un affichage ET un fichier log // ca sera pour la version 2 - if (httrack_logmode==1) { - opt->log=stdout; - opt->errlog=stderr; - } else if (httrack_logmode>=2) { + if (httrack_logmode == 1) { + opt->log = stdout; + opt->errlog = stderr; + } else if (httrack_logmode >= 2) { // deux fichiers log structcheck(StringBuff(opt->path_log)); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-log.txt"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-log.txt")); - if (fexist(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-err.txt"))) - remove(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-err.txt")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-log.txt"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-log.txt")); + if (fexist + (fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-err.txt"))) + remove(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-err.txt")); /* Check FS directory structure created */ structcheck(StringBuff(opt->path_log)); - opt->log=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-log.txt"),"w"); - if (httrack_logmode==2) - opt->errlog=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-err.txt"),"w"); + opt->log = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-log.txt"), + "w"); + if (httrack_logmode == 2) + opt->errlog = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-err.txt"), + "w"); else - opt->errlog=opt->log; - if (opt->log==NULL) { + opt->errlog = opt->log; + if (opt->log == NULL) { char s[HTS_CDLMAXSIZE + 256]; - sprintf(s,"Unable to create log file %s",fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-log.txt")); + + sprintf(s, "Unable to create log file %s", + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-log.txt")); HTS_PANIC_PRINTF(s); htsmain_free(); return -1; - } else if (opt->errlog==NULL) { + } else if (opt->errlog == NULL) { char s[HTS_CDLMAXSIZE + 256]; - sprintf(s,"Unable to create log file %s",fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-err.txt")); + + sprintf(s, "Unable to create log file %s", + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-err.txt")); HTS_PANIC_PRINTF(s); htsmain_free(); return -1; } } else { - opt->log=NULL; - opt->errlog=NULL; + opt->log = NULL; + opt->errlog = NULL; } - + // un petit lock-file pour indiquer un miroir en cours, ainsi qu'un éventuel fichier log { - FILE* fp=NULL; + FILE *fp = NULL; + //int n=0; char t[256]; - time_local_rfc822(t); // faut bien que ca serve quelque part l'heure RFC1945 arf' - + + time_local_rfc822(t); // faut bien que ca serve quelque part l'heure RFC1945 arf' + /* readme for information purpose */ { - FILE* fp=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/readme.txt"),"wb"); + FILE *fp = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/readme.txt"), "wb"); if (fp) { - fprintf(fp,"What's in this folder?"LF); - fprintf(fp,""LF); - fprintf(fp,"This folder (hts-cache) has been generated by WinHTTrack "HTTRACK_VERSION"%s"LF, hts_get_version_info(opt)); - fprintf(fp,"and is used for updating this website."LF); - fprintf(fp,"(The HTML website structure is stored here to allow fast updates)"LF""LF); - fprintf(fp,"DO NOT delete this folder unless you do not want to update the mirror in the future!!"LF); - fprintf(fp,"(you can safely delete old.zip and old.lst files, however)"LF); - fprintf(fp,""LF); - fprintf(fp,HTS_LOG_SECURITY_WARNING); + fprintf(fp, "What's in this folder?" LF); + fprintf(fp, "" LF); + fprintf(fp, + "This folder (hts-cache) has been generated by WinHTTrack " + HTTRACK_VERSION "%s" LF, hts_get_version_info(opt)); + fprintf(fp, "and is used for updating this website." LF); + fprintf(fp, + "(The HTML website structure is stored here to allow fast updates)" + LF "" LF); + fprintf(fp, + "DO NOT delete this folder unless you do not want to update the mirror in the future!!" + LF); + fprintf(fp, + "(you can safely delete old.zip and old.lst files, however)" + LF); + fprintf(fp, "" LF); + fprintf(fp, HTS_LOG_SECURITY_WARNING); fclose(fp); } } - strcpy(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-in_progress.lock")); + strcpy(n_lock, + fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-in_progress.lock")); //sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n); /*do { - if (!n) - sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n); - else - sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress%d.lock"),n); - n++; - } while((fexist(n_lock)) && opt->quiet); - if (fexist(n_lock)) { - if (!recuperer) { - remove(n_lock); - } - }*/ + if (!n) + sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n); + else + sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress%d.lock"),n); + n++; + } while((fexist(n_lock)) && opt->quiet); + if (fexist(n_lock)) { + if (!recuperer) { + remove(n_lock); + } + } */ // vérifier existence de la structure structcheck(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_html), "/")); structcheck(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), "/")); - + // reprise/update if (opt->cache) { - FILE* fp; + FILE *fp; int i; + #ifdef _WIN32 - mkdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache")); + mkdir(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache")); #else - mkdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache"),HTS_PROTECT_FOLDER); + mkdir(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), "hts-cache"), + HTS_PROTECT_FOLDER); #endif - fp=fopen(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-cache/doit.log"),"wb"); + fp = + fopen(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + "hts-cache/doit.log"), "wb"); if (fp) { - for(i=0+1;i<argc;i++) { - if ( ((strchr(argv[i],' ')!=NULL) || (strchr(argv[i],'"')!=NULL) || (strchr(argv[i],'\\')!=NULL)) && (argv[i][0]!='"') ) { + for(i = 0 + 1; i < argc; i++) { + if (((strchr(argv[i], ' ') != NULL) + || (strchr(argv[i], '"') != NULL) + || (strchr(argv[i], '\\') != NULL)) && (argv[i][0] != '"')) { int j; - fprintf(fp,"\""); - for(j=0;j<(int) strlen(argv[i]);j++) { - if (argv[i][j]==34) - fprintf(fp,"\\\""); - else if (argv[i][j]=='\\') - fprintf(fp,"\\\\"); + + fprintf(fp, "\""); + for(j = 0; j < (int) strlen(argv[i]); j++) { + if (argv[i][j] == 34) + fprintf(fp, "\\\""); + else if (argv[i][j] == '\\') + fprintf(fp, "\\\\"); else - fprintf(fp,"%c",argv[i][j]); + fprintf(fp, "%c", argv[i][j]); } - fprintf(fp,"\""); - } else if (strnotempty(argv[i])==0) { // "" - fprintf(fp,"\"\""); - } else { // non critique - fprintf(fp,"%s",argv[i]); + fprintf(fp, "\""); + } else if (strnotempty(argv[i]) == 0) { // "" + fprintf(fp, "\"\""); + } else { // non critique + fprintf(fp, "%s", argv[i]); } - if (i<argc-1) - fprintf(fp," "); + if (i < argc - 1) + fprintf(fp, " "); } - fprintf(fp,LF); - fprintf(fp,"File generated automatically on %s, do NOT edit"LF,t); - fprintf(fp,LF); - fprintf(fp,"To update a mirror, just launch httrack without any parameters"LF); - fprintf(fp,"The existing cache will be used (and modified)"LF); - fprintf(fp,"To have other options, retype all parameters and launch HTTrack"LF); - fprintf(fp,"To continue an interrupted mirror, just launch httrack without any parameters"LF); - fprintf(fp,LF); - fclose(fp); fp=NULL; - //} else if (opt->debug>1) { - // printf("! FileOpen error, \"%s\"\n",strerror(errno)); + fprintf(fp, LF); + fprintf(fp, "File generated automatically on %s, do NOT edit" LF, t); + fprintf(fp, LF); + fprintf(fp, + "To update a mirror, just launch httrack without any parameters" + LF); + fprintf(fp, "The existing cache will be used (and modified)" LF); + fprintf(fp, + "To have other options, retype all parameters and launch HTTrack" + LF); + fprintf(fp, + "To continue an interrupted mirror, just launch httrack without any parameters" + LF); + fprintf(fp, LF); + fclose(fp); + fp = NULL; + //} else if (opt->debug>1) { + // printf("! FileOpen error, \"%s\"\n",strerror(errno)); } } - // petit message dans le lock - if ( (fp=fopen(n_lock,"wb"))!=NULL) { + if ((fp = fopen(n_lock, "wb")) != NULL) { int i; - fprintf(fp,"Mirror in progress since %s .. please wait!"LF,t); - for(i=0;i<argc;i++) { - if (strchr(argv[i],' ')==NULL) - fprintf(fp,"%s ",argv[i]); - else // entre "" - fprintf(fp,"\"%s\" ",argv[i]); + + fprintf(fp, "Mirror in progress since %s .. please wait!" LF, t); + for(i = 0; i < argc; i++) { + if (strchr(argv[i], ' ') == NULL) + fprintf(fp, "%s ", argv[i]); + else // entre "" + fprintf(fp, "\"%s\" ", argv[i]); } - fprintf(fp,LF); - fprintf(fp, "To pause the engine: create an empty file named 'hts-stop.lock'"LF); + fprintf(fp, LF); + fprintf(fp, + "To pause the engine: create an empty file named 'hts-stop.lock'" + LF); #if USE_BEGINTHREAD - fprintf(fp, "PID=%d\n", (int)getpid()); + fprintf(fp, "PID=%d\n", (int) getpid()); #ifndef _WIN32 - fprintf(fp, "UID=%d\n", (int)getuid()); - fprintf(fp, "GID=%d\n", (int)getuid()); + fprintf(fp, "UID=%d\n", (int) getuid()); + fprintf(fp, "GID=%d\n", (int) getuid()); #endif - fprintf(fp, "START=%d\n", (int)time(NULL)); + fprintf(fp, "START=%d\n", (int) time(NULL)); #endif - fclose(fp); fp=NULL; + fclose(fp); + fp = NULL; } - // fichier log if (opt->log) { int i; - fprintf(opt->log,"HTTrack"HTTRACK_VERSION"%s launched on %s at %s"LF, - hts_get_version_info(opt), - t, url); - fprintf(opt->log,"("); - for(i=0;i<argc;i++) { + + fprintf(opt->log, + "HTTrack" HTTRACK_VERSION "%s launched on %s at %s" LF, + hts_get_version_info(opt), t, url); + fprintf(opt->log, "("); + for(i = 0; i < argc; i++) { #ifdef _WIN32 - char *carg = hts_convertStringSystemToUTF8(argv[i], (int) strlen(argv[i])); + char *carg = + hts_convertStringSystemToUTF8(argv[i], (int) strlen(argv[i])); char *arg = carg != NULL ? carg : argv[i]; #else const char *arg = argv[i]; #endif if (strchr(arg, ' ') == NULL || strchr(arg, '\"') != NULL) - fprintf(opt->log,"%s ", arg); - else // entre "" (si espace(s) et pas déja de ") - fprintf(opt->log,"\"%s\" ", arg); + fprintf(opt->log, "%s ", arg); + else // entre "" (si espace(s) et pas déja de ") + fprintf(opt->log, "\"%s\" ", arg); #ifdef _WIN32 if (carg != NULL) free(carg); #endif } - fprintf(opt->log,")"LF); - fprintf(opt->log,LF); - fprintf(opt->log,"Information, Warnings and Errors reported for this mirror:"LF); - fprintf(opt->log,HTS_LOG_SECURITY_WARNING ); - fprintf(opt->log,LF); + fprintf(opt->log, ")" LF); + fprintf(opt->log, LF); + fprintf(opt->log, + "Information, Warnings and Errors reported for this mirror:" + LF); + fprintf(opt->log, HTS_LOG_SECURITY_WARNING); + fprintf(opt->log, LF); } if (httrack_logmode) { - printf("Mirror launched on %s by HTTrack Website Copier/"HTTRACK_VERSION"%s "HTTRACK_AFF_AUTHORS""LF,t,hts_get_version_info(opt)); - if (opt->wizard==0) { - printf("mirroring %s with %d levels, %d sockets,t=%d,s=%d,logm=%d,lnk=%d,mdg=%d\n",url,opt->depth,opt->maxsoc,opt->travel,opt->seeker,httrack_logmode,opt->urlmode,opt->getmode); - } else { // the magic wizard - printf("mirroring %s with the wizard help..\n",url); + printf("Mirror launched on %s by HTTrack Website Copier/" + HTTRACK_VERSION "%s " HTTRACK_AFF_AUTHORS "" LF, t, + hts_get_version_info(opt)); + if (opt->wizard == 0) { + printf + ("mirroring %s with %d levels, %d sockets,t=%d,s=%d,logm=%d,lnk=%d,mdg=%d\n", + url, opt->depth, opt->maxsoc, opt->travel, opt->seeker, + httrack_logmode, opt->urlmode, opt->getmode); + } else { // the magic wizard + printf("mirroring %s with the wizard help..\n", url); } } } - + io_flush; /* Enforce limits to avoid bandwith abuse. The bypass_limits should only be used by administrators and experts. */ if (!opt->bypass_limits) { if (opt->maxsoc <= 0 || opt->maxsoc > 8) { opt->maxsoc = 8; - hts_log_print(opt, LOG_WARNING, "* security warning: maximum number of simultaneous connections limited to %d to avoid server overload", (int)opt->maxsoc); + hts_log_print(opt, LOG_WARNING, + "* security warning: maximum number of simultaneous connections limited to %d to avoid server overload", + (int) opt->maxsoc); } if (opt->maxrate <= 0 || opt->maxrate > 250000) { opt->maxrate = 250000; - hts_log_print(opt, LOG_WARNING, "* security warning: maximum bandwidth limited to %d to avoid server overload", (int)opt->maxrate); + hts_log_print(opt, LOG_WARNING, + "* security warning: maximum bandwidth limited to %d to avoid server overload", + (int) opt->maxrate); } if (opt->maxconn <= 0 || opt->maxconn > 5.0) { opt->maxconn = 5.0; - hts_log_print(opt, LOG_WARNING, "* security warning: maximum number of connections per second limited to %f to avoid server overload", (float)opt->maxconn); + hts_log_print(opt, LOG_WARNING, + "* security warning: maximum number of connections per second limited to %f to avoid server overload", + (float) opt->maxconn); } } else { - hts_log_print(opt, LOG_WARNING, "* security warning: !!! BYPASSING SECURITY LIMITS - MONITOR THIS SESSION WITH EXTREME CARE !!!"); + hts_log_print(opt, LOG_WARNING, + "* security warning: !!! BYPASSING SECURITY LIMITS - MONITOR THIS SESSION WITH EXTREME CARE !!!"); } - /* Info for wrappers */ - hts_log_print(opt, LOG_DEBUG, "engine: init"); + /* Info for wrappers */ + hts_log_print(opt, LOG_DEBUG, "engine: init"); - /* Init external */ - RUN_CALLBACK_NOARG(opt, init); + /* Init external */ + RUN_CALLBACK_NOARG(opt, init); - // détourner SIGHUP etc. + // détourner SIGHUP etc. #if DEBUG_STEPS - printf("Launching the mirror\n"); + printf("Launching the mirror\n"); #endif // Lancement du miroir // ------------------------------------------------------------ opt->state._hts_in_mirror = 1; - if (httpmirror(url, opt)==0) { - printf("Error during operation (see log file), site has not been successfully mirrored\n"); + if (httpmirror(url, opt) == 0) { + printf + ("Error during operation (see log file), site has not been successfully mirrored\n"); } else { - if (opt->shell) { + if (opt->shell) { HTT_REQUEST_START; - HT_PRINT("TRANSFER DONE"LF); - HTT_REQUEST_END - } else { + HT_PRINT("TRANSFER DONE" LF); + HTT_REQUEST_END} else { printf("Done.\n"); } } @@ -2231,24 +2969,26 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { // // Build top index if (opt->dir_topindex) { - char BIGSTK rpath[1024*2]; - char* a; - strcpybuff(rpath,StringBuff(opt->path_html)); + char BIGSTK rpath[1024 * 2]; + char *a; + + strcpybuff(rpath, StringBuff(opt->path_html)); if (rpath[0]) { - if (rpath[strlen(rpath)-1]=='/') - rpath[strlen(rpath)-1]='\0'; + if (rpath[strlen(rpath) - 1] == '/') + rpath[strlen(rpath) - 1] = '\0'; } - a=strrchr(rpath,'/'); + a = strrchr(rpath, '/'); if (a) { - *a='\0'; - hts_buildtopindex(opt,rpath,StringBuff(opt->path_bin)); + *a = '\0'; + hts_buildtopindex(opt, rpath, StringBuff(opt->path_bin)); hts_log_print(opt, LOG_INFO, "Top index rebuilt (done)"); } } - if (opt->state.exit_xh ==1) { + if (opt->state.exit_xh == 1) { if (opt->log) { - fprintf(opt->log,"* * MIRROR ABORTED! * *\nThe current temporary cache is required for any update operation and only contains data downloaded during the present aborted session.\nThe former cache might contain more complete information; if you do not want to lose that information, you have to restore it and delete the current cache.\nThis can easily be done here by erasing the hts-cache/new.* files]\n"); + fprintf(opt->log, + "* * MIRROR ABORTED! * *\nThe current temporary cache is required for any update operation and only contains data downloaded during the present aborted session.\nThe former cache might contain more complete information; if you do not want to lose that information, you have to restore it and delete the current cache.\nThis can easily be done here by erasing the hts-cache/new.* files]\n"); } } @@ -2257,20 +2997,26 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { // erase ref files if not interrupted DIR *dir; struct dirent *entry; - for(dir = opendir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME)) - ; dir != NULL && ( entry = readdir(dir) ) != NULL - ; ) - { + + for(dir = + opendir(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), + CACHE_REFNAME)); + dir != NULL && (entry = readdir(dir)) != NULL;) { if (entry->d_name[0] != '\0' && entry->d_name[0] != '.') { char *f = OPT_GET_BUFF(opt); + sprintf(f, "%s/%s", CACHE_REFNAME, entry->d_name); - (void)unlink(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), f)); + (void) + unlink(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), f)); } } if (dir != NULL) { (void) closedir(dir); } - (void)rmdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME)); + (void) + rmdir(fconcat + (OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME)); } /* Info for wrappers */ @@ -2279,19 +3025,24 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { /* UnInit */ RUN_CALLBACK_NOARG(opt, uninit); - if (httrack_logmode!=1) { - if (opt->errlog == opt->log) opt->errlog=NULL; - if (opt->log) { fclose(opt->log); opt->log=NULL; } - if (opt->errlog) { fclose(opt->errlog); opt->errlog=NULL; } - } - + if (httrack_logmode != 1) { + if (opt->errlog == opt->log) + opt->errlog = NULL; + if (opt->log) { + fclose(opt->log); + opt->log = NULL; + } + if (opt->errlog) { + fclose(opt->errlog); + opt->errlog = NULL; + } + } // Débuggage des en têtes if (_DEBUG_HEAD) { if (ioinfo) { fclose(ioinfo); } } - // supprimer lock remove(n_lock); } @@ -2310,61 +3061,60 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { printf("Thanks for using HTTrack!\n"); io_flush; htsmain_free(); - return 0; // OK + return 0; // OK } - // main() subroutines // vérifier chemin path -int check_path(String* s, char* defaultname) { +int check_path(String * s, char *defaultname) { int i; - int return_value=0; + int return_value = 0; // Replace name: ~/mywebsites/# -> /home/foo/mywebsites/# expand_home(s); - for(i = 0 ; i < (int) StringLength(*s) ; i++) // conversion \ -> / + for(i = 0; i < (int) StringLength(*s); i++) // conversion \ -> / if (StringSub(*s, i) == '\\') StringSubRW(*s, i) = '/'; - + // remove ending / if (StringNotEmpty(*s) && StringRight(*s, 1) == '/') StringPopRight(*s); - // Replace name: /home/foo/mywebsites/# -> /home/foo/mywebsites/wonderfulsite + // Replace name: /home/foo/mywebsites/# -> /home/foo/mywebsites/wonderfulsite if (StringNotEmpty(*s)) { if (StringRight(*s, 1) == '#') { - if (strnotempty((defaultname?defaultname:""))) { - char* a = strchr(defaultname,'#'); // we never know.. + if (strnotempty((defaultname ? defaultname : ""))) { + char *a = strchr(defaultname, '#'); // we never know.. + if (a) - *a='\0'; + *a = '\0'; StringPopRight(*s); StringCat(*s, defaultname); } else { - StringClear(*s); // Clear path (no name/default url given) + StringClear(*s); // Clear path (no name/default url given) } - return_value=1; // expanded + return_value = 1; // expanded } } - // ending / - if (StringNotEmpty(*s) && StringRight(*s, 1) != '/') // ajouter slash à la fin + if (StringNotEmpty(*s) && StringRight(*s, 1) != '/') // ajouter slash à la fin StringCat(*s, "/"); return return_value; } // détermine si l'argument est une option -int cmdl_opt(char* s) { - if (s[0]=='-') { // c'est peut être une option - if (strchr(s,'.')!=NULL && strchr(s,'%')==NULL) - return 0; // sans doute un -www.truc.fr (note: -www n'est pas compris) - else if (strchr(s,'/')!=NULL) - return 0; // idem, -*cgi-bin/ - else if (strchr(s,'*')!=NULL) - return 0; // joker, idem +int cmdl_opt(char *s) { + if (s[0] == '-') { // c'est peut être une option + if (strchr(s, '.') != NULL && strchr(s, '%') == NULL) + return 0; // sans doute un -www.truc.fr (note: -www n'est pas compris) + else if (strchr(s, '/') != NULL) + return 0; // idem, -*cgi-bin/ + else if (strchr(s, '*') != NULL) + return 0; // joker, idem else return 1; - } else return 0; + } else + return 0; } - diff --git a/src/htscoremain.h b/src/htscoremain.h index 1de4c36..b6c435e 100644 --- a/src/htscoremain.h +++ b/src/htscoremain.h @@ -17,28 +17,23 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* main routine (first called) */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - - #ifndef HTSMAINHSR_DEFH -#define HTSMAINHSR_DEFH +#define HTSMAINHSR_DEFH // --assume standard #define HTS_ASSUME_STANDARD \ @@ -52,13 +47,12 @@ Please visit our Website: http://www.httrack.com // Main, récupère les paramètres et appelle le robot #ifndef HTTRACK_DEFLIB HTSEXT_API int hts_main(int argc, char **argv); -HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt); +HTSEXT_API int hts_main2(int argc, char **argv, httrackp * opt); #endif -int cmdl_opt(char* s); -int check_path(String* s,char* defaultname); +int cmdl_opt(char *s); +int check_path(String * s, char *defaultname); #endif - #endif diff --git a/src/htsdefines.h b/src/htsdefines.h index 8ca9c67..3d6db0a 100644 --- a/src/htsdefines.h +++ b/src/htsdefines.h @@ -23,11 +23,9 @@ Important notes: emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Some defines for httrack.c and others */ /* Author: Xavier Roche */ @@ -72,71 +70,94 @@ typedef struct t_hts_callbackarg t_hts_callbackarg; #ifdef _WIN32 #define EXTERNAL_FUNCTION __declspec(dllexport) #else -#define EXTERNAL_FUNCTION +#define EXTERNAL_FUNCTION #endif #endif /* --wrapper plug function prototype */ -typedef int (*t_hts_plug)(httrackp *opt, const char* argv); -typedef int (*t_hts_unplug)(httrackp *opt); +typedef int (*t_hts_plug) (httrackp * opt, const char *argv); +typedef int (*t_hts_unplug) (httrackp * opt); /* htsopt function callbacks definitions */ -typedef void (* t_hts_htmlcheck_init)(t_hts_callbackarg *carg); -typedef void (* t_hts_htmlcheck_uninit)(t_hts_callbackarg *carg); -typedef int (* t_hts_htmlcheck_start)(t_hts_callbackarg *carg, httrackp* opt); -typedef int (* t_hts_htmlcheck_end)(t_hts_callbackarg *carg, httrackp *opt); -typedef int (* t_hts_htmlcheck_chopt)(t_hts_callbackarg *carg, httrackp* opt); -typedef int (* t_hts_htmlcheck_process)(t_hts_callbackarg *carg, httrackp *opt, - char** html, int* len, const char* url_adresse, const char* url_fichier); +typedef void (*t_hts_htmlcheck_init) (t_hts_callbackarg * carg); +typedef void (*t_hts_htmlcheck_uninit) (t_hts_callbackarg * carg); +typedef int (*t_hts_htmlcheck_start) (t_hts_callbackarg * carg, httrackp * opt); +typedef int (*t_hts_htmlcheck_end) (t_hts_callbackarg * carg, httrackp * opt); +typedef int (*t_hts_htmlcheck_chopt) (t_hts_callbackarg * carg, httrackp * opt); +typedef int (*t_hts_htmlcheck_process) (t_hts_callbackarg * carg, + httrackp * opt, char **html, int *len, + const char *url_adresse, + const char *url_fichier); typedef t_hts_htmlcheck_process t_hts_htmlcheck_preprocess; typedef t_hts_htmlcheck_process t_hts_htmlcheck_postprocess; -typedef int (* t_hts_htmlcheck_check_html)(t_hts_callbackarg *carg, httrackp *opt, - char* html, int len, const char* url_adresse, const char* url_fichier); -typedef const char* (* t_hts_htmlcheck_query)(t_hts_callbackarg *carg, httrackp *opt, - const char* question); -typedef const char* (* t_hts_htmlcheck_query2)(t_hts_callbackarg *carg, httrackp *opt, - const char* question); -typedef const char* (* t_hts_htmlcheck_query3)(t_hts_callbackarg *carg, httrackp *opt, - const char* question); -typedef int (* t_hts_htmlcheck_loop)(t_hts_callbackarg *carg, httrackp *opt, - lien_back* back, int back_max, int back_index, - int lien_tot, int lien_ntot, - int stat_time, hts_stat_struct* stats); -typedef int (* t_hts_htmlcheck_check_link)(t_hts_callbackarg *carg, httrackp *opt, - const char* adr, const char* fil, int status); -typedef int (* t_hts_htmlcheck_check_mime)(t_hts_callbackarg *carg, httrackp *opt, - const char* adr, const char* fil, const char* mime, int status); -typedef void (* t_hts_htmlcheck_pause)(t_hts_callbackarg *carg, httrackp *opt, - const char* lockfile); -typedef void (* t_hts_htmlcheck_filesave)(t_hts_callbackarg *carg, httrackp *opt, - const char* file); -typedef void (* t_hts_htmlcheck_filesave2)(t_hts_callbackarg *carg, httrackp *opt, - const char* hostname, const char* filename, const char* localfile, - int is_new, int is_modified, int not_updated); -typedef int (* t_hts_htmlcheck_linkdetected)(t_hts_callbackarg *carg, httrackp *opt, - char* link); -typedef int (* t_hts_htmlcheck_linkdetected2)(t_hts_callbackarg *carg, httrackp *opt, - char* link, const char* tag_start); -typedef int (* t_hts_htmlcheck_xfrstatus)(t_hts_callbackarg *carg, httrackp *opt, - lien_back* back); -typedef int (* t_hts_htmlcheck_savename)(t_hts_callbackarg *carg, httrackp *opt, - const char* adr_complete, const char* fil_complete, - const char* referer_adr, const char* referer_fil, char* save); +typedef int (*t_hts_htmlcheck_check_html) (t_hts_callbackarg * carg, + httrackp * opt, char *html, int len, + const char *url_adresse, + const char *url_fichier); +typedef const char *(*t_hts_htmlcheck_query) (t_hts_callbackarg * carg, + httrackp * opt, + const char *question); +typedef const char *(*t_hts_htmlcheck_query2) (t_hts_callbackarg * carg, + httrackp * opt, + const char *question); +typedef const char *(*t_hts_htmlcheck_query3) (t_hts_callbackarg * carg, + httrackp * opt, + const char *question); +typedef int (*t_hts_htmlcheck_loop) (t_hts_callbackarg * carg, httrackp * opt, + lien_back * back, int back_max, + int back_index, int lien_tot, + int lien_ntot, int stat_time, + hts_stat_struct * stats); +typedef int (*t_hts_htmlcheck_check_link) (t_hts_callbackarg * carg, + httrackp * opt, const char *adr, + const char *fil, int status); +typedef int (*t_hts_htmlcheck_check_mime) (t_hts_callbackarg * carg, + httrackp * opt, const char *adr, + const char *fil, const char *mime, + int status); +typedef void (*t_hts_htmlcheck_pause) (t_hts_callbackarg * carg, httrackp * opt, + const char *lockfile); +typedef void (*t_hts_htmlcheck_filesave) (t_hts_callbackarg * carg, + httrackp * opt, const char *file); +typedef void (*t_hts_htmlcheck_filesave2) (t_hts_callbackarg * carg, + httrackp * opt, const char *hostname, + const char *filename, + const char *localfile, int is_new, + int is_modified, int not_updated); +typedef int (*t_hts_htmlcheck_linkdetected) (t_hts_callbackarg * carg, + httrackp * opt, char *link); +typedef int (*t_hts_htmlcheck_linkdetected2) (t_hts_callbackarg * carg, + httrackp * opt, char *link, + const char *tag_start); +typedef int (*t_hts_htmlcheck_xfrstatus) (t_hts_callbackarg * carg, + httrackp * opt, lien_back * back); +typedef int (*t_hts_htmlcheck_savename) (t_hts_callbackarg * carg, + httrackp * opt, + const char *adr_complete, + const char *fil_complete, + const char *referer_adr, + const char *referer_fil, char *save); typedef t_hts_htmlcheck_savename t_hts_htmlcheck_extsavename; -typedef int (* t_hts_htmlcheck_sendhead)(t_hts_callbackarg *carg, httrackp *opt, - char* buff, const char* adr, const char* fil, - const char* referer_adr, const char* referer_fil, - htsblk* outgoing); -typedef int (* t_hts_htmlcheck_receivehead)(t_hts_callbackarg *carg, httrackp *opt, - char* buff, const char* adr, const char* fil, - const char* referer_adr, const char* referer_fil, - htsblk* incoming); +typedef int (*t_hts_htmlcheck_sendhead) (t_hts_callbackarg * carg, + httrackp * opt, char *buff, + const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, + htsblk * outgoing); +typedef int (*t_hts_htmlcheck_receivehead) (t_hts_callbackarg * carg, + httrackp * opt, char *buff, + const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, + htsblk * incoming); /* External additional parsing module(s) */ -typedef int (*t_hts_htmlcheck_detect)(t_hts_callbackarg *carg, httrackp *opt, htsmoduleStruct* str); -typedef int (*t_hts_htmlcheck_parse)(t_hts_callbackarg *carg, httrackp *opt, htsmoduleStruct* str); +typedef int (*t_hts_htmlcheck_detect) (t_hts_callbackarg * carg, httrackp * opt, + htsmoduleStruct * str); +typedef int (*t_hts_htmlcheck_parse) (t_hts_callbackarg * carg, httrackp * opt, + htsmoduleStruct * str); /* Callbacks */ #ifndef HTS_DEF_FWSTRUCT_t_hts_htmlcheck_callbacks @@ -152,7 +173,7 @@ typedef struct t_hts_htmlcheck_callbacks t_hts_htmlcheck_callbacks; } NAME /* Callback items */ -typedef void* t_hts_htmlcheck_t_hts_htmlcheck_callbacks_item; +typedef void *t_hts_htmlcheck_t_hts_htmlcheck_callbacks_item; typedef DEFCALLBACK(t_hts_htmlcheck_callbacks_item); /* Linked list, which should be used for the 'arg' user-defined argument */ @@ -170,33 +191,33 @@ struct t_hts_callbackarg { /* Callback structure */ struct t_hts_htmlcheck_callbacks { /* v3.41 */ - DEFCALLBACK(init); - DEFCALLBACK(uninit); - DEFCALLBACK(start); - DEFCALLBACK(end); - DEFCALLBACK(chopt); - DEFCALLBACK(preprocess); - DEFCALLBACK(postprocess); - DEFCALLBACK(check_html); - DEFCALLBACK(query); - DEFCALLBACK(query2); - DEFCALLBACK(query3); - DEFCALLBACK(loop); - DEFCALLBACK(check_link); - DEFCALLBACK(check_mime); - DEFCALLBACK(pause); - DEFCALLBACK(filesave); - DEFCALLBACK(filesave2); - DEFCALLBACK(linkdetected); - DEFCALLBACK(linkdetected2); - DEFCALLBACK(xfrstatus); - DEFCALLBACK(savename); - DEFCALLBACK(sendhead); - DEFCALLBACK(receivehead); + DEFCALLBACK(init); + DEFCALLBACK(uninit); + DEFCALLBACK(start); + DEFCALLBACK(end); + DEFCALLBACK(chopt); + DEFCALLBACK(preprocess); + DEFCALLBACK(postprocess); + DEFCALLBACK(check_html); + DEFCALLBACK(query); + DEFCALLBACK(query2); + DEFCALLBACK(query3); + DEFCALLBACK(loop); + DEFCALLBACK(check_link); + DEFCALLBACK(check_mime); + DEFCALLBACK(pause); + DEFCALLBACK(filesave); + DEFCALLBACK(filesave2); + DEFCALLBACK(linkdetected); + DEFCALLBACK(linkdetected2); + DEFCALLBACK(xfrstatus); + DEFCALLBACK(savename); + DEFCALLBACK(sendhead); + DEFCALLBACK(receivehead); DEFCALLBACK(detect); DEFCALLBACK(parse); /* >3.41 */ - DEFCALLBACK(extsavename); + DEFCALLBACK(extsavename); }; /* Library internal definitions */ @@ -207,8 +228,8 @@ struct t_hts_htmlcheck_callbacks { typedef struct t_hts_callback_ref t_hts_callback_ref; #endif struct t_hts_callback_ref { - const char *name; - size_t offset; + const char *name; + size_t offset; }; extern const t_hts_htmlcheck_callbacks default_callbacks; @@ -216,14 +237,13 @@ extern const t_hts_callback_ref default_callbacks_ref[]; #define HT_PRINT(A) strcatbuff(opt->state.HTbuff,A); #define HT_REQUEST_START opt->state.HTbuff[0]='\0'; -#define HT_REQUEST_END +#define HT_REQUEST_END #define HTT_REQUEST_START opt->state.HTbuff[0]='\0'; -#define HTT_REQUEST_END +#define HTT_REQUEST_END #define HTS_REQUEST_START opt->state.HTbuff[0]='\0'; -#define HTS_REQUEST_END +#define HTS_REQUEST_END #define HTS_PANIC_PRINTF(S) strcpybuff(opt->state._hts_errmsg,S); #endif #endif - diff --git a/src/htsfilters.c b/src/htsfilters.c index 4cc740b..d7b6737 100644 --- a/src/htsfilters.c +++ b/src/htsfilters.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* filters ("regexp") */ @@ -38,7 +35,6 @@ Please visit our Website: http://www.httrack.com /* Internal engine bytecode */ #define HTS_INTERNAL_BYTECODE - // *.gif match all gif files // *[file]/*[file].exe match all exe files with one folder structure // *[A-Z,a-z,0-9,/,?] match letters, nums, / and ? @@ -59,43 +55,45 @@ Please visit our Website: http://www.httrack.com // optionnel: taille à contrôller (ou numéro, etc) en pointeur // (en de détection de *size, la taille limite est écrite par dessus *size) // exemple: +-*.gif*[<5] == supprimer GIF si <5KB -int fa_strjoker(int type,char** filters,int nfil,char* nom,LLint* size,int* size_flag,int* depth) { - int verdict = 0; // on sait pas +int fa_strjoker(int type, char **filters, int nfil, char *nom, LLint * size, + int *size_flag, int *depth) { + int verdict = 0; // on sait pas int i; - LLint sizelimit=0; + LLint sizelimit = 0; + if (size) - sizelimit=*size; - for(i=0;i<nfil;i++) { + sizelimit = *size; + for(i = 0; i < nfil; i++) { LLint sz; int filteroffs = 1; + if (strncmp(filters[i] + filteroffs, "mime:", 5) == 0) { - if (type == 0) // regular filters + if (type == 0) // regular filters continue; - filteroffs += 5; // +mime:text/html - } else { // mime filters + filteroffs += 5; // +mime:text/html + } else { // mime filters if (type != 0) continue; } if (size) - sz=*size; - if (strjoker(nom, filters[i] + filteroffs, &sz, size_flag)) { // reconnu + sz = *size; + if (strjoker(nom, filters[i] + filteroffs, &sz, size_flag)) { // reconnu if (size) - if (sz != *size) - sizelimit=sz; - if (filters[i][0]=='+') - verdict = 1; // autorisé + if (sz != *size) + sizelimit = sz; + if (filters[i][0] == '+') + verdict = 1; // autorisé else - verdict = -1; // interdit + verdict = -1; // interdit if (depth) - *depth=i; + *depth = i; } } if (size) - *size=sizelimit; + *size = sizelimit; return verdict; } - // supercomparateur joker (tm) // compare a et b (b=avec joker dedans), case insensitive [voir CI] // renvoi l'adresse de la première lettre de la chaine @@ -104,211 +102,232 @@ int fa_strjoker(int type,char** filters,int nfil,char* nom,LLint* size,int* size // cet algo est 'un peu' récursif mais ne consomme pas trop de tm // * = toute lettre // --?-- : spécifique à HTTrack et aux ? -HTS_INLINE char* strjoker(char* chaine,char* joker,LLint* size,int* size_flag) { +HTS_INLINE char *strjoker(char *chaine, char *joker, LLint * size, + int *size_flag) { //int err=0; - if (strnotempty(joker)==0) { // fin de chaine joker - if (strnotempty(chaine)==0) // fin aussi pour la chaine: ok + if (strnotempty(joker) == 0) { // fin de chaine joker + if (strnotempty(chaine) == 0) // fin aussi pour la chaine: ok return chaine; - else if (chaine[0]=='?') - return chaine; // --?-- pour les index.html?Choix=2 + else if (chaine[0] == '?') + return chaine; // --?-- pour les index.html?Choix=2 else - return NULL; // non trouvé + return NULL; // non trouvé } - // on va progresser en suivant les 'mots' contenus dans le joker // un mot peut être un * ou bien toute autre séquence de lettres - - if (strcmp(joker,"*")==0) { // ok, rien après + + if (strcmp(joker, "*") == 0) { // ok, rien après return chaine; } - // 1er cas: jokers * ou jokers multiples *[..] - if (joker[0]=='*') { // comparer joker+reste (*toto/..) - int jmp; // nombre de caractères pour le prochain mot dans joker - int cut = 0; // interdire tout caractère superflu + if (joker[0] == '*') { // comparer joker+reste (*toto/..) + int jmp; // nombre de caractères pour le prochain mot dans joker + int cut = 0; // interdire tout caractère superflu char pass[256]; - char LEFT='[',RIGHT=']'; - int unique=0; - - switch(joker[1]) { - case '[': - LEFT='['; - RIGHT=']'; - unique=0; + char LEFT = '[', RIGHT = ']'; + int unique = 0; + + switch (joker[1]) { + case '[': + LEFT = '['; + RIGHT = ']'; + unique = 0; break; - case '(': - LEFT='('; - RIGHT=')'; - unique=1; + case '(': + LEFT = '('; + RIGHT = ')'; + unique = 1; break; } - if ((joker[1]==LEFT) && (joker[2]!=LEFT)) { // multijoker (tm) + if ((joker[1] == LEFT) && (joker[2] != LEFT)) { // multijoker (tm) int i; - for(i=0;i<256;i++) pass[i]=0; + + for(i = 0; i < 256; i++) + pass[i] = 0; // noms réservés - if ((strfield(joker+2,"file")) || (strfield(joker+2,"name"))) { - for(i=0;i<256;i++) pass[i]=1; + if ((strfield(joker + 2, "file")) || (strfield(joker + 2, "name"))) { + for(i = 0; i < 256; i++) + pass[i] = 1; pass[(int) '?'] = 0; //pass[(int) ';'] = 0; pass[(int) '/'] = 0; - i=2; - { int len=(int) strlen(joker); - while ((joker[i]!=RIGHT) && (joker[i]) && (i<len)) i++; + i = 2; + { + int len = (int) strlen(joker); + + while((joker[i] != RIGHT) && (joker[i]) && (i < len)) + i++; } - } else if (strfield(joker+2,"path")) { - for(i=0;i<256;i++) pass[i]=1; + } else if (strfield(joker + 2, "path")) { + for(i = 0; i < 256; i++) + pass[i] = 1; pass[(int) '?'] = 0; //pass[(int) ';'] = 0; - i=2; - { int len=(int) strlen(joker); - while ((joker[i]!=RIGHT) && (joker[i]) && (i<len)) i++; + i = 2; + { + int len = (int) strlen(joker); + + while((joker[i] != RIGHT) && (joker[i]) && (i < len)) + i++; } - } else if (strfield(joker+2,"param")) { - if (chaine[0]=='?') { // il y a un paramètre juste là - for(i=0;i<256;i++) pass[i]=1; - } // sinon synonyme de 'rien' - i=2; - { int len=(int) strlen(joker); - while ((joker[i]!=RIGHT) && (joker[i]) && (i<len)) i++; + } else if (strfield(joker + 2, "param")) { + if (chaine[0] == '?') { // il y a un paramètre juste là + for(i = 0; i < 256; i++) + pass[i] = 1; + } // sinon synonyme de 'rien' + i = 2; + { + int len = (int) strlen(joker); + + while((joker[i] != RIGHT) && (joker[i]) && (i < len)) + i++; } } else { // décode les directives comme *[A-Z,âêîôû,0-9] - i=2; - if (joker[i] == RIGHT) { // *[] signifie "plus rien après" - cut = 1; // caractère supplémentaire interdit + i = 2; + if (joker[i] == RIGHT) { // *[] signifie "plus rien après" + cut = 1; // caractère supplémentaire interdit } else { - int len=(int) strlen(joker); - while ((joker[i]!=RIGHT) && (joker[i]) && (i<len)) { - if ( (joker[i]=='<') || (joker[i]=='>') ) { // *[<10] - int lsize=0; + int len = (int) strlen(joker); + + while((joker[i] != RIGHT) && (joker[i]) && (i < len)) { + if ((joker[i] == '<') || (joker[i] == '>')) { // *[<10] + int lsize = 0; int lverdict; + i++; - if (sscanf(joker+i,"%d",&lsize) == 1) { + if (sscanf(joker + i, "%d", &lsize) == 1) { if (size) { - if (*size>=0) { + if (*size >= 0) { if (size_flag) - *size_flag=1; /* a joué */ - if (joker[i-1]=='<') - lverdict=(*size<lsize); + *size_flag = 1; /* a joué */ + if (joker[i - 1] == '<') + lverdict = (*size < lsize); else - lverdict=(*size>lsize); + lverdict = (*size > lsize); if (!lverdict) { - return NULL; // ne correspond pas + return NULL; // ne correspond pas } else { - *size=lsize; - return chaine; // ok + *size = lsize; + return chaine; // ok } } else - return NULL; // ne correspond pas + return NULL; // ne correspond pas } else - return NULL; // ne correspond pas (test impossible) + return NULL; // ne correspond pas (test impossible) // jump - while(isdigit((unsigned char)joker[i])) i++; + while(isdigit((unsigned char) joker[i])) + i++; } - } - else if (joker[i+1]=='-') { // 2 car, ex: *[A-Z] - if ((int) (unsigned char) joker[i+2]>(int) (unsigned char) joker[i]) { + } else if (joker[i + 1] == '-') { // 2 car, ex: *[A-Z] + if ((int) (unsigned char) joker[i + 2] > + (int) (unsigned char) joker[i]) { int j; - for(j=(int) (unsigned char) joker[i];j<=(int) (unsigned char) joker[i+2];j++) - pass[j]=1; - + + for(j = (int) (unsigned char) joker[i]; + j <= (int) (unsigned char) joker[i + 2]; j++) + pass[j] = 1; + } // else err=1; - i+=3; + i += 3; } else { // 1 car, ex: *[ ] - if (joker[i+2]=='\\' && joker[i+3] != 0) { // escaped char, such as *[\[] or *[\]] + if (joker[i + 2] == '\\' && joker[i + 3] != 0) { // escaped char, such as *[\[] or *[\]] i++; } - pass[(int) (unsigned char) joker[i]]=1; + pass[(int) (unsigned char) joker[i]] = 1; i++; } - if ((joker[i]==',') || (joker[i]==';')) i++; + if ((joker[i] == ',') || (joker[i] == ';')) + i++; } } } // à sauter dans joker - jmp=i; - if (joker[i]) jmp++; - + jmp = i; + if (joker[i]) + jmp++; + // - } else { // tout autoriser + } else { // tout autoriser // int i; - for(i=0;i<256;i++) pass[i]=1; // tout autoriser - jmp=1; + + for(i = 0; i < 256; i++) + pass[i] = 1; // tout autoriser + jmp = 1; ////if (joker[2]==LEFT) jmp=3; // permet de recher *<crochet ouvrant> } - + { - int i,max; - char* adr; + int i, max; + char *adr; // la chaine doit se terminer exactement if (cut) { if (strnotempty(chaine)) - return NULL; // perdu + return NULL; // perdu else - return chaine; // ok + return chaine; // ok } - // comparaison en boucle, c'est ca qui consomme huhu.. // le tableau pass[256] indique les caractères ASCII autorisés // tester sans le joker (pas ()+ mais ()*) if (!unique) { - if ( (adr=strjoker(chaine,joker+jmp,size,size_flag)) ) { + if ((adr = strjoker(chaine, joker + jmp, size, size_flag))) { return adr; } } - // tester - i=0; + i = 0; if (!unique) max = (int) strlen(chaine); - else /* *(a) only match a (not aaaaa) */ - max=1; - while(i<(int) max) { - if (pass[(int) (unsigned char) chaine[i]]) { // caractère autorisé - if ( (adr=strjoker(chaine+i+1,joker+jmp,size,size_flag)) ) { + else /* *(a) only match a (not aaaaa) */ + max = 1; + while(i < (int) max) { + if (pass[(int) (unsigned char) chaine[i]]) { // caractère autorisé + if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) { return adr; } i++; - } else i=max+2; // sortir + } else + i = max + 2; // sortir } // tester chaîne vide - if (i!=max+2) // avant c'est ok - if ( (adr=strjoker(chaine+max,joker+jmp,size,size_flag)) ) - return adr; - - return NULL; // perdu + if (i != max + 2) // avant c'est ok + if ((adr = strjoker(chaine + max, joker + jmp, size, size_flag))) + return adr; + + return NULL; // perdu } - } else { // comparer mot+reste (toto*..) + } else { // comparer mot+reste (toto*..) if (strnotempty(chaine)) { - int jmp=0,ok=1; - + int jmp = 0, ok = 1; + // comparer début de joker et début de chaine - while((joker[jmp]!='*') && (joker[jmp]) && (ok)) { + while((joker[jmp] != '*') && (joker[jmp]) && (ok)) { // CI : remplacer streql par une comparaison != - if (!streql(chaine[jmp],joker[jmp])) { - ok=0; // quitter + if (!streql(chaine[jmp], joker[jmp])) { + ok = 0; // quitter } jmp++; } - + // comparaison ok? if (ok) { // continuer la comparaison. - if (strjoker(chaine+jmp,joker+jmp,size,size_flag)) - return chaine; // retourner 1e lettre + if (strjoker(chaine + jmp, joker + jmp, size, size_flag)) + return chaine; // retourner 1e lettre } - - } // strlen(a) - return NULL; - } // * ou mot + + } // strlen(a) + return NULL; + } // * ou mot return NULL; } @@ -317,10 +336,11 @@ HTS_INLINE char* strjoker(char* chaine,char* joker,LLint* size,int* size_flag) { // exemple: find dans un texte de strcpybuff(*[A-Z,a-z],"*[0-9]"); va rechercher la première occurence // d'un strcpy sur une variable ayant un nom en lettres et copiant une chaine de chiffres // ATTENTION!! Eviter les jokers en début, où gare au temps machine! -char* strjokerfind(char* chaine,char* joker) { - char* adr; +char *strjokerfind(char *chaine, char *joker) { + char *adr; + while(*chaine) { - if ( (adr=strjoker(chaine,joker,NULL,NULL)) ) { // ok trouvé + if ((adr = strjoker(chaine, joker, NULL, NULL))) { // ok trouvé return adr; } chaine++; diff --git a/src/htsfilters.h b/src/htsfilters.h index d49ef95..73e1880 100644 --- a/src/htsfilters.h +++ b/src/htsfilters.h @@ -17,37 +17,34 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* filters ("regexp") */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - - #ifndef HTSFILT_DEFH -#define HTSFILT_DEFH +#define HTSFILT_DEFH /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE #include "htsbase.h" -int fa_strjoker(int type,char** filters,int nfil,char* nom,LLint* size,int* size_flag,int* depth); -HTS_INLINE char* strjoker(char* chaine,char* joker,LLint* size,int* size_flag); -char* strjokerfind(char* chaine,char* joker); +int fa_strjoker(int type, char **filters, int nfil, char *nom, LLint * size, + int *size_flag, int *depth); +HTS_INLINE char *strjoker(char *chaine, char *joker, LLint * size, + int *size_flag); +char *strjokerfind(char *chaine, char *joker); #endif #endif diff --git a/src/htsftp.c b/src/htsftp.c index d8ed03d..1a9ff77 100644 --- a/src/htsftp.c +++ b/src/htsftp.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: basic FTP protocol manager */ /* Author: Xavier Roche */ @@ -54,7 +51,7 @@ Please visit our Website: http://www.httrack.com #ifndef __cplusplus // DOS #ifndef _WIN32_WCE -#include <process.h> /* _beginthread, _endthread */ +#include <process.h> /* _beginthread, _endthread */ #endif #endif #endif @@ -72,19 +69,20 @@ Please visit our Website: http://www.httrack.com #if USE_BEGINTHREAD -void back_launch_ftp( void* pP ) { - FTPDownloadStruct *pStruct = (FTPDownloadStruct*)pP; +void back_launch_ftp(void *pP) { + FTPDownloadStruct *pStruct = (FTPDownloadStruct *) pP; + if (pStruct == NULL) - return ; + return; - if (pStruct == NULL) { + if (pStruct == NULL) { #if FTP_DEBUG printf("[ftp error: no args]\n"); #endif - return ; + return; } - /* Initialize */ + /* Initialize */ hts_init(); // lancer ftp @@ -93,22 +91,23 @@ void back_launch_ftp( void* pP ) { #endif run_launch_ftp(pStruct); // prêt - pStruct->pBack->status=STATUS_FTP_READY; + pStruct->pBack->status = STATUS_FTP_READY; - /* Delete structure */ - free(pP); + /* Delete structure */ + free(pP); /* Uninitialize */ hts_uninit(); - return ; + return; } + // lancer en back -void launch_ftp(FTPDownloadStruct* params) { +void launch_ftp(FTPDownloadStruct * params) { // DOS #if FTP_DEBUG printf("[Launching main ftp thread]\n"); #endif - hts_newthread(back_launch_ftp, (void*) params); + hts_newthread(back_launch_ftp, (void *) params); } #else @@ -133,71 +132,81 @@ void launch_ftp(FTPDownloadStruct* params) { } // la véritable fonction une fois lancées les routines thread/fork -int run_launch_ftp(FTPDownloadStruct *pStruct) { - lien_back* back = pStruct->pBack; - httrackp *opt = pStruct->pOpt; - char user[256]="anonymous"; - char pass[256]="user@"; +int run_launch_ftp(FTPDownloadStruct * pStruct) { + lien_back *back = pStruct->pBack; + httrackp *opt = pStruct->pOpt; + char user[256] = "anonymous"; + char pass[256] = "user@"; char line_retr[2048]; - int port=21; + int port = 21; + #if FTP_PASV - int port_pasv=0; + int port_pasv = 0; #endif char BIGSTK adr_ip[1024]; - char *adr,*real_adr; - char* ftp_filename=""; - int timeout = 300; // timeout - int timeout_onfly=8; // attente réponse supplémentaire - int transfer_list=0; // directory - int rest_understood=0; // rest command understood - t_fullhostent fullhostent_buffer; // buffer pour resolver + char *adr, *real_adr; + char *ftp_filename = ""; + int timeout = 300; // timeout + int timeout_onfly = 8; // attente réponse supplémentaire + int transfer_list = 0; // directory + int rest_understood = 0; // rest command understood + t_fullhostent fullhostent_buffer; // buffer pour resolver + // - T_SOC soc_ctl=INVALID_SOCKET; - T_SOC soc_servdat=INVALID_SOCKET; - T_SOC soc_dat=INVALID_SOCKET; + T_SOC soc_ctl = INVALID_SOCKET; + T_SOC soc_servdat = INVALID_SOCKET; + T_SOC soc_dat = INVALID_SOCKET; + // SOCaddr server_data; - int server_data_size=sizeof(server_data); + int server_data_size = sizeof(server_data); + // - line_retr[0]=adr_ip[0]='\0'; - - timeout=300; - + line_retr[0] = adr_ip[0] = '\0'; + + timeout = 300; + // effacer - strcpybuff(back->r.msg,""); - back->r.statuscode=0; - back->r.size=0; - + strcpybuff(back->r.msg, ""); + back->r.statuscode = 0; + back->r.size = 0; + // récupérer user et pass si présents, et sauter user:id@ dans adr - real_adr = strchr(back->url_adr,':'); - if (real_adr) real_adr++; - else real_adr=back->url_adr; - while(*real_adr=='/') real_adr++; // sauter / - if ( (adr = jump_identification(real_adr)) != real_adr) { // user - int i=-1; - pass[0]='\0'; + real_adr = strchr(back->url_adr, ':'); + if (real_adr) + real_adr++; + else + real_adr = back->url_adr; + while(*real_adr == '/') + real_adr++; // sauter / + if ((adr = jump_identification(real_adr)) != real_adr) { // user + int i = -1; + + pass[0] = '\0'; do { i++; - user[i]=real_adr[i]; - } while( (real_adr[i]!=':') && (real_adr[i]) ); - user[i]='\0'; - if (real_adr[i]==':') { // pass - int j=-1; - i++; // oui on saute aussi le : + user[i] = real_adr[i]; + } while((real_adr[i] != ':') && (real_adr[i])); + user[i] = '\0'; + if (real_adr[i] == ':') { // pass + int j = -1; + + i++; // oui on saute aussi le : do { j++; - pass[j]=real_adr[i+j]; - } while( ((&real_adr[i+j+1]) < adr) && (real_adr[i+j]) ); - pass[j]='\0'; + pass[j] = real_adr[i + j]; + } while(((&real_adr[i + j + 1]) < adr) && (real_adr[i + j])); + pass[j] = '\0'; } } - // Calculer RETR <nom> { - char* a; + char *a; + #if 0 - a=back->url_fil + strlen(back->url_fil)-1; - while( (a > back->url_fil) && (*a!='/')) a--; + a = back->url_fil + strlen(back->url_fil) - 1; + while((a > back->url_fil) && (*a != '/')) + a--; if (*a != '/') { a = NULL; } @@ -206,349 +215,359 @@ int run_launch_ftp(FTPDownloadStruct *pStruct) { #endif if (a != NULL && *a != '\0') { #if 0 - a++; // sauter / + a++; // sauter / #endif - ftp_filename=a; + ftp_filename = a; if (strnotempty(a)) { - char catbuff[CATBUFF_SIZE]; - char* ua=unescape_http(catbuff,a); + char catbuff[CATBUFF_SIZE]; + char *ua = unescape_http(catbuff, a); int len_a = (int) strlen(ua); - if (len_a > 0 && ua[len_a -1] == '/') { /* obviously a directory listing */ - transfer_list=1; - sprintf(line_retr,"LIST -A %s",ua); - } else if ( - (strchr(ua, ' ')) - || - (strchr(ua, '\"')) - || - (strchr(ua, '\'')) + + if (len_a > 0 && ua[len_a - 1] == '/') { /* obviously a directory listing */ + transfer_list = 1; + sprintf(line_retr, "LIST -A %s", ua); + } else if ((strchr(ua, ' ')) + || (strchr(ua, '\"')) + || (strchr(ua, '\'')) ) { - sprintf(line_retr,"RETR \"%s\"",ua); - } else { /* Regular one */ - sprintf(line_retr,"RETR %s",ua); + sprintf(line_retr, "RETR \"%s\"", ua); + } else { /* Regular one */ + sprintf(line_retr, "RETR %s", ua); } } else { - transfer_list=1; - sprintf(line_retr,"LIST -A"); + transfer_list = 1; + sprintf(line_retr, "LIST -A"); } } else { - strcpybuff(back->r.msg,"Unexpected PORT error"); + strcpybuff(back->r.msg, "Unexpected PORT error"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } } - + #if FTP_DEBUG - printf("Connecting to %s...\n",adr); + printf("Connecting to %s...\n", adr); #endif - + // connexion { SOCaddr server; - int server_size=sizeof(server); - t_hostent* hp; - char * a; + int server_size = sizeof(server); + t_hostent *hp; + char *a; char _adr[256]; - _adr[0]='\0'; + + _adr[0] = '\0'; //T_SOC soc_ctl; // effacer structure memset(&server, 0, sizeof(server)); - + // port - a=strchr(adr,':'); // port + a = strchr(adr, ':'); // port if (a) { - sscanf(a+1,"%d",&port); - strncatbuff(_adr,adr,(int) (a - adr)); + sscanf(a + 1, "%d", &port); + strncatbuff(_adr, adr, (int) (a - adr)); } else - strcpybuff(_adr,adr); - + strcpybuff(_adr, adr); + // récupérer adresse résolue - strcpybuff(back->info,"host name"); - hp = hts_gethostbyname(opt,_adr, &fullhostent_buffer); + strcpybuff(back->info, "host name"); + hp = hts_gethostbyname(opt, _adr, &fullhostent_buffer); if (hp == NULL) { - strcpybuff(back->r.msg,"Unable to get server's address"); + strcpybuff(back->r.msg, "Unable to get server's address"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_NON_FATAL; - _HALT_FTP - return 0; + back->r.statuscode = STATUSCODE_NON_FATAL; + _HALT_FTP return 0; } _CHECK_HALT_FTP; - + // copie adresse SOCaddr_copyaddr(server, server_size, hp->h_addr_list[0], hp->h_length); // copie adresse pour cnx data - SOCaddr_copyaddr(server_data, server_data_size, hp->h_addr_list[0], hp->h_length); + SOCaddr_copyaddr(server_data, server_data_size, hp->h_addr_list[0], + hp->h_length); // memcpy(&server.sin_addr, hp->h_addr, hp->h_length); - + // créer ("attachement") une socket (point d'accès) internet,en flot soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0); - if (soc_ctl==INVALID_SOCKET) { - strcpybuff(back->r.msg,"Unable to create a socket"); + if (soc_ctl == INVALID_SOCKET) { + strcpybuff(back->r.msg, "Unable to create a socket"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - _HALT_FTP - return 0; + back->r.statuscode = STATUSCODE_INVALID; + _HALT_FTP return 0; } SOCaddr_initport(server, port); // server.sin_port = htons((unsigned short int) port); - + // connexion (bloquante, on est en thread) - strcpybuff(back->info,"connect"); + strcpybuff(back->info, "connect"); #ifdef _WIN32 - if (connect(soc_ctl, (const struct sockaddr FAR *)&server, server_size) != 0) { + if (connect(soc_ctl, (const struct sockaddr FAR *) &server, server_size) != + 0) { #else - if (connect(soc_ctl, (struct sockaddr *)&server, server_size) == -1) { + if (connect(soc_ctl, (struct sockaddr *) &server, server_size) == -1) { #endif - strcpybuff(back->r.msg,"Unable to connect to the server"); - // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - _HALT_FTP - return 0; + strcpybuff(back->r.msg, "Unable to connect to the server"); + // back->status=STATUS_FTP_READY; // fini + back->r.statuscode = STATUSCODE_INVALID; + _HALT_FTP return 0; #ifdef _WIN32 - } + } #else } #endif _CHECK_HALT_FTP; - + { char BIGSTK line[1024]; + // envoi du login - + // --USER-- - get_ftp_line(soc_ctl,line,timeout); // en tête + get_ftp_line(soc_ctl, line, timeout); // en tête _CHECK_HALT_FTP; - - if (line[0]=='2') { // ok, connecté - strcpybuff(back->info,"login: user"); - sprintf(line,"USER %s",user); - send_line(soc_ctl,line); - get_ftp_line(soc_ctl,line,timeout); - _CHECK_HALT_FTP; - if ((line[0]=='3') || (line[0]=='2')) { + + if (line[0] == '2') { // ok, connecté + strcpybuff(back->info, "login: user"); + sprintf(line, "USER %s", user); + send_line(soc_ctl, line); + get_ftp_line(soc_ctl, line, timeout); + _CHECK_HALT_FTP; + if ((line[0] == '3') || (line[0] == '2')) { // --PASS-- if (line[0] == '3') { - strcpybuff(back->info,"login: pass"); - sprintf(line,"PASS %s",pass); - send_line(soc_ctl,line); - get_ftp_line(soc_ctl,line,timeout); + strcpybuff(back->info, "login: pass"); + sprintf(line, "PASS %s", pass); + send_line(soc_ctl, line); + get_ftp_line(soc_ctl, line, timeout); _CHECK_HALT_FTP; } - if (line[0]=='2') { // ok - send_line(soc_ctl,"TYPE I"); - get_ftp_line(soc_ctl,line,timeout); - _CHECK_HALT_FTP; - if (line[0]=='2') { + if (line[0] == '2') { // ok + send_line(soc_ctl, "TYPE I"); + get_ftp_line(soc_ctl, line, timeout); + _CHECK_HALT_FTP; + if (line[0] == '2') { // ok } else { - strcpybuff(back->r.msg,"TYPE I error"); + strcpybuff(back->r.msg, "TYPE I error"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } #if 0 // --CWD-- - char* a; - a=back->url_fil + strlen(back->url_fil)-1; - while( (a > back->url_fil) && (*a!='/')) a--; + char *a; + + a = back->url_fil + strlen(back->url_fil) - 1; + while((a > back->url_fil) && (*a != '/')) + a--; if (*a == '/') { // ok repéré char BIGSTK target[1024]; - target[0]='\0'; - strncatbuff(target,back->url_fil,(int) (a - back->url_fil)); - if (strnotempty(target)==0) - strcatbuff(target,"/"); - strcpybuff(back->info,"cwd"); - sprintf(line,"CWD %s",target); - send_line(soc_ctl,line); - get_ftp_line(soc_ctl,line,timeout); - _CHECK_HALT_FTP; - if (line[0]=='2') { - send_line(soc_ctl,"TYPE I"); - get_ftp_line(soc_ctl,line,timeout); - _CHECK_HALT_FTP; - if (line[0]=='2') { + + target[0] = '\0'; + strncatbuff(target, back->url_fil, (int) (a - back->url_fil)); + if (strnotempty(target) == 0) + strcatbuff(target, "/"); + strcpybuff(back->info, "cwd"); + sprintf(line, "CWD %s", target); + send_line(soc_ctl, line); + get_ftp_line(soc_ctl, line, timeout); + _CHECK_HALT_FTP; + if (line[0] == '2') { + send_line(soc_ctl, "TYPE I"); + get_ftp_line(soc_ctl, line, timeout); + _CHECK_HALT_FTP; + if (line[0] == '2') { // ok.. } else { - strcpybuff(back->r.msg,"TYPE I error"); + strcpybuff(back->r.msg, "TYPE I error"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } } else { - sprintf(back->r.msg,"CWD error: %s",linejmp(line)); + sprintf(back->r.msg, "CWD error: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - } // sinon on est prêts + back->r.statuscode = STATUSCODE_INVALID; + } // sinon on est prêts } else { - strcpybuff(back->r.msg,"Unexpected ftp error"); + strcpybuff(back->r.msg, "Unexpected ftp error"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } #endif - + } else { - sprintf(back->r.msg,"Bad password: %s",linejmp(line)); + sprintf(back->r.msg, "Bad password: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } } else { - sprintf(back->r.msg,"Bad user name: %s",linejmp(line)); + sprintf(back->r.msg, "Bad user name: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } } else { - sprintf(back->r.msg,"Connection refused: %s",linejmp(line)); + sprintf(back->r.msg, "Connection refused: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } - + // ok, si on est prêts on écoute sur un port et on demande la sauce if (back->r.statuscode != -1) { - // // Pré-REST // #if FTP_PASV if (SOCaddr_getproto(server, server_size) == '1') { - strcpybuff(back->info,"pasv"); - sprintf(line,"PASV"); - send_line(soc_ctl,line); - get_ftp_line(soc_ctl,line,timeout); - } else { /* ipv6 */ - line[0]='\0'; + strcpybuff(back->info, "pasv"); + sprintf(line, "PASV"); + send_line(soc_ctl, line); + get_ftp_line(soc_ctl, line, timeout); + } else { /* ipv6 */ + line[0] = '\0'; } - _CHECK_HALT_FTP; - if (line[0]=='2') { - char *a,*b,*c; - a=strchr(line,'('); // exemple: 227 Entering Passive Mode (123,45,67,89,177,27) + _CHECK_HALT_FTP; + if (line[0] == '2') { + char *a, *b, *c; + + a = strchr(line, '('); // exemple: 227 Entering Passive Mode (123,45,67,89,177,27) if (a) { - + // -- analyse de l'adresse IP et du port -- a++; - b=strchr(a,','); - if (b) b=strchr(b+1,','); - if (b) b=strchr(b+1,','); - if (b) b=strchr(b+1,','); - c=a; while( (c=strchr(c,',')) ) *c='.'; // remplacer , par . - if (b) *b='\0'; + b = strchr(a, ','); + if (b) + b = strchr(b + 1, ','); + if (b) + b = strchr(b + 1, ','); + if (b) + b = strchr(b + 1, ','); + c = a; + while((c = strchr(c, ','))) + *c = '.'; // remplacer , par . + if (b) + *b = '\0'; // - strcpybuff(adr_ip,a); // copier adresse ip + strcpybuff(adr_ip, a); // copier adresse ip // if (b) { - a=b+1; // début du port - b=strchr(a,'.'); + a = b + 1; // début du port + b = strchr(a, '.'); if (b) { - int n1,n2; + int n1, n2; + // - *b='\0'; + *b = '\0'; b++; - c=strchr(b,')'); + c = strchr(b, ')'); if (c) { - *c='\0'; - if ( (sscanf(a,"%d",&n1)==1) && (sscanf(b,"%d",&n2)==1) && (strlen(adr_ip)<=16)) { - port_pasv=n2+(n1<<8); + *c = '\0'; + if ((sscanf(a, "%d", &n1) == 1) && (sscanf(b, "%d", &n2) == 1) + && (strlen(adr_ip) <= 16)) { + port_pasv = n2 + (n1 << 8); } } else { - deletesoc(soc_dat); soc_dat=INVALID_SOCKET; - } // sinon on est prêts + deletesoc(soc_dat); + soc_dat = INVALID_SOCKET; + } // sinon on est prêts } } // -- fin analyse de l'adresse IP et du port -- } else { - sprintf(back->r.msg,"PASV incorrect: %s",linejmp(line)); + sprintf(back->r.msg, "PASV incorrect: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - } // sinon on est prêts + back->r.statuscode = STATUSCODE_INVALID; + } // sinon on est prêts } else { /* - * try epsv (ipv6) * - */ - strcpybuff(back->info,"pasv"); - sprintf(line,"EPSV"); - send_line(soc_ctl,line); - get_ftp_line(soc_ctl,line,timeout); - _CHECK_HALT_FTP; - if (line[0]=='2') { /* got it */ + * try epsv (ipv6) * + */ + strcpybuff(back->info, "pasv"); + sprintf(line, "EPSV"); + send_line(soc_ctl, line); + get_ftp_line(soc_ctl, line, timeout); + _CHECK_HALT_FTP; + if (line[0] == '2') { /* got it */ char *a; - a=strchr(line,'('); // exemple: 229 Entering Extended Passive Mode (|||6446|) - if ( - (a != NULL) - && - (*a == '(') - && (*(a+1)) - && (*(a+1) == *(a+2)) && (*(a+1) == *(a+3)) - && (isdigit(*(a+4))) - && (*(a+5)) + + a = strchr(line, '('); // exemple: 229 Entering Extended Passive Mode (|||6446|) + if ((a != NULL) + && (*a == '(') + && (*(a + 1)) + && (*(a + 1) == *(a + 2)) && (*(a + 1) == *(a + 3)) + && (isdigit(*(a + 4))) + && (*(a + 5)) ) { unsigned int n1 = 0; - if (sscanf(a+4,"%d",&n1)==1) { + + if (sscanf(a + 4, "%d", &n1) == 1) { if ((n1 < 65535) && (n1 > 0)) { - port_pasv=n1; + port_pasv = n1; } } } else { - sprintf(back->r.msg,"EPSV incorrect: %s",linejmp(line)); + sprintf(back->r.msg, "EPSV incorrect: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } } else { - sprintf(back->r.msg,"PASV/EPSV error: %s",linejmp(line)); + sprintf(back->r.msg, "PASV/EPSV error: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - } // sinon on est prêts + back->r.statuscode = STATUSCODE_INVALID; + } // sinon on est prêts } #else // rien à faire avant #endif - + #if FTP_PASV if (port_pasv) { #endif // SIZE if (back->r.statuscode != -1) { if (!transfer_list) { - char catbuff[CATBUFF_SIZE]; - char* ua=unescape_http(catbuff,ftp_filename); - if ( - (strchr(ua, ' ')) - || - (strchr(ua, '\"')) - || - (strchr(ua, '\'')) + char catbuff[CATBUFF_SIZE]; + char *ua = unescape_http(catbuff, ftp_filename); + + if ((strchr(ua, ' ')) + || (strchr(ua, '\"')) + || (strchr(ua, '\'')) ) { - sprintf(line,"SIZE \"%s\"", ua); + sprintf(line, "SIZE \"%s\"", ua); } else { - sprintf(line,"SIZE %s", ua); + sprintf(line, "SIZE %s", ua); } - + // SIZE? - strcpybuff(back->info,"size"); - send_line(soc_ctl,line); - get_ftp_line(soc_ctl,line,timeout); - _CHECK_HALT_FTP; - if (line[0]=='2') { // SIZE compris, ALORS tester REST (sinon pas tester: cf probleme des txt.gz decompresses a la volee) - char* szstr = strchr(line, ' '); + strcpybuff(back->info, "size"); + send_line(soc_ctl, line); + get_ftp_line(soc_ctl, line, timeout); + _CHECK_HALT_FTP; + if (line[0] == '2') { // SIZE compris, ALORS tester REST (sinon pas tester: cf probleme des txt.gz decompresses a la volee) + char *szstr = strchr(line, ' '); + if (szstr) { LLint size = 0; + szstr++; if (sscanf(szstr, LLintP, &size) == 1) { back->r.totalsize = size; } } - // REST? - if (fexist(back->url_sav) && (transfer_list==0)) { - strcpybuff(back->info,"rest"); - sprintf(line,"REST "LLintP,(LLint)fsize(back->url_sav)); - send_line(soc_ctl,line); - get_ftp_line(soc_ctl,line,timeout); - _CHECK_HALT_FTP; - if ((line[0]=='3') || (line[0]=='2')) { // ok - rest_understood=1; - } // sinon tant pis - } - } // sinon tant pis + if (fexist(back->url_sav) && (transfer_list == 0)) { + strcpybuff(back->info, "rest"); + sprintf(line, "REST " LLintP, (LLint) fsize(back->url_sav)); + send_line(soc_ctl, line); + get_ftp_line(soc_ctl, line, timeout); + _CHECK_HALT_FTP; + if ((line[0] == '3') || (line[0] == '2')) { // ok + rest_understood = 1; + } // sinon tant pis + } + } // sinon tant pis } } #if FTP_PASV @@ -562,29 +581,31 @@ int run_launch_ftp(FTPDownloadStruct *pStruct) { // Ok, se connecter if (port_pasv) { SOCaddr server; - int server_size=sizeof(server); - t_hostent* hp; + int server_size = sizeof(server); + t_hostent *hp; + // effacer structure memset(&server, 0, sizeof(server)); - + // infos - strcpybuff(back->info,"resolv"); - + strcpybuff(back->info, "resolv"); + // résoudre if (adr_ip[0]) { - hp = hts_gethostbyname(opt,adr_ip, &fullhostent_buffer); + hp = hts_gethostbyname(opt, adr_ip, &fullhostent_buffer); if (hp) { - SOCaddr_copyaddr(server, server_size, hp->h_addr_list[0], hp->h_length); + SOCaddr_copyaddr(server, server_size, hp->h_addr_list[0], + hp->h_length); } else { - server_size=0; + server_size = 0; } } else { memcpy(&server, &server_data, sizeof(server_data)); - server_size=server_data_size; + server_size = server_data_size; } - + // infos - strcpybuff(back->info,"cnxdata"); + strcpybuff(back->info, "cnxdata"); #if FTP_DEBUG printf("Data: Connecting to %s:%d...\n", adr_ip, port_pasv); #endif @@ -596,79 +617,85 @@ int run_launch_ftp(FTPDownloadStruct *pStruct) { SOCaddr_initport(server, port_pasv); // server.sin_port = htons((unsigned short int) port_pasv); #ifdef _WIN32 - if (connect(soc_dat, (const struct sockaddr FAR *)&server, server_size) == 0) { + if (connect + (soc_dat, (const struct sockaddr FAR *) &server, + server_size) == 0) { #else - if (connect(soc_dat, (struct sockaddr *)&server, server_size) != -1) { + if (connect(soc_dat, (struct sockaddr *) &server, server_size) != + -1) { #endif - strcpybuff(back->info,"retr"); - strcpybuff(line,line_retr); - send_line(soc_ctl,line); - get_ftp_line(soc_ctl,line,timeout); - _CHECK_HALT_FTP; - if (line[0]=='1') { + strcpybuff(back->info, "retr"); + strcpybuff(line, line_retr); + send_line(soc_ctl, line); + get_ftp_line(soc_ctl, line, timeout); + _CHECK_HALT_FTP; + if (line[0] == '1') { // OK } else { - deletesoc(soc_dat); soc_dat=INVALID_SOCKET; + deletesoc(soc_dat); + soc_dat = INVALID_SOCKET; // - sprintf(back->r.msg,"RETR command errror: %s",linejmp(line)); + sprintf(back->r.msg, "RETR command errror: %s", + linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - } // sinon on est prêts + back->r.statuscode = STATUSCODE_INVALID; + } // sinon on est prêts } else { #if FTP_DEBUG printf("Data: unable to connect\n"); #endif - deletesoc(soc_dat); soc_dat=INVALID_SOCKET; + deletesoc(soc_dat); + soc_dat = INVALID_SOCKET; // - strcpybuff(back->r.msg,"Unable to connect"); + strcpybuff(back->r.msg, "Unable to connect"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - } // sinon on est prêts + back->r.statuscode = STATUSCODE_INVALID; + } // sinon on est prêts } else { - strcpybuff(back->r.msg,"Unable to create a socket"); + strcpybuff(back->r.msg, "Unable to create a socket"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - } // sinon on est prêts + back->r.statuscode = STATUSCODE_INVALID; + } // sinon on est prêts } else { - sprintf(back->r.msg,"Unable to resolve IP %s",adr_ip); + sprintf(back->r.msg, "Unable to resolve IP %s", adr_ip); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - } // sinon on est prêts + back->r.statuscode = STATUSCODE_INVALID; + } // sinon on est prêts } else { - sprintf(back->r.msg,"PASV incorrect: %s",linejmp(line)); + sprintf(back->r.msg, "PASV incorrect: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - } // sinon on est prêts + back->r.statuscode = STATUSCODE_INVALID; + } // sinon on est prêts #else //T_SOC soc_servdat; - strcpybuff(back->info,"listening"); - if ( (soc_servdat = get_datasocket(line)) != INVALID_SOCKET) { - _CHECK_HALT_FTP; - send_line(soc_ctl,line); // envoi du RETR - get_ftp_line(soc_ctl,line,timeout); - _CHECK_HALT_FTP; - if (line[0]=='2') { // ok - strcpybuff(back->info,"retr"); - strcpybuff(line,line_retr); - send_line(soc_ctl,line); - get_ftp_line(soc_ctl,line,timeout); - _CHECK_HALT_FTP; - if (line[0]=='1') { + strcpybuff(back->info, "listening"); + if ((soc_servdat = get_datasocket(line)) != INVALID_SOCKET) { + _CHECK_HALT_FTP; + send_line(soc_ctl, line); // envoi du RETR + get_ftp_line(soc_ctl, line, timeout); + _CHECK_HALT_FTP; + if (line[0] == '2') { // ok + strcpybuff(back->info, "retr"); + strcpybuff(line, line_retr); + send_line(soc_ctl, line); + get_ftp_line(soc_ctl, line, timeout); + _CHECK_HALT_FTP; + if (line[0] == '1') { //T_SOC soc_dat; - if ( (soc_dat=accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) { - strcpybuff(back->r.msg,"Unable to accept connection"); + if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) { + strcpybuff(back->r.msg, "Unable to accept connection"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } } else { - sprintf(back->r.msg,"RETR command errror: %s",linejmp(line)); + sprintf(back->r.msg, "RETR command errror: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } } else { - sprintf(back->r.msg,"PORT command error: %s",linejmp(line)); + sprintf(back->r.msg, "PORT command error: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } #ifdef _WIN32 closesocket(soc_servdat); @@ -676,356 +703,374 @@ int run_launch_ftp(FTPDownloadStruct *pStruct) { close(soc_servdat); #endif } else { - strcpybuff(back->r.msg,"Unable to listen to a port"); + strcpybuff(back->r.msg, "Unable to listen to a port"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } #endif - + // // Ok, connexion initiée // if (soc_dat != INVALID_SOCKET) { - if (rest_understood) { // REST envoyée et comprise - file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 0, 1, 0); - back->r.fp = fileappend(&opt->state.strc, back->url_sav); + if (rest_understood) { // REST envoyée et comprise + file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 0, 1, + 0); + back->r.fp = fileappend(&opt->state.strc, back->url_sav); } else { - file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 1, 1, 0); + file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 1, 1, + 0); back->r.fp = filecreate(&opt->state.strc, back->url_sav); } - strcpybuff(back->info,"receiving"); + strcpybuff(back->info, "receiving"); if (back->r.fp != NULL) { char BIGSTK buff[1024]; - int len=1; - int read_len=1024; + int len = 1; + int read_len = 1024; + //HTS_TOTAL_RECV_CHECK(read_len); // Diminuer au besoin si trop de données reçues - - while( (len>0) && (!stop_ftp(back)) ) { + + while((len > 0) && (!stop_ftp(back))) { // attendre les données - len=1; // pas d'erreur pour le moment - switch(wait_socket_receive(soc_dat,timeout)) { + len = 1; // pas d'erreur pour le moment + switch (wait_socket_receive(soc_dat, timeout)) { case -1: - strcpybuff(back->r.msg,"FTP read error"); + strcpybuff(back->r.msg, "FTP read error"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - len=0; // fin + back->r.statuscode = STATUSCODE_INVALID; + len = 0; // fin break; case 0: - sprintf(back->r.msg,"Time out (%d)",timeout); + sprintf(back->r.msg, "Time out (%d)", timeout); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - len=0; // fin + back->r.statuscode = STATUSCODE_INVALID; + len = 0; // fin break; } - + // réception if (len) { - len=recv(soc_dat,buff,read_len,0); - if (len>0) { - back->r.size+=len; - HTS_STAT.HTS_TOTAL_RECV+=len; + len = recv(soc_dat, buff, read_len, 0); + if (len > 0) { + back->r.size += len; + HTS_STAT.HTS_TOTAL_RECV += len; if (back->r.fp) { - if ((INTsys)fwrite(buff,1,(INTsys)len,back->r.fp) != len) { + if ((INTsys) fwrite(buff, 1, (INTsys) len, back->r.fp) != + len) { /* - int fcheck; - if ((fcheck=check_fatal_io_errno())) { - opt->state.exit_xh=-1; - } - */ - strcpybuff(back->r.msg,"Write error"); + int fcheck; + if ((fcheck=check_fatal_io_errno())) { + opt->state.exit_xh=-1; + } + */ + strcpybuff(back->r.msg, "Write error"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; - len=0; // error + back->r.statuscode = STATUSCODE_INVALID; + len = 0; // error } } else { - strcpybuff(back->r.msg,"Unexpected write error"); + strcpybuff(back->r.msg, "Unexpected write error"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } } else { // Erreur ou terminé // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=0; - if (back->r.totalsize > 0 && back->r.size != back->r.totalsize) { - back->r.statuscode=STATUSCODE_INVALID; - strcpybuff(back->r.msg,"FTP file incomplete"); + back->r.statuscode = 0; + if (back->r.totalsize > 0 + && back->r.size != back->r.totalsize) { + back->r.statuscode = STATUSCODE_INVALID; + strcpybuff(back->r.msg, "FTP file incomplete"); } } - read_len=1024; + read_len = 1024; //HTS_TOTAL_RECV_CHECK(read_len); // Diminuer au besoin si trop de données reçues } } - if (back->r.fp) { - fclose(back->r.fp); - back->r.fp=NULL; + if (back->r.fp) { + fclose(back->r.fp); + back->r.fp = NULL; } } else { - strcpybuff(back->r.msg,"Unable to write file"); + strcpybuff(back->r.msg, "Unable to write file"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } #ifdef _WIN32 closesocket(soc_dat); #else close(soc_dat); #endif - + // 226 Transfer complete? if (back->r.statuscode != -1) { - if (wait_socket_receive(soc_ctl,timeout_onfly)>0) { + if (wait_socket_receive(soc_ctl, timeout_onfly) > 0) { // récupérer 226 transfer complete - get_ftp_line(soc_ctl,line,timeout); - if (line[0]=='2') { // OK - strcpybuff(back->r.msg,"OK"); + get_ftp_line(soc_ctl, line, timeout); + if (line[0] == '2') { // OK + strcpybuff(back->r.msg, "OK"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=HTTP_OK; + back->r.statuscode = HTTP_OK; } else { - sprintf(back->r.msg,"RETR incorrect: %s",linejmp(line)); + sprintf(back->r.msg, "RETR incorrect: %s", linejmp(line)); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } } else { - strcpybuff(back->r.msg,"FTP read error"); + strcpybuff(back->r.msg, "FTP read error"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; } } - + } - - - + } - - + } - + _CHECK_HALT_FTP; - strcpybuff(back->info,"quit"); - send_line(soc_ctl,"QUIT"); // bye bye - get_ftp_line(soc_ctl,NULL,timeout); + strcpybuff(back->info, "quit"); + send_line(soc_ctl, "QUIT"); // bye bye + get_ftp_line(soc_ctl, NULL, timeout); #ifdef _WIN32 closesocket(soc_ctl); #else close(soc_ctl); #endif } - - if (back->r.statuscode!=-1) { - back->r.statuscode=HTTP_OK; - strcpybuff(back->r.msg,"OK"); + + if (back->r.statuscode != -1) { + back->r.statuscode = HTTP_OK; + strcpybuff(back->r.msg, "OK"); } // back->status=STATUS_FTP_READY; // fini return 0; } - - // ouverture d'un port -T_SOC get_datasocket(char* to_send) { +T_SOC get_datasocket(char *to_send) { T_SOC soc = INVALID_SOCKET; - char h_loc[256+2]; - - to_send[0]='\0'; - if (gethostname(h_loc,256)==0) { // host name + char h_loc[256 + 2]; + + to_send[0] = '\0'; + if (gethostname(h_loc, 256) == 0) { // host name SOCaddr server; - int server_size=sizeof(server); - t_hostent* hp_loc; + int server_size = sizeof(server); + t_hostent *hp_loc; t_fullhostent buffer; // effacer structure memset(&server, 0, sizeof(server)); - if ( (hp_loc=vxgethostbyname(h_loc, &buffer)) ) { // notre host + if ((hp_loc = vxgethostbyname(h_loc, &buffer))) { // notre host // copie adresse - SOCaddr_copyaddr(server, server_size, hp_loc->h_addr_list[0], hp_loc->h_length); + SOCaddr_copyaddr(server, server_size, hp_loc->h_addr_list[0], + hp_loc->h_length); - if ( (soc = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0)) != INVALID_SOCKET) { + if ((soc = + (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, + 0)) != INVALID_SOCKET) { - if ( bind(soc,(struct sockaddr*) &server, server_size) == 0 ) { + if (bind(soc, (struct sockaddr *) &server, server_size) == 0) { SOCaddr server2; SOClen len; - len=sizeof(server2); + + len = sizeof(server2); // effacer structure memset(&server2, 0, sizeof(server2)); - if (getsockname(soc,(struct sockaddr*) &server2, &len) == 0) { + if (getsockname(soc, (struct sockaddr *) &server2, &len) == 0) { // *port=ntohs(server.sin_port); // récupérer port - if (listen(soc,10)>=0) { // au pif le 10 + if (listen(soc, 10) >= 0) { // au pif le 10 #if HTS_INET6==0 - unsigned short int a,n1,n2; + unsigned short int a, n1, n2; + // calculer port - a = SOCaddr_sinport(server2); + a = SOCaddr_sinport(server2); n1 = (a & 0xff); - n2 = ((a>>8) & 0xff); + n2 = ((a >> 8) & 0xff); { - char dots[256+2]; - char dot[256+2]; - char* a; + char dots[256 + 2]; + char dot[256 + 2]; + char *a; + SOCaddr_inetntoa(dot, 256, server2, sizeof(server2)); // - dots[0]='\0'; + dots[0] = '\0'; strncatbuff(dots, dot, 128); - while( (a=strchr(dots,'.')) ) *a=','; // virgules! - while( (a=strchr(dots,':')) ) *a=','; // virgules! - sprintf(to_send,"PORT %s,%d,%d",dots,n1,n2); + while((a = strchr(dots, '.'))) + *a = ','; // virgules! + while((a = strchr(dots, ':'))) + *a = ','; // virgules! + sprintf(to_send, "PORT %s,%d,%d", dots, n1, n2); } #else /* - EPRT |1|132.235.1.2|6275| - EPRT |2|1080::8:800:200C:417A|5282| - */ + EPRT |1|132.235.1.2|6275| + EPRT |2|1080::8:800:200C:417A|5282| + */ { - char dot[256+2]; + char dot[256 + 2]; + SOCaddr_inetntoa(dot, 256, server2, len); - sprintf(to_send,"EPRT |%c|%s|%d|", SOCaddr_getproto(server2, len), dot, SOCaddr_sinport(server2)); + sprintf(to_send, "EPRT |%c|%s|%d|", + SOCaddr_getproto(server2, len), dot, + SOCaddr_sinport(server2)); } #endif - + } else { #ifdef _WIN32 closesocket(soc); #else close(soc); #endif - soc=INVALID_SOCKET; + soc = INVALID_SOCKET; } - - + } else { #ifdef _WIN32 closesocket(soc); #else close(soc); #endif - soc=INVALID_SOCKET; + soc = INVALID_SOCKET; } - - + } else { #ifdef _WIN32 closesocket(soc); #else close(soc); #endif - soc=INVALID_SOCKET; + soc = INVALID_SOCKET; } } } } - - + return soc; } #if FTP_DEBUG -FILE* dd=NULL; +FILE *dd = NULL; #endif // routines de réception/émission // 0 = ERROR -int send_line(T_SOC soc,char* data) { +int send_line(T_SOC soc, char *data) { char BIGSTK line[1024]; + if (_DEBUG_HEAD) { if (ioinfo) { - fprintf(ioinfo,"---> %s\x0d\x0a",data); + fprintf(ioinfo, "---> %s\x0d\x0a", data); fflush(ioinfo); } } #if FTP_DEBUG - if (dd == NULL) dd = fopen("toto.txt","w"); - fprintf(dd,"---> %s\x0d\x0a",data); fflush(dd); - printf("---> %s",data); fflush(stdout); + if (dd == NULL) + dd = fopen("toto.txt", "w"); + fprintf(dd, "---> %s\x0d\x0a", data); + fflush(dd); + printf("---> %s", data); + fflush(stdout); #endif - sprintf(line,"%s\x0d\x0a",data); + sprintf(line, "%s\x0d\x0a", data); if (check_socket_connect(soc) != 1) { #if FTP_DEBUG printf("!SOC WRITE ERROR\n"); #endif - return 0; // erreur, plus connecté! + return 0; // erreur, plus connecté! } #if FTP_DEBUG { - int r = (send(soc,line,strlen(line),0) == (int) strlen(line)); - printf("%s\x0d\x0a",data); fflush(stdout); + int r = (send(soc, line, strlen(line), 0) == (int) strlen(line)); + + printf("%s\x0d\x0a", data); + fflush(stdout); return r; } #else - return (send(soc,line,(int)strlen(line),0) == (int) strlen(line)); + return (send(soc, line, (int) strlen(line), 0) == (int) strlen(line)); #endif } -int get_ftp_line(T_SOC soc,char* line,int timeout) { +int get_ftp_line(T_SOC soc, char *line, int timeout) { char BIGSTK data[1024]; - int i,ok,multiline; + int i, ok, multiline; + #if FTP_DEBUG - if (dd == NULL) dd = fopen("toto.txt","w"); + if (dd == NULL) + dd = fopen("toto.txt", "w"); #endif - - data[0]='\0'; - i=ok=multiline=0; data[3]='\0'; + + data[0] = '\0'; + i = ok = multiline = 0; + data[3] = '\0'; do { - char b; - + char b; + // vérifier données - switch(wait_socket_receive(soc,timeout)) { - case -1: // erreur de lecture - if (line) strcpybuff(line,"500 *read error"); + switch (wait_socket_receive(soc, timeout)) { + case -1: // erreur de lecture + if (line) + strcpybuff(line, "500 *read error"); return 0; break; case 0: - if (line) sprintf(line,"500 *read timeout (%d)",timeout); + if (line) + sprintf(line, "500 *read timeout (%d)", timeout); return 0; break; } - + //HTS_TOTAL_RECV_CHECK(dummy); // Diminuer au besoin si trop de données reçues - switch(recv(soc,&b,1,0)) { + switch (recv(soc, &b, 1, 0)) { //case 0: break; // pas encore --> erreur (on attend)! case 1: - HTS_STAT.HTS_TOTAL_RECV+=1; // compter flux entrant - if ((b!=10) && (b!=13)) - data[i++]=b; + HTS_STAT.HTS_TOTAL_RECV += 1; // compter flux entrant + if ((b != 10) && (b != 13)) + data[i++] = b; break; default: - if (line) strcpybuff(line,"500 *read error"); - return 0; // error + if (line) + strcpybuff(line, "500 *read error"); + return 0; // error break; } - if ( ((b==13) || (b==10)) && (i>0) ){ // CR/LF - if ( - (data[3] == '-') - || - ((multiline) && (!isdigit((unsigned char)data[0]))) - ) - { - data[3]='\0'; - i=0; - multiline=1; - } - else - ok=1; // sortir + if (((b == 13) || (b == 10)) && (i > 0)) { // CR/LF + if ((data[3] == '-') + || ((multiline) && (!isdigit((unsigned char) data[0]))) + ) { + data[3] = '\0'; + i = 0; + multiline = 1; + } else + ok = 1; // sortir } } while(!ok); - data[i++]='\0'; - + data[i++] = '\0'; + if (_DEBUG_HEAD) { if (ioinfo) { - fprintf(ioinfo,"<--- %s\x0d\x0a",data); + fprintf(ioinfo, "<--- %s\x0d\x0a", data); fflush(ioinfo); } } #if FTP_DEBUG - fprintf(dd,"<--- %s\n",data); fflush(dd); - printf("<--- %s\n",data); + fprintf(dd, "<--- %s\n", data); + fflush(dd); + printf("<--- %s\n", data); #endif - if (line) strcpybuff(line,data); + if (line) + strcpybuff(line, data); return (strnotempty(data)); } // sauter NNN -char* linejmp(char* line) { - if (strlen(line)>4) - return line+4; +char *linejmp(char *line) { + if (strlen(line) > 4) + return line + 4; else return line; } @@ -1035,78 +1080,82 @@ char* linejmp(char* line) { // 1 : data detected // -1: error int check_socket(T_SOC soc) { - fd_set fds,fds_e; // poll structures - struct timeval tv; // structure for select + fd_set fds, fds_e; // poll structures + struct timeval tv; // structure for select + FD_ZERO(&fds); - FD_ZERO(&fds_e); + FD_ZERO(&fds_e); // socket read - FD_SET(soc,&fds); + FD_SET(soc, &fds); // socket error - FD_SET(soc,&fds_e); - tv.tv_sec=0; - tv.tv_usec=0; + FD_SET(soc, &fds_e); + tv.tv_sec = 0; + tv.tv_usec = 0; // poll! - select(soc + 1,&fds,NULL,&fds_e,&tv); - if (FD_ISSET(soc,&fds_e)) { // error detected + select(soc + 1, &fds, NULL, &fds_e, &tv); + if (FD_ISSET(soc, &fds_e)) { // error detected return -1; - } else if (FD_ISSET(soc,&fds)) { + } else if (FD_ISSET(soc, &fds)) { return 1; } return 0; } + // check if connected int check_socket_connect(T_SOC soc) { - fd_set fds,fds_e; // poll structures - struct timeval tv; // structure for select + fd_set fds, fds_e; // poll structures + struct timeval tv; // structure for select + FD_ZERO(&fds); - FD_ZERO(&fds_e); + FD_ZERO(&fds_e); // socket write - FD_SET(soc,&fds); + FD_SET(soc, &fds); // socket error - FD_SET(soc,&fds_e); - tv.tv_sec=0; - tv.tv_usec=0; + FD_SET(soc, &fds_e); + tv.tv_sec = 0; + tv.tv_usec = 0; // poll! - select(soc + 1,NULL,&fds,&fds_e,&tv); - if (FD_ISSET(soc,&fds_e)) { // error detected + select(soc + 1, NULL, &fds, &fds_e, &tv); + if (FD_ISSET(soc, &fds_e)) { // error detected return -1; - } else if (FD_ISSET(soc,&fds)) { + } else if (FD_ISSET(soc, &fds)) { return 1; } return 0; } + // attendre des données -int wait_socket_receive(T_SOC soc,int timeout) { +int wait_socket_receive(T_SOC soc, int timeout) { // attendre les données - TStamp ltime=time_local(); + TStamp ltime = time_local(); int r; + #if FTP_DEBUG - printf("\x0dWaiting for data "); fflush(stdout); + printf("\x0dWaiting for data "); + fflush(stdout); #endif - while( (!(r = check_socket(soc))) && ( ((int) ((TStamp) (time_local()-ltime))) < timeout )) { + while((!(r = check_socket(soc))) + && (((int) ((TStamp) (time_local() - ltime))) < timeout)) { Sleep(100); #if FTP_DEBUG - printf("."); fflush(stdout); + printf("."); + fflush(stdout); #endif } #if FTP_DEBUG - printf("\x0dreturn: %d\x0d",r); fflush(stdout); + printf("\x0dreturn: %d\x0d", r); + fflush(stdout); #endif return r; } - // cancel reçu? -int stop_ftp(lien_back* back) { +int stop_ftp(lien_back * back) { if (back->stop_ftp) { strcpybuff(back->r.msg, "Cancelled by User"); // back->status=STATUS_FTP_READY; // fini - back->r.statuscode=STATUSCODE_INVALID; + back->r.statuscode = STATUSCODE_INVALID; return 1; } return 0; } - - - - diff --git a/src/htsftp.h b/src/htsftp.h index 2cd46fe..d7e4bfb 100644 --- a/src/htsftp.h +++ b/src/htsftp.h @@ -17,26 +17,22 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: basic FTP protocol manager .h */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - #ifndef HTSFTP_DEFH -#define HTSFTP_DEFH +#define HTSFTP_DEFH #include "htsbase.h" #include "htsbasenet.h" @@ -58,30 +54,29 @@ typedef struct httrackp httrackp; typedef struct FTPDownloadStruct FTPDownloadStruct; #endif struct FTPDownloadStruct { - lien_back *pBack; - httrackp *pOpt; + lien_back *pBack; + httrackp *pOpt; }; /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE #if USE_BEGINTHREAD -void launch_ftp(FTPDownloadStruct *params); -void back_launch_ftp( void* pP ); +void launch_ftp(FTPDownloadStruct * params); +void back_launch_ftp(void *pP); #else -void launch_ftp(FTPDownloadStruct *params,char* path,char* exec); -int back_launch_ftp(FTPDownloadStruct *params); +void launch_ftp(FTPDownloadStruct * params, char *path, char *exec); +int back_launch_ftp(FTPDownloadStruct * params); #endif -int run_launch_ftp(FTPDownloadStruct *params); -int send_line(T_SOC soc,char* data); -int get_ftp_line(T_SOC soc,char* line,int timeout); -T_SOC get_datasocket(char* to_send); -int stop_ftp(lien_back* back); -char* linejmp(char* line); +int run_launch_ftp(FTPDownloadStruct * params); +int send_line(T_SOC soc, char *data); +int get_ftp_line(T_SOC soc, char *line, int timeout); +T_SOC get_datasocket(char *to_send); +int stop_ftp(lien_back * back); +char *linejmp(char *line); int check_socket(T_SOC soc); int check_socket_connect(T_SOC soc); -int wait_socket_receive(T_SOC soc,int timeout); +int wait_socket_receive(T_SOC soc, int timeout); #endif #endif - diff --git a/src/htsglobal.h b/src/htsglobal.h index 8d13c43..f18c14b 100644 --- a/src/htsglobal.h +++ b/src/htsglobal.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Global #define file */ /* Author: Xavier Roche */ @@ -71,7 +68,6 @@ Please visit our Website: http://www.httrack.com #endif #endif - // config.h #ifdef _WIN32 @@ -101,7 +97,7 @@ Please visit our Website: http://www.httrack.com #define HTS_SPARE_MEMORY 1 #define HTS_ALIGN 8 #define BIGSTK static -#undef DLLIB // LoadLibrary(libssl) crashes +#undef DLLIB // LoadLibrary(libssl) crashes #define NOSTRDEBUG 1 #undef HTS_MAKE_KEYWORD_INDEX #ifdef HTS_CECOMPAT @@ -175,7 +171,7 @@ Please visit our Website: http://www.httrack.com #define HTS_SPARE_MEMORY 0 #endif -#ifndef BIGSTK +#ifndef BIGSTK #define BIGSTK #endif @@ -304,7 +300,7 @@ Please visit our Website: http://www.httrack.com #define HTSEXT_API __declspec(dllimport) #endif #else -#define HTSEXT_API +#define HTSEXT_API #endif #ifndef HTS_LONGLONG @@ -320,41 +316,48 @@ Please visit our Website: http://www.httrack.com #if HTS_LONGLONG #ifdef LLINT_FORMAT - typedef LLINT_TYPE LLint; - typedef LLINT_TYPE TStamp; - #define LLintP LLINT_FORMAT +typedef LLINT_TYPE LLint; +typedef LLINT_TYPE TStamp; + +#define LLintP LLINT_FORMAT #else #ifdef _WIN32 - typedef __int64 LLint; - typedef __int64 TStamp; - #define LLintP "%I64d" +typedef __int64 LLint; +typedef __int64 TStamp; + +#define LLintP "%I64d" #elif (defined(__x86_64__) || defined(_LP64) || defined(__64BIT__)) - typedef unsigned long int LLint; - typedef unsigned long int TStamp; - #define LLintP "%ld" +typedef unsigned long int LLint; +typedef unsigned long int TStamp; + +#define LLintP "%ld" #else - typedef long long int LLint; - typedef long long int TStamp; - #define LLintP "%lld" +typedef long long int LLint; +typedef long long int TStamp; + +#define LLintP "%lld" #endif -#endif /* HTS_LONGLONG */ +#endif /* HTS_LONGLONG */ #else - typedef int LLint; - #define LLintP "%d" - typedef double TStamp; +typedef int LLint; + +#define LLintP "%d" +typedef double TStamp; #endif #ifdef LFS_FLAG typedef LLint INTsys; + #define INTsysP LLintP #ifdef __linux #define HTS_FSEEKO #endif #else typedef int INTsys; + #define INTsysP "%d" #endif @@ -378,7 +381,7 @@ typedef int T_SOC; #ifdef _WIN32 #else -#define __cdecl +#define __cdecl #endif /* mode pour mkdir ET chmod (accès aux fichiers) */ @@ -408,8 +411,6 @@ typedef int T_SOC; /* fflush sur stdout */ #define io_flush { fflush(stdout); fflush(stdin); } - - /* HTSLib */ // Cache DNS, accélère les résolution d'adresses @@ -435,15 +436,16 @@ typedef int T_SOC; //#define HTS_TRACE_MALLOC #ifdef HTS_TRACE_MALLOC typedef unsigned long int t_htsboundary; + #ifndef HTS_DEF_FWSTRUCT_mlink #define HTS_DEF_FWSTRUCT_mlink typedef struct mlink mlink; #endif struct mlink { - char* adr; + char *adr; int len; int id; - struct mlink* next; + struct mlink *next; }; static const t_htsboundary htsboundary = 0xDEADBEEF; #endif @@ -454,7 +456,6 @@ static const t_htsboundary htsboundary = 0xDEADBEEF; #define STRDEBUG 1 #endif - /* ------------------------------------------------------------ */ /* Debugging */ /* ------------------------------------------------------------ */ @@ -498,7 +499,6 @@ static const t_htsboundary htsboundary = 0xDEADBEEF; // htsmain #define DEBUG_STEPS 0 - // Débuggage de contrôle #if HTS_DEBUG_CLOSESOCK #define _HTS_WIDE 1 @@ -507,13 +507,11 @@ static const t_htsboundary htsboundary = 0xDEADBEEF; #define _HTS_WIDE 1 #endif #if _HTS_WIDE -extern FILE* DEBUG_fp; +extern FILE *DEBUG_fp; + #define DEBUG_W(A) { if (DEBUG_fp==NULL) DEBUG_fp=fopen("bug.out","wb"); fprintf(DEBUG_fp,":>"A); fflush(DEBUG_fp); } #undef _ #define _ , #endif - - #endif - diff --git a/src/htshash.c b/src/htshash.c index 034ee81..de673b5 100644 --- a/src/htshash.c +++ b/src/htshash.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* hash table system (fast index) */ @@ -63,68 +60,70 @@ Please visit our Website: http://www.httrack.com // type: numero enregistrement - 0 est case insensitive (sav) 1 (adr+fil) 2 (former_adr+former_fil) // recherche dans la table selon nom1,nom2 et le no d'enregistrement // retour: position ou -1 si non trouvé -int hash_read(const hash_struct* hash,const char* nom1,const char* nom2,int type,int normalized) { - char BIGSTK normfil_[HTS_URLMAXSIZE*2]; - char catbuff[CATBUFF_SIZE]; - const char* normfil; - const char* normadr; +int hash_read(const hash_struct * hash, const char *nom1, const char *nom2, + int type, int normalized) { + char BIGSTK normfil_[HTS_URLMAXSIZE * 2]; + char catbuff[CATBUFF_SIZE]; + const char *normfil; + const char *normadr; unsigned int cle; - int pos; + int pos; + // calculer la clé de recherche, non modulée if (type) - cle = hash_cle(nom1,nom2); + cle = hash_cle(nom1, nom2); else - cle = hash_cle(convtolower(catbuff,nom1),nom2); // case insensitive + cle = hash_cle(convtolower(catbuff, nom1), nom2); // case insensitive // la position se calcule en modulant - pos = (int) (cle%HTS_HASH_SIZE); + pos = (int) (cle % HTS_HASH_SIZE); // entrée trouvée? - if (hash->hash[type][pos] >= 0) { // un ou plusieurs enregistrement(s) avec une telle clé existe.. + if (hash->hash[type][pos] >= 0) { // un ou plusieurs enregistrement(s) avec une telle clé existe.. // tester table de raccourcis (hash) // pos est maintenant la position recherchée dans liens pos = hash->hash[type][pos]; - while (pos>=0) { // parcourir la chaine + while(pos >= 0) { // parcourir la chaine switch (type) { - case 0: // sav - if (strfield2(nom1,hash->liens[pos]->sav)) { // case insensitive + case 0: // sav + if (strfield2(nom1, hash->liens[pos]->sav)) { // case insensitive #if DEBUG_HASH==2 - printf("hash: found shortcut at %d\n",pos); + printf("hash: found shortcut at %d\n", pos); #endif return pos; } break; - case 1: // adr+fil + case 1: // adr+fil { if (!normalized) - normfil=hash->liens[pos]->fil; + normfil = hash->liens[pos]->fil; else - normfil=fil_normalized(hash->liens[pos]->fil,normfil_); + normfil = fil_normalized(hash->liens[pos]->fil, normfil_); if (!normalized) normadr = jump_identification(hash->liens[pos]->adr); else normadr = jump_normalized(hash->liens[pos]->adr); - if ((strfield2(nom1,normadr)!=0) && (strcmp(nom2,normfil)==0)) { + if ((strfield2(nom1, normadr) != 0) && (strcmp(nom2, normfil) == 0)) { #if DEBUG_HASH==2 - printf("hash: found shortcut at %d\n",pos); + printf("hash: found shortcut at %d\n", pos); #endif return pos; } } break; - case 2: // former_adr+former_fil + case 2: // former_adr+former_fil { if (hash->liens[pos]->former_adr) { if (!normalized) - normfil=hash->liens[pos]->former_fil; + normfil = hash->liens[pos]->former_fil; else - normfil=fil_normalized(hash->liens[pos]->former_fil,normfil_); + normfil = fil_normalized(hash->liens[pos]->former_fil, normfil_); if (!normalized) normadr = jump_identification(hash->liens[pos]->former_adr); else normadr = jump_normalized(hash->liens[pos]->former_adr); - - if ((strfield2(nom1,normadr)!=0) && (strcmp(nom2,normfil)==0)) { + + if ((strfield2(nom1, normadr) != 0) && (strcmp(nom2, normfil) == 0)) { #if DEBUG_HASH==2 - printf("hash: found shortcut at %d\n",pos); + printf("hash: found shortcut at %d\n", pos); #endif return pos; } @@ -134,135 +133,140 @@ int hash_read(const hash_struct* hash,const char* nom1,const char* nom2,int type } // calculer prochaine position dans la chaine { - int old=pos; - pos=hash->liens[pos]->hash_next[type]; // sinon prochain dans la chaine - if (old==pos) - pos=-1; // erreur de bouclage (ne devrait pas arriver) + int old = pos; + + pos = hash->liens[pos]->hash_next[type]; // sinon prochain dans la chaine + if (old == pos) + pos = -1; // erreur de bouclage (ne devrait pas arriver) } } - + // Ok va falloir chercher alors.. /*pos=hash->max_lien; // commencer à max_lien - switch (type) { - case 0: // sav - while(pos>=0) { - if (hash->liens[pos]->hash_sav == cle ) { - if (strcmp(nom1,hash->liens[pos]->sav)==0) { - hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table -#if DEBUG_HASH==2 - printf("hash: found long search at %d\n",pos); -#endif - return pos; - } - } - pos--; - } - break; - case 1: // adr+fil - while(pos>=0) { - if (hash->liens[pos]->hash_adrfil == cle ) { - if ((strcmp(nom1,hash->liens[pos]->adr)==0) && (strcmp(nom2,hash->liens[pos]->fil)==0)) { - hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table -#if DEBUG_HASH==2 - printf("hash: found long search at %d\n",pos); -#endif - return pos; - } - } - pos--; - } - break; - case 2: // former_adr+former_fil - while(pos>=0) { - if (hash->liens[pos]->hash_fadrfil == cle ) { - if (hash->liens[pos]->former_adr) - if ((strcmp(nom1,hash->liens[pos]->former_adr)==0) && (strcmp(nom2,hash->liens[pos]->former_fil)==0)) { - hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table -#if DEBUG_HASH==2 - printf("hash: found long search at %d\n",pos); -#endif - return pos; - } - } - pos--; - } - }*/ + switch (type) { + case 0: // sav + while(pos>=0) { + if (hash->liens[pos]->hash_sav == cle ) { + if (strcmp(nom1,hash->liens[pos]->sav)==0) { + hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table + #if DEBUG_HASH==2 + printf("hash: found long search at %d\n",pos); + #endif + return pos; + } + } + pos--; + } + break; + case 1: // adr+fil + while(pos>=0) { + if (hash->liens[pos]->hash_adrfil == cle ) { + if ((strcmp(nom1,hash->liens[pos]->adr)==0) && (strcmp(nom2,hash->liens[pos]->fil)==0)) { + hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table + #if DEBUG_HASH==2 + printf("hash: found long search at %d\n",pos); + #endif + return pos; + } + } + pos--; + } + break; + case 2: // former_adr+former_fil + while(pos>=0) { + if (hash->liens[pos]->hash_fadrfil == cle ) { + if (hash->liens[pos]->former_adr) + if ((strcmp(nom1,hash->liens[pos]->former_adr)==0) && (strcmp(nom2,hash->liens[pos]->former_fil)==0)) { + hash->hash[type][(int) (cle%HTS_HASH_SIZE)] = pos; // noter plus récent dans shortcut table + #if DEBUG_HASH==2 + printf("hash: found long search at %d\n",pos); + #endif + return pos; + } + } + pos--; + } + } */ #if DEBUG_HASH==1 - printf("hash: not found after test %s%s\n",nom1,nom2); + printf("hash: not found after test %s%s\n", nom1, nom2); #endif - return -1; // non trouvé + return -1; // non trouvé } else { #if DEBUG_HASH==2 - printf("hash: not found %s%s\n",nom1,nom2); + printf("hash: not found %s%s\n", nom1, nom2); #endif - return -1; // non trouvé : clé non entrée (même une fois) + return -1; // non trouvé : clé non entrée (même une fois) } } // enregistrement lien lpos dans les 3 tables hash1..3 -void hash_write(hash_struct* hash,int lpos,int normalized) { - char BIGSTK normfil_[HTS_URLMAXSIZE*2]; - char catbuff[CATBUFF_SIZE]; - const char* normfil; +void hash_write(hash_struct * hash, int lpos, int normalized) { + char BIGSTK normfil_[HTS_URLMAXSIZE * 2]; + char catbuff[CATBUFF_SIZE]; + const char *normfil; unsigned int cle; - int pos; - int* ptr; + int pos; + int *ptr; + // - if (hash->liens[lpos]) { // on sait jamais.. - hash->max_lien = max(hash->max_lien,lpos); + if (hash->liens[lpos]) { // on sait jamais.. + hash->max_lien = max(hash->max_lien, lpos); #if DEBUG_HASH - hashnumber=hash->max_lien; + hashnumber = hash->max_lien; #endif // élément actuel sur -1 (fin de chaine) - hash->liens[lpos]->hash_next[0]=hash->liens[lpos]->hash_next[1]=hash->liens[lpos]->hash_next[2]=-1; + hash->liens[lpos]->hash_next[0] = hash->liens[lpos]->hash_next[1] = + hash->liens[lpos]->hash_next[2] = -1; // - cle = hash_cle(convtolower(catbuff,hash->liens[lpos]->sav),""); // CASE INSENSITIVE - pos = (int) (cle%HTS_HASH_SIZE); - ptr = hash_calc_chaine(hash,0,pos); // calculer adresse chaine - *ptr = lpos; // noter dernier enregistré + cle = hash_cle(convtolower(catbuff, hash->liens[lpos]->sav), ""); // CASE INSENSITIVE + pos = (int) (cle % HTS_HASH_SIZE); + ptr = hash_calc_chaine(hash, 0, pos); // calculer adresse chaine + *ptr = lpos; // noter dernier enregistré #if DEBUG_HASH==3 - printf("[%d",pos); + printf("[%d", pos); #endif // if (!normalized) - normfil=hash->liens[lpos]->fil; + normfil = hash->liens[lpos]->fil; else - normfil=fil_normalized(hash->liens[lpos]->fil,normfil_); + normfil = fil_normalized(hash->liens[lpos]->fil, normfil_); if (!normalized) - cle = hash_cle(jump_identification(hash->liens[lpos]->adr),normfil); + cle = hash_cle(jump_identification(hash->liens[lpos]->adr), normfil); else - cle = hash_cle(jump_normalized(hash->liens[lpos]->adr),normfil); - pos = (int) (cle%HTS_HASH_SIZE); - ptr = hash_calc_chaine(hash,1,pos); // calculer adresse chaine - *ptr = lpos; // noter dernier enregistré + cle = hash_cle(jump_normalized(hash->liens[lpos]->adr), normfil); + pos = (int) (cle % HTS_HASH_SIZE); + ptr = hash_calc_chaine(hash, 1, pos); // calculer adresse chaine + *ptr = lpos; // noter dernier enregistré #if DEBUG_HASH==3 - printf(",%d",pos); + printf(",%d", pos); #endif // - if (hash->liens[lpos]->former_adr) { // former_adr existe? + if (hash->liens[lpos]->former_adr) { // former_adr existe? if (!normalized) - normfil=hash->liens[lpos]->former_fil; + normfil = hash->liens[lpos]->former_fil; else - normfil=fil_normalized(hash->liens[lpos]->former_fil,normfil_); + normfil = fil_normalized(hash->liens[lpos]->former_fil, normfil_); if (!normalized) - cle = hash_cle(jump_identification(hash->liens[lpos]->former_adr),normfil); + cle = + hash_cle(jump_identification(hash->liens[lpos]->former_adr), normfil); else - cle = hash_cle(jump_normalized(hash->liens[lpos]->former_adr),normfil); - pos = (int) (cle%HTS_HASH_SIZE); - ptr = hash_calc_chaine(hash,2,pos); // calculer adresse chaine - *ptr = lpos; // noter dernier enregistré + cle = hash_cle(jump_normalized(hash->liens[lpos]->former_adr), normfil); + pos = (int) (cle % HTS_HASH_SIZE); + ptr = hash_calc_chaine(hash, 2, pos); // calculer adresse chaine + *ptr = lpos; // noter dernier enregistré #if DEBUG_HASH==3 - printf(",%d",pos); + printf(",%d", pos); #endif } #if DEBUG_HASH==3 - printf("] "); fflush(stdout); + printf("] "); + fflush(stdout); #endif } #if DEBUT_HASH else { printf("* hash_write=0!!\n"); - abortLogFmt("unexpected error in hash_write (pos=%d)" _ pos); + abortLogFmt("unexpected error in hash_write (pos=%d)" _pos); abort(); } #endif @@ -271,37 +275,37 @@ void hash_write(hash_struct* hash,int lpos,int normalized) { // calcul clé // il n'y a pas de formule de hashage universelle, celle-ci semble acceptable.. -unsigned long int hash_cle(const char* nom1, const char* nom2) { +unsigned long int hash_cle(const char *nom1, const char *nom2) { /* - unsigned int sum=0; - int i=0; - while(*nom1) { - sum += 1; - sum += (unsigned int) *(nom1); - sum *= (unsigned int) *(nom1++); - sum += (unsigned int) i; - i++; - } - while(*nom2) { - sum += 1; - sum += (unsigned int) *(nom2); - sum *= (unsigned int) *(nom2++); - sum += (unsigned int) i; - i++; - } - */ + unsigned int sum=0; + int i=0; + while(*nom1) { + sum += 1; + sum += (unsigned int) *(nom1); + sum *= (unsigned int) *(nom1++); + sum += (unsigned int) i; + i++; + } + while(*nom2) { + sum += 1; + sum += (unsigned int) *(nom2); + sum *= (unsigned int) *(nom2++); + sum += (unsigned int) i; + i++; + } + */ return md5sum32(nom1) - +md5sum32(nom2); + + md5sum32(nom2); } // calcul de la position finale dans la chaine des elements ayant la même clé -int* hash_calc_chaine(hash_struct* hash,int type,int pos) { +int *hash_calc_chaine(hash_struct * hash, int type, int pos) { #if DEBUG_HASH - int count=0; + int count = 0; #endif if (hash->hash[type][pos] == -1) return &(hash->hash[type][pos]); // premier élément dans la chaine - pos=hash->hash[type][pos]; + pos = hash->hash[type][pos]; while(hash->liens[pos]->hash_next[type] != -1) { pos = hash->liens[pos]->hash_next[type]; #if DEBUG_HASH @@ -310,9 +314,9 @@ int* hash_calc_chaine(hash_struct* hash,int type,int pos) { } #if DEBUG_HASH count++; - longest_hash[type]=max(longest_hash[type],count); + longest_hash[type] = max(longest_hash[type], count); #endif return &(hash->liens[pos]->hash_next[type]); } -// FIN GESTION DES TABLES DE HACHAGE +// FIN GESTION DES TABLES DE HACHAGE diff --git a/src/htshash.h b/src/htshash.h index 20399e6..36e0ae8 100644 --- a/src/htshash.h +++ b/src/htshash.h @@ -17,28 +17,23 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* hash table system (fast index) */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - - #ifndef HTSHASH_DEFH -#define HTSHASH_DEFH +#define HTSHASH_DEFH /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE @@ -50,10 +45,11 @@ typedef struct hash_struct hash_struct; #endif // tables de hachage -int hash_read(const hash_struct* hash,const char* nom1,const char* nom2,int type,int normalized); -void hash_write(hash_struct* hash,int lpos,int normalized); -int* hash_calc_chaine(hash_struct* hash,int type,int pos); -unsigned long int hash_cle(const char* nom1,const char* nom2); +int hash_read(const hash_struct * hash, const char *nom1, const char *nom2, + int type, int normalized); +void hash_write(hash_struct * hash, int lpos, int normalized); +int *hash_calc_chaine(hash_struct * hash, int type, int pos); +unsigned long int hash_cle(const char *nom1, const char *nom2); #endif #endif diff --git a/src/htshelp.c b/src/htshelp.c index 37aee84..c6e91d8 100644 --- a/src/htshelp.c +++ b/src/htshelp.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* command-line help system */ @@ -56,50 +53,53 @@ Please visit our Website: http://www.httrack.com /* END specific definitions */ #define waitkey if (more) { char s[4]; printf("\nMORE.. q to quit\n"); linput(stdin,s,4); if (strcmp(s,"q")==0) quit=1; else printf("Page %d\n\n",++m); } -void infomsg(char* msg) { - int l=0; - int m=0; - int more=0; - int quit=0; - int done=0; +void infomsg(char *msg) { + int l = 0; + int m = 0; + int more = 0; + int quit = 0; + int done = 0; + // - if (msg==NULL) - quit=0; + if (msg == NULL) + quit = 0; if (msg) { if (!quit) { - if (strlen(msg)==1) { - if (msg[0]=='1') { - more=1; + if (strlen(msg) == 1) { + if (msg[0] == '1') { + more = 1; return; } } /* afficher alias? */ - if (((int)strlen(msg)) > 4) { - if (msg[0]==' ') { - if (msg[2]!=' ') { - if ((msg[3]==' ') || (msg[4]==' ')) { - char cmd[32]="-"; - int p=0; - while(cmd[p]==' ') p++; - sscanf(msg+p,"%s",cmd+strlen(cmd)); + if (((int) strlen(msg)) > 4) { + if (msg[0] == ' ') { + if (msg[2] != ' ') { + if ((msg[3] == ' ') || (msg[4] == ' ')) { + char cmd[32] = "-"; + int p = 0; + + while(cmd[p] == ' ') + p++; + sscanf(msg + p, "%s", cmd + strlen(cmd)); /* clears cN -> c */ - if ((p = (int) strlen(cmd))>2) - if (cmd[p-1]=='N') - cmd[p-1]='\0'; + if ((p = (int) strlen(cmd)) > 2) + if (cmd[p - 1] == 'N') + cmd[p - 1] = '\0'; /* finds alias (if any) */ - p=optreal_find(cmd); - if (p>=0) { + p = optreal_find(cmd); + if (p >= 0) { /* fings type of parameter: number,param,param concatenated,single cmd */ - if (strcmp(opttype_value(p),"param") == 0) - printf("%s (--%s[=N])\n",msg,optalias_value(p)); - else if (strcmp(opttype_value(p),"param1") == 0) - printf("%s (--%s <param>)\n",msg,optalias_value(p)); - else if (strcmp(opttype_value(p),"param0") == 0) - printf("%s (--%s<param>)\n",msg,optalias_value(p)); + if (strcmp(opttype_value(p), "param") == 0) + printf("%s (--%s[=N])\n", msg, optalias_value(p)); + else if (strcmp(opttype_value(p), "param1") == 0) + printf("%s (--%s <param>)\n", msg, optalias_value(p)); + else if (strcmp(opttype_value(p), "param0") == 0) + printf("%s (--%s<param>)\n", msg, optalias_value(p)); else - printf("%s (--%s)\n",msg,optalias_value(p)); - done=1; + printf("%s (--%s)\n", msg, optalias_value(p)); + done = 1; } } } @@ -108,182 +108,214 @@ void infomsg(char* msg) { /* sinon */ if (!done) - printf("%s\n",msg); + printf("%s\n", msg); l++; - if (l>20) { - l=0; + if (l > 20) { + l = 0; waitkey; } } } } -void help_wizard(httrackp* opt) { - char* urls = (char*) malloct(HTS_URLMAXSIZE*2); - char* mainpath = (char*) malloct(256); - char* projname = (char*) malloct(256); - char* stropt = (char*) malloct(2048); // options - char* stropt2 = (char*) malloct(2048); // options longues - char* strwild = (char*) malloct(2048); // wildcards - char* cmd = (char*) malloct(4096); - char* str = (char*) malloct(256); - char** argv = (char**) malloct(256 * sizeof(char*)); +void help_wizard(httrackp * opt) { + char *urls = (char *) malloct(HTS_URLMAXSIZE * 2); + char *mainpath = (char *) malloct(256); + char *projname = (char *) malloct(256); + char *stropt = (char *) malloct(2048); // options + char *stropt2 = (char *) malloct(2048); // options longues + char *strwild = (char *) malloct(2048); // wildcards + char *cmd = (char *) malloct(4096); + char *str = (char *) malloct(256); + char **argv = (char **) malloct(256 * sizeof(char *)); + // - char* a; + char *a; + // - if (urls == NULL || mainpath == NULL || projname == NULL || stropt == NULL - || stropt2 == NULL || strwild == NULL || cmd == NULL || str == NULL || argv == NULL) { + if (urls == NULL || mainpath == NULL || projname == NULL || stropt == NULL + || stropt2 == NULL || strwild == NULL || cmd == NULL || str == NULL + || argv == NULL) { fprintf(stderr, "* memory exhausted in %s, line %d\n", __FILE__, __LINE__); return; } - urls[0] = mainpath[0] = projname[0] = stropt[0] = stropt2[0] = strwild[0] = cmd[0] = str[0] = '\0'; + urls[0] = mainpath[0] = projname[0] = stropt[0] = stropt2[0] = strwild[0] = + cmd[0] = str[0] = '\0'; // - strcpybuff(stropt,"-"); - mainpath[0]=projname[0]=stropt2[0]=strwild[0]='\0'; + strcpybuff(stropt, "-"); + mainpath[0] = projname[0] = stropt2[0] = strwild[0] = '\0'; // - + printf("\n"); - printf("Welcome to HTTrack Website Copier (Offline Browser) "HTTRACK_VERSION"%s\n", hts_get_version_info(opt)); + printf("Welcome to HTTrack Website Copier (Offline Browser) " HTTRACK_VERSION + "%s\n", hts_get_version_info(opt)); printf("Copyright (C) Xavier Roche and other contributors\n"); #ifdef _WIN32 printf("Note: You are running the commandline version,\n"); printf("run 'WinHTTrack.exe' to get the GUI version.\n"); #endif #ifdef HTTRACK_AFF_WARNING - printf("NOTE: "HTTRACK_AFF_WARNING"\n"); + printf("NOTE: " HTTRACK_AFF_WARNING "\n"); #endif #ifdef HTS_PLATFORM_NAME #if USE_BEGINTHREAD - printf("[compiled: "HTS_PLATFORM_NAME" - MT]\n"); + printf("[compiled: " HTS_PLATFORM_NAME " - MT]\n"); #else - printf("[compiled: "HTS_PLATFORM_NAME"]\n"); + printf("[compiled: " HTS_PLATFORM_NAME "]\n"); #endif #endif printf("To see the option list, enter a blank line or try httrack --help\n"); // // Project name - while(strnotempty(projname)==0) { + while(strnotempty(projname) == 0) { printf("\n"); printf("Enter project name :"); fflush(stdout); - linput(stdin,projname,250); - if (strnotempty(projname)==0) - help("httrack",1); + linput(stdin, projname, 250); + if (strnotempty(projname) == 0) + help("httrack", 1); } // // Path if (strnotempty(hts_gethome())) - printf("\nBase path (return=%s/websites/) :",hts_gethome()); + printf("\nBase path (return=%s/websites/) :", hts_gethome()); else printf("\nBase path (return=current directory) :"); - linput(stdin,str,250); + linput(stdin, str, 250); if (!strnotempty(str)) { - strcatbuff(str,hts_gethome()); - strcatbuff(str,"/websites/"); + strcatbuff(str, hts_gethome()); + strcatbuff(str, "/websites/"); } if (strnotempty(str)) - if ((str[strlen(str)-1]!='/') && (str[strlen(str)-1]!='\\')) - strcatbuff(str,"/"); - strcatbuff(stropt2,"-O \""); strcatbuff(stropt2,str); strcatbuff(stropt2,projname); strcatbuff(stropt2,"\" "); + if ((str[strlen(str) - 1] != '/') && (str[strlen(str) - 1] != '\\')) + strcatbuff(str, "/"); + strcatbuff(stropt2, "-O \""); + strcatbuff(stropt2, str); + strcatbuff(stropt2, projname); + strcatbuff(stropt2, "\" "); // Créer si ce n'est fait un index.html 1er niveau make_empty_index(str); // printf("\n"); printf("Enter URLs (separated by commas or blank spaces) :"); fflush(stdout); - linput(stdin,urls,250); + linput(stdin, urls, 250); if (strnotempty(urls)) { - while( (a=strchr(urls,',')) ) *a=' '; - while( (a=strchr(urls,'\t')) ) *a=' '; - + while((a = strchr(urls, ','))) + *a = ' '; + while((a = strchr(urls, '\t'))) + *a = ' '; + // Action printf("\nAction:\n"); - switch(help_query("Mirror Web Site(s)|Mirror Web Site(s) with Wizard|Just Get Files Indicated|Mirror ALL links in URLs (Multiple Mirror)|Test Links In URLs (Bookmark Test)|Update/Continue a Mirror",1)) { - case 1: break; - case 2: strcatbuff(stropt,"W"); break; - case 3: strcatbuff(stropt2,"--get "); break; - case 4: strcatbuff(stropt2,"--mirrorlinks "); break; - case 5: strcatbuff(stropt2,"--testlinks "); break; - case 6: strcatbuff(stropt2,"--update "); break; - case 0: return; break; + switch (help_query + ("Mirror Web Site(s)|Mirror Web Site(s) with Wizard|Just Get Files Indicated|Mirror ALL links in URLs (Multiple Mirror)|Test Links In URLs (Bookmark Test)|Update/Continue a Mirror", + 1)) { + case 1: + break; + case 2: + strcatbuff(stropt, "W"); + break; + case 3: + strcatbuff(stropt2, "--get "); + break; + case 4: + strcatbuff(stropt2, "--mirrorlinks "); + break; + case 5: + strcatbuff(stropt2, "--testlinks "); + break; + case 6: + strcatbuff(stropt2, "--update "); + break; + case 0: + return; + break; } - + // Proxy printf("\nProxy (return=none) :"); - linput(stdin,str,250); + linput(stdin, str, 250); if (strnotempty(str)) { - while( (a=strchr(str,' ')) ) *a=':'; // port - if (!strchr(jump_identification(str),':')) { + while((a = strchr(str, ' '))) + *a = ':'; // port + if (!strchr(jump_identification(str), ':')) { char str2[256]; + printf("\nProxy port (return=8080) :"); - linput(stdin,str2,250); - strcatbuff(str,":"); - if (strnotempty(str2)==0) - strcatbuff(str,"8080"); + linput(stdin, str2, 250); + strcatbuff(str, ":"); + if (strnotempty(str2) == 0) + strcatbuff(str, "8080"); else - strcatbuff(str,str2); + strcatbuff(str, str2); } - strcatbuff(stropt2,"-P "); strcatbuff(stropt2,str); strcatbuff(stropt2," "); + strcatbuff(stropt2, "-P "); + strcatbuff(stropt2, str); + strcatbuff(stropt2, " "); } - // Display - strcatbuff(stropt2," -%v "); + strcatbuff(stropt2, " -%v "); // Wildcards - printf("\nYou can define wildcards, like: -*.gif +www.*.com/*.zip -*img_*.zip\n"); + printf + ("\nYou can define wildcards, like: -*.gif +www.*.com/*.zip -*img_*.zip\n"); printf("Wildcards (return=none) :"); - linput(stdin,strwild,250); - + linput(stdin, strwild, 250); + // Options do { - printf("\nYou can define additional options, such as recurse level (-r<number>), separed by blank spaces\n"); + printf + ("\nYou can define additional options, such as recurse level (-r<number>), separed by blank spaces\n"); printf("To see the option list, type help\n"); printf("Additional options (return=none) :"); - linput(stdin,str,250); - if (strfield2(str,"help")) { - help("httrack",2); + linput(stdin, str, 250); + if (strfield2(str, "help")) { + help("httrack", 2); } else if (strnotempty(str)) { - strcatbuff(stropt2,str); - strcatbuff(stropt2," "); + strcatbuff(stropt2, str); + strcatbuff(stropt2, " "); } - } while(strfield2(str,"help")); - + } while(strfield2(str, "help")); + { - int argc=1; - int g=0; - int i=0; + int argc = 1; + int g = 0; + int i = 0; + // printf("\n"); - if (strlen(stropt)==1) - stropt[0]='\0'; // aucune - sprintf(cmd,"%s %s %s %s",urls,stropt,stropt2,strwild); - printf("---> Wizard command line: httrack %s\n\n",cmd); + if (strlen(stropt) == 1) + stropt[0] = '\0'; // aucune + sprintf(cmd, "%s %s %s %s", urls, stropt, stropt2, strwild); + printf("---> Wizard command line: httrack %s\n\n", cmd); printf("Ready to launch the mirror? (Y/n) :"); fflush(stdout); - linput(stdin,str,250); + linput(stdin, str, 250); if (strnotempty(str)) { - if (!((str[0]=='y') || (str[0]=='Y'))) - return ; + if (!((str[0] == 'y') || (str[0] == 'Y'))) + return; } printf("\n"); // couper en morceaux - argv[0]="winhttrack"; - argv[1]=cmd; - argc++; - while(cmd[i]) { - if(cmd[i]=='\"') g=!g; - if(cmd[i]==' '){ - if(!g){ - cmd[i]='\0'; - argv[argc++]=cmd+i+1; + argv[0] = "winhttrack"; + argv[1] = cmd; + argc++; + while(cmd[i]) { + if (cmd[i] == '\"') + g = !g; + if (cmd[i] == ' ') { + if (!g) { + cmd[i] = '\0'; + argv[argc++] = cmd + i + 1; } - } + } i++; } - hts_main(argc,argv); + hts_main(argc, argv); } - //} else { - // help("httrack",1); + //} else { + // help("httrack",1); } /* Free buffers */ @@ -296,33 +328,36 @@ void help_wizard(httrackp* opt) { freet(cmd); freet(str); } -int help_query(char* list,int def) { +int help_query(char *list, int def) { char s[256]; - char* a; + char *a; int opt; - int n=1; - a=list; + int n = 1; + + a = list; while(strnotempty(a)) { - char* b = strchr(a,'|'); + char *b = strchr(a, '|'); + if (b) { char str[256]; - str[0]='\0'; + + str[0] = '\0'; // - strncatbuff(str,a,(int) (b - a)); - if (n==def) - printf("(enter)\t%d\t%s\n",n++,str); + strncatbuff(str, a, (int) (b - a)); + if (n == def) + printf("(enter)\t%d\t%s\n", n++, str); else - printf("\t%d\t%s\n",n++,str); - a=b+1; + printf("\t%d\t%s\n", n++, str); + a = b + 1; } else - a=list+strlen(list); + a = list + strlen(list); } printf("\t0\tQuit"); do { printf("\n: "); fflush(stdout); - linput(stdin,s,250); - } while ((strnotempty(s)!=0) && (sscanf(s,"%d",&opt)!=1)); + linput(stdin, s, 250); + } while((strnotempty(s) != 0) && (sscanf(s, "%d", &opt) != 1)); if (strnotempty(s)) return opt; else @@ -330,40 +365,47 @@ int help_query(char* list,int def) { } // Capture d'URL -void help_catchurl(const char* dest_path) { - char BIGSTK adr_prox[HTS_URLMAXSIZE*2]; +void help_catchurl(const char *dest_path) { + char BIGSTK adr_prox[HTS_URLMAXSIZE * 2]; int port_prox; - T_SOC soc=catch_url_init_std(&port_prox,adr_prox); - if (soc!=INVALID_SOCKET) { - char BIGSTK url[HTS_URLMAXSIZE*2]; + T_SOC soc = catch_url_init_std(&port_prox, adr_prox); + + if (soc != INVALID_SOCKET) { + char BIGSTK url[HTS_URLMAXSIZE * 2]; char method[32]; char BIGSTK data[32768]; - url[0]=method[0]=data[0]='\0'; + + url[0] = method[0] = data[0] = '\0'; // - printf("Okay, temporary proxy installed.\nSet your browser's preferences to:\n\n"); - printf("\tProxy's address: \t%s\n\tProxy's port: \t%d\n",adr_prox,port_prox); + printf + ("Okay, temporary proxy installed.\nSet your browser's preferences to:\n\n"); + printf("\tProxy's address: \t%s\n\tProxy's port: \t%d\n", adr_prox, + port_prox); // - if (catch_url(soc,url,method,data)) { - char BIGSTK dest[HTS_URLMAXSIZE*2]; - int i=0; + if (catch_url(soc, url, method, data)) { + char BIGSTK dest[HTS_URLMAXSIZE * 2]; + int i = 0; + do { - sprintf(dest,"%s%s%d",dest_path,"hts-post",i); + sprintf(dest, "%s%s%d", dest_path, "hts-post", i); i++; } while(fexist(dest)); { - FILE* fp=fopen(dest,"wb"); + FILE *fp = fopen(dest, "wb"); + if (fp) { - fwrite(data,strlen(data),1,fp); + fwrite(data, strlen(data), 1, fp); fclose(fp); } } // former URL! { - char BIGSTK finalurl[HTS_URLMAXSIZE*2]; + char BIGSTK finalurl[HTS_URLMAXSIZE * 2]; + escape_check_url(dest); - sprintf(finalurl,"%s"POSTTOK"file:%s",url,dest); - printf("\nThe URL is: \"%s\"\n",finalurl); - printf("You can capture it through: httrack \"%s\"\n",finalurl); + sprintf(finalurl, "%s" POSTTOK "file:%s", url, dest); + printf("\nThe URL is: \"%s\"\n", finalurl); + printf("You can capture it through: httrack \"%s\"\n", finalurl); } } else printf("Unable to analyse the URL\n"); @@ -377,13 +419,16 @@ void help_catchurl(const char* dest_path) { } // Créer un index.html vide -void make_empty_index(char* str) { +void make_empty_index(char *str) { #if 0 - if (!fexist(fconcat(str,"index.html"))) { - FILE* fp=fopen(fconcat(str,"index.html"),"wb"); + if (!fexist(fconcat(str, "index.html"))) { + FILE *fp = fopen(fconcat(str, "index.html"), "wb"); + if (fp) { - fprintf(fp,"<!-- "HTS_TOPINDEX" -->"CRLF); - fprintf(fp,"<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"CRLF); + fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF); + fprintf(fp, + "<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>" + CRLF); fclose(fp); } } @@ -392,24 +437,30 @@ void make_empty_index(char* str) { // mini-aide (h: help) // y -void help(char* app,int more) { +void help(char *app, int more) { char info[2048]; + infomsg(""); if (more) infomsg("1"); if (more != 2) { - sprintf(info, "HTTrack version "HTTRACK_VERSION"%s (compiled "__DATE__")", hts_is_available()); + sprintf(info, + "HTTrack version " HTTRACK_VERSION "%s (compiled " __DATE__ ")", + hts_is_available()); infomsg(info); #ifdef HTTRACK_AFF_WARNING - infomsg("NOTE: "HTTRACK_AFF_WARNING); + infomsg("NOTE: " HTTRACK_AFF_WARNING); #endif - sprintf(info,"\tusage: %s <URLs> [-option] [+<URL_FILTER>] [-<URL_FILTER>] [+<mime:MIME_FILTER>] [-<mime:MIME_FILTER>]",app); + sprintf(info, + "\tusage: %s <URLs> [-option] [+<URL_FILTER>] [-<URL_FILTER>] [+<mime:MIME_FILTER>] [-<mime:MIME_FILTER>]", + app); infomsg(info); infomsg("\twith options listed below: (* is the default value)"); infomsg(""); } infomsg("General options:"); - infomsg(" O path for mirror/logfiles+cache (-O path_mirror[,path_cache_and_logfiles])"); + infomsg + (" O path for mirror/logfiles+cache (-O path_mirror[,path_cache_and_logfiles])"); #ifndef _WIN32 infomsg(" %O chroot path to, must be r00t (-%O root_path)"); #endif @@ -419,7 +470,8 @@ void help(char* app,int more) { infomsg(" W mirror web sites, semi-automatic (asks questions)"); infomsg(" g just get files (saved in the current directory)"); infomsg(" i continue an interrupted mirror using the cache"); - infomsg(" Y mirror ALL links located in the first level pages (mirror links)"); + infomsg + (" Y mirror ALL links located in the first level pages (mirror links)"); infomsg(""); infomsg("Proxy options:"); infomsg(" P proxy use (-P proxy:port or -P user:pass@proxy:port)"); @@ -435,69 +487,103 @@ void help(char* app,int more) { infomsg(" EN maximum mirror time in seconds (60=1 minute, 3600=1 hour)"); infomsg(" AN maximum transfer rate in bytes/seconds (1000=1KB/s max)"); infomsg(" %cN maximum number of connections/seconds (*%c10)"); - infomsg(" GN pause transfer if N bytes reached, and wait until lock file is deleted"); + infomsg + (" GN pause transfer if N bytes reached, and wait until lock file is deleted"); #if HTS_USEMMS - infomsg(" %mN maximum mms stream download time in seconds (60=1 minute, 3600=1 hour)"); + infomsg + (" %mN maximum mms stream download time in seconds (60=1 minute, 3600=1 hour)"); #endif infomsg(""); infomsg("Flow control:"); infomsg(" cN number of multiple connections (*c8)"); - infomsg(" TN timeout, number of seconds after a non-responding link is shutdown"); - infomsg(" RN number of retries, in case of timeout or non-fatal errors (*R1)"); - infomsg(" JN traffic jam control, minimum transfert rate (bytes/seconds) tolerated for a link"); - infomsg(" HN host is abandonned if: 0=never, 1=timeout, 2=slow, 3=timeout or slow"); + infomsg + (" TN timeout, number of seconds after a non-responding link is shutdown"); + infomsg + (" RN number of retries, in case of timeout or non-fatal errors (*R1)"); + infomsg + (" JN traffic jam control, minimum transfert rate (bytes/seconds) tolerated for a link"); + infomsg + (" HN host is abandonned if: 0=never, 1=timeout, 2=slow, 3=timeout or slow"); infomsg(""); infomsg("Links options:"); - infomsg(" %P *extended parsing, attempt to parse all links, even in unknown tags or Javascript (%P0 don't use)"); - infomsg(" n get non-html files 'near' an html file (ex: an image located outside)"); + infomsg + (" %P *extended parsing, attempt to parse all links, even in unknown tags or Javascript (%P0 don't use)"); + infomsg + (" n get non-html files 'near' an html file (ex: an image located outside)"); infomsg(" t test all URLs (even forbidden ones)"); - infomsg(" %L <file> add all URL located in this text file (one URL per line)"); - infomsg(" %S <file> add all scan rules located in this text file (one scan rule per line)"); + infomsg + (" %L <file> add all URL located in this text file (one URL per line)"); + infomsg + (" %S <file> add all scan rules located in this text file (one scan rule per line)"); infomsg(""); infomsg("Build options:"); infomsg(" NN structure type (0 *original structure, 1+: see below)"); infomsg(" or user defined structure (-N \"%h%p/%n%q.%t\")"); - infomsg(" %N delayed type check, don't make any link test but wait for files download to start instead (experimental) (%N0 don't use, %N1 use for unknown extensions, * %N2 always use)"); - infomsg(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)"); + infomsg + (" %N delayed type check, don't make any link test but wait for files download to start instead (experimental) (%N0 don't use, %N1 use for unknown extensions, * %N2 always use)"); + infomsg + (" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)"); infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)"); - infomsg(" LN long names (L1 *long names / L0 8-3 conversion / L2 ISO9660 compatible)"); - infomsg(" KN keep original links (e.g. http://www.adr/link) (K0 *relative link, K absolute links, K4 original links, K3 absolute URI links, K5 transparent proxy link)"); + infomsg + (" LN long names (L1 *long names / L0 8-3 conversion / L2 ISO9660 compatible)"); + infomsg + (" KN keep original links (e.g. http://www.adr/link) (K0 *relative link, K absolute links, K4 original links, K3 absolute URI links, K5 transparent proxy link)"); infomsg(" x replace external html links by error pages"); - infomsg(" %x do not include any password for external password protected websites (%x0 include)"); - infomsg(" %q *include query string for local files (useless, for information purpose only) (%q0 don't include)"); - infomsg(" o *generate output html file in case of error (404..) (o0 don't generate)"); + infomsg + (" %x do not include any password for external password protected websites (%x0 include)"); + infomsg + (" %q *include query string for local files (useless, for information purpose only) (%q0 don't include)"); + infomsg + (" o *generate output html file in case of error (404..) (o0 don't generate)"); infomsg(" X *purge old files after update (X0 keep delete)"); infomsg(" %p preserve html files 'as is' (identical to '-K4 -%F \"\"')"); infomsg(" %T links conversion to UTF-8"); infomsg(""); infomsg("Spider options:"); infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)"); - infomsg(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)"); - infomsg(" j *parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive)"); - infomsg(" sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules))"); - infomsg(" %h force HTTP/1.0 requests (reduce update features, only for old servers or proxies)"); - infomsg(" %k use keep-alive if possible, greately reducing latency for small files and test requests (%k0 don't use)"); - infomsg(" %B tolerant requests (accept bogus responses on some servers, but not standard!)"); - infomsg(" %s update hacks: various hacks to limit re-transfers when updating (identical size, bogus response..)"); - infomsg(" %u url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..)"); - infomsg(" %A assume that a type (cgi,asp..) is always linked with a mime type (-%A php3,cgi=text/html;dat,bin=application/x-zip)"); - infomsg(" shortcut: '--assume standard' is equivalent to -%A "HTS_ASSUME_STANDARD); - infomsg(" can also be used to force a specific file type: --assume foo.cgi=text/html"); - infomsg(" @iN internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only)"); - infomsg(" %w disable a specific external mime module (-%w htsswf -%w htsjava)"); + infomsg + (" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)"); + infomsg + (" j *parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive)"); + infomsg + (" sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules))"); + infomsg + (" %h force HTTP/1.0 requests (reduce update features, only for old servers or proxies)"); + infomsg + (" %k use keep-alive if possible, greately reducing latency for small files and test requests (%k0 don't use)"); + infomsg + (" %B tolerant requests (accept bogus responses on some servers, but not standard!)"); + infomsg + (" %s update hacks: various hacks to limit re-transfers when updating (identical size, bogus response..)"); + infomsg + (" %u url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..)"); + infomsg + (" %A assume that a type (cgi,asp..) is always linked with a mime type (-%A php3,cgi=text/html;dat,bin=application/x-zip)"); + infomsg(" shortcut: '--assume standard' is equivalent to -%A " + HTS_ASSUME_STANDARD); + infomsg + (" can also be used to force a specific file type: --assume foo.cgi=text/html"); + infomsg + (" @iN internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only)"); + infomsg + (" %w disable a specific external mime module (-%w htsswf -%w htsjava)"); infomsg(""); infomsg("Browser ID:"); - infomsg(" F user-agent field sent in HTTP headers (-F \"user-agent name\")"); + infomsg + (" F user-agent field sent in HTTP headers (-F \"user-agent name\")"); infomsg(" %R default referer field sent in HTTP headers"); infomsg(" %E from email address sent in HTTP headers"); - infomsg(" %F footer string in Html code (-%F \"Mirrored [from host %s [file %s [at %s]]]\""); + infomsg + (" %F footer string in Html code (-%F \"Mirrored [from host %s [file %s [at %s]]]\""); infomsg(" %l preffered language (-%l \"fr, en, jp, *\""); infomsg(""); infomsg("Log, index, cache"); - infomsg(" C create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)"); + infomsg + (" C create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)"); infomsg(" k store all files in cache (not useful if files on disk)"); infomsg(" %n do not re-download locally erased files"); - infomsg(" %v display on screen filenames downloaded (in realtime) - * %v1 short version - %v2 full animation"); + infomsg + (" %v display on screen filenames downloaded (in realtime) - * %v1 short version - %v2 full animation"); infomsg(" Q no log - quiet mode"); infomsg(" q no questions - quiet mode"); infomsg(" z log - extra infos"); @@ -517,7 +603,7 @@ void help(char* app,int more) { infomsg(" *p3 save all files"); infomsg(" p7 get html files before, then treat other files"); infomsg(" S stay on the same directory"); - infomsg(" D *can only go down into subdirs"); + infomsg(" D *can only go down into subdirs"); infomsg(" U can only go to upper directories"); infomsg(" B can both go up&down into the directory structure"); infomsg(" a *stay on the same address"); @@ -548,15 +634,19 @@ void help(char* app,int more) { infomsg(" #Z generate transfer rate statictics every minutes"); infomsg(" #! execute a shell command (-#! \"echo hello\")"); infomsg(""); - infomsg("Dangerous options: (do NOT use unless you exactly know what you are doing)"); - infomsg(" %! bypass built-in security limits aimed to avoid bandwith abuses (bandwidth, simultaneous connections)"); + infomsg + ("Dangerous options: (do NOT use unless you exactly know what you are doing)"); + infomsg + (" %! bypass built-in security limits aimed to avoid bandwith abuses (bandwidth, simultaneous connections)"); infomsg(" IMPORTANT NOTE: DANGEROUS OPTION, ONLY SUITABLE FOR EXPERTS"); infomsg(" USE IT WITH EXTREME CARE"); infomsg(""); infomsg("Command-line specific options:"); - infomsg(" V execute system command after each files ($0 is the filename: -V \"rm \\$0\")"); + infomsg + (" V execute system command after each files ($0 is the filename: -V \"rm \\$0\")"); infomsg(" %U run the engine with another id when called as root (-%U smith)"); - infomsg(" %W use an external library function as a wrapper (-%W myfoo.so[,myparameters])"); + infomsg + (" %W use an external library function as a wrapper (-%W myfoo.so[,myparameters])"); /* infomsg(" %O do a chroot before setuid"); */ infomsg(""); infomsg("Details: Option N"); @@ -564,19 +654,27 @@ void help(char* app,int more) { infomsg(" N1 HTML in web/, images/other files in web/images/"); infomsg(" N2 HTML in web/HTML, images/other in web/images"); infomsg(" N3 HTML in web/, images/other in web/"); - infomsg(" N4 HTML in web/, images/other in web/xxx, where xxx is the file extension (all gif will be placed onto web/gif, for example)"); + infomsg + (" N4 HTML in web/, images/other in web/xxx, where xxx is the file extension (all gif will be placed onto web/gif, for example)"); infomsg(" N5 Images/other in web/xxx and HTML in web/HTML"); infomsg(" N99 All files in web/, with random names (gadget !)"); infomsg(" N100 Site-structure, without www.domain.xxx/"); - infomsg(" N101 Identical to N1 exept that \"web\" is replaced by the site's name"); - infomsg(" N102 Identical to N2 exept that \"web\" is replaced by the site's name"); - infomsg(" N103 Identical to N3 exept that \"web\" is replaced by the site's name"); - infomsg(" N104 Identical to N4 exept that \"web\" is replaced by the site's name"); - infomsg(" N105 Identical to N5 exept that \"web\" is replaced by the site's name"); - infomsg(" N199 Identical to N99 exept that \"web\" is replaced by the site's name"); + infomsg + (" N101 Identical to N1 exept that \"web\" is replaced by the site's name"); + infomsg + (" N102 Identical to N2 exept that \"web\" is replaced by the site's name"); + infomsg + (" N103 Identical to N3 exept that \"web\" is replaced by the site's name"); + infomsg + (" N104 Identical to N4 exept that \"web\" is replaced by the site's name"); + infomsg + (" N105 Identical to N5 exept that \"web\" is replaced by the site's name"); + infomsg + (" N199 Identical to N99 exept that \"web\" is replaced by the site's name"); infomsg(" N1001 Identical to N1 exept that there is no \"web\" directory"); infomsg(" N1002 Identical to N2 exept that there is no \"web\" directory"); - infomsg(" N1003 Identical to N3 exept that there is no \"web\" directory (option set for g option)"); + infomsg + (" N1003 Identical to N3 exept that there is no \"web\" directory (option set for g option)"); infomsg(" N1004 Identical to N4 exept that there is no \"web\" directory"); infomsg(" N1005 Identical to N5 exept that there is no \"web\" directory"); infomsg(" N1099 Identical to N99 exept that there is no \"web\" directory"); @@ -593,36 +691,46 @@ void help(char* app,int more) { infomsg(" '%q' small query string MD5 (16 bits, 4 ascii bytes)"); infomsg(" '%s?' Short name version (ex: %sN)"); infomsg(" '%[param]' param variable in query string"); - infomsg(" '%[param:before:after:empty:notfound]' advanced variable extraction"); + infomsg + (" '%[param:before:after:empty:notfound]' advanced variable extraction"); infomsg("Details: User-defined option N and advanced variable extraction"); - infomsg(" %[param:before:after:empty:notfound]"); + infomsg(" %[param:before:after:empty:notfound]"); infomsg(" param : parameter name"); infomsg(" before : string to prepend if the parameter was found"); infomsg(" after : string to append if the parameter was found"); - infomsg(" notfound : string replacement if the parameter could not be found"); + infomsg + (" notfound : string replacement if the parameter could not be found"); infomsg(" empty : string replacement if the parameter was empty"); - infomsg(" all fields, except the first one (the parameter name), can be empty"); + infomsg + (" all fields, except the first one (the parameter name), can be empty"); infomsg(""); infomsg("Details: Option K"); infomsg(" K0 foo.cgi?q=45 -> foo4B54.html?q=45 (relative URI, default)"); - infomsg(" K -> http://www.foobar.com/folder/foo.cgi?q=45 (absolute URL)"); + infomsg + (" K -> http://www.foobar.com/folder/foo.cgi?q=45 (absolute URL)"); infomsg(" K3 -> /folder/foo.cgi?q=45 (absolute URI)"); infomsg(" K4 -> foo.cgi?q=45 (original URL)"); - infomsg(" K5 -> http://www.foobar.com/folder/foo4B54.html?q=45 (transparent proxy URL)"); + infomsg + (" K5 -> http://www.foobar.com/folder/foo4B54.html?q=45 (transparent proxy URL)"); infomsg(""); infomsg("Shortcuts:"); infomsg("--mirror <URLs> *make a mirror of site(s) (default)"); - infomsg("--get <URLs> get the files indicated, do not seek other URLs (-qg)"); + infomsg + ("--get <URLs> get the files indicated, do not seek other URLs (-qg)"); infomsg("--list <text file> add all URL located in this text file (-%L)"); infomsg("--mirrorlinks <URLs> mirror all links in 1st level pages (-Y)"); infomsg("--testlinks <URLs> test links in pages (-r1p0C0I0t)"); - infomsg("--spider <URLs> spider site(s), to test links: reports Errors & Warnings (-p0C0I0t)"); + infomsg + ("--spider <URLs> spider site(s), to test links: reports Errors & Warnings (-p0C0I0t)"); infomsg("--testsite <URLs> identical to --spider"); - infomsg("--skeleton <URLs> make a mirror, but gets only html files (-p1)"); + infomsg + ("--skeleton <URLs> make a mirror, but gets only html files (-p1)"); infomsg("--update update a mirror, without confirmation (-iC2)"); - infomsg("--continue continue a mirror, without confirmation (-iC1)"); + infomsg + ("--continue continue a mirror, without confirmation (-iC1)"); infomsg(""); - infomsg("--catchurl create a temporary proxy to capture an URL or a form post URL"); + infomsg + ("--catchurl create a temporary proxy to capture an URL or a form post URL"); infomsg("--clean erase cache & log files"); infomsg(""); infomsg("--http10 force http/1.0 requests (-%h)"); @@ -633,13 +741,17 @@ void help(char* app,int more) { infomsg("example: httrack www.someweb.com/bob/"); infomsg("means: mirror site www.someweb.com/bob/ and only this site"); infomsg(""); - infomsg("example: httrack www.someweb.com/bob/ www.anothertest.com/mike/ +*.com/*.jpg -mime:application/*"); - infomsg("means: mirror the two sites together (with shared links) and accept any .jpg files on .com sites"); + infomsg + ("example: httrack www.someweb.com/bob/ www.anothertest.com/mike/ +*.com/*.jpg -mime:application/*"); + infomsg + ("means: mirror the two sites together (with shared links) and accept any .jpg files on .com sites"); infomsg(""); infomsg("example: httrack www.someweb.com/bob/bobby.html +* -r6"); - infomsg("means get all files starting from bobby.html, with 6 link-depth, and possibility of going everywhere on the web"); + infomsg + ("means get all files starting from bobby.html, with 6 link-depth, and possibility of going everywhere on the web"); infomsg(""); - infomsg("example: httrack www.someweb.com/bob/bobby.html --spider -P proxy.myhost.com:8080"); + infomsg + ("example: httrack www.someweb.com/bob/bobby.html --spider -P proxy.myhost.com:8080"); infomsg("runs the spider on www.someweb.com/bob/bobby.html using a proxy"); infomsg(""); infomsg("example: httrack --update"); @@ -651,16 +763,15 @@ void help(char* app,int more) { infomsg("example: httrack --continue"); infomsg("continues a mirror in the current folder"); infomsg(""); - sprintf(info, "HTTrack version "HTTRACK_VERSION"%s (compiled "__DATE__")", hts_is_available()); + sprintf(info, "HTTrack version " HTTRACK_VERSION "%s (compiled " __DATE__ ")", + hts_is_available()); infomsg(info); infomsg("Copyright (C) Xavier Roche and other contributors"); #ifdef HTS_PLATFORM_NAME - infomsg("[compiled: "HTS_PLATFORM_NAME"]"); + infomsg("[compiled: " HTS_PLATFORM_NAME "]"); #endif infomsg(NULL); // infomsg(" R *relative links (e.g ../link)\n"); // infomsg(" A absolute links (e.g /www.adr/link)\n"); } - - diff --git a/src/htshelp.h b/src/htshelp.h index fd937cd..a8fbd17 100644 --- a/src/htshelp.h +++ b/src/htshelp.h @@ -17,28 +17,23 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* command-line help system */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - - #ifndef HTSHELP_DEFH -#define HTSHELP_DEFH +#define HTSHELP_DEFH /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE @@ -49,12 +44,12 @@ Please visit our Website: http://www.httrack.com typedef struct httrackp httrackp; #endif -void infomsg(char* msg); -void help(char* app,int more); -void make_empty_index(char* str); -void help_wizard(httrackp* opt); -int help_query(char* list,int def); -void help_catchurl(const char* dest_path); +void infomsg(char *msg); +void help(char *app, int more); +void make_empty_index(char *str); +void help_wizard(httrackp * opt); +int help_query(char *list, int def); +void help_catchurl(const char *dest_path); #endif diff --git a/src/htsindex.c b/src/htsindex.c index 48d50e2..1fa453e 100644 --- a/src/htsindex.c +++ b/src/htsindex.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsindex.c */ /* keyword indexing system (search index) */ @@ -38,7 +35,6 @@ Please visit our Website: http://www.httrack.com /* Internal engine bytecode */ #define HTS_INTERNAL_BYTECODE - #include "htsindex.h" #include "htsglobal.h" #include "htslib.h" @@ -47,7 +43,6 @@ Please visit our Website: http://www.httrack.com #include "htshash.h" #include "htsinthash.h" - /* Keyword Indexer Parameters */ // Maximum length for a keyword @@ -109,33 +104,32 @@ Please visit our Website: http://www.httrack.com /* End of Keyword Indexer Parameters */ -int strcpos(char* adr,char c); -int mystrcmp(const void* _e1,const void* _e2); +int strcpos(char *adr, char c); +int mystrcmp(const void *_e1, const void *_e2); // Global variables -int hts_index_init=1; -int hts_primindex_size=0; -FILE* fp_tmpproject=NULL; -int hts_primindex_words=0; +int hts_index_init = 1; +int hts_primindex_size = 0; +FILE *fp_tmpproject = NULL; +int hts_primindex_words = 0; #endif /* Init index */ -void index_init(const char* indexpath) { +void index_init(const char *indexpath) { #if HTS_MAKE_KEYWORD_INDEX #ifndef _WIN32_WCE /* remove(concat(indexpath,"index.txt")); */ - hts_index_init=1; - hts_primindex_size=0; - hts_primindex_words=0; - fp_tmpproject=tmpfile(); + hts_index_init = 1; + hts_primindex_size = 0; + hts_primindex_words = 0; + fp_tmpproject = tmpfile(); #endif #endif } - /* Indexing system A little bit dirty, (quick'n dirty, in fact) @@ -143,16 +137,19 @@ void index_init(const char* indexpath) { Tags and javascript handled (ignored) */ /* Note: utf-8 */ -int index_keyword(const char* html_data,LLint size,const char* mime,const char* filename,const char* indexpath) { +int index_keyword(const char *html_data, LLint size, const char *mime, + const char *filename, const char *indexpath) { #if HTS_MAKE_KEYWORD_INDEX - char catbuff[CATBUFF_SIZE]; - int intag=0,inscript=0,incomment=0; - char keyword[KEYW_LEN+32]; - int i=0; + char catbuff[CATBUFF_SIZE]; + int intag = 0, inscript = 0, incomment = 0; + char keyword[KEYW_LEN + 32]; + int i = 0; + // - int WordIndexSize=1024; - inthash WordIndexHash=NULL; - FILE *tmpfp=NULL; + int WordIndexSize = 1024; + inthash WordIndexHash = NULL; + FILE *tmpfp = NULL; + // // Check parameters @@ -167,35 +164,29 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char* // Init ? if (hts_index_init) { - UNLINK(concat(catbuff,indexpath,"index.txt")); - UNLINK(concat(catbuff,indexpath,"sindex.html")); - hts_index_init=0; + UNLINK(concat(catbuff, indexpath, "index.txt")); + UNLINK(concat(catbuff, indexpath, "sindex.html")); + hts_index_init = 0; } - // Check MIME type if (is_html_mime_type(mime)) { - inscript=0; - } + inscript = 0; + } // FIXME - temporary fix for image/svg+xml (svg) // "IN XML" (html like, in fact :) ) - else if ( - (strfield2(mime,"image/svg+xml")) - || - (strfield2(mime,"image/svg-xml")) + else if ((strfield2(mime, "image/svg+xml")) + || (strfield2(mime, "image/svg-xml")) #if HTS_USEMMS - || - strfield2(mime,"video/x-ms-asf") + || strfield2(mime, "video/x-ms-asf") #endif ) { - inscript=0; - } - else if ( - (strfield2(mime,"application/x-javascript")) - || (strfield2(mime,"text/css")) + inscript = 0; + } else if ((strfield2(mime, "application/x-javascript")) + || (strfield2(mime, "text/css")) ) { - inscript=1; - //} else if (strfield2(mime, "text/vnd.wap.wml")) { // humm won't work in many cases - // inscript=0; + inscript = 1; + //} else if (strfield2(mime, "text/vnd.wap.wml")) { // humm won't work in many cases + // inscript=0; } else return 0; @@ -206,94 +197,88 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char* // Create hash structure // Hash tables rulez da world! - WordIndexHash=inthash_new(WordIndexSize); + WordIndexHash = inthash_new(WordIndexSize); if (!WordIndexHash) return 0; // Start indexing this page - keyword[0]='\0'; - while(i<size) { - if (strfield(html_data + i , "<script")) { - inscript=1; - } - else if (strfield(html_data + i , "<!--")) { - incomment=1; - } - else if (strfield(html_data + i , "</script")) { + keyword[0] = '\0'; + while(i < size) { + if (strfield(html_data + i, "<script")) { + inscript = 1; + } else if (strfield(html_data + i, "<!--")) { + incomment = 1; + } else if (strfield(html_data + i, "</script")) { if (!incomment) - inscript=0; - } - else if (strfield(html_data + i , "-->")) { - incomment=0; - } - else if (html_data[i]=='<') { + inscript = 0; + } else if (strfield(html_data + i, "-->")) { + incomment = 0; + } else if (html_data[i] == '<') { if (!inscript) - intag=1; - } - else if (html_data[i]=='>') { - intag=0; - } - else { + intag = 1; + } else if (html_data[i] == '>') { + intag = 0; + } else { // Okay, parse keywords - if ( (!inscript) && (!incomment) && (!intag) ) { - char cchar=html_data[i]; + if ((!inscript) && (!incomment) && (!intag)) { + char cchar = html_data[i]; int pos; int len = (int) strlen(keyword); - + // Replace (ignore case, and so on..) - if ((pos=strcpos(KEYW_TRANSCODE_FROM,cchar))>=0) - cchar=KEYW_TRANSCODE_TO[pos]; - - if (strchr(KEYW_ACCEPT,cchar)) { + if ((pos = strcpos(KEYW_TRANSCODE_FROM, cchar)) >= 0) + cchar = KEYW_TRANSCODE_TO[pos]; + + if (strchr(KEYW_ACCEPT, cchar)) { /* Ignore some characters at beginning */ - if ((len>0) || (!strchr(KEYW_IGNORE_BEG,cchar))) { - keyword[len++]=cchar; - keyword[len]='\0'; + if ((len > 0) || (!strchr(KEYW_IGNORE_BEG, cchar))) { + keyword[len++] = cchar; + keyword[len] = '\0'; } - } else if ( (strchr(KEYW_SPACE,cchar)) || (!cchar) ) { - + } else if ((strchr(KEYW_SPACE, cchar)) || (!cchar)) { /* Avoid these words */ - if (len>0) { - if (strchr(KEYW_NOT_BEG,keyword[0])) { - keyword[(len=0)]='\0'; + if (len > 0) { + if (strchr(KEYW_NOT_BEG, keyword[0])) { + keyword[(len = 0)] = '\0'; } } /* Strip ending . and so */ { - int ok=0; + int ok = 0; + while((len = (int) strlen(keyword)) && (!ok)) { - if (strchr(KEYW_STRIP_END,keyword[len-1])) { /* strip it */ - keyword[len-1]='\0'; + if (strchr(KEYW_STRIP_END, keyword[len - 1])) { /* strip it */ + keyword[len - 1] = '\0'; } else - ok=1; + ok = 1; } } - + /* Store it ? */ - if (len >= KEYW_MIN_LEN ) { + if (len >= KEYW_MIN_LEN) { hts_primindex_words++; - if (inthash_inc(WordIndexHash,keyword)) { /* added new */ - fprintf(tmpfp,"%s\n",keyword); + if (inthash_inc(WordIndexHash, keyword)) { /* added new */ + fprintf(tmpfp, "%s\n", keyword); } } - keyword[(len=0)]='\0'; - } else /* Invalid */ - keyword[(len=0)]='\0'; + keyword[(len = 0)] = '\0'; + } else /* Invalid */ + keyword[(len = 0)] = '\0'; - if (len>KEYW_LEN) { - keyword[(len=0)]='\0'; + if (len > KEYW_LEN) { + keyword[(len = 0)] = '\0'; } } - + } - + i++; } // Reset temp file - fseek(tmpfp,0,SEEK_SET); + fseek(tmpfp, 0, SEEK_SET); // Process indexing for this page { @@ -302,21 +287,25 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char* if (fp_tmpproject) { while(!feof(tmpfp)) { char line[KEYW_LEN + 32]; - linput(tmpfp,line,KEYW_LEN + 2); + + linput(tmpfp, line, KEYW_LEN + 2); if (strnotempty(line)) { - intptr_t e=0; - if (inthash_read(WordIndexHash,line,&e)) { + intptr_t e = 0; + + if (inthash_read(WordIndexHash, line, &e)) { //if (e) { - char BIGSTK savelst[HTS_URLMAXSIZE*2]; - e++; /* 0 means "once" */ - - if (strncmp((const char*)fslash(catbuff,(char*)indexpath),filename,strlen(indexpath))==0) // couper - strcpybuff(savelst,filename+strlen(indexpath)); + char BIGSTK savelst[HTS_URLMAXSIZE * 2]; + + e++; /* 0 means "once" */ + + if (strncmp((const char *) fslash(catbuff, (char *) indexpath), filename, strlen(indexpath)) == 0) // couper + strcpybuff(savelst, filename + strlen(indexpath)); else - strcpybuff(savelst,filename); - + strcpybuff(savelst, filename); + // Add entry for this file and word - fprintf(fp_tmpproject,"%s %d %s\n",line,(int) (KEYW_SORT_MAXCOUNT - e),savelst); + fprintf(fp_tmpproject, "%s %d %s\n", line, + (int) (KEYW_SORT_MAXCOUNT - e), savelst); hts_primindex_size++; //} } @@ -328,7 +317,7 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char* // Delete temp file fclose(tmpfp); - tmpfp=NULL; + tmpfp = NULL; // Clear hash table inthash_delete(&WordIndexHash); @@ -340,125 +329,134 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char* Sort index! */ /* Note: NOT utf-8 */ -void index_finish(const char* indexpath,int mode) { +void index_finish(const char *indexpath, int mode) { #if HTS_MAKE_KEYWORD_INDEX - char catbuff[CATBUFF_SIZE]; - char** tab; - char* blk; + char catbuff[CATBUFF_SIZE]; + char **tab; + char *blk; off_t size = fpsize(fp_tmpproject); - if (size>0) { + + if (size > 0) { //FILE* fp=fopen(concat(indexpath,"index.txt"),"rb"); if (fp_tmpproject) { - tab=(char**)malloct(sizeof(char*) * (hts_primindex_size+2) ); + tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2)); if (tab) { - blk = malloct(size+4); + blk = malloct(size + 4); if (blk) { - fseek(fp_tmpproject,0,SEEK_SET); - if ((INTsys)fread(blk,1,size,fp_tmpproject) == size) { - char *a=blk,*b; - int index=0; + fseek(fp_tmpproject, 0, SEEK_SET); + if ((INTsys) fread(blk, 1, size, fp_tmpproject) == size) { + char *a = blk, *b; + int index = 0; int i; - FILE* fp; + FILE *fp; - while( (b=strchr(a,'\n')) && (index < hts_primindex_size) ) { - tab[index++]=a; - *b='\0'; - a=b+1; + while((b = strchr(a, '\n')) && (index < hts_primindex_size)) { + tab[index++] = a; + *b = '\0'; + a = b + 1; } - + // Sort it! - qsort(tab,index,sizeof(char*),mystrcmp); + qsort(tab, index, sizeof(char *), mystrcmp); // Delete fp_tmpproject fclose(fp_tmpproject); - fp_tmpproject=NULL; + fp_tmpproject = NULL; // Write new file if (mode == 1) // TEXT - fp=fopen(concat(catbuff,indexpath,"index.txt"),"wb"); + fp = fopen(concat(catbuff, indexpath, "index.txt"), "wb"); else // HTML - fp=fopen(concat(catbuff,indexpath,"sindex.html"),"wb"); + fp = fopen(concat(catbuff, indexpath, "sindex.html"), "wb"); if (fp) { char current_word[KEYW_LEN + 32]; char word[KEYW_LEN + 32]; int hit; - int total_hit=0; - int total_line=0; - int last_pos=0; - char word0='\0'; - current_word[0]='\0'; - - if (mode == 2) { // HTML - for(i=0;i<index;i++) { + int total_hit = 0; + int total_line = 0; + int last_pos = 0; + char word0 = '\0'; + + current_word[0] = '\0'; + + if (mode == 2) { // HTML + for(i = 0; i < index; i++) { if (word0 != tab[i][0]) { word0 = tab[i][0]; - fprintf(fp," <a href=\"#%c\">%c</a>\r\n",word0,word0); + fprintf(fp, " <a href=\"#%c\">%c</a>\r\n", word0, word0); } } - word0='\0'; - fprintf(fp,"<br><br>\r\n"); - fprintf(fp,"<table width=\"100%%\" border=\"0\">\r\n<tr>\r\n<td>word</td>\r\n<td>location\r\n"); + word0 = '\0'; + fprintf(fp, "<br><br>\r\n"); + fprintf(fp, + "<table width=\"100%%\" border=\"0\">\r\n<tr>\r\n<td>word</td>\r\n<td>location\r\n"); } - for(i=0;i<index;i++) { - if (sscanf(tab[i],"%s %d",word,&hit) == 2) { - char* a=strchr(tab[i],' '); - if (a) a=strchr(a+1,' '); - if (a++) { /* Yes, a++, not ++a :) */ - hit=KEYW_SORT_MAXCOUNT-hit; - if (strcmp(word,current_word)) { /* New word */ + for(i = 0; i < index; i++) { + if (sscanf(tab[i], "%s %d", word, &hit) == 2) { + char *a = strchr(tab[i], ' '); + + if (a) + a = strchr(a + 1, ' '); + if (a++) { /* Yes, a++, not ++a :) */ + hit = KEYW_SORT_MAXCOUNT - hit; + if (strcmp(word, current_word)) { /* New word */ if (total_hit) { - if (mode == 1) // TEXT - fprintf(fp,"\t=%d\r\n",total_hit); + if (mode == 1) // TEXT + fprintf(fp, "\t=%d\r\n", total_hit); //else // HTML // fprintf(fp,"<br>(%d total hits)\r\n",total_hit); - if ( - ( ((total_hit*1000 ) / hts_primindex_words) >= KEYW_USELESS1K ) - || - ( ((total_line*1000) / index ) >= KEYW_USELESS1KPG ) + if ((((total_hit * 1000) / hts_primindex_words) >= + KEYW_USELESS1K) + || (((total_line * 1000) / index) >= + KEYW_USELESS1KPG) ) { - fseek(fp,last_pos,SEEK_SET); - if (mode == 1) // TEXT - fprintf(fp,"\tignored (%d)\r\n",((total_hit*1000)/hts_primindex_words)); + fseek(fp, last_pos, SEEK_SET); + if (mode == 1) // TEXT + fprintf(fp, "\tignored (%d)\r\n", + ((total_hit * 1000) / hts_primindex_words)); else - fprintf(fp,"(ignored) [%d hits]<br>\r\n",total_hit); - } - else { - if (mode == 1) // TEXT - fprintf(fp,"\t(%d)\r\n",((total_hit*1000)/hts_primindex_words)); + fprintf(fp, "(ignored) [%d hits]<br>\r\n", + total_hit); + } else { + if (mode == 1) // TEXT + fprintf(fp, "\t(%d)\r\n", + ((total_hit * 1000) / hts_primindex_words)); //else // HTML // fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words)); } } - if (mode == 1) // TEXT - fprintf(fp,"%s\r\n",word); - else { // HTML - fprintf(fp,"</td></tr>\r\n"); + if (mode == 1) // TEXT + fprintf(fp, "%s\r\n", word); + else { // HTML + fprintf(fp, "</td></tr>\r\n"); if (word0 != word[0]) { word0 = word[0]; - fprintf(fp,"<th>%c</th>\r\n",word0); - fprintf(fp,"<a name=\"%c\"></a>\r\n",word0); + fprintf(fp, "<th>%c</th>\r\n", word0); + fprintf(fp, "<a name=\"%c\"></a>\r\n", word0); } - fprintf(fp,"<tr>\r\n<td>%s</td>\r\n<td>\r\n",word); + fprintf(fp, "<tr>\r\n<td>%s</td>\r\n<td>\r\n", word); } - fflush(fp); last_pos=ftell(fp); - strcpybuff(current_word,word); - total_hit=total_line=0; + fflush(fp); + last_pos = ftell(fp); + strcpybuff(current_word, word); + total_hit = total_line = 0; } - total_hit+=hit; + total_hit += hit; total_line++; if (mode == 1) // TEXT - fprintf(fp,"\t%d %s\r\n",hit,a); - else // HTML - fprintf(fp,"<a href=\"%s\">%s</a> [%d hits]<br>\r\n",a,a,hit); + fprintf(fp, "\t%d %s\r\n", hit, a); + else // HTML + fprintf(fp, "<a href=\"%s\">%s</a> [%d hits]<br>\r\n", a, + a, hit); } } } - if (mode == 2) // HTML - fprintf(fp,"</td></tr>\r\n</table>\r\n"); + if (mode == 2) // HTML + fprintf(fp, "</td></tr>\r\n</table>\r\n"); fclose(fp); } - + } freet(blk); } @@ -470,26 +468,26 @@ void index_finish(const char* indexpath,int mode) { } if (fp_tmpproject) fclose(fp_tmpproject); - fp_tmpproject=NULL; + fp_tmpproject = NULL; #endif } - /* Subroutines */ #if HTS_MAKE_KEYWORD_INDEX -int strcpos(char* adr,char c) { - char* apos=strchr(adr,c); +int strcpos(char *adr, char c) { + char *apos = strchr(adr, c); + if (apos) - return (int)(apos-adr); + return (int) (apos - adr); else return -1; } -int mystrcmp(const void* _e1,const void* _e2) { - char** e1=(char**)_e1; - char** e2=(char**)_e2; - return strcmp(*e1,*e2); +int mystrcmp(const void *_e1, const void *_e2) { + char **e1 = (char **) _e1; + char **e2 = (char **) _e2; + + return strcmp(*e1, *e2); } #endif - diff --git a/src/htsindex.h b/src/htsindex.h index 1fc3072..4997386 100644 --- a/src/htsindex.h +++ b/src/htsindex.h @@ -17,25 +17,21 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsindex.h */ /* keyword indexing system (search index) */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - #ifndef HTSKINDEX_DEFH #define HTSKINDEX_DEFH @@ -44,9 +40,10 @@ Please visit our Website: http://www.httrack.com #include "htsglobal.h" -int index_keyword(const char* html_data,LLint size,const char* mime,const char* filename,const char* indexpath); -void index_init(const char* indexpath); -void index_finish(const char* indexpath,int mode); +int index_keyword(const char *html_data, LLint size, const char *mime, + const char *filename, const char *indexpath); +void index_init(const char *indexpath); +void index_finish(const char *indexpath, int mode); #endif #endif diff --git a/src/htsinthash.c b/src/htsinthash.c index 9ec9fbd..426f815 100644 --- a/src/htsinthash.c +++ b/src/htsinthash.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* hash table system (fast index) */ @@ -61,53 +58,60 @@ Please visit our Website: http://www.httrack.com // static functions -static void inthash_delchain(inthash_chain* hash,t_inthash_freehandler free_handler); -static void inthash_default_free_handler(void* value); -static unsigned long int inthash_key(const char* value); +static void inthash_delchain(inthash_chain * hash, + t_inthash_freehandler free_handler); +static void inthash_default_free_handler(void *value); +static unsigned long int inthash_key(const char *value); static void inthash_init(inthash hashtable); - // inthash -- simple hash table, using a key (char[]) and a value (ulong int) -static unsigned long int inthash_key(const char* value) { +static unsigned long int inthash_key(const char *value) { return md5sum32(value); } -int inthash_read_pvoid(inthash hashtable,const char* name, void** pvalue) { +int inthash_read_pvoid(inthash hashtable, const char *name, void **pvalue) { inthash_value value = INTHASH_VALUE_NULL; - int ret = inthash_read_value(hashtable, name, (pvalue != NULL) ? &value : NULL); + int ret = + inthash_read_value(hashtable, name, (pvalue != NULL) ? &value : NULL); if (pvalue != NULL) *pvalue = value.ptr; return ret; } -int inthash_write_pvoid(inthash hashtable,const char* name, void* pvalue) { +int inthash_write_pvoid(inthash hashtable, const char *name, void *pvalue) { inthash_value value = INTHASH_VALUE_NULL; + value.ptr = pvalue; return inthash_write_value(hashtable, name, value); } -void inthash_add_pvoid(inthash hashtable, const char* name, void* pvalue) { +void inthash_add_pvoid(inthash hashtable, const char *name, void *pvalue) { inthash_value value = INTHASH_VALUE_NULL; + value.ptr = pvalue; inthash_add_value(hashtable, name, value); } // Check for duplicate entry (==1 : added) -int inthash_write(inthash hashtable,const char* name,intptr_t intvalue) { +int inthash_write(inthash hashtable, const char *name, intptr_t intvalue) { inthash_value value = INTHASH_VALUE_NULL; + value.intg = intvalue; return inthash_write_value(hashtable, name, value); } -int inthash_write_value(inthash hashtable,const char* name,inthash_value value) { +int inthash_write_value(inthash hashtable, const char *name, + inthash_value value) { int pos = (inthash_key(name) % hashtable->hash_size); - inthash_chain* h=hashtable->hash[pos]; - while (h) { - if (strcmp(h->name,name)==0) { + inthash_chain *h = hashtable->hash[pos]; + + while(h) { + if (strcmp(h->name, name) == 0) { /* Delete element */ if (hashtable->flag_valueismalloc) { - void* ptr = h->value.ptr; + void *ptr = h->value.ptr; + if (ptr != NULL) { if (hashtable->free_handler) hashtable->free_handler(ptr); @@ -116,147 +120,147 @@ int inthash_write_value(inthash hashtable,const char* name,inthash_value value) } } /* Insert */ - h->value=value; + h->value = value; return 0; } - h=h->next; + h = h->next; } // Not found, add it! - inthash_add_value(hashtable,name,value); + inthash_add_value(hashtable, name, value); return 1; } // Increment pos value, create one if necessary (=0) // (==1 : created) -int inthash_inc(inthash hashtable,const char* name) { - intptr_t value=0; - int r=0; - if (inthash_read(hashtable,name,&value)) { +int inthash_inc(inthash hashtable, const char *name) { + intptr_t value = 0; + int r = 0; + + if (inthash_read(hashtable, name, &value)) { value++; + } else { /* create new value */ + value = 0; + r = 1; } - else { /* create new value */ - value=0; - r=1; - } - inthash_write(hashtable,name,value); + inthash_write(hashtable, name, value); return (r); } - // Does not check for duplicate entry -void inthash_add(inthash hashtable, const char* name, intptr_t intvalue) { +void inthash_add(inthash hashtable, const char *name, intptr_t intvalue) { inthash_value value = INTHASH_VALUE_NULL; + memset(&value, 0, sizeof(value)); value.intg = intvalue; inthash_add_value(hashtable, name, value); } -void inthash_add_value(inthash hashtable, const char* name, inthash_value value) { +void inthash_add_value(inthash hashtable, const char *name, inthash_value value) { int pos = (inthash_key(name) % hashtable->hash_size); - inthash_chain** h=&hashtable->hash[pos]; - - while (*h) - h=&((*h)->next); - *h=(inthash_chain*)calloct(1, - sizeof(inthash_chain) - + - strlen(name)+2 - ); + inthash_chain **h = &hashtable->hash[pos]; + + while(*h) + h = &((*h)->next); + *h = (inthash_chain *) calloct(1, sizeof(inthash_chain) + + strlen(name) + 2); if (*h) { - (*h)->name=((char*)(*h)) + sizeof(inthash_chain); - (*h)->next=NULL; - strcpybuff((*h)->name,name); - (*h)->value=value; + (*h)->name = ((char *) (*h)) + sizeof(inthash_chain); + (*h)->next = NULL; + strcpybuff((*h)->name, name); + (*h)->value = value; hashtable->nitems++; } } -void* inthash_addblk(inthash hashtable,const char* name,int blksize) { +void *inthash_addblk(inthash hashtable, const char *name, int blksize) { int pos = (inthash_key(name) % hashtable->hash_size); - inthash_chain** h=&hashtable->hash[pos]; - - while (*h) - h=&((*h)->next); - *h=(inthash_chain*)calloct(1, - sizeof(inthash_chain) - + - strlen(name)+2 - + - blksize - ); + inthash_chain **h = &hashtable->hash[pos]; + + while(*h) + h = &((*h)->next); + *h = (inthash_chain *) calloct(1, sizeof(inthash_chain) + + strlen(name) + 2 + blksize); if (*h) { - (*h)->name = ((char*)(*h)) + sizeof(inthash_chain); - (*h)->next=NULL; - strcpybuff((*h)->name,name); - (*h)->value.ptr = (void*) ( ((char*)(*h)) + sizeof(inthash_chain) + strlen(name) + 2 ); + (*h)->name = ((char *) (*h)) + sizeof(inthash_chain); + (*h)->next = NULL; + strcpybuff((*h)->name, name); + (*h)->value.ptr = + (void *) (((char *) (*h)) + sizeof(inthash_chain) + strlen(name) + 2); hashtable->nitems++; return (*h)->value.ptr; } return NULL; } -int inthash_read(inthash hashtable,const char* name,intptr_t* intvalue) { +int inthash_read(inthash hashtable, const char *name, intptr_t * intvalue) { inthash_value value = INTHASH_VALUE_NULL; - int ret = inthash_read_value(hashtable, name, (intvalue != NULL) ? &value : NULL); + int ret = + inthash_read_value(hashtable, name, (intvalue != NULL) ? &value : NULL); if (intvalue != NULL) *intvalue = value.intg; return ret; } -int inthash_read_value(inthash hashtable,const char* name,inthash_value* value) { +int inthash_read_value(inthash hashtable, const char *name, + inthash_value * value) { int pos = (inthash_key(name) % hashtable->hash_size); - inthash_chain* h=hashtable->hash[pos]; - while (h) { - if (strcmp(h->name,name)==0) { + inthash_chain *h = hashtable->hash[pos]; + + while(h) { + if (strcmp(h->name, name) == 0) { if (value != NULL) - *value=h->value; + *value = h->value; return 1; } - h=h->next; + h = h->next; } return 0; } -int inthash_exists(inthash hashtable, const char* name) { +int inthash_exists(inthash hashtable, const char *name) { return inthash_read_value(hashtable, name, NULL); } -int inthash_remove(inthash hashtable,const char* name) { +int inthash_remove(inthash hashtable, const char *name) { int pos = (inthash_key(name) % hashtable->hash_size); - inthash_chain** h=&hashtable->hash[pos]; - t_inthash_freehandler free_handler=NULL; - if ( hashtable->flag_valueismalloc ) { - if ( hashtable->free_handler ) - free_handler=hashtable->free_handler; + inthash_chain **h = &hashtable->hash[pos]; + t_inthash_freehandler free_handler = NULL; + + if (hashtable->flag_valueismalloc) { + if (hashtable->free_handler) + free_handler = hashtable->free_handler; else - free_handler=inthash_default_free_handler; + free_handler = inthash_default_free_handler; } - while (*h) { - if (strcmp((*h)->name,name)==0) { - inthash_chain* next; + while(*h) { + if (strcmp((*h)->name, name) == 0) { + inthash_chain *next; + if (free_handler) { if ((*h)->value.ptr) { - void* ptr = (*h)->value.ptr; + void *ptr = (*h)->value.ptr; + if (free_handler) free_handler(ptr); else freet(ptr); - (*h)->value.ptr=0; + (*h)->value.ptr = 0; } } - next=(*h)->next; + next = (*h)->next; freet(*h); - *h=next; + *h = next; hashtable->nitems--; return 1; } - h=&((*h)->next); + h = &((*h)->next); } return 0; } -int inthash_readptr(inthash hashtable,const char* name,intptr_t* value) { +int inthash_readptr(inthash hashtable, const char *name, intptr_t * value) { int ret; + *value = 0; ret = inthash_read(hashtable, name, value); if (*value == 0) @@ -266,30 +270,34 @@ int inthash_readptr(inthash hashtable,const char* name,intptr_t* value) { static void inthash_init(inthash hashtable) { unsigned int i; - for(i=0;i<hashtable->hash_size;i++) { - hashtable->hash[i]=NULL; + + for(i = 0; i < hashtable->hash_size; i++) { + hashtable->hash[i] = NULL; } } -static void inthash_delchain(inthash_chain* hash,t_inthash_freehandler free_handler) { +static void inthash_delchain(inthash_chain * hash, + t_inthash_freehandler free_handler) { while(hash != NULL) { - inthash_chain* next=hash->next; - if (free_handler) { // pos is a malloc() block, delete it! + inthash_chain *next = hash->next; + + if (free_handler) { // pos is a malloc() block, delete it! if (hash->value.ptr) { - void* ptr = hash->value.ptr; + void *ptr = hash->value.ptr; + if (free_handler) free_handler(ptr); else freet(ptr); - hash->value.ptr=0; + hash->value.ptr = 0; } } freet(hash); - hash=next; + hash = next; } } -static void inthash_default_free_handler(void* value) { +static void inthash_default_free_handler(void *value) { if (value) freet(value); } @@ -297,12 +305,14 @@ static void inthash_default_free_handler(void* value) { // -- inthash inthash_new(int size) { - inthash hashtable=(inthash)calloct(1,sizeof(struct_inthash)); + inthash hashtable = (inthash) calloct(1, sizeof(struct_inthash)); + if (hashtable) { - hashtable->hash_size=0; - hashtable->flag_valueismalloc=0; - if ((hashtable->hash=(inthash_chain**)calloct(size,sizeof(inthash_chain*)))) { - hashtable->hash_size=size; + hashtable->hash_size = 0; + hashtable->flag_valueismalloc = 0; + if ((hashtable->hash = + (inthash_chain **) calloct(size, sizeof(inthash_chain *)))) { + hashtable->hash_size = size; inthash_init(hashtable); } hashtable->nitems = 0; @@ -317,41 +327,43 @@ int inthash_created(inthash hashtable) { return 0; } -void inthash_value_is_malloc(inthash hashtable,int flag) { - hashtable->flag_valueismalloc=flag; +void inthash_value_is_malloc(inthash hashtable, int flag) { + hashtable->flag_valueismalloc = flag; } -void inthash_value_set_free_handler(inthash hashtable, t_inthash_freehandler free_handler) { +void inthash_value_set_free_handler(inthash hashtable, + t_inthash_freehandler free_handler) { hashtable->free_handler = free_handler; } unsigned int inthash_nitems(inthash hashtable) { - if (hashtable!= NULL) + if (hashtable != NULL) return hashtable->nitems; return 0; } -void inthash_delete(inthash* hashtable) { +void inthash_delete(inthash * hashtable) { if (hashtable) { if (*hashtable) { if ((*hashtable)->hash) { unsigned int i; - t_inthash_freehandler free_handler=NULL; - if ( (*hashtable)->flag_valueismalloc ) { - if ( (*hashtable)->free_handler ) - free_handler=(*hashtable)->free_handler; + t_inthash_freehandler free_handler = NULL; + + if ((*hashtable)->flag_valueismalloc) { + if ((*hashtable)->free_handler) + free_handler = (*hashtable)->free_handler; else - free_handler=inthash_default_free_handler; + free_handler = inthash_default_free_handler; } - for(i=0;i<(*hashtable)->hash_size;i++) { + for(i = 0; i < (*hashtable)->hash_size; i++) { inthash_delchain((*hashtable)->hash[i], free_handler); - (*hashtable)->hash[i]=NULL; + (*hashtable)->hash[i] = NULL; } freet((*hashtable)->hash); (*hashtable)->hash = NULL; } freet(*hashtable); - *hashtable=NULL; + *hashtable = NULL; } } } @@ -360,6 +372,7 @@ void inthash_delete(inthash* hashtable) { struct_inthash_enum inthash_enum_new(inthash hashtable) { struct_inthash_enum e; + memset(&e, 0, sizeof(e)); e.index = 0; e.item = NULL; @@ -367,8 +380,9 @@ struct_inthash_enum inthash_enum_new(inthash hashtable) { return e; } -inthash_chain* inthash_enum_next(struct_inthash_enum* e) { - inthash_chain* item = NULL; +inthash_chain *inthash_enum_next(struct_inthash_enum * e) { + inthash_chain *item = NULL; + if (e != NULL) { while(e->item == NULL && e->index < (int) e->table->hash_size) { e->item = e->table->hash[e->index]; diff --git a/src/htsinthash.h b/src/htsinthash.h index 9cfc4b9..a085b4b 100644 --- a/src/htsinthash.h +++ b/src/htsinthash.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* hash table system (fast index) */ @@ -38,7 +35,7 @@ Please visit our Website: http://www.httrack.com // inthash -- simple hash table, using a key (char[]) and a value (uintptr_t) #ifndef HTSINTHASH_DEFH -#define HTSINTHASH_DEFH +#define HTSINTHASH_DEFH /* Includes */ #ifdef _WIN32 @@ -52,8 +49,8 @@ Please visit our Website: http://www.httrack.com // value typedef union inthash_value { - uintptr_t intg; /* integer value */ - void* ptr; /* ptr value */ + uintptr_t intg; /* integer value */ + void *ptr; /* ptr value */ } inthash_value; #define INTHASH_VALUE_NULL { 0 } @@ -64,12 +61,12 @@ typedef union inthash_value { typedef struct inthash_chain inthash_chain; #endif struct inthash_chain { - char* name; /* key (name) */ - inthash_value value; /* value */ - struct inthash_chain* next; /* next element */ + char *name; /* key (name) */ + inthash_value value; /* value */ + struct inthash_chain *next; /* next element */ }; -typedef void (* t_inthash_freehandler)(void* value); +typedef void (*t_inthash_freehandler) (void *value); /* inthash structure */ #ifndef HTS_DEF_FWSTRUCT_struct_inthash @@ -77,7 +74,7 @@ typedef void (* t_inthash_freehandler)(void* value); typedef struct struct_inthash struct_inthash, *inthash; #endif struct struct_inthash { - inthash_chain** hash; + inthash_chain **hash; unsigned int nitems; t_inthash_freehandler free_handler; unsigned int hash_size; @@ -92,7 +89,7 @@ typedef struct struct_inthash_enum struct_inthash_enum; struct struct_inthash_enum { inthash table; int index; - inthash_chain* item; + inthash_chain *item; }; /* Library internal definictions */ @@ -101,34 +98,42 @@ struct struct_inthash_enum { // main functions: /* Hash functions: */ -inthash inthash_new(int size); /* Create a new hash table */ -int inthash_created(inthash hashtable); /* Test if the hash table was successfully created */ -unsigned int inthash_nitems(inthash hashtable); /* Number of items */ -void inthash_delete(inthash* hashtable); /* Delete an hash table */ -void inthash_value_is_malloc(inthash hashtable,int flag); /* Is the 'value' member a value that needs to be free()'ed ? */ -void inthash_value_set_free_handler(inthash hashtable, /* value free() handler (default one is 'free') */ - t_inthash_freehandler free_handler); +inthash inthash_new(int size); /* Create a new hash table */ +int inthash_created(inthash hashtable); /* Test if the hash table was successfully created */ +unsigned int inthash_nitems(inthash hashtable); /* Number of items */ +void inthash_delete(inthash * hashtable); /* Delete an hash table */ +void inthash_value_is_malloc(inthash hashtable, int flag); /* Is the 'value' member a value that needs to be free()'ed ? */ +void inthash_value_set_free_handler(inthash hashtable, /* value free() handler (default one is 'free') */ + t_inthash_freehandler free_handler); + /* */ -int inthash_read(inthash hashtable,const char* name,intptr_t* intvalue); /* Read entry from the hash table */ -int inthash_readptr(inthash hashtable,const char* name,intptr_t* intvalue); /* Same function, but returns 0 upon null ptr */ -int inthash_exists(inthash hashtable, const char* name); /* Is the key existing ? */ +int inthash_read(inthash hashtable, const char *name, intptr_t * intvalue); /* Read entry from the hash table */ +int inthash_readptr(inthash hashtable, const char *name, intptr_t * intvalue); /* Same function, but returns 0 upon null ptr */ +int inthash_exists(inthash hashtable, const char *name); /* Is the key existing ? */ + /* */ -int inthash_read_value(inthash hashtable,const char* name,inthash_value* value); -int inthash_write_value(inthash hashtable,const char* name,inthash_value value); -void inthash_add_value(inthash hashtable, const char* name, inthash_value value); +int inthash_read_value(inthash hashtable, const char *name, + inthash_value * value); +int inthash_write_value(inthash hashtable, const char *name, + inthash_value value); +void inthash_add_value(inthash hashtable, const char *name, + inthash_value value); /* */ -int inthash_read_pvoid(inthash hashtable,const char* name, void** value); -int inthash_write_pvoid(inthash hashtable,const char* name, void* value); -void inthash_add_pvoid(inthash hashtable, const char* name, void* value); +int inthash_read_pvoid(inthash hashtable, const char *name, void **value); +int inthash_write_pvoid(inthash hashtable, const char *name, void *value); +void inthash_add_pvoid(inthash hashtable, const char *name, void *value); + /* */ -void inthash_add(inthash hashtable,const char* name,intptr_t value); /* Add entry in the hash table */ -void* inthash_addblk(inthash hashtable,const char* name,int blksize); /* Add entry in the hash table and set value to a new memory block */ -int inthash_write(inthash hashtable,const char* name,intptr_t value); /* Overwrite/add entry in the hash table */ -int inthash_inc(inthash hashtable,const char* name); /* Increment entry in the hash table */ -int inthash_remove(inthash hashtable,const char* name); /* Remove an entry from the hashtable */ +void inthash_add(inthash hashtable, const char *name, intptr_t value); /* Add entry in the hash table */ +void *inthash_addblk(inthash hashtable, const char *name, int blksize); /* Add entry in the hash table and set value to a new memory block */ +int inthash_write(inthash hashtable, const char *name, intptr_t value); /* Overwrite/add entry in the hash table */ +int inthash_inc(inthash hashtable, const char *name); /* Increment entry in the hash table */ +int inthash_remove(inthash hashtable, const char *name); /* Remove an entry from the hashtable */ + /* */ -struct_inthash_enum inthash_enum_new(inthash hashtable); /* Start a new enumerator */ -inthash_chain* inthash_enum_next(struct_inthash_enum* e); /* Fetch an item in the enumerator */ +struct_inthash_enum inthash_enum_new(inthash hashtable); /* Start a new enumerator */ +inthash_chain *inthash_enum_next(struct_inthash_enum * e); /* Fetch an item in the enumerator */ + /* End of hash functions: */ #endif diff --git a/src/htsjava.c b/src/htsjava.c index 2a3d648..16ec55c 100644 --- a/src/htsjava.c +++ b/src/htsjava.c @@ -17,24 +17,20 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Java classes parser */ /* Author: Yann Philippot */ /* ------------------------------------------------------------ */ - /* Version: Oct/2000 */ /* Fixed: problems with class structure (10/2000) */ @@ -69,8 +65,9 @@ Please visit our Website: http://www.httrack.com #include "htsjava.h" static int reverse_endian(void) { - int endian = 1; - return ( * ( (char*) &endian) == 1); + int endian = 1; + + return (*((char *) &endian) == 1); } /* big/little endian swap */ @@ -78,10 +75,11 @@ static int reverse_endian(void) { #define hts_swap32(A) ( (( (hts_swap16(A)) & 0xFFFF)<<16) | (( (hts_swap16(A>>16)) & 0xFFFF)) ) /* Static definitions */ -static RESP_STRUCT readtable(htsmoduleStruct* str,FILE *fp,RESP_STRUCT,int*); -static unsigned short int readshort(FILE *fp); -static int tris(httrackp *opt,char*); -static char * printname(char [1024],char [1024]); +static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp, RESP_STRUCT, + int *); +static unsigned short int readshort(FILE * fp); +static int tris(httrackp * opt, char *); +static char *printname(char[1024], char[1024]); // ** HTS_xx sinon pas pris par VC++ #define HTS_CLASS 7 @@ -106,38 +104,41 @@ static const char *libName = "htsjava"; #define strncasecmp(a,b,n) strnicmp(a,b,n) #endif -static int detect_mime(htsmoduleStruct* str) { - const char* savename = str->filename; +static int detect_mime(htsmoduleStruct * str) { + const char *savename = str->filename; + if (savename) { int len = (int) strlen(savename); - if (len > 6 && strcasecmp(savename + len - 6,".class") == 0) { + + if (len > 6 && strcasecmp(savename + len - 6, ".class") == 0) { return 1; } } return 0; } -static int hts_detect_java(t_hts_callbackarg *carg, httrackp *opt, - htsmoduleStruct* str) -{ +static int hts_detect_java(t_hts_callbackarg * carg, httrackp * opt, + htsmoduleStruct * str) { /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, detect) != NULL) { - if (CALLBACKARG_PREV_FUN(carg, detect)(CALLBACKARG_PREV_CARG(carg), opt, str)) { - return 1; /* Found before us, let them have the priority */ + if (CALLBACKARG_PREV_FUN(carg, detect) + (CALLBACKARG_PREV_CARG(carg), opt, str)) { + return 1; /* Found before us, let them have the priority */ } } /* Check MIME */ if (detect_mime(str)) { - str->wrapper_name = libName; /* Our ID */ - return 1; /* Known format, we take it */ + str->wrapper_name = libName; /* Our ID */ + return 1; /* Known format, we take it */ } - return 0; /* Unknown format */ + return 0; /* Unknown format */ } -static off_t fsize(const char* s) { +static off_t fsize(const char *s) { STRUCT_STAT st; + if (STAT(s, &st) == 0 && S_ISREG(st.st_mode)) { return st.st_size; } else { @@ -145,17 +146,18 @@ static off_t fsize(const char* s) { } } -static int hts_parse_java(t_hts_callbackarg *carg, httrackp *opt, - htsmoduleStruct* str) -{ +static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt, + htsmoduleStruct * str) { /* The wrapper_name memebr has changed: not for us anymore */ if (str->wrapper_name == NULL || strcmp(str->wrapper_name, libName) != 0) { /* Call parent functions if multiple callbacks are chained. */ if (CALLBACKARG_PREV_FUN(carg, parse) != NULL) { - return CALLBACKARG_PREV_FUN(carg, parse)(CALLBACKARG_PREV_CARG(carg), opt, str); + return CALLBACKARG_PREV_FUN(carg, parse) (CALLBACKARG_PREV_CARG(carg), + opt, str); } - strcpy(str->err_msg, "unexpected error: bad wrapper_name and no previous wrapper"); - return 0; /* Unexpected error */ + strcpy(str->err_msg, + "unexpected error: bad wrapper_name and no previous wrapper"); + return 0; /* Unexpected error */ } else { if (detect_mime(str)) { @@ -164,74 +166,82 @@ static int hts_parse_java(t_hts_callbackarg *carg, httrackp *opt, FILE *fpout; JAVA_HEADER header; RESP_STRUCT *tab; - const char* file = str->filename; + const char *file = str->filename; str->relativeToHtmlLink = 1; #if JAVADEBUG printf("fopen\n"); #endif - if ((fpout = FOPEN(fconv(catbuff, file), "r+b")) == NULL) - { + if ((fpout = FOPEN(fconv(catbuff, file), "r+b")) == NULL) { //fprintf(stderr, "Cannot open input file.\n"); - sprintf(str->err_msg,"Unable to open file %s",file); - return 0; // une erreur.. + sprintf(str->err_msg, "Unable to open file %s", file); + return 0; // une erreur.. } - #if JAVADEBUG printf("fread\n"); #endif //if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) { // pas complet.. - if (fread(&header,1,10,fpout) != 10) { // pas complet.. + if (fread(&header, 1, 10, fpout) != 10) { // pas complet.. fclose(fpout); - sprintf(str->err_msg,"File header too small (file len = "LLintP")",(LLint)fsize(file)); + sprintf(str->err_msg, "File header too small (file len = " LLintP ")", + (LLint) fsize(file)); return 0; } - #if JAVADEBUG printf("header\n"); #endif // tester en tête if (reverse_endian()) { header.magic = hts_swap32(header.magic); - header.count = hts_swap16(header.count); + header.count = hts_swap16(header.count); } - if(header.magic!=0xCAFEBABE) { - sprintf(str->err_msg,"non java file"); - if (fpout) { fclose(fpout); fpout=NULL; } + if (header.magic != 0xCAFEBABE) { + sprintf(str->err_msg, "non java file"); + if (fpout) { + fclose(fpout); + fpout = NULL; + } return 0; } - tab =(RESP_STRUCT*)calloc(header.count,sizeof(RESP_STRUCT)); + tab = (RESP_STRUCT *) calloc(header.count, sizeof(RESP_STRUCT)); if (!tab) { - sprintf(str->err_msg,"Unable to alloc %d bytes",(int)sizeof(RESP_STRUCT)); - if (fpout) { fclose(fpout); fpout=NULL; } - return 0; // erreur.. + sprintf(str->err_msg, "Unable to alloc %d bytes", + (int) sizeof(RESP_STRUCT)); + if (fpout) { + fclose(fpout); + fpout = NULL; + } + return 0; // erreur.. } - #if JAVADEBUG printf("calchead\n"); #endif { int i; - for (i = 1; i < header.count; i++) { - int err=0; // ++ - tab[i]=readtable(str,fpout,tab[i],&err); + for(i = 1; i < header.count; i++) { + int err = 0; // ++ + + tab[i] = readtable(str, fpout, tab[i], &err); if (!err) { - if ((tab[i].type == HTS_LONG) ||(tab[i].type == HTS_DOUBLE)) i++; //2 element si double ou float - } else { // ++ une erreur est survenue! - if (strnotempty(str->err_msg)==0) - strcpy(str->err_msg,"Internal readtable error"); + if ((tab[i].type == HTS_LONG) || (tab[i].type == HTS_DOUBLE)) + i++; //2 element si double ou float + } else { // ++ une erreur est survenue! + if (strnotempty(str->err_msg) == 0) + strcpy(str->err_msg, "Internal readtable error"); free(tab); - if (fpout) { fclose(fpout); fpout=NULL; } + if (fpout) { + fclose(fpout); + fpout = NULL; + } return 0; } } } - #if JAVADEBUG printf("addfiles\n"); #endif @@ -240,251 +250,266 @@ static int hts_parse_java(t_hts_callbackarg *carg, httrackp *opt, unsigned int Class; unsigned int SClass; int i; + //acess = readshort(fpout); Class = readshort(fpout); SClass = readshort(fpout); - for (i = 1; i <header.count; i++) { + for(i = 1; i < header.count; i++) { if (tab[i].type == HTS_CLASS) { - if ((tab[i].index1<header.count) && (tab[i].index1>=0)) { - + if ((tab[i].index1 < header.count) && (tab[i].index1 >= 0)) { - if((tab[i].index1!=SClass) && (tab[i].index1!=Class) && (tab[tab[i].index1].name[0]!='[')) { + if ((tab[i].index1 != SClass) && (tab[i].index1 != Class) + && (tab[tab[i].index1].name[0] != '[')) { - if(!strstr(tab[tab[i].index1].name,"java/")) { + if (!strstr(tab[tab[i].index1].name, "java/")) { char BIGSTK tempo[1024]; - tempo[0]='\0'; - sprintf(tempo,"%s.class",tab[tab[i].index1].name); + tempo[0] = '\0'; + + sprintf(tempo, "%s.class", tab[tab[i].index1].name); #if JAVADEBUG - printf("add %s\n",tempo); + printf("add %s\n", tempo); #endif if (tab[tab[i].index1].file_position >= 0) - str->addLink(str,tempo); /* tab[tab[i].index1].file_position */ + str->addLink(str, tempo); /* tab[tab[i].index1].file_position */ } } - } else { - i=header.count; // exit + } else { + i = header.count; // exit } } } } - #if JAVADEBUG printf("end\n"); #endif free(tab); - if (fpout) { fclose(fpout); fpout=NULL; } + if (fpout) { + fclose(fpout); + fpout = NULL; + } return 1; } else { strcpy(str->err_msg, "bad MIME type"); } } - return 0; /* Error */ + return 0; /* Error */ } /* module entry point */ -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv); -EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) { +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv); +EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) { /* Plug callback functions */ CHAIN_FUNCTION(opt, detect, hts_detect_java, NULL); CHAIN_FUNCTION(opt, parse, hts_parse_java, NULL); - return 1; /* success */ + return 1; /* success */ } // error: !=0 si erreur fatale -static RESP_STRUCT readtable(htsmoduleStruct* str, - FILE *fp, RESP_STRUCT trans, int* error) -{ - char rname[1024]; +static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp, + RESP_STRUCT trans, int *error) { + char rname[1024]; unsigned short int length; int j; - *error = 0; // pas d'erreur - trans.file_position=-1; - trans.type = (int)(unsigned char)fgetc(fp); + + *error = 0; // pas d'erreur + trans.file_position = -1; + trans.type = (int) (unsigned char) fgetc(fp); switch (trans.type) { case HTS_CLASS: - strcpy(trans.name,"Class"); + strcpy(trans.name, "Class"); trans.index1 = readshort(fp); break; - + case HTS_FIELDREF: - strcpy(trans.name,"Field Reference"); + strcpy(trans.name, "Field Reference"); trans.index1 = readshort(fp); readshort(fp); break; - + case HTS_METHODREF: - strcpy(trans.name,"Method Reference"); + strcpy(trans.name, "Method Reference"); trans.index1 = readshort(fp); readshort(fp); break; - + case HTS_INTERFACE: - strcpy(trans.name,"Interface Method Reference"); - trans.index1 =readshort(fp); + strcpy(trans.name, "Interface Method Reference"); + trans.index1 = readshort(fp); readshort(fp); break; case HTS_NAMEANDTYPE: - strcpy(trans.name,"Name and Type"); + strcpy(trans.name, "Name and Type"); trans.index1 = readshort(fp); readshort(fp); break; - - case HTS_STRING: // CONSTANT_String - strcpy(trans.name,"String"); + + case HTS_STRING: // CONSTANT_String + strcpy(trans.name, "String"); trans.index1 = readshort(fp); break; - + case HTS_INTEGER: - strcpy(trans.name,"Integer"); - for(j=0;j<4;j++) fgetc(fp); + strcpy(trans.name, "Integer"); + for(j = 0; j < 4; j++) + fgetc(fp); break; - + case HTS_FLOAT: - strcpy(trans.name,"Float"); - for(j=0;j<4;j++) fgetc(fp); + strcpy(trans.name, "Float"); + for(j = 0; j < 4; j++) + fgetc(fp); break; - + case HTS_LONG: - strcpy(trans.name,"Long"); - for(j=0;j<8;j++) fgetc(fp); + strcpy(trans.name, "Long"); + for(j = 0; j < 8; j++) + fgetc(fp); break; case HTS_DOUBLE: - strcpy(trans.name,"Double"); - for(j=0;j<8;j++) fgetc(fp); + strcpy(trans.name, "Double"); + for(j = 0; j < 8; j++) + fgetc(fp); break; - + case HTS_ASCIZ: case HTS_UNICODE: - + if (trans.type == HTS_ASCIZ) - strcpy(trans.name,"HTS_ASCIZ"); + strcpy(trans.name, "HTS_ASCIZ"); else - strcpy(trans.name,"HTS_UNICODE"); - + strcpy(trans.name, "HTS_UNICODE"); + { - char BIGSTK buffer[1024]; + char BIGSTK buffer[1024]; char *p; - - p=&buffer[0]; + + p = &buffer[0]; //fflush(fp); - trans.file_position=ftell(fp); + trans.file_position = ftell(fp); length = readshort(fp); - if (length<HTS_URLMAXSIZE) { + if (length < HTS_URLMAXSIZE) { // while ((length > 0) && (length<500)) { - while (length > 0) { - *p++ =fgetc(fp); - + while(length > 0) { + *p++ = fgetc(fp); + length--; } - *p='\0'; - + *p = '\0'; + //#if JDEBUG // if(tris(buffer)==1) printf("%s\n ",buffer); // if(tris(buffer)==2) printf("%s\n ",printname(buffer)); //#endif - if(tris(str->opt,buffer)==1) str->addLink(str, buffer); /* trans.file_position */ - else if(tris(str->opt,buffer)==2) str->addLink(str, printname(rname,buffer)); - - strcpy(trans.name,buffer); - } else { // gros pb - while ( (length > 0) && (!feof(fp))) { + if (tris(str->opt, buffer) == 1) + str->addLink(str, buffer); /* trans.file_position */ + else if (tris(str->opt, buffer) == 2) + str->addLink(str, printname(rname, buffer)); + + strcpy(trans.name, buffer); + } else { // gros pb + while((length > 0) && (!feof(fp))) { fgetc(fp); length--; } if (!feof(fp)) { - trans.type=-1; + trans.type = -1; } else { - sprintf(str->err_msg,"Internal stucture error (ASCII)"); + sprintf(str->err_msg, "Internal stucture error (ASCII)"); *error = 1; } - return(trans); + return (trans); } } break; default: // printf("Type inconnue\n"); // on arrête tout - sprintf(str->err_msg,"Internal structure unknown (type %d)",trans.type); + sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type); *error = 1; - return(trans); + return (trans); break; - } - return(trans); + } + return (trans); } - -static unsigned short int readshort(FILE *fp) -{ +static unsigned short int readshort(FILE * fp) { unsigned short int valint; - fread(&valint,sizeof(valint),1,fp); + + fread(&valint, sizeof(valint), 1, fp); if (reverse_endian()) return hts_swap16(valint); else return valint; - + } -static int tris(httrackp *opt,char * buffer) -{ - char catbuff[CATBUFF_SIZE]; +static int tris(httrackp * opt, char *buffer) { + char catbuff[CATBUFF_SIZE]; + // // Java - if((buffer[0]=='[') && buffer[1]=='L' && (!strstr(buffer,"java/")) ) + if ((buffer[0] == '[') && buffer[1] == 'L' && (!strstr(buffer, "java/"))) return 2; - if (strstr(buffer,".gif") || strstr(buffer,".jpg") || strstr(buffer,".jpeg") || strstr(buffer,".au") ) + if (strstr(buffer, ".gif") || strstr(buffer, ".jpg") + || strstr(buffer, ".jpeg") || strstr(buffer, ".au")) return 1; // Ajouts R.X: test type // Autres fichiers { char type[256]; - type[0]='\0'; - get_httptype(opt,type,buffer,0); - if (strnotempty(type)) // type reconnu! + + type[0] = '\0'; + get_httptype(opt, type, buffer, 0); + if (strnotempty(type)) // type reconnu! return 1; // ajout RX 05/2001 - else if (is_dyntype(get_ext(catbuff, buffer))) // asp,cgi... + else if (is_dyntype(get_ext(catbuff, buffer))) // asp,cgi... return 1; } return 0; } -static char * printname(char rname[1024], char name[1024]) -{ +static char *printname(char rname[1024], char name[1024]) { char *p; char *p1; int j; - rname[0]='\0'; + + rname[0] = '\0'; // - - p=&name[0]; - - if(*p!='[') return ""; - p+=2; + + p = &name[0]; + + if (*p != '[') + return ""; + p += 2; //rname=(char*)calloct(strlen(name)+8,sizeof(char)); - p1=rname; - for (j = 0; j < (int) strlen(name); j++,p++) { - if (*p == '/') *p1='.'; - if (*p==';'){*p1='\0'; - strcat(rname,".class"); - return (rname);} - else *p1=*p; + p1 = rname; + for(j = 0; j < (int) strlen(name); j++, p++) { + if (*p == '/') + *p1 = '.'; + if (*p == ';') { + *p1 = '\0'; + strcat(rname, ".class"); + return (rname); + } else + *p1 = *p; p1++; } - p1-=3; - *p1='\0'; + p1 -= 3; + *p1 = '\0'; return (rname); - + } diff --git a/src/htsjava.h b/src/htsjava.h index 212d400..5f45a97 100644 --- a/src/htsjava.h +++ b/src/htsjava.h @@ -17,33 +17,29 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Java classes parser .h */ /* Author: Yann Philippot */ /* ------------------------------------------------------------ */ - #ifndef HTSJAVA_DEFH -#define HTSJAVA_DEFH +#define HTSJAVA_DEFH #ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER #define HTS_DEF_FWSTRUCT_JAVA_HEADER typedef struct JAVA_HEADER JAVA_HEADER; #endif struct JAVA_HEADER { - unsigned long int magic; + unsigned long int magic; unsigned short int minor; unsigned short int major; unsigned short int count; @@ -61,11 +57,10 @@ struct RESP_STRUCT { char name[1024]; }; - /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -EXTERNAL_FUNCTION int hts_plug_java(httrackp *opt, const char* argv); +EXTERNAL_FUNCTION int hts_plug_java(httrackp * opt, const char *argv); #endif diff --git a/src/htslib.c b/src/htslib.c index 870badf..bf6577e 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Subroutines */ /* Author: Xavier Roche */ @@ -43,7 +40,7 @@ Please visit our Website: http://www.httrack.com #ifdef _WIN32_WCE #ifndef HTS_CECOMPAT -#pragma comment(lib, "celib.lib") //link with celib +#pragma comment(lib, "celib.lib") //link with celib #endif #endif @@ -72,7 +69,7 @@ Please visit our Website: http://www.httrack.com #ifdef HAVE_UNISTD_H #include <unistd.h> #endif -#endif /* _WIN32 */ +#endif /* _WIN32 */ #include <string.h> #include <time.h> @@ -85,7 +82,7 @@ Please visit our Website: http://www.httrack.com #ifndef HTS_CECOMPAT #include <sys/timeb.h> #endif -#endif /* _WIN32_WCE */ +#endif /* _WIN32_WCE */ // pour utimbuf #ifdef _WIN32 @@ -98,54 +95,53 @@ Please visit our Website: http://www.httrack.com #endif #else #include <utime.h> -#endif /* _WIN32 */ +#endif /* _WIN32 */ #ifndef _WIN32_WCE #include <sys/stat.h> #endif /* END specific definitions */ - // Debugging #if _HTS_WIDE -FILE* DEBUG_fp=NULL; +FILE *DEBUG_fp = NULL; #endif /* variables globales */ int _DEBUG_HEAD; -FILE* ioinfo; +FILE *ioinfo; #if HTS_USEOPENSSL - SSL_CTX *openssl_ctx = NULL; +SSL_CTX *openssl_ctx = NULL; #endif int IPV6_resolver = 0; /* détection complémentaire */ -const char* hts_detect[] = { +const char *hts_detect[] = { "archive", "background", - "data", // OBJECT + "data", // OBJECT "dynsrc", "lowsrc", - "profile", // element META + "profile", // element META "src", "swurl", "url", "usemap", - "longdesc", // accessibility - "xlink:href", // xml/svg tag - "poster", // HTML5 + "longdesc", // accessibility + "xlink:href", // xml/svg tag + "poster", // HTML5 "" }; /* détecter début */ -const char* hts_detectbeg[] = { - "hotspot", /* hotspot1=..,hotspot2=.. */ +const char *hts_detectbeg[] = { + "hotspot", /* hotspot1=..,hotspot2=.. */ "" }; /* ne pas détcter de liens dedans */ -const char* hts_nodetect[] = { +const char *hts_nodetect[] = { "accept-charset", "accesskey", "action", @@ -184,10 +180,9 @@ const char* hts_nodetect[] = { "" }; - /* détection de mini-code javascript */ /* ALSO USED: detection based on the name: onXXX="<tag>" where XXX starts with upper case letter */ -const char* hts_detect_js[] = { +const char *hts_detect_js[] = { "onAbort", "onBlur", "onChange", @@ -211,11 +206,11 @@ const char* hts_detect_js[] = { "onSelect", "onSubmit", "onUnload", - "style", /* hack for CSS code data */ + "style", /* hack for CSS code data */ "" }; -const char* hts_main_mime[] = { +const char *hts_main_mime[] = { "application", "audio", "image", @@ -227,19 +222,19 @@ const char* hts_main_mime[] = { }; /* détection "...URL=<url>" */ -const char* hts_detectURL[] = { +const char *hts_detectURL[] = { "content", "" }; /* tags où l'URL doit être réécrite mais non capturée */ -const char* hts_detectandleave[] = { +const char *hts_detectandleave[] = { "action", "" }; /* ne pas renommer les types renvoyés (souvent types inconnus) */ -const char* hts_mime_keep[] = { +const char *hts_mime_keep[] = { "application/octet-stream", "text/plain", "application/xml", @@ -248,13 +243,13 @@ const char* hts_mime_keep[] = { }; /* bogus servers returns these mime types when the extension is seen within the filename */ -const char* hts_mime_bogus_multiple[] = { +const char *hts_mime_bogus_multiple[] = { "application/x-wais-source", /* src (src.rpm) */ "" }; /* pas de type mime connu, mais extension connue */ -const char* hts_ext_dynamic[] = { +const char *hts_ext_dynamic[] = { "php3", "php", "php4", @@ -263,290 +258,290 @@ const char* hts_ext_dynamic[] = { "asp", "jsp", "pl", - /*"exe",*/ + /*"exe", */ "cfm", - "nsf", /* lotus */ + "nsf", /* lotus */ "" }; /* types MIME note: application/octet-stream should not be used here */ -const char* hts_mime[][2] = { - {"application/acad","dwg"}, - {"application/arj","arj"}, - {"application/clariscad","ccad"}, - {"application/drafting","drw"}, - {"application/dxf","dxf"}, - {"application/excel","xls"}, - {"application/i-deas","unv"}, - {"application/iges","isg"}, - {"application/iges","iges"}, - {"application/mac-binhex40","hqx"}, - {"application/mac-compactpro","cpt"}, - {"application/msword","doc"}, - {"application/msword","w6w"}, - {"application/msword","word"}, - {"application/mswrite","wri"}, - /*{"application/octet-stream","dms"},*/ - /*{"application/octet-stream","lzh"},*/ - /*{"application/octet-stream","lha"},*/ - /*{"application/octet-stream","bin"},*/ - {"application/oda","oda"}, - {"application/pdf","pdf"}, - {"application/postscript","ps"}, - {"application/postscript","ai"}, - {"application/postscript","eps"}, - {"application/powerpoint","ppt"}, - {"application/pro_eng","prt"}, - {"application/pro_eng","part"}, - {"application/rtf","rtf"}, - {"application/set","set"}, - {"application/sla","stl"}, - {"application/smil","smi"}, - {"application/smil","smil"}, - {"application/smil","sml"}, - {"application/solids","sol"}, - {"application/STEP","stp"}, - {"application/STEP","step"}, - {"application/vda","vda"}, - {"application/x-authorware-map","aam"}, - {"application/x-authorware-seg","aas"}, - {"application/x-authorware-bin","aab"}, - {"application/x-bzip2","bz2"}, - {"application/x-cocoa","cco"}, - {"application/x-csh","csh"}, - {"application/x-director","dir"}, - {"application/x-director","dcr"}, - {"application/x-director","dxr"}, - {"application/x-mif","mif"}, - {"application/x-dvi","dvi"}, - {"application/x-gzip","gz"}, - {"application/x-gzip","gzip"}, - {"application/x-hdf","hdf"}, - {"application/x-javascript","js"}, - {"application/x-koan","skp"}, - {"application/x-koan","skd"}, - {"application/x-koan","skt"}, - {"application/x-koan","skm"}, - {"application/x-latex","latex"}, - {"application/x-netcdf","nc"}, - {"application/x-netcdf","cdf"}, +const char *hts_mime[][2] = { + {"application/acad", "dwg"}, + {"application/arj", "arj"}, + {"application/clariscad", "ccad"}, + {"application/drafting", "drw"}, + {"application/dxf", "dxf"}, + {"application/excel", "xls"}, + {"application/i-deas", "unv"}, + {"application/iges", "isg"}, + {"application/iges", "iges"}, + {"application/mac-binhex40", "hqx"}, + {"application/mac-compactpro", "cpt"}, + {"application/msword", "doc"}, + {"application/msword", "w6w"}, + {"application/msword", "word"}, + {"application/mswrite", "wri"}, + /*{"application/octet-stream","dms"}, */ + /*{"application/octet-stream","lzh"}, */ + /*{"application/octet-stream","lha"}, */ + /*{"application/octet-stream","bin"}, */ + {"application/oda", "oda"}, + {"application/pdf", "pdf"}, + {"application/postscript", "ps"}, + {"application/postscript", "ai"}, + {"application/postscript", "eps"}, + {"application/powerpoint", "ppt"}, + {"application/pro_eng", "prt"}, + {"application/pro_eng", "part"}, + {"application/rtf", "rtf"}, + {"application/set", "set"}, + {"application/sla", "stl"}, + {"application/smil", "smi"}, + {"application/smil", "smil"}, + {"application/smil", "sml"}, + {"application/solids", "sol"}, + {"application/STEP", "stp"}, + {"application/STEP", "step"}, + {"application/vda", "vda"}, + {"application/x-authorware-map", "aam"}, + {"application/x-authorware-seg", "aas"}, + {"application/x-authorware-bin", "aab"}, + {"application/x-bzip2", "bz2"}, + {"application/x-cocoa", "cco"}, + {"application/x-csh", "csh"}, + {"application/x-director", "dir"}, + {"application/x-director", "dcr"}, + {"application/x-director", "dxr"}, + {"application/x-mif", "mif"}, + {"application/x-dvi", "dvi"}, + {"application/x-gzip", "gz"}, + {"application/x-gzip", "gzip"}, + {"application/x-hdf", "hdf"}, + {"application/x-javascript", "js"}, + {"application/x-koan", "skp"}, + {"application/x-koan", "skd"}, + {"application/x-koan", "skt"}, + {"application/x-koan", "skm"}, + {"application/x-latex", "latex"}, + {"application/x-netcdf", "nc"}, + {"application/x-netcdf", "cdf"}, /* {"application/x-sh","sh"}, */ /* {"application/x-csh","csh"}, */ /* {"application/x-ksh","ksh"}, */ - {"application/x-shar","shar"}, - {"application/x-stuffit","sit"}, - {"application/x-tcl","tcl"}, - {"application/x-tex","tex"}, - {"application/x-texinfo","texinfo"}, - {"application/x-texinfo","texi"}, - {"application/x-troff","t"}, - {"application/x-troff","tr"}, - {"application/x-troff","roff"}, - {"application/x-troff-man","man"}, - {"application/x-troff-me","ms"}, - {"application/x-wais-source","src"}, - {"application/zip","zip"}, - {"application/x-zip-compressed","zip"}, - {"application/x-bcpio","bcpio"}, - {"application/x-cdlink","vcd"}, - {"application/x-cpio","cpio"}, - {"application/x-gtar","tgz"}, - {"application/x-gtar","gtar"}, - {"application/x-shar","shar"}, - {"application/x-shockwave-flash","swf"}, - {"application/x-sv4cpio","sv4cpio"}, - {"application/x-sv4crc","sv4crc"}, - {"application/x-tar","tar"}, - {"application/x-ustar","ustar"}, - {"application/x-winhelp","hlp"}, - {"application/xml","xml"}, - {"audio/midi","mid"}, - {"audio/midi","midi"}, - {"audio/midi","kar"}, - {"audio/mpeg","mp3"}, - {"audio/mpeg","mpga"}, - {"audio/mpeg","mp2"}, - {"audio/basic","au"}, - {"audio/basic","snd"}, - {"audio/x-aiff","aif"}, - {"audio/x-aiff","aiff"}, - {"audio/x-aiff","aifc"}, - {"audio/x-pn-realaudio","rm"}, - {"audio/x-pn-realaudio","ram"}, - {"audio/x-pn-realaudio","ra"}, - {"audio/x-pn-realaudio-plugin","rpm"}, - {"audio/x-wav","wav"}, - {"chemical/x-pdb","pdb"}, - {"chemical/x-pdb","xyz"}, - {"drawing/x-dwf","dwf"}, - {"image/gif","gif"}, - {"image/ief","ief"}, - {"image/jpeg","jpg"}, - {"image/jpeg","jpe"}, - {"image/jpeg","jpeg"}, - {"image/pict","pict"}, - {"image/png","png"}, - {"image/tiff","tiff"}, - {"image/tiff","tif"}, - {"image/svg+xml","svg"}, - {"image/svg-xml","svg"}, - {"image/x-cmu-raster","ras"}, - {"image/x-freehand","fh4"}, - {"image/x-freehand","fh7"}, - {"image/x-freehand","fh5"}, - {"image/x-freehand","fhc"}, - {"image/x-freehand","fh"}, - {"image/x-portable-anymap","pnm"}, - {"image/x-portable-bitmap","pgm"}, - {"image/x-portable-pixmap","ppm"}, - {"image/x-rgb","rgb"}, - {"image/x-xbitmap","xbm"}, - {"image/x-xpixmap","xpm"}, - {"image/x-xwindowdump","xwd"}, - {"model/mesh","msh"}, - {"model/mesh","mesh"}, - {"model/mesh","silo"}, - {"multipart/x-zip","zip"}, - {"multipart/x-gzip","gzip"}, - {"text/css","css"}, - {"text/html","html"}, - {"text/html","htm"}, - {"text/plain","txt"}, - {"text/plain","g"}, - {"text/plain","h"}, - {"text/plain","c"}, - {"text/plain","cc"}, - {"text/plain","hh"}, - {"text/plain","m"}, - {"text/plain","f90"}, - {"text/richtext","rtx"}, - {"text/tab-separated-values","tsv"}, - {"text/x-setext","etx"}, - {"text/x-sgml","sgml"}, - {"text/x-sgml","sgm"}, - {"text/xml","xml"}, - {"text/xml","dtd"}, - {"video/mpeg","mpeg"}, - {"video/mpeg","mpg"}, - {"video/mpeg","mpe"}, - {"video/quicktime","qt"}, - {"video/quicktime","mov"}, - {"video/x-msvideo","avi"}, - {"video/x-sgi-movie","movie"}, - {"x-conference/x-cooltalk","ice"}, - /*{"application/x-httpd-cgi","cgi"},*/ - {"x-world/x-vrml","wrl"}, + {"application/x-shar", "shar"}, + {"application/x-stuffit", "sit"}, + {"application/x-tcl", "tcl"}, + {"application/x-tex", "tex"}, + {"application/x-texinfo", "texinfo"}, + {"application/x-texinfo", "texi"}, + {"application/x-troff", "t"}, + {"application/x-troff", "tr"}, + {"application/x-troff", "roff"}, + {"application/x-troff-man", "man"}, + {"application/x-troff-me", "ms"}, + {"application/x-wais-source", "src"}, + {"application/zip", "zip"}, + {"application/x-zip-compressed", "zip"}, + {"application/x-bcpio", "bcpio"}, + {"application/x-cdlink", "vcd"}, + {"application/x-cpio", "cpio"}, + {"application/x-gtar", "tgz"}, + {"application/x-gtar", "gtar"}, + {"application/x-shar", "shar"}, + {"application/x-shockwave-flash", "swf"}, + {"application/x-sv4cpio", "sv4cpio"}, + {"application/x-sv4crc", "sv4crc"}, + {"application/x-tar", "tar"}, + {"application/x-ustar", "ustar"}, + {"application/x-winhelp", "hlp"}, + {"application/xml", "xml"}, + {"audio/midi", "mid"}, + {"audio/midi", "midi"}, + {"audio/midi", "kar"}, + {"audio/mpeg", "mp3"}, + {"audio/mpeg", "mpga"}, + {"audio/mpeg", "mp2"}, + {"audio/basic", "au"}, + {"audio/basic", "snd"}, + {"audio/x-aiff", "aif"}, + {"audio/x-aiff", "aiff"}, + {"audio/x-aiff", "aifc"}, + {"audio/x-pn-realaudio", "rm"}, + {"audio/x-pn-realaudio", "ram"}, + {"audio/x-pn-realaudio", "ra"}, + {"audio/x-pn-realaudio-plugin", "rpm"}, + {"audio/x-wav", "wav"}, + {"chemical/x-pdb", "pdb"}, + {"chemical/x-pdb", "xyz"}, + {"drawing/x-dwf", "dwf"}, + {"image/gif", "gif"}, + {"image/ief", "ief"}, + {"image/jpeg", "jpg"}, + {"image/jpeg", "jpe"}, + {"image/jpeg", "jpeg"}, + {"image/pict", "pict"}, + {"image/png", "png"}, + {"image/tiff", "tiff"}, + {"image/tiff", "tif"}, + {"image/svg+xml", "svg"}, + {"image/svg-xml", "svg"}, + {"image/x-cmu-raster", "ras"}, + {"image/x-freehand", "fh4"}, + {"image/x-freehand", "fh7"}, + {"image/x-freehand", "fh5"}, + {"image/x-freehand", "fhc"}, + {"image/x-freehand", "fh"}, + {"image/x-portable-anymap", "pnm"}, + {"image/x-portable-bitmap", "pgm"}, + {"image/x-portable-pixmap", "ppm"}, + {"image/x-rgb", "rgb"}, + {"image/x-xbitmap", "xbm"}, + {"image/x-xpixmap", "xpm"}, + {"image/x-xwindowdump", "xwd"}, + {"model/mesh", "msh"}, + {"model/mesh", "mesh"}, + {"model/mesh", "silo"}, + {"multipart/x-zip", "zip"}, + {"multipart/x-gzip", "gzip"}, + {"text/css", "css"}, + {"text/html", "html"}, + {"text/html", "htm"}, + {"text/plain", "txt"}, + {"text/plain", "g"}, + {"text/plain", "h"}, + {"text/plain", "c"}, + {"text/plain", "cc"}, + {"text/plain", "hh"}, + {"text/plain", "m"}, + {"text/plain", "f90"}, + {"text/richtext", "rtx"}, + {"text/tab-separated-values", "tsv"}, + {"text/x-setext", "etx"}, + {"text/x-sgml", "sgml"}, + {"text/x-sgml", "sgm"}, + {"text/xml", "xml"}, + {"text/xml", "dtd"}, + {"video/mpeg", "mpeg"}, + {"video/mpeg", "mpg"}, + {"video/mpeg", "mpe"}, + {"video/quicktime", "qt"}, + {"video/quicktime", "mov"}, + {"video/x-msvideo", "avi"}, + {"video/x-sgi-movie", "movie"}, + {"x-conference/x-cooltalk", "ice"}, + /*{"application/x-httpd-cgi","cgi"}, */ + {"x-world/x-vrml", "wrl"}, /* More from w3schools.com */ - { "application/envoy", "evy" }, - { "application/fractals", "fif" }, - { "application/futuresplash", "spl" }, - { "application/hta", "hta" }, - { "application/internet-property-stream", "acx" }, - { "application/msword", "dot" }, - { "application/olescript", "axs" }, - { "application/pics-rules", "prf" }, - { "application/pkcs10", "p10" }, - { "application/pkix-crl", "crl" }, - { "application/set-payment-initiation", "setpay" }, - { "application/set-registration-initiation", "setreg" }, - { "application/vnd.ms-excel", "xls" }, - { "application/vnd.ms-excel", "xla" }, - { "application/vnd.ms-excel", "xlc" }, - { "application/vnd.ms-excel", "xlm" }, - { "application/vnd.ms-excel", "xlt" }, - { "application/vnd.ms-excel", "xlw" }, - { "application/vnd.ms-pkicertstore", "sst" }, - { "application/vnd.ms-pkiseccat", "cat" }, - { "application/vnd.ms-powerpoint", "ppt" }, - { "application/vnd.ms-powerpoint", "pot" }, - { "application/vnd.ms-powerpoint", "pps" }, - { "application/vnd.ms-project", "mpp" }, - { "application/vnd.ms-works", "wcm" }, - { "application/vnd.ms-works", "wdb" }, - { "application/vnd.ms-works", "wks" }, - { "application/vnd.ms-works", "wps" }, - { "application/x-compress", "z" }, - { "application/x-compressed", "tgz" }, - { "application/x-internet-signup", "ins" }, - { "application/x-internet-signup", "isp" }, - { "application/x-iphone", "iii" }, - { "application/x-javascript", "js" }, - { "application/x-msaccess", "mdb" }, - { "application/x-mscardfile", "crd" }, - { "application/x-msclip", "clp" }, - { "application/x-msmediaview", "m13" }, - { "application/x-msmediaview", "m14" }, - { "application/x-msmediaview", "mvb" }, - { "application/x-msmetafile", "wmf" }, - { "application/x-msmoney", "mny" }, - { "application/x-mspublisher", "pub" }, - { "application/x-msschedule", "scd" }, - { "application/x-msterminal", "trm" }, - { "application/x-perfmon", "pma" }, - { "application/x-perfmon", "pmc" }, - { "application/x-perfmon", "pml" }, - { "application/x-perfmon", "pmr" }, - { "application/x-perfmon", "pmw" }, - { "application/x-pkcs12", "p12" }, - { "application/x-pkcs12", "pfx" }, - { "application/x-pkcs7-certificates", "p7b" }, - { "application/x-pkcs7-certificates", "spc" }, - { "application/x-pkcs7-certreqresp", "p7r" }, - { "application/x-pkcs7-mime", "p7c" }, - { "application/x-pkcs7-mime", "p7m" }, - { "application/x-pkcs7-signature", "p7s" }, - { "application/x-troff-me", "me" }, - { "application/x-x509-ca-cert", "cer" }, - { "application/x-x509-ca-cert", "crt" }, - { "application/x-x509-ca-cert", "der" }, - { "application/ynd.ms-pkipko", "pko" }, - { "audio/mid", "mid" }, - { "audio/mid", "rmi" }, - { "audio/mpeg", "mp3" }, - { "audio/x-mpegurl", "m3u" }, - { "image/bmp", "bmp" }, - { "image/cis-cod", "cod" }, - { "image/pipeg", "jfif" }, - { "image/x-cmx", "cmx" }, - { "image/x-icon", "ico" }, - { "image/x-portable-bitmap", "pbm" }, - { "message/rfc822", "mht" }, - { "message/rfc822", "mhtml" }, - { "message/rfc822", "nws" }, - { "text/css", "css" }, - { "text/h323", "323" }, - { "text/html", "stm" }, - { "text/iuls", "uls" }, - { "text/plain", "bas" }, - { "text/scriptlet", "sct" }, - { "text/webviewhtml", "htt" }, - { "text/x-component", "htc" }, - { "text/x-vcard", "vcf" }, - { "video/mpeg", "mp2" }, - { "video/mpeg", "mpa" }, - { "video/mpeg", "mpv2" }, - { "video/x-la-asf", "lsf" }, - { "video/x-la-asf", "lsx" }, - { "video/x-ms-asf", "asf" }, - { "video/x-ms-asf", "asr" }, - { "video/x-ms-asf", "asx" }, - { "video/x-ms-wmv", "wmv" }, - { "x-world/x-vrml", "flr" }, - { "x-world/x-vrml", "vrml" }, - { "x-world/x-vrml", "wrz" }, - { "x-world/x-vrml", "xaf" }, - { "x-world/x-vrml", "xof" }, + {"application/envoy", "evy"}, + {"application/fractals", "fif"}, + {"application/futuresplash", "spl"}, + {"application/hta", "hta"}, + {"application/internet-property-stream", "acx"}, + {"application/msword", "dot"}, + {"application/olescript", "axs"}, + {"application/pics-rules", "prf"}, + {"application/pkcs10", "p10"}, + {"application/pkix-crl", "crl"}, + {"application/set-payment-initiation", "setpay"}, + {"application/set-registration-initiation", "setreg"}, + {"application/vnd.ms-excel", "xls"}, + {"application/vnd.ms-excel", "xla"}, + {"application/vnd.ms-excel", "xlc"}, + {"application/vnd.ms-excel", "xlm"}, + {"application/vnd.ms-excel", "xlt"}, + {"application/vnd.ms-excel", "xlw"}, + {"application/vnd.ms-pkicertstore", "sst"}, + {"application/vnd.ms-pkiseccat", "cat"}, + {"application/vnd.ms-powerpoint", "ppt"}, + {"application/vnd.ms-powerpoint", "pot"}, + {"application/vnd.ms-powerpoint", "pps"}, + {"application/vnd.ms-project", "mpp"}, + {"application/vnd.ms-works", "wcm"}, + {"application/vnd.ms-works", "wdb"}, + {"application/vnd.ms-works", "wks"}, + {"application/vnd.ms-works", "wps"}, + {"application/x-compress", "z"}, + {"application/x-compressed", "tgz"}, + {"application/x-internet-signup", "ins"}, + {"application/x-internet-signup", "isp"}, + {"application/x-iphone", "iii"}, + {"application/x-javascript", "js"}, + {"application/x-msaccess", "mdb"}, + {"application/x-mscardfile", "crd"}, + {"application/x-msclip", "clp"}, + {"application/x-msmediaview", "m13"}, + {"application/x-msmediaview", "m14"}, + {"application/x-msmediaview", "mvb"}, + {"application/x-msmetafile", "wmf"}, + {"application/x-msmoney", "mny"}, + {"application/x-mspublisher", "pub"}, + {"application/x-msschedule", "scd"}, + {"application/x-msterminal", "trm"}, + {"application/x-perfmon", "pma"}, + {"application/x-perfmon", "pmc"}, + {"application/x-perfmon", "pml"}, + {"application/x-perfmon", "pmr"}, + {"application/x-perfmon", "pmw"}, + {"application/x-pkcs12", "p12"}, + {"application/x-pkcs12", "pfx"}, + {"application/x-pkcs7-certificates", "p7b"}, + {"application/x-pkcs7-certificates", "spc"}, + {"application/x-pkcs7-certreqresp", "p7r"}, + {"application/x-pkcs7-mime", "p7c"}, + {"application/x-pkcs7-mime", "p7m"}, + {"application/x-pkcs7-signature", "p7s"}, + {"application/x-troff-me", "me"}, + {"application/x-x509-ca-cert", "cer"}, + {"application/x-x509-ca-cert", "crt"}, + {"application/x-x509-ca-cert", "der"}, + {"application/ynd.ms-pkipko", "pko"}, + {"audio/mid", "mid"}, + {"audio/mid", "rmi"}, + {"audio/mpeg", "mp3"}, + {"audio/x-mpegurl", "m3u"}, + {"image/bmp", "bmp"}, + {"image/cis-cod", "cod"}, + {"image/pipeg", "jfif"}, + {"image/x-cmx", "cmx"}, + {"image/x-icon", "ico"}, + {"image/x-portable-bitmap", "pbm"}, + {"message/rfc822", "mht"}, + {"message/rfc822", "mhtml"}, + {"message/rfc822", "nws"}, + {"text/css", "css"}, + {"text/h323", "323"}, + {"text/html", "stm"}, + {"text/iuls", "uls"}, + {"text/plain", "bas"}, + {"text/scriptlet", "sct"}, + {"text/webviewhtml", "htt"}, + {"text/x-component", "htc"}, + {"text/x-vcard", "vcf"}, + {"video/mpeg", "mp2"}, + {"video/mpeg", "mpa"}, + {"video/mpeg", "mpv2"}, + {"video/x-la-asf", "lsf"}, + {"video/x-la-asf", "lsx"}, + {"video/x-ms-asf", "asf"}, + {"video/x-ms-asf", "asr"}, + {"video/x-ms-asf", "asx"}, + {"video/x-ms-wmv", "wmv"}, + {"x-world/x-vrml", "flr"}, + {"x-world/x-vrml", "vrml"}, + {"x-world/x-vrml", "wrz"}, + {"x-world/x-vrml", "xaf"}, + {"x-world/x-vrml", "xof"}, /* Various */ - { "application/ogg", "ogg" }, + {"application/ogg", "ogg"}, - {"application/x-java-vm","class"}, - - {"",""}}; + {"application/x-java-vm", "class"}, + {"", ""} +}; // Reserved (RFC2396) #define CIS(c,ch) ( ((unsigned char)(c)) == (ch) ) @@ -601,29 +596,29 @@ const char* hts_mime[][2] = { || CIS(c,')') ) //#define CHAR_MARK(c) ( strchr("-_.!~*'()",(unsigned char)(c)) != 0 ) - - // conversion éventuelle / vers antislash #ifdef _WIN32 -char* antislash(char *catbuff, const char* s) { - char* a; - strcpybuff(catbuff,s); - while(a=strchr(catbuff,'/')) *a='\\'; +char *antislash(char *catbuff, const char *s) { + char *a; + + strcpybuff(catbuff, s); + while(a = strchr(catbuff, '/')) + *a = '\\'; return catbuff; } #endif #ifdef _WIN32_WCE -char cwd[MAX_PATH+1] = ""; +char cwd[MAX_PATH + 1] = ""; #endif // Initialize a htsblk structure -void hts_init_htsblk(htsblk* r) { - memset(r, 0, sizeof(htsblk)); // effacer - r->soc=INVALID_SOCKET; - r->msg[0]='\0'; - r->statuscode=STATUSCODE_INVALID; - r->totalsize=-1; +void hts_init_htsblk(htsblk * r) { + memset(r, 0, sizeof(htsblk)); // effacer + r->soc = INVALID_SOCKET; + r->msg[0] = '\0'; + r->statuscode = STATUSCODE_INVALID; + r->totalsize = -1; } // Récupération d'un fichier http sur le net. @@ -634,32 +629,33 @@ void hts_init_htsblk(htsblk* r) { // suivre l'évolution du chargement si le process a été lancé // en background -htsblk httpget(httrackp *opt,char* url) { - char BIGSTK adr[HTS_URLMAXSIZE*2]; // adresse - char BIGSTK fil[HTS_URLMAXSIZE*2]; // chemin - +htsblk httpget(httrackp * opt, char *url) { + char BIGSTK adr[HTS_URLMAXSIZE * 2]; // adresse + char BIGSTK fil[HTS_URLMAXSIZE * 2]; // chemin + // séparer URL en adresse+chemin - if (ident_url_absolute(url,adr,fil)==-1) { + if (ident_url_absolute(url, adr, fil) == -1) { htsblk retour; + hts_init_htsblk(&retour); //memset(&retour, 0, sizeof(htsblk)); // effacer // retour prédéfini: erreur - retour.adr=NULL; - retour.size=0; - retour.msg[0]='\0'; - retour.statuscode=STATUSCODE_INVALID; - strcpybuff(retour.msg,"Error invalid URL"); + retour.adr = NULL; + retour.size = 0; + retour.msg[0] = '\0'; + retour.statuscode = STATUSCODE_INVALID; + strcpybuff(retour.msg, "Error invalid URL"); return retour; } - - return xhttpget(opt,adr,fil); + + return xhttpget(opt, adr, fil); } // ouvre une liaison http, envoie une requète GET et réceptionne le header // retour: socket -T_SOC http_fopen(httrackp *opt,char* adr,char* fil,htsblk* retour) { +T_SOC http_fopen(httrackp * opt, char *adr, char *fil, htsblk * retour) { // / GET, traiter en-tête - return http_xfopen(opt,0,1,1,NULL,adr,fil,retour); + return http_xfopen(opt, 0, 1, 1, NULL, adr, fil, retour); } // ouverture d'une liaison http, envoi d'une requète @@ -667,27 +663,28 @@ T_SOC http_fopen(httrackp *opt,char* adr,char* fil,htsblk* retour) { // treat: traiter header? // waitconnect: attendre le connect() // note: dans retour, on met les params du proxy -T_SOC http_xfopen(httrackp *opt,int mode,int treat,int waitconnect,char* xsend,char* adr,char* fil,htsblk* retour) { +T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect, + char *xsend, char *adr, char *fil, htsblk * retour) { //htsblk retour; //int bufl=TAILLE_BUFFER; // 8Ko de buffer - T_SOC soc=INVALID_SOCKET; + T_SOC soc = INVALID_SOCKET; + //char *p,*q; - + // retour prédéfini: erreur if (retour) { - retour->adr=NULL; - retour->size=0; - retour->msg[0]='\0'; - retour->statuscode=STATUSCODE_NON_FATAL; // a priori erreur non fatale + retour->adr = NULL; + retour->size = 0; + retour->msg[0] = '\0'; + retour->statuscode = STATUSCODE_NON_FATAL; // a priori erreur non fatale } - #if HDEBUG - printf("adr=%s\nfichier=%s\n",adr,fil); + printf("adr=%s\nfichier=%s\n", adr, fil); #endif - + // ouvrir liaison #if HDEBUG - printf("Création d'une socket sur %s\n",adr); + printf("Création d'une socket sur %s\n", adr); #endif #if CNXDEBUG @@ -696,24 +693,22 @@ T_SOC http_xfopen(httrackp *opt,int mode,int treat,int waitconnect,char* xsend,c /* connexion */ if (retour) { - if ( (!(retour->req.proxy.active)) - || - ( - (strcmp(adr,"file://")==0) - || - (strncmp(adr,"https://", 8)==0) - ) - ) { /* pas de proxy, ou non utilisable ici */ - soc=newhttp(opt,adr,retour,-1,waitconnect); + if ((!(retour->req.proxy.active)) + || ((strcmp(adr, "file://") == 0) + || (strncmp(adr, "https://", 8) == 0) + ) + ) { /* pas de proxy, ou non utilisable ici */ + soc = newhttp(opt, adr, retour, -1, waitconnect); } else { - soc=newhttp(opt, retour->req.proxy.name, retour,retour->req.proxy.port, waitconnect); // ouvrir sur le proxy à la place + soc = newhttp(opt, retour->req.proxy.name, retour, retour->req.proxy.port, waitconnect); // ouvrir sur le proxy à la place } } else { - soc=newhttp(opt,adr,NULL,-1,waitconnect); + soc = newhttp(opt, adr, NULL, -1, waitconnect); } // copier index socket retour - if (retour) retour->soc=soc; + if (retour) + retour->soc = soc; /* Check for errors */ if (soc == INVALID_SOCKET) { @@ -722,141 +717,149 @@ T_SOC http_xfopen(httrackp *opt,int mode,int treat,int waitconnect,char* xsend,c if (!strnotempty(retour->msg)) { #ifdef _WIN32 int last_errno = WSAGetLastError(); - sprintf(retour->msg,"Connect error: %s", strerror(last_errno)); + + sprintf(retour->msg, "Connect error: %s", strerror(last_errno)); #else int last_errno = errno; - sprintf(retour->msg,"Connect error: %s", strerror(last_errno)); + + sprintf(retour->msg, "Connect error: %s", strerror(last_errno)); #endif } } } } - // -------------------- // court-circuit (court circuite aussi le proxy..) // LOCAL_SOCKET_ID est une pseudo-socket locale - if (soc==LOCAL_SOCKET_ID) { - retour->is_file=1; // fichier local - if (mode==0) { // GET + if (soc == LOCAL_SOCKET_ID) { + retour->is_file = 1; // fichier local + if (mode == 0) { // GET // Test en cas de file:///C|... - if (!fexist(fconv(OPT_GET_BUFF(opt), unescape_http(OPT_GET_BUFF(opt),fil)))) - if (fexist(fconv(OPT_GET_BUFF(opt), unescape_http(OPT_GET_BUFF(opt),fil+1)))) { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - strcpybuff(tempo,fil+1); - strcpybuff(fil,tempo); + if (!fexist + (fconv(OPT_GET_BUFF(opt), unescape_http(OPT_GET_BUFF(opt), fil)))) + if (fexist + (fconv + (OPT_GET_BUFF(opt), unescape_http(OPT_GET_BUFF(opt), fil + 1)))) { + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + + strcpybuff(tempo, fil + 1); + strcpybuff(fil, tempo); } - // Ouvrir - retour->totalsize=fsize(fconv(OPT_GET_BUFF(opt), unescape_http(OPT_GET_BUFF(opt),fil))); // taille du fichier - retour->msg[0]='\0'; - soc=INVALID_SOCKET; - if (retour->totalsize<0) - strcpybuff(retour->msg,"Unable to open local file"); - else if (retour->totalsize==0) - strcpybuff(retour->msg,"File empty"); + retour->totalsize = fsize(fconv(OPT_GET_BUFF(opt), unescape_http(OPT_GET_BUFF(opt), fil))); // taille du fichier + retour->msg[0] = '\0'; + soc = INVALID_SOCKET; + if (retour->totalsize < 0) + strcpybuff(retour->msg, "Unable to open local file"); + else if (retour->totalsize == 0) + strcpybuff(retour->msg, "File empty"); else { // Note: On passe par un FILE* (plus propre) //soc=open(fil,O_RDONLY,0); // en lecture seule! - retour->fp=FOPEN(fconv(OPT_GET_BUFF(opt), unescape_http(OPT_GET_BUFF(opt),fil)),"rb"); // ouvrir - if (retour->fp==NULL) - soc=INVALID_SOCKET; + retour->fp = FOPEN(fconv(OPT_GET_BUFF(opt), unescape_http(OPT_GET_BUFF(opt), fil)), "rb"); // ouvrir + if (retour->fp == NULL) + soc = INVALID_SOCKET; else - soc=LOCAL_SOCKET_ID; + soc = LOCAL_SOCKET_ID; } - retour->soc=soc; - if (soc!=INVALID_SOCKET) { - retour->statuscode=HTTP_OK; // OK - strcpybuff(retour->msg,"OK"); - guess_httptype(opt,retour->contenttype,fil); - } else if (strnotempty(retour->msg)==0) - strcpybuff(retour->msg,"Unable to open local file"); - return soc; // renvoyer - } else { // HEAD ou POST : interdit sur un local!!!! (c'est idiot!) - strcpybuff(retour->msg,"Unexpected Head/Post local request"); - soc=INVALID_SOCKET; // erreur - retour->soc=soc; + retour->soc = soc; + if (soc != INVALID_SOCKET) { + retour->statuscode = HTTP_OK; // OK + strcpybuff(retour->msg, "OK"); + guess_httptype(opt, retour->contenttype, fil); + } else if (strnotempty(retour->msg) == 0) + strcpybuff(retour->msg, "Unable to open local file"); + return soc; // renvoyer + } else { // HEAD ou POST : interdit sur un local!!!! (c'est idiot!) + strcpybuff(retour->msg, "Unexpected Head/Post local request"); + soc = INVALID_SOCKET; // erreur + retour->soc = soc; return soc; } - } + } // -------------------- - if (soc!=INVALID_SOCKET) { + if (soc != INVALID_SOCKET) { char rcvd[1100]; - rcvd[0]='\0'; + + rcvd[0] = '\0'; #if HDEBUG - printf("Ok, connexion réussie, id=%d\n",soc); + printf("Ok, connexion réussie, id=%d\n", soc); #endif - + // connecté? if (waitconnect) { - http_sendhead(opt,NULL,mode,xsend,adr,fil,NULL,NULL,retour); - } - - if (soc!=INVALID_SOCKET) { - + http_sendhead(opt, NULL, mode, xsend, adr, fil, NULL, NULL, retour); + } + + if (soc != INVALID_SOCKET) { + #if HDEBUG printf("Attente de la réponse:\n"); #endif - + // si GET (réception d'un fichier), réceptionner en-tête d'abord, // et ensuite le corps // si POST on ne réceptionne rien du tout, c'est après que l'on fera // une réception standard pour récupérer l'en tête - if ((treat) && (waitconnect)) { // traiter (attendre!) en-tête + if ((treat) && (waitconnect)) { // traiter (attendre!) en-tête // Réception de la status line et de l'en-tête (norme RFC1945) - + // status-line à récupérer - finput(soc,rcvd,1024); - if (strnotempty(rcvd)==0) - finput(soc,rcvd,1024); // "certains serveurs buggés envoient un \n au début" (RFC) + finput(soc, rcvd, 1024); + if (strnotempty(rcvd) == 0) + finput(soc, rcvd, 1024); // "certains serveurs buggés envoient un \n au début" (RFC) // traiter status-line - treatfirstline(retour,rcvd); + treatfirstline(retour, rcvd); #if HDEBUG - printf("Status-Code=%d\n",retour->statuscode); + printf("Status-Code=%d\n", retour->statuscode); #endif - + // en-tête - + // header // ** !attention! HTTP/0.9 non supporté do { - finput(soc,rcvd,1024); + finput(soc, rcvd, 1024); #if HDEBUG - printf(">%s\n",rcvd); + printf(">%s\n", rcvd); #endif if (strnotempty(rcvd)) - treathead(NULL,NULL,NULL,retour,rcvd); // traiter + treathead(NULL, NULL, NULL, retour, rcvd); // traiter } while(strnotempty(rcvd)); - + //rcvsize=-1; // forCER CHARGEMENT INCONNU - + //if (retour) // retour->totalsize=rcvsize; - - } else { // si GET, on recevra l'en tête APRES + + } else { // si GET, on recevra l'en tête APRES //rcvsize=-1; // on ne connait pas la taille de l'en-tête if (retour) - retour->totalsize=-1; + retour->totalsize = -1; } - + } - + } - + return soc; } - // envoi d'une requète -int http_sendhead(httrackp *opt,t_cookie* cookie,int mode,char* xsend,char* adr,char* fil,char* referer_adr,char* referer_fil,htsblk* retour) { +int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, char *xsend, + char *adr, char *fil, char *referer_adr, char *referer_fil, + htsblk * retour) { char BIGSTK buff[8192]; + //int use_11=0; // HTTP 1.1 utilisé - int direct_url=0; // ne pas analyser l'url (exemple: ftp://) - char* search_tag=NULL; - buff[0]='\0'; + int direct_url = 0; // ne pas analyser l'url (exemple: ftp://) + char *search_tag = NULL; + + buff[0] = '\0'; // header Date //strcatbuff(buff,"Date: "); @@ -869,24 +872,28 @@ int http_sendhead(httrackp *opt,t_cookie* cookie,int mode,char* xsend,char* adr, // exemple: http://www.someweb.com/test.cgi?foo>post:posteddata=10&foo=5 // si présence d'un tag >postfile: alors envoyer en tête brut contenu dans le fichier en question // exemple: http://www.someweb.com/test.cgi?foo>postfile:post0.txt - search_tag=strstr(fil,POSTTOK":"); + search_tag = strstr(fil, POSTTOK ":"); if (!search_tag) { - search_tag=strstr(fil,POSTTOK"file:"); - if (search_tag) { // postfile - if (mode==0) { // GET! - FILE* fp=FOPEN(unescape_http(OPT_GET_BUFF(opt),search_tag+strlen(POSTTOK)+5),"rb"); + search_tag = strstr(fil, POSTTOK "file:"); + if (search_tag) { // postfile + if (mode == 0) { // GET! + FILE *fp = + FOPEN(unescape_http + (OPT_GET_BUFF(opt), search_tag + strlen(POSTTOK) + 5), "rb"); if (fp) { char BIGSTK line[1100]; - char BIGSTK protocol[256],url[HTS_URLMAXSIZE*2],method[256]; - linput(fp,line,1000); - if (sscanf(line,"%s %s %s",method,url,protocol) == 3) { + char BIGSTK protocol[256], url[HTS_URLMAXSIZE * 2], method[256]; + + linput(fp, line, 1000); + if (sscanf(line, "%s %s %s", method, url, protocol) == 3) { // selon que l'on a ou pas un proxy if (retour->req.proxy.active) - sprintf(buff,"%s http://%s%s %s\r\n",method,adr,url,protocol); + sprintf(buff, "%s http://%s%s %s\r\n", method, adr, url, + protocol); else - sprintf(buff,"%s %s %s\r\n",method,url,protocol); + sprintf(buff, "%s %s %s\r\n", method, url, protocol); // lire le reste en brut - if (fread(buff+strlen(buff),8000-strlen(buff),1,fp) < 0) { + if (fread(buff + strlen(buff), 8000 - strlen(buff), 1, fp) < 0) { return -1; } } @@ -896,458 +903,490 @@ int http_sendhead(httrackp *opt,t_cookie* cookie,int mode,char* xsend,char* adr, } } // Fin postfile - - if (strnotempty(buff)==0) { // PAS POSTFILE + + if (strnotempty(buff) == 0) { // PAS POSTFILE // Type de requète? - if ((search_tag) && (mode==0)) { - strcatbuff(buff,"POST "); - } else if (mode==0) { // GET - strcatbuff(buff,"GET "); - } else { // if (mode==1) { - if (!retour->req.http11) // forcer HTTP/1.0 - strcatbuff(buff,"GET "); // certains serveurs (cgi) buggent avec HEAD + if ((search_tag) && (mode == 0)) { + strcatbuff(buff, "POST "); + } else if (mode == 0) { // GET + strcatbuff(buff, "GET "); + } else { // if (mode==1) { + if (!retour->req.http11) // forcer HTTP/1.0 + strcatbuff(buff, "GET "); // certains serveurs (cgi) buggent avec HEAD else - strcatbuff(buff,"HEAD "); + strcatbuff(buff, "HEAD "); } - + // si on gère un proxy, il faut une Absolute URI: on ajoute avant http://www.adr.dom - if ( retour->req.proxy.active && (strncmp(adr,"https://", 8) != 0) ) { - if (!link_has_authority(adr)) { // default http + if (retour->req.proxy.active && (strncmp(adr, "https://", 8) != 0)) { + if (!link_has_authority(adr)) { // default http #if HDEBUG - printf("Proxy Use: for %s%s proxy %d port %d\n",adr,fil,retour->req.proxy.name,retour->req.proxy.port); + printf("Proxy Use: for %s%s proxy %d port %d\n", adr, fil, + retour->req.proxy.name, retour->req.proxy.port); #endif - strcatbuff(buff,"http://"); - strcatbuff(buff,jump_identification(adr)); - } else { // ftp:// en proxy http + strcatbuff(buff, "http://"); + strcatbuff(buff, jump_identification(adr)); + } else { // ftp:// en proxy http #if HDEBUG - printf("Proxy Use for ftp: for %s%s proxy %d port %d\n",adr,fil,retour->req.proxy.name,retour->req.proxy.port); + printf("Proxy Use for ftp: for %s%s proxy %d port %d\n", adr, fil, + retour->req.proxy.name, retour->req.proxy.port); #endif - direct_url=1; // ne pas analyser user/pass - strcatbuff(buff,adr); + direct_url = 1; // ne pas analyser user/pass + strcatbuff(buff, adr); } - } - + } // NOM DU FICHIER // on slash doit être présent en début, sinon attention aux bad request! (400) - if (*fil!='/') strcatbuff(buff,"/"); + if (*fil != '/') + strcatbuff(buff, "/"); { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - tempo[0]='\0'; + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + + tempo[0] = '\0'; if (search_tag) - strncatbuff(tempo,fil,(int) (search_tag - fil)); + strncatbuff(tempo, fil, (int) (search_tag - fil)); else - strcpybuff(tempo,fil); + strcpybuff(tempo, fil); escape_check_url(tempo); - strcatbuff(buff,tempo); // avec échappement + strcatbuff(buff, tempo); // avec échappement } - + // protocole - if (!retour->req.http11) { // forcer HTTP/1.0 + if (!retour->req.http11) { // forcer HTTP/1.0 //use_11=0; - strcatbuff(buff," HTTP/1.0\x0d\x0a"); - } else { // Requète 1.1 + strcatbuff(buff, " HTTP/1.0\x0d\x0a"); + } else { // Requète 1.1 //use_11=1; - strcatbuff(buff," HTTP/1.1\x0d\x0a"); + strcatbuff(buff, " HTTP/1.1\x0d\x0a"); } /* supplemental data */ - if (xsend) strcatbuff(buff,xsend); // éventuelles autres lignes + if (xsend) + strcatbuff(buff, xsend); // éventuelles autres lignes // tester proxy authentication if (retour->req.proxy.active) { - if (link_has_authorization(retour->req.proxy.name)) { // et hop, authentification proxy! - const char* a = jump_identification(retour->req.proxy.name); - const char* astart = jump_protocol(retour->req.proxy.name); + if (link_has_authorization(retour->req.proxy.name)) { // et hop, authentification proxy! + const char *a = jump_identification(retour->req.proxy.name); + const char *astart = jump_protocol(retour->req.proxy.name); char autorisation[1100]; - char user_pass[256]; - autorisation[0]=user_pass[0]='\0'; + char user_pass[256]; + + autorisation[0] = user_pass[0] = '\0'; // - strncatbuff(user_pass,astart,(int) (a - astart) - 1); - strcpybuff(user_pass,unescape_http(OPT_GET_BUFF(opt),user_pass)); - code64((unsigned char*)user_pass,(int)strlen(user_pass),(unsigned char*)autorisation,0); - strcatbuff(buff,"Proxy-Authorization: Basic "); - strcatbuff(buff,autorisation); - strcatbuff(buff,H_CRLF); + strncatbuff(user_pass, astart, (int) (a - astart) - 1); + strcpybuff(user_pass, unescape_http(OPT_GET_BUFF(opt), user_pass)); + code64((unsigned char *) user_pass, (int) strlen(user_pass), + (unsigned char *) autorisation, 0); + strcatbuff(buff, "Proxy-Authorization: Basic "); + strcatbuff(buff, autorisation); + strcatbuff(buff, H_CRLF); #if HDEBUG - printf("Proxy-Authenticate, %s (code: %s)\n",user_pass,autorisation); + printf("Proxy-Authenticate, %s (code: %s)\n", user_pass, autorisation); #endif } } - // Referer? - if (referer_adr != NULL && referer_fil != NULL - && strnotempty(referer_adr) && strnotempty(referer_fil) - ) { // non vide - if ( - (strcmp(referer_adr,"file://") != 0) - && - ( /* no https referer to http urls */ - (strncmp(referer_adr, "https://", 8) != 0) /* referer is not https */ - || - (strncmp(adr, "https://", 8) == 0) /* or referer AND addresses are https */ - ) - ) { // PAS file:// - strcatbuff(buff,"Referer: "); - strcatbuff(buff,"http://"); - strcatbuff(buff,jump_identification(referer_adr)); - strcatbuff(buff,referer_fil); - strcatbuff(buff,H_CRLF); + if (referer_adr != NULL && referer_fil != NULL && strnotempty(referer_adr) + && strnotempty(referer_fil) + ) { // non vide + if ((strcmp(referer_adr, "file://") != 0) + && ( /* no https referer to http urls */ + (strncmp(referer_adr, "https://", 8) != 0) /* referer is not https */ + ||(strncmp(adr, "https://", 8) == 0) /* or referer AND addresses are https */ + ) + ) { // PAS file:// + strcatbuff(buff, "Referer: "); + strcatbuff(buff, "http://"); + strcatbuff(buff, jump_identification(referer_adr)); + strcatbuff(buff, referer_fil); + strcatbuff(buff, H_CRLF); } } // HTTP field: referer else if (retour->req.referer[0] != '\0') { - strcatbuff(buff,"Referer: "); + strcatbuff(buff, "Referer: "); strcatbuff(buff, retour->req.referer); - strcatbuff(buff, H_CRLF); + strcatbuff(buff, H_CRLF); } - // POST? - if (mode==0) { // GET! + if (mode == 0) { // GET! if (search_tag) { char clen[256]; - sprintf(clen,"Content-length: %d"H_CRLF,(int)(strlen(unescape_http(OPT_GET_BUFF(opt),search_tag+strlen(POSTTOK)+1)))); - strcatbuff(buff,clen); + + sprintf(clen, "Content-length: %d" H_CRLF, + (int) (strlen + (unescape_http + (OPT_GET_BUFF(opt), + search_tag + strlen(POSTTOK) + 1)))); + strcatbuff(buff, clen); } } - // gestion cookies? if (cookie) { - char buffer[8192]; - char* b=cookie->data; - int cook=0; - int max_cookies=8; - size_t max_size=2048; - max_size+=strlen(buff); + char buffer[8192]; + char *b = cookie->data; + int cook = 0; + int max_cookies = 8; + size_t max_size = 2048; + + max_size += strlen(buff); do { - b=cookie_find(b,"",jump_identification(adr),fil); // prochain cookie satisfaisant aux conditions + b = cookie_find(b, "", jump_identification(adr), fil); // prochain cookie satisfaisant aux conditions if (b) { max_cookies--; if (!cook) { - strcatbuff(buff,"Cookie: "); - strcatbuff(buff,"$Version=1; "); - cook=1; + strcatbuff(buff, "Cookie: "); + strcatbuff(buff, "$Version=1; "); + cook = 1; } else - strcatbuff(buff,"; "); - strcatbuff(buff,cookie_get(buffer,b,5)); - strcatbuff(buff,"="); - strcatbuff(buff,cookie_get(buffer,b,6)); - strcatbuff(buff,"; $Path="); - strcatbuff(buff,cookie_get(buffer,b,2)); - b=cookie_nextfield(b); + strcatbuff(buff, "; "); + strcatbuff(buff, cookie_get(buffer, b, 5)); + strcatbuff(buff, "="); + strcatbuff(buff, cookie_get(buffer, b, 6)); + strcatbuff(buff, "; $Path="); + strcatbuff(buff, cookie_get(buffer, b, 2)); + b = cookie_nextfield(b); } - } while( (b) && (max_cookies>0) && ((int)strlen(buff)<max_size)); - if (cook) { // on a envoyé un (ou plusieurs) cookie? - strcatbuff(buff,H_CRLF); + } while((b) && (max_cookies > 0) && ((int) strlen(buff) < max_size)); + if (cook) { // on a envoyé un (ou plusieurs) cookie? + strcatbuff(buff, H_CRLF); #if DEBUG_COOK - printf("Header:\n%s\n",buff); + printf("Header:\n%s\n", buff); #endif } } - // gérer le keep-alive (garder socket) if (retour->req.http11 && !retour->req.nokeepalive) { - strcatbuff(buff,"Connection: Keep-Alive"H_CRLF); - } else { - strcatbuff(buff,"Connection: close"H_CRLF); - } - + strcatbuff(buff, "Connection: Keep-Alive" H_CRLF); + } else { + strcatbuff(buff, "Connection: close" H_CRLF); + } + { - char* real_adr=jump_identification(adr); + char *real_adr = jump_identification(adr); + //if ((use_11) || (retour->user_agent_send)) { // Pour le 1.1 on utilise un Host: - if (!direct_url) { // pas ftp:// par exemple + if (!direct_url) { // pas ftp:// par exemple //if (!retour->req.proxy.active) { - strcatbuff(buff,"Host: "); strcatbuff(buff,real_adr); strcatbuff(buff,H_CRLF); + strcatbuff(buff, "Host: "); + strcatbuff(buff, real_adr); + strcatbuff(buff, H_CRLF); //} } //} // HTTP field: from - if (retour->req.from[0] != '\0') { // HTTP from - strcatbuff(buff,"From: "); + if (retour->req.from[0] != '\0') { // HTTP from + strcatbuff(buff, "From: "); strcatbuff(buff, retour->req.from); strcatbuff(buff, H_CRLF); } - // Présence d'un user-agent? - if (retour->req.user_agent_send) { // ohh un user-agent + if (retour->req.user_agent_send) { // ohh un user-agent char s[256]; + // HyperTextSeeker/"HTSVERSION - sprintf(s,"User-Agent: %s"H_CRLF,retour->req.user_agent); - strcatbuff(buff,s); - + sprintf(s, "User-Agent: %s" H_CRLF, retour->req.user_agent); + strcatbuff(buff, s); + // pour les serveurs difficiles - strcatbuff(buff,"Accept: " - "image/png, image/jpeg, image/pjpeg, image/x-xbitmap, image/svg+xml" /* Accepted */ - ", " - "image/gif;q=0.9" /* also accepted but with lower preference */ - ", " - "*/*;q=0.1" /* also accepted but with even lower preference */ - H_CRLF); + strcatbuff(buff, "Accept: " "image/png, image/jpeg, image/pjpeg, image/x-xbitmap, image/svg+xml" /* Accepted */ + ", " "image/gif;q=0.9" /* also accepted but with lower preference */ + ", " "*/*;q=0.1" /* also accepted but with even lower preference */ + H_CRLF); if (strnotempty(retour->req.lang_iso)) { - strcatbuff(buff,"Accept-Language: "); strcatbuff(buff,retour->req.lang_iso); strcatbuff(buff,H_CRLF); + strcatbuff(buff, "Accept-Language: "); + strcatbuff(buff, retour->req.lang_iso); + strcatbuff(buff, H_CRLF); } if (retour->req.http11) { #if HTS_USEZLIB //strcatbuff(buff,"Accept-Encoding: gzip, deflate, compress, identity"H_CRLF); - if (gz_is_available && (!retour->req.range_used) && (!retour->req.nocompression)) - strcatbuff(buff,"Accept-Encoding: " - "gzip" /* gzip if the preffered encoding */ - ", " - "identity;q=0.9" - H_CRLF); + if (gz_is_available && (!retour->req.range_used) + && (!retour->req.nocompression)) + strcatbuff(buff, "Accept-Encoding: " "gzip" /* gzip if the preffered encoding */ + ", " "identity;q=0.9" H_CRLF); else - strcatbuff(buff,"Accept-Encoding: identity"H_CRLF); /* no compression */ + strcatbuff(buff, "Accept-Encoding: identity" H_CRLF); /* no compression */ #else - strcatbuff(buff,"Accept-Encoding: identity"H_CRLF); /* no compression */ + strcatbuff(buff, "Accept-Encoding: identity" H_CRLF); /* no compression */ #endif } } else { - strcatbuff(buff,"Accept: */*"H_CRLF); // le minimum + strcatbuff(buff, "Accept: */*" H_CRLF); // le minimum } /* Authentification */ { char autorisation[1100]; - char* a; - autorisation[0]='\0'; - if (link_has_authorization(adr)) { // ohh une authentification! - char* a=jump_identification(adr); - char* astart=jump_protocol(adr); - if (!direct_url) { // pas ftp:// par exemple + char *a; + + autorisation[0] = '\0'; + if (link_has_authorization(adr)) { // ohh une authentification! + char *a = jump_identification(adr); + char *astart = jump_protocol(adr); + + if (!direct_url) { // pas ftp:// par exemple char user_pass[256]; - user_pass[0]='\0'; - strncatbuff(user_pass,astart,(int) (a - astart) - 1); - strcpybuff(user_pass,unescape_http(OPT_GET_BUFF(opt),user_pass)); - code64((unsigned char*)user_pass,(int)strlen(user_pass),(unsigned char*)autorisation,0); - if (strcmp(fil,"/robots.txt")) /* pas robots.txt */ - bauth_add(cookie,astart,fil,autorisation); + + user_pass[0] = '\0'; + strncatbuff(user_pass, astart, (int) (a - astart) - 1); + strcpybuff(user_pass, unescape_http(OPT_GET_BUFF(opt), user_pass)); + code64((unsigned char *) user_pass, (int) strlen(user_pass), + (unsigned char *) autorisation, 0); + if (strcmp(fil, "/robots.txt")) /* pas robots.txt */ + bauth_add(cookie, astart, fil, autorisation); } - } else if ( (a=bauth_check(cookie,real_adr,fil)) ) - strcpybuff(autorisation,a); + } else if ((a = bauth_check(cookie, real_adr, fil))) + strcpybuff(autorisation, a); /* On a une autorisation a donner? */ if (strnotempty(autorisation)) { - strcatbuff(buff,"Authorization: Basic "); - strcatbuff(buff,autorisation); - strcatbuff(buff,H_CRLF); + strcatbuff(buff, "Authorization: Basic "); + strcatbuff(buff, autorisation); + strcatbuff(buff, H_CRLF); } } } //strcatbuff(buff,"Accept-Language: en\n"); //strcatbuff(buff,"Accept-Charset: iso-8859-1,*,utf-8\n"); - + // CRLF de fin d'en tête - strcatbuff(buff,H_CRLF); - + strcatbuff(buff, H_CRLF); + // données complémentaires? if (search_tag) - if (mode==0) // GET! - strcatbuff(buff,unescape_http(OPT_GET_BUFF(opt),search_tag+strlen(POSTTOK)+1)); + if (mode == 0) // GET! + strcatbuff(buff, + unescape_http(OPT_GET_BUFF(opt), + search_tag + strlen(POSTTOK) + 1)); } - #if HDEBUG #endif if (_DEBUG_HEAD) { if (ioinfo) { - fprintf(ioinfo,"[%d] request for %s%s:\r\n",retour->debugid,jump_identification(adr),fil); - fprintfio(ioinfo,buff,"<<< "); - fprintf(ioinfo,"\r\n"); + fprintf(ioinfo, "[%d] request for %s%s:\r\n", retour->debugid, + jump_identification(adr), fil); + fprintfio(ioinfo, buff, "<<< "); + fprintf(ioinfo, "\r\n"); fflush(ioinfo); } - } // Fin test pas postfile + } // Fin test pas postfile // // Callback { - int test_head = RUN_CALLBACK6(opt, sendhead, buff, adr, fil, referer_adr, referer_fil, retour); - if (test_head!=1) { + int test_head = + RUN_CALLBACK6(opt, sendhead, buff, adr, fil, referer_adr, referer_fil, + retour); + if (test_head != 1) { deletesoc_r(retour); - strcpybuff(retour->msg,"Header refused by external wrapper"); - retour->soc=INVALID_SOCKET; + strcpybuff(retour->msg, "Header refused by external wrapper"); + retour->soc = INVALID_SOCKET; } } // Envoi HTS_STAT.last_request = mtime_local(); - if (sendc(retour, buff)<0) { // ERREUR, socket rompue?... - //if (sendc(retour->soc,buff) != strlen(buff)) { // ERREUR, socket rompue?... - deletesoc_r(retour); // fermer tout de même + if (sendc(retour, buff) < 0) { // ERREUR, socket rompue?... + //if (sendc(retour->soc,buff) != strlen(buff)) { // ERREUR, socket rompue?... + deletesoc_r(retour); // fermer tout de même // et tenter de reconnecter - + strcpybuff(retour->msg, "Write error"); - retour->soc=INVALID_SOCKET; + retour->soc = INVALID_SOCKET; } - // RX'98 return 0; } - - - // traiter 1ere ligne d'en tête -void treatfirstline(htsblk* retour,char* rcvd) { - char* a=rcvd; +void treatfirstline(htsblk * retour, char *rcvd) { + char *a = rcvd; + // exemple: // HTTP/1.0 200 OK if (*a) { // note: certains serveurs buggés renvoient HTTP/1.0\n200 OK ou " HTTP/1.0 200 OK" - while ((*a==' ') || (*a==10) || (*a==13) || (*a==9)) a++; // épurer espaces au début + while((*a == ' ') || (*a == 10) || (*a == 13) || (*a == 9)) + a++; // épurer espaces au début if (strfield(a, "HTTP/")) { // sauter HTTP/1.x - while ((*a!=' ') && (*a!='\0') && (*a!=10) && (*a!=13) && (*a!=9)) a++; + while((*a != ' ') && (*a != '\0') && (*a != 10) && (*a != 13) + && (*a != 9)) + a++; if (*a != '\0') { - while ((*a==' ') || (*a==10) || (*a==13) || (*a==9)) a++; // épurer espaces - if ((*a>='0') && (*a<='9')) { - sscanf(a,"%d",&(retour->statuscode)); + while((*a == ' ') || (*a == 10) || (*a == 13) || (*a == 9)) + a++; // épurer espaces + if ((*a >= '0') && (*a <= '9')) { + sscanf(a, "%d", &(retour->statuscode)); // sauter 200 - while ((*a!=' ') && (*a!='\0') && (*a!=10) && (*a!=13) && (*a!=9)) a++; - while ((*a==' ') || (*a==10) || (*a==13) || (*a==9)) a++; // épurer espaces - if ((strlen(a) > 1) && (strlen(a) < 64) ) // message retour - strcpybuff(retour->msg,a); + while((*a != ' ') && (*a != '\0') && (*a != 10) && (*a != 13) + && (*a != 9)) + a++; + while((*a == ' ') || (*a == 10) || (*a == 13) || (*a == 9)) + a++; // épurer espaces + if ((strlen(a) > 1) && (strlen(a) < 64)) // message retour + strcpybuff(retour->msg, a); else - infostatuscode(retour->msg,retour->statuscode); + infostatuscode(retour->msg, retour->statuscode); // type MIME par défaut2 - strcpybuff(retour->contenttype,HTS_HYPERTEXT_DEFAULT_MIME); - } else { // pas de code! - retour->statuscode=STATUSCODE_INVALID; - strcpybuff(retour->msg,"Unknown response structure"); + strcpybuff(retour->contenttype, HTS_HYPERTEXT_DEFAULT_MIME); + } else { // pas de code! + retour->statuscode = STATUSCODE_INVALID; + strcpybuff(retour->msg, "Unknown response structure"); } - } else { // euhh?? - retour->statuscode=STATUSCODE_INVALID; - strcpybuff(retour->msg,"Unknown response structure"); + } else { // euhh?? + retour->statuscode = STATUSCODE_INVALID; + strcpybuff(retour->msg, "Unknown response structure"); } } else { - if (*a == '<') { + if (*a == '<') { /* This is dirty .. */ - retour->statuscode=HTTP_OK; - retour->keep_alive=0; + retour->statuscode = HTTP_OK; + retour->keep_alive = 0; strcpybuff(retour->msg, "Unknown, assuming junky server"); - strcpybuff(retour->contenttype,HTS_HYPERTEXT_DEFAULT_MIME); - } else if (strnotempty(a)) { - retour->statuscode=STATUSCODE_INVALID; - strcpybuff(retour->msg,"Unknown (not HTTP/xx) response structure"); + strcpybuff(retour->contenttype, HTS_HYPERTEXT_DEFAULT_MIME); + } else if (strnotempty(a)) { + retour->statuscode = STATUSCODE_INVALID; + strcpybuff(retour->msg, "Unknown (not HTTP/xx) response structure"); } else { /* This is dirty .. */ - retour->statuscode=HTTP_OK; - retour->keep_alive=0; + retour->statuscode = HTTP_OK; + retour->keep_alive = 0; strcpybuff(retour->msg, "Unknown, assuming junky server"); - strcpybuff(retour->contenttype,HTS_HYPERTEXT_DEFAULT_MIME); + strcpybuff(retour->contenttype, HTS_HYPERTEXT_DEFAULT_MIME); } } - } else { // vide! + } else { // vide! /* - retour->statuscode=STATUSCODE_INVALID; - strcpybuff(retour->msg,"Empty reponse or internal error"); - */ + retour->statuscode=STATUSCODE_INVALID; + strcpybuff(retour->msg,"Empty reponse or internal error"); + */ /* This is dirty .. */ - retour->statuscode=HTTP_OK; + retour->statuscode = HTTP_OK; strcpybuff(retour->msg, "Unknown, assuming junky server"); - strcpybuff(retour->contenttype,HTS_HYPERTEXT_DEFAULT_MIME); + strcpybuff(retour->contenttype, HTS_HYPERTEXT_DEFAULT_MIME); } } // traiter ligne par ligne l'en tête // gestion des cookies -void treathead(t_cookie* cookie,char* adr,char* fil,htsblk* retour,char* rcvd) { +void treathead(t_cookie * cookie, char *adr, char *fil, htsblk * retour, + char *rcvd) { int p; - if ((p=strfield(rcvd,"Content-length:"))!=0) { + + if ((p = strfield(rcvd, "Content-length:")) != 0) { #if HDEBUG printf("ok, Content-length: détecté\n"); #endif - if (sscanf(rcvd+p,LLintP,&(retour->totalsize)) == 1) { + if (sscanf(rcvd + p, LLintP, &(retour->totalsize)) == 1) { if (retour->totalsize == 0) { retour->empty = 1; } } - } - else if ((p=strfield(rcvd,"Content-Disposition:"))!=0) { - while(is_realspace(*(rcvd+p))) p++; // sauter espaces - if ((int) strlen(rcvd+p)<250) { // pas trop long? + } else if ((p = strfield(rcvd, "Content-Disposition:")) != 0) { + while(is_realspace(*(rcvd + p))) + p++; // sauter espaces + if ((int) strlen(rcvd + p) < 250) { // pas trop long? char tmp[256]; - char *a=NULL,*b=NULL; - strcpybuff(tmp,rcvd+p); - a=strstr(tmp,"filename="); + char *a = NULL, *b = NULL; + + strcpybuff(tmp, rcvd + p); + a = strstr(tmp, "filename="); if (a) { - a+=strlen("filename="); - while(is_space(*a)) a++; + a += strlen("filename="); + while(is_space(*a)) + a++; //a=strchr(a,'"'); if (a) { - char *c=NULL; + char *c = NULL; + //a++; /* jump " */ - while((c=strchr(a,'/'))) /* skip all / (see RFC2616) */ - a=c+1; + while((c = strchr(a, '/'))) /* skip all / (see RFC2616) */ + a = c + 1; //b=strchr(a+1,'"'); - b=a+strlen(a)-1; - while(is_space(*b)) b--; + b = a + strlen(a) - 1; + while(is_space(*b)) + b--; b++; if (b) { - *b='\0'; - if ((int) strlen(a) < 200) { // pas trop long? - strcpybuff(retour->cdispo,a); + *b = '\0'; + if ((int) strlen(a) < 200) { // pas trop long? + strcpybuff(retour->cdispo, a); } } } - } + } } - } - else if ((p=strfield(rcvd,"Last-Modified:"))!=0) { - while(is_realspace(*(rcvd+p))) p++; // sauter espaces - if ((int) strlen(rcvd+p)<64) { // pas trop long? + } else if ((p = strfield(rcvd, "Last-Modified:")) != 0) { + while(is_realspace(*(rcvd + p))) + p++; // sauter espaces + if ((int) strlen(rcvd + p) < 64) { // pas trop long? //struct tm* tm_time=convert_time_rfc822(rcvd+p); - strcpybuff(retour->lastmodified,rcvd+p); + strcpybuff(retour->lastmodified, rcvd + p); } - } - else if ((p=strfield(rcvd,"Date:"))!=0) { - if (strnotempty(retour->lastmodified)==0) { /* pas encore de last-modified */ - while(is_realspace(*(rcvd+p))) p++; // sauter espaces - if ((int) strlen(rcvd+p)<64) { // pas trop long? + } else if ((p = strfield(rcvd, "Date:")) != 0) { + if (strnotempty(retour->lastmodified) == 0) { /* pas encore de last-modified */ + while(is_realspace(*(rcvd + p))) + p++; // sauter espaces + if ((int) strlen(rcvd + p) < 64) { // pas trop long? //struct tm* tm_time=convert_time_rfc822(rcvd+p); - strcpybuff(retour->lastmodified,rcvd+p); + strcpybuff(retour->lastmodified, rcvd + p); } } - } - else if ((p=strfield(rcvd,"Etag:"))!=0) { /* Etag */ + } else if ((p = strfield(rcvd, "Etag:")) != 0) { /* Etag */ if (retour) { - while(is_realspace(*(rcvd+p))) p++; // sauter espaces - if ((int) strlen(rcvd+p)<64) // pas trop long? - strcpybuff(retour->etag,rcvd+p); - else // erreur.. ignorer - retour->etag[0]='\0'; + while(is_realspace(*(rcvd + p))) + p++; // sauter espaces + if ((int) strlen(rcvd + p) < 64) // pas trop long? + strcpybuff(retour->etag, rcvd + p); + else // erreur.. ignorer + retour->etag[0] = '\0'; } } // else if ((p=strfield(rcvd,"Transfer-Encoding: chunked"))!=0) { // chunk! - else if ((p=strfield(rcvd,"Transfer-Encoding:"))!=0) { // chunk! - while(is_realspace(*(rcvd+p))) p++; // sauter espaces - if (strfield(rcvd+p,"chunked")) { - retour->is_chunk=1; // chunked + else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk! + while(is_realspace(*(rcvd + p))) + p++; // sauter espaces + if (strfield(rcvd + p, "chunked")) { + retour->is_chunk = 1; // chunked //retour->http11=2; // chunked #if HDEBUG printf("ok, Transfer-Encoding: détecté\n"); #endif } - } - else if ((p=strfield(rcvd,"Content-type:"))!=0) { + } else if ((p = strfield(rcvd, "Content-type:")) != 0) { if (retour) { char tempo[1100]; + // éviter les text/html; charset=foo { - char* a=strchr(rcvd+p,';'); - if (a) { // extended information - *a='\0'; + char *a = strchr(rcvd + p, ';'); + + if (a) { // extended information + *a = '\0'; a++; - while(is_space(*a)) a++; + while(is_space(*a)) + a++; if (strfield(a, "charset")) { a += 7; - while(is_space(*a)) a++; + while(is_space(*a)) + a++; if (*a == '=') { a++; - while(is_space(*a)) a++; - if (*a == '\"') a++; - while(is_space(*a)) a++; + while(is_space(*a)) + a++; + if (*a == '\"') + a++; + while(is_space(*a)) + a++; if (*a) { - char* chs = a; - while(*a && !is_space(*a) && *a != '\"' && *a != ';') a++; + char *chs = a; + + while(*a && !is_space(*a) && *a != '\"' && *a != ';') + a++; *a = '\0'; if (*chs) { if (strlen(chs) < sizeof(retour->charset) - 2) { @@ -1359,217 +1398,229 @@ void treathead(t_cookie* cookie,char* adr,char* fil,htsblk* retour,char* rcvd) { } } } - sscanf(rcvd+p,"%s",tempo); - if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!! - strcpybuff(retour->contenttype,tempo); + sscanf(rcvd + p, "%s", tempo); + if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!! + strcpybuff(retour->contenttype, tempo); else - strcpybuff(retour->contenttype,"application/octet-stream-unknown"); // erreur + strcpybuff(retour->contenttype, "application/octet-stream-unknown"); // erreur } - } - else if ((p=strfield(rcvd,"Content-Range:"))!=0) { + } else if ((p = strfield(rcvd, "Content-Range:")) != 0) { // Content-Range: bytes 0-70870/70871 - char* a; - for(a = rcvd+p ; is_space(*a) ; a++) ; + char *a; + + for(a = rcvd + p; is_space(*a); a++) ; if (strncasecmp(a, "bytes ", 6) == 0) { - for(a += 6 ; is_space(*a) ; a++) ; - if (sscanf(a, LLintP "-" LLintP "/" LLintP, &retour->crange_start, &retour->crange_end, &retour->crange) != 3) { + for(a += 6; is_space(*a); a++) ; + if (sscanf + (a, LLintP "-" LLintP "/" LLintP, &retour->crange_start, + &retour->crange_end, &retour->crange) != 3) { retour->crange_start = 0; retour->crange_end = 0; retour->crange = 0; - a = strchr(rcvd+p, '/'); + a = strchr(rcvd + p, '/'); if (a != NULL) { a++; - if (sscanf(a,LLintP,&retour->crange) == 1) { + if (sscanf(a, LLintP, &retour->crange) == 1) { retour->crange_start = 0; retour->crange_end = retour->crange - 1; } else { - retour->crange=0; + retour->crange = 0; } } } } - } - else if ((p=strfield(rcvd,"Connection:"))!=0) { - char* a = rcvd + p; - while(is_space(*a)) a++; - if (*a) { - if (strfield(a, "Keep-Alive")) { + } else if ((p = strfield(rcvd, "Connection:")) != 0) { + char *a = rcvd + p; + + while(is_space(*a)) + a++; + if (*a) { + if (strfield(a, "Keep-Alive")) { if (!retour->keep_alive) { retour->keep_alive_max = 10; retour->keep_alive_t = 15; } retour->keep_alive = 1; } else { - retour->keep_alive = 0; + retour->keep_alive = 0; } - } - } - else if ((p=strfield(rcvd,"Keep-Alive:"))!=0) { - char* a = rcvd + p; - while(is_space(*a)) a++; - if (*a) { - char* p; + } + } else if ((p = strfield(rcvd, "Keep-Alive:")) != 0) { + char *a = rcvd + p; + + while(is_space(*a)) + a++; + if (*a) { + char *p; + retour->keep_alive = 1; retour->keep_alive_max = 10; retour->keep_alive_t = 15; - if ((p=strstr(a, "timeout="))) { - p+=strlen("timeout="); + if ((p = strstr(a, "timeout="))) { + p += strlen("timeout="); sscanf(p, "%d", &retour->keep_alive_t); } - if ((p=strstr(a, "max="))) { - p+=strlen("max="); + if ((p = strstr(a, "max="))) { + p += strlen("max="); sscanf(p, "%d", &retour->keep_alive_max); } if (retour->keep_alive_max <= 1 || retour->keep_alive_t < 3) { retour->keep_alive = 0; } } - } - else if ((p=strfield(rcvd,"TE:"))!=0) { - char* a = rcvd + p; - while(is_space(*a)) a++; - if (*a) { + } else if ((p = strfield(rcvd, "TE:")) != 0) { + char *a = rcvd + p; + + while(is_space(*a)) + a++; + if (*a) { if (strfield(a, "trailers")) { - retour->keep_alive_trailers=1; + retour->keep_alive_trailers = 1; } } - } - else if ((p=strfield(rcvd,"Content-Encoding:"))!=0) { - if (retour) { - char tempo[1100]; - char* a = rcvd + p; - while(is_space(*a)) a++; - { - char* a=strchr(rcvd+p,';'); - if (a) *a='\0'; - } - sscanf(a,"%s",tempo); - if (strlen(tempo)<64) // pas trop long!! - strcpybuff(retour->contentencoding,tempo); + } else if ((p = strfield(rcvd, "Content-Encoding:")) != 0) { + if (retour) { + char tempo[1100]; + char *a = rcvd + p; + + while(is_space(*a)) + a++; + { + char *a = strchr(rcvd + p, ';'); + + if (a) + *a = '\0'; + } + sscanf(a, "%s", tempo); + if (strlen(tempo) < 64) // pas trop long!! + strcpybuff(retour->contentencoding, tempo); else - retour->contentencoding[0]='\0'; // erreur + retour->contentencoding[0] = '\0'; // erreur #if HTS_USEZLIB /* Check known encodings */ if (retour->contentencoding[0]) { - if ( - (strfield2(retour->contentencoding, "gzip")) - || (strfield2(retour->contentencoding, "x-gzip")) - /* - || (strfield2(retour->contentencoding, "compress")) - || (strfield2(retour->contentencoding, "x-compress")) - */ - || (strfield2(retour->contentencoding, "deflate")) - || (strfield2(retour->contentencoding, "x-deflate")) + if ((strfield2(retour->contentencoding, "gzip")) + || (strfield2(retour->contentencoding, "x-gzip")) + /* + || (strfield2(retour->contentencoding, "compress")) + || (strfield2(retour->contentencoding, "x-compress")) + */ + || (strfield2(retour->contentencoding, "deflate")) + || (strfield2(retour->contentencoding, "x-deflate")) ) { - retour->compressed=1; + retour->compressed = 1; } } #endif } - } - else if ((p=strfield(rcvd,"Location:"))!=0) { + } else if ((p = strfield(rcvd, "Location:")) != 0) { if (retour) { if (retour->location) { - while(is_realspace(*(rcvd+p))) p++; // sauter espaces - if ((int) strlen(rcvd+p)<HTS_URLMAXSIZE) // pas trop long? - strcpybuff(retour->location,rcvd+p); - else // erreur.. ignorer - retour->location[0]='\0'; + while(is_realspace(*(rcvd + p))) + p++; // sauter espaces + if ((int) strlen(rcvd + p) < HTS_URLMAXSIZE) // pas trop long? + strcpybuff(retour->location, rcvd + p); + else // erreur.. ignorer + retour->location[0] = '\0'; } } - } - else if ( ((p=strfield(rcvd,"Set-Cookie:"))!=0) && (cookie) ) { // ohh un cookie - char* a = rcvd+p; // pointeur + } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) && (cookie)) { // ohh un cookie + char *a = rcvd + p; // pointeur char domain[256]; // domaine cookie (.netscape.com) char path[256]; // chemin (/) char cook_name[256]; // nom cookie (MYCOOK) - char BIGSTK cook_value[8192]; // valeur (ID=toto,S=1234) + char BIGSTK cook_value[8192]; // valeur (ID=toto,S=1234) + #if DEBUG_COOK printf("set-cookie detected\n"); #endif while(*a) { - char *token_st,*token_end; - char *value_st,*value_end; + char *token_st, *token_end; + char *value_st, *value_end; char name[256]; char BIGSTK value[8192]; - int next=0; - name[0]=value[0]='\0'; + int next = 0; + + name[0] = value[0] = '\0'; // // initialiser cookie lu actuellement if (adr) - strcpybuff(domain,jump_identification(adr)); // domaine - strcpybuff(path,"/"); // chemin (/) - strcpybuff(cook_name,""); // nom cookie (MYCOOK) - strcpybuff(cook_value,""); // valeur (ID=toto,S=1234) + strcpybuff(domain, jump_identification(adr)); // domaine + strcpybuff(path, "/"); // chemin (/) + strcpybuff(cook_name, ""); // nom cookie (MYCOOK) + strcpybuff(cook_value, ""); // valeur (ID=toto,S=1234) // boucler jusqu'au prochain cookie ou la fin do { - char* start_loop=a; - while(is_space(*a)) a++; // sauter espaces - token_st=a; // départ token - while((!is_space(*a)) && (*a) && (*a!=';') && (*a!='=')) a++; // arrêter si espace, point virgule - token_end=a; - while(is_space(*a)) a++; // sauter espaces - if (*a=='=') { // name=value + char *start_loop = a; + + while(is_space(*a)) + a++; // sauter espaces + token_st = a; // départ token + while((!is_space(*a)) && (*a) && (*a != ';') && (*a != '=')) + a++; // arrêter si espace, point virgule + token_end = a; + while(is_space(*a)) + a++; // sauter espaces + if (*a == '=') { // name=value a++; - while(is_space(*a)) a++; // sauter espaces - value_st=a; - while( (*a!=';') && (*a)) a++; // prochain ; + while(is_space(*a)) + a++; // sauter espaces + value_st = a; + while((*a != ';') && (*a)) + a++; // prochain ; //while( ((*a!='"') || (*(a-1)=='\\')) && (*a)) a++; // prochain " (et pas \") - value_end=a; + value_end = a; //if (*a==';') { // finit par un ; // vérifier débordements - if ( (((int) (token_end - token_st))<200) && (((int) (value_end - value_st))<8000) - && (((int) (token_end - token_st))>0) && (((int) (value_end - value_st))>0) ) - { + if ((((int) (token_end - token_st)) < 200) + && (((int) (value_end - value_st)) < 8000) + && (((int) (token_end - token_st)) > 0) + && (((int) (value_end - value_st)) > 0)) { int name_len = (int) (token_end - token_st); int value_len = (int) (value_end - value_st); - name[0]='\0'; - value[0]='\0'; - strncatbuff(name,token_st,name_len); - strncatbuff(value,value_st,value_len); + + name[0] = '\0'; + value[0] = '\0'; + strncatbuff(name, token_st, name_len); + strncatbuff(value, value_st, value_len); #if DEBUG_COOK - printf("detected cookie-av: name=\"%s\" value=\"%s\"\n",name,value); + printf("detected cookie-av: name=\"%s\" value=\"%s\"\n", name, + value); #endif - if (strfield2(name,"domain")) { + if (strfield2(name, "domain")) { if (value_len < sizeof(domain) - 1) { - strcpybuff(domain,value); + strcpybuff(domain, value); } else { cook_name[0] = 0; break; } - } - else if (strfield2(name,"path")) { + } else if (strfield2(name, "path")) { if (value_len < sizeof(path) - 1) { - strcpybuff(path,value); + strcpybuff(path, value); } else { cook_name[0] = 0; break; } - } - else if (strfield2(name,"max-age")) { + } else if (strfield2(name, "max-age")) { // ignoré.. - } - else if (strfield2(name,"expires")) { + } else if (strfield2(name, "expires")) { // ignoré.. - } - else if (strfield2(name,"version")) { + } else if (strfield2(name, "version")) { // ignoré.. - } - else if (strfield2(name,"comment")) { + } else if (strfield2(name, "comment")) { // ignoré - } - else if (strfield2(name,"secure")) { // ne devrait pas arriver ici + } else if (strfield2(name, "secure")) { // ne devrait pas arriver ici // ignoré - } - else { - if (value_len < sizeof(cook_value) - 1 && name_len < sizeof(cook_name) - 1) { - if (strnotempty(cook_name)==0) { // noter premier: nom et valeur cookie - strcpybuff(cook_name,name); - strcpybuff(cook_value,value); - } else { // prochain cookie - a=start_loop; // on devra recommencer à cette position - next=1; // enregistrer + } else { + if (value_len < sizeof(cook_value) - 1 + && name_len < sizeof(cook_name) - 1) { + if (strnotempty(cook_name) == 0) { // noter premier: nom et valeur cookie + strcpybuff(cook_name, name); + strcpybuff(cook_value, value); + } else { // prochain cookie + a = start_loop; // on devra recommencer à cette position + next = 1; // enregistrer } } else { cook_name[0] = 0; @@ -1579,88 +1630,176 @@ void treathead(t_cookie* cookie,char* adr,char* fil,htsblk* retour,char* rcvd) { } } if (!next) { - while((*a!=';') && (*a)) a++; // prochain - while(*a==';') a++; // sauter ; + while((*a != ';') && (*a)) + a++; // prochain + while(*a == ';') + a++; // sauter ; } } while((*a) && (!next)); - if (strnotempty(cook_name)) { // cookie? + if (strnotempty(cook_name)) { // cookie? #if DEBUG_COOK - printf("new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",cook_name,cook_value,domain,path); + printf + ("new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n", + cook_name, cook_value, domain, path); #endif - cookie_add(cookie,cook_name,cook_value,domain,path); + cookie_add(cookie, cook_name, cook_value, domain, path); } } } } - // transforme le message statuscode en chaîne -HTSEXT_API void infostatuscode(char* msg,int statuscode) { - switch( statuscode) { +HTSEXT_API void infostatuscode(char *msg, int statuscode) { + switch (statuscode) { // Erreurs HTTP, selon RFC - case 100: strcpybuff( msg,"Continue"); break; - case 101: strcpybuff( msg,"Switching Protocols"); break; - case 200: strcpybuff( msg,"OK"); break; - case 201: strcpybuff( msg,"Created"); break; - case 202: strcpybuff( msg,"Accepted"); break; - case 203: strcpybuff( msg,"Non-Authoritative Information"); break; - case 204: strcpybuff( msg,"No Content"); break; - case 205: strcpybuff( msg,"Reset Content"); break; - case 206: strcpybuff( msg,"Partial Content"); break; - case 300: strcpybuff( msg,"Multiple Choices"); break; - case 301: strcpybuff( msg,"Moved Permanently"); break; - case 302: strcpybuff( msg,"Moved Temporarily"); break; - case 303: strcpybuff( msg,"See Other"); break; - case 304: strcpybuff( msg,"Not Modified"); break; - case 305: strcpybuff( msg,"Use Proxy"); break; - case 306: strcpybuff( msg,"Undefined 306 error"); break; - case 307: strcpybuff( msg,"Temporary Redirect"); break; - case 400: strcpybuff( msg,"Bad Request"); break; - case 401: strcpybuff( msg,"Unauthorized"); break; - case 402: strcpybuff( msg,"Payment Required"); break; - case 403: strcpybuff( msg,"Forbidden"); break; - case 404: strcpybuff( msg,"Not Found"); break; - case 405: strcpybuff( msg,"Method Not Allowed"); break; - case 406: strcpybuff( msg,"Not Acceptable"); break; - case 407: strcpybuff( msg,"Proxy Authentication Required"); break; - case 408: strcpybuff( msg,"Request Time-out"); break; - case 409: strcpybuff( msg,"Conflict"); break; - case 410: strcpybuff( msg,"Gone"); break; - case 411: strcpybuff( msg,"Length Required"); break; - case 412: strcpybuff( msg,"Precondition Failed"); break; - case 413: strcpybuff( msg,"Request Entity Too Large"); break; - case 414: strcpybuff( msg,"Request-URI Too Large"); break; - case 415: strcpybuff( msg,"Unsupported Media Type"); break; - case 416: strcpybuff( msg,"Requested Range Not Satisfiable"); break; - case 417: strcpybuff( msg,"Expectation Failed"); break; - case 500: strcpybuff( msg,"Internal Server Error"); break; - case 501: strcpybuff( msg,"Not Implemented"); break; - case 502: strcpybuff( msg,"Bad Gateway"); break; - case 503: strcpybuff( msg,"Service Unavailable"); break; - case 504: strcpybuff( msg,"Gateway Time-out"); break; - case 505: strcpybuff( msg,"HTTP Version Not Supported"); break; + case 100: + strcpybuff(msg, "Continue"); + break; + case 101: + strcpybuff(msg, "Switching Protocols"); + break; + case 200: + strcpybuff(msg, "OK"); + break; + case 201: + strcpybuff(msg, "Created"); + break; + case 202: + strcpybuff(msg, "Accepted"); + break; + case 203: + strcpybuff(msg, "Non-Authoritative Information"); + break; + case 204: + strcpybuff(msg, "No Content"); + break; + case 205: + strcpybuff(msg, "Reset Content"); + break; + case 206: + strcpybuff(msg, "Partial Content"); + break; + case 300: + strcpybuff(msg, "Multiple Choices"); + break; + case 301: + strcpybuff(msg, "Moved Permanently"); + break; + case 302: + strcpybuff(msg, "Moved Temporarily"); + break; + case 303: + strcpybuff(msg, "See Other"); + break; + case 304: + strcpybuff(msg, "Not Modified"); + break; + case 305: + strcpybuff(msg, "Use Proxy"); + break; + case 306: + strcpybuff(msg, "Undefined 306 error"); + break; + case 307: + strcpybuff(msg, "Temporary Redirect"); + break; + case 400: + strcpybuff(msg, "Bad Request"); + break; + case 401: + strcpybuff(msg, "Unauthorized"); + break; + case 402: + strcpybuff(msg, "Payment Required"); + break; + case 403: + strcpybuff(msg, "Forbidden"); + break; + case 404: + strcpybuff(msg, "Not Found"); + break; + case 405: + strcpybuff(msg, "Method Not Allowed"); + break; + case 406: + strcpybuff(msg, "Not Acceptable"); + break; + case 407: + strcpybuff(msg, "Proxy Authentication Required"); + break; + case 408: + strcpybuff(msg, "Request Time-out"); + break; + case 409: + strcpybuff(msg, "Conflict"); + break; + case 410: + strcpybuff(msg, "Gone"); + break; + case 411: + strcpybuff(msg, "Length Required"); + break; + case 412: + strcpybuff(msg, "Precondition Failed"); + break; + case 413: + strcpybuff(msg, "Request Entity Too Large"); + break; + case 414: + strcpybuff(msg, "Request-URI Too Large"); + break; + case 415: + strcpybuff(msg, "Unsupported Media Type"); + break; + case 416: + strcpybuff(msg, "Requested Range Not Satisfiable"); + break; + case 417: + strcpybuff(msg, "Expectation Failed"); + break; + case 500: + strcpybuff(msg, "Internal Server Error"); + break; + case 501: + strcpybuff(msg, "Not Implemented"); + break; + case 502: + strcpybuff(msg, "Bad Gateway"); + break; + case 503: + strcpybuff(msg, "Service Unavailable"); + break; + case 504: + strcpybuff(msg, "Gateway Time-out"); + break; + case 505: + strcpybuff(msg, "HTTP Version Not Supported"); + break; // - default: if (strnotempty(msg)==0) strcpybuff( msg,"Unknown error"); break; + default: + if (strnotempty(msg) == 0) + strcpybuff(msg, "Unknown error"); + break; } } - // identique au précédent, sauf que l'on donne adr+fil et non url complète -htsblk xhttpget(httrackp *opt,char* adr,char* fil) { +htsblk xhttpget(httrackp * opt, char *adr, char *fil) { T_SOC soc; htsblk retour; - + hts_init_htsblk(&retour); //memset(&retour, 0, sizeof(htsblk)); - soc=http_fopen(opt,adr,fil,&retour); + soc = http_fopen(opt, adr, fil, &retour); - if (soc!=INVALID_SOCKET) { - http_fread(soc,&retour); + if (soc != INVALID_SOCKET) { + http_fread(soc, &retour); #if HTS_DEBUG_CLOSESOCK DEBUG_W("xhttpget: deletehttp\n"); #endif - if (retour.soc!=INVALID_SOCKET) deletehttp(&retour); // fermer - retour.soc=INVALID_SOCKET; + if (retour.soc != INVALID_SOCKET) + deletehttp(&retour); // fermer + retour.soc = INVALID_SOCKET; } return retour; } @@ -1668,68 +1807,70 @@ htsblk xhttpget(httrackp *opt,char* adr,char* fil) { // variation sur un thème... // réceptionne uniquement un en-tête (HEAD) // retourne dans xx.adr l'adresse pointant sur le bloc de mémoire de l'en tête -htsblk http_gethead(httrackp *opt,char* adr,char* fil) { +htsblk http_gethead(httrackp * opt, char *adr, char *fil) { T_SOC soc; htsblk retour; hts_init_htsblk(&retour); //memset(&retour, 0, sizeof(htsblk)); - soc=http_xfopen(opt,1,0,1,NULL,adr,fil,&retour); // HEAD, pas de traitement en-tête + soc = http_xfopen(opt, 1, 0, 1, NULL, adr, fil, &retour); // HEAD, pas de traitement en-tête - if (soc!=INVALID_SOCKET) { - http_fread(soc,&retour); // réception en-tête + if (soc != INVALID_SOCKET) { + http_fread(soc, &retour); // réception en-tête #if HTS_DEBUG_CLOSESOCK DEBUG_W("http_gethead: deletehttp\n"); #endif - if (retour.soc!=INVALID_SOCKET) deletehttp(&retour); // fermer - retour.soc=INVALID_SOCKET; + if (retour.soc != INVALID_SOCKET) + deletehttp(&retour); // fermer + retour.soc = INVALID_SOCKET; } return retour; } -// oui ca ressemble vachement à xhttpget - en étant sobre on peut voir LA différence.. +// oui ca ressemble vachement à xhttpget - en étant sobre on peut voir LA différence.. // lecture sur une socket ouverte, le header a déja été envoyé dans le cas de GET // il ne reste plus qu'à lire les données // (pour HEAD le header est lu ici!) -void http_fread(T_SOC soc,htsblk* retour) { +void http_fread(T_SOC soc, htsblk * retour) { //int bufl=TAILLE_BUFFER; // 8Ko de buffer - - if (retour) retour->soc=soc; - if (soc!=INVALID_SOCKET) { + + if (retour) + retour->soc = soc; + if (soc != INVALID_SOCKET) { // fonction de lecture d'une socket (plus propre) - while(http_fread1(retour)!=-1); - soc=retour->soc; - if (retour->adr==NULL) { - if (strnotempty(retour->msg)==0) - sprintf(retour->msg,"Unable to read"); - return ; // erreur - } - + while(http_fread1(retour) != -1) ; + soc = retour->soc; + if (retour->adr == NULL) { + if (strnotempty(retour->msg) == 0) + sprintf(retour->msg, "Unable to read"); + return; // erreur + } #if HDEBUG printf("Ok, données reçues\n"); -#endif +#endif + + return; + + } - return ; - - } - - return ; + return; } // check if data is available -int check_readinput(htsblk* r) { +int check_readinput(htsblk * r) { if (r->soc != INVALID_SOCKET) { - fd_set fds; // poll structures + fd_set fds; // poll structures struct timeval tv; // structure for select const int soc = (int) r->soc; + assertf(soc == r->soc); FD_ZERO(&fds); - FD_SET(soc,&fds); - tv.tv_sec=0; - tv.tv_usec=0; - select(soc + 1,&fds,NULL,NULL,&tv); - if (FD_ISSET(soc,&fds)) + FD_SET(soc, &fds); + tv.tv_sec = 0; + tv.tv_usec = 0; + select(soc + 1, &fds, NULL, NULL, &tv); + if (FD_ISSET(soc, &fds)) return 1; else return 0; @@ -1740,16 +1881,17 @@ int check_readinput(htsblk* r) { // check if data is available int check_readinput_t(T_SOC soc, int timeout) { if (soc != INVALID_SOCKET) { - fd_set fds; // poll structures + fd_set fds; // poll structures struct timeval tv; // structure for select const int isoc = (int) soc; + assertf(isoc == soc); FD_ZERO(&fds); - FD_SET(isoc,&fds); - tv.tv_sec=timeout; - tv.tv_usec=0; - select(isoc + 1,&fds,NULL,NULL,&tv); - if (FD_ISSET(isoc,&fds)) + FD_SET(isoc, &fds); + tv.tv_sec = timeout; + tv.tv_usec = 0; + select(isoc + 1, &fds, NULL, NULL, &tv); + if (FD_ISSET(isoc, &fds)) return 1; else return 0; @@ -1757,13 +1899,12 @@ int check_readinput_t(T_SOC soc, int timeout) { return 0; } - // lecture d'un bloc sur une socket (ou un fichier!) // >=0 : nombre d'octets lus // <0 : fin ou erreur -HTS_INLINE LLint http_fread1(htsblk* r) { +HTS_INLINE LLint http_fread1(htsblk * r) { //int bufl=TAILLE_BUFFER; // taille d'un buffer max. - return http_xfread1(r,TAILLE_BUFFER); + return http_xfread1(r, TAILLE_BUFFER); } // idem, sauf qu'ici on peut choisir la taille max de données à recevoir @@ -1772,163 +1913,165 @@ HTS_INLINE LLint http_fread1(htsblk* r) { // SI bufl==-1 alors le buffer est censé être de 8kos, et on recoit ligne par ligne // en éliminant les cr (ex: header), arrêt si double-lf // Note: les +1 dans les malloc sont dûs à l'octet nul rajouté en fin de fichier -LLint http_xfread1(htsblk* r,int bufl) { - int nl=-1; +LLint http_xfread1(htsblk * r, int bufl) { + int nl = -1; // EOF if (r->totalsize >= 0 && r->size == r->totalsize) { return READ_EOF; } - if (bufl>0) { - if (!r->is_write) { // stocker en mémoire - if (r->totalsize>=0) { // totalsize déterminé ET ALLOUE - if (r->adr==NULL) { - r->adr = (char*) malloct((size_t) r->totalsize + 1); + if (bufl > 0) { + if (!r->is_write) { // stocker en mémoire + if (r->totalsize >= 0) { // totalsize déterminé ET ALLOUE + if (r->adr == NULL) { + r->adr = (char *) malloct((size_t) r->totalsize + 1); r->size = 0; } if (r->adr != NULL) { // lecture - const size_t req_size = r->totalsize-r->size; - nl = req_size > 0 ? hts_read(r,r->adr + ((int) r->size),(int) req_size ) : 0; /* NO 32 bit overlow possible here (no 4GB html!) */ + const size_t req_size = r->totalsize - r->size; + + nl = req_size > 0 ? hts_read(r, r->adr + ((int) r->size), (int) req_size) : 0; /* NO 32 bit overlow possible here (no 4GB html!) */ // nouvelle taille - if (nl >= 0) r->size+=nl; + if (nl >= 0) + r->size += nl; /* - if (r->size >= r->totalsize) - nl = -1; // break - */ - - r->adr[r->size]='\0'; // caractère NULL en fin au cas où l'on traite des HTML + if (r->size >= r->totalsize) + nl = -1; // break + */ + + r->adr[r->size] = '\0'; // caractère NULL en fin au cas où l'on traite des HTML } - - } else { // inconnu.. + + } else { // inconnu.. // réserver de la mémoire? - if (r->adr==NULL) { + if (r->adr == NULL) { #if HDEBUG printf("..alloc xfread\n"); #endif - r->adr=(char*) malloct(bufl + 1); - r->size=0; - } - else { + r->adr = (char *) malloct(bufl + 1); + r->size = 0; + } else { #if HDEBUG printf("..realloc xfread1\n"); #endif - r->adr=(char*) realloct(r->adr,(int)r->size+bufl + 1); + r->adr = (char *) realloct(r->adr, (int) r->size + bufl + 1); } - - if (r->adr!=NULL) { + + if (r->adr != NULL) { // lecture - nl = hts_read(r,r->adr+(int)r->size,bufl); + nl = hts_read(r, r->adr + (int) r->size, bufl); if (nl > 0) { // resize - r->adr=(char*) realloct(r->adr,(int)r->size+nl + 1); + r->adr = (char *) realloct(r->adr, (int) r->size + nl + 1); // nouvelle taille - r->size+=nl; + r->size += nl; // octet nul - if (r->adr) r->adr[r->size]='\0'; + if (r->adr) + r->adr[r->size] = '\0'; - } // sinon on a fini + } // sinon on a fini #if HDEBUG else if (nl < 0) printf("..end read (%d)\n", nl); #endif } #if HDEBUG - else printf("..-> error\n"); + else + printf("..-> error\n"); #endif } // pas de adr=erreur - if (r->adr == NULL) nl = READ_ERROR; + if (r->adr == NULL) + nl = READ_ERROR; + + } else { // stocker sur disque + char *buff; - } else { // stocker sur disque - char* buff; - buff=(char*) malloct(bufl); - if (buff!=NULL) { + buff = (char *) malloct(bufl); + if (buff != NULL) { // lecture - nl = hts_read(r,buff,bufl); + nl = hts_read(r, buff, bufl); // nouvelle taille - if (nl > 0) { - r->size+=nl; - if (fwrite(buff,1,nl,r->out)!=nl) { - r->statuscode=STATUSCODE_INVALID; - strcpybuff(r->msg,"Write error on disk"); - nl=READ_ERROR; + if (nl > 0) { + r->size += nl; + if (fwrite(buff, 1, nl, r->out) != nl) { + r->statuscode = STATUSCODE_INVALID; + strcpybuff(r->msg, "Write error on disk"); + nl = READ_ERROR; } } - //if ((nl < 0) || ((r->totalsize>0) && (r->size >= r->totalsize))) // nl=-1; // break // libérer bloc tempo freet(buff); } else - nl=READ_ERROR; - - if ((nl < 0) && (r->out!=NULL)) { - fflush(r->out); + nl = READ_ERROR; + + if ((nl < 0) && (r->out != NULL)) { + fflush(r->out); } - - - } // stockage disque ou mémoire - - } else if (bufl == -2) { // force reserve - if (r->adr==NULL) { - r->adr=(char*) malloct(8192); - r->size=0; + + } // stockage disque ou mémoire + + } else if (bufl == -2) { // force reserve + if (r->adr == NULL) { + r->adr = (char *) malloct(8192); + r->size = 0; return 0; } return -1; - } else { // réception d'un en-tête octet par octet - int count=256; - int tot_nl=0; - int lf_detected=0; - int at_beginning=1; + } else { // réception d'un en-tête octet par octet + int count = 256; + int tot_nl = 0; + int lf_detected = 0; + int at_beginning = 1; + do { nl = READ_INTERNAL_ERROR; count--; - if (r->adr==NULL) { - r->adr=(char*) malloct(8192); - r->size=0; + if (r->adr == NULL) { + r->adr = (char *) malloct(8192); + r->size = 0; } - if (r->adr!=NULL) { + if (r->adr != NULL) { if (r->size < 8190) { // lecture - nl = hts_read(r,r->adr+r->size,1); + nl = hts_read(r, r->adr + r->size, 1); if (nl > 0) { // exit if: // lf detected AND already detected before // or // lf detected AND first character read - if (*(r->adr+r->size) == 10) { - if (lf_detected || (at_beginning) || (bufl<0)) - count=-1; - lf_detected=1; + if (*(r->adr + r->size) == 10) { + if (lf_detected || (at_beginning) || (bufl < 0)) + count = -1; + lf_detected = 1; } - if (*(r->adr+r->size) != 13) { // sauter caractères 13 - if ( - (*(r->adr+r->size) != 10) - && - (*(r->adr+r->size) != 13) + if (*(r->adr + r->size) != 13) { // sauter caractères 13 + if ((*(r->adr + r->size) != 10) + && (*(r->adr + r->size) != 13) ) { // restart for new line - lf_detected=0; + lf_detected = 0; } (r->size)++; - at_beginning=0; + at_beginning = 0; } - *(r->adr+r->size)='\0'; // terminer par octet nul + *(r->adr + r->size) = '\0'; // terminer par octet nul } } } if (nl >= 0) { - tot_nl+=nl; + tot_nl += nl; if (!check_readinput(r)) - count=-1; + count = -1; } - } while((nl >= 0) && (count>0)); + } while((nl >= 0) && (count > 0)); if (nl >= 0) { nl = tot_nl; } @@ -1941,207 +2084,213 @@ LLint http_xfread1(htsblk* r,int bufl) { } } - // teste une adresse, et suit l'éventuel chemin "moved" // retourne 200 ou le code d'erreur (404=NOT FOUND, etc) // copie dans loc la véritable adresse si celle-ci est différente -htsblk http_location(httrackp *opt,char* adr,char* fil,char* loc) { +htsblk http_location(httrackp * opt, char *adr, char *fil, char *loc) { htsblk retour; - int retry=0; + int retry = 0; int tryagain; + // note: "RFC says" // 5 boucles au plus, on en teste au plus 8 ici // sinon abandon.. do { - tryagain=0; - switch ((retour=http_test(opt,adr,fil,loc)).statuscode) { + tryagain = 0; + switch ((retour = http_test(opt, adr, fil, loc)).statuscode) { case HTTP_OK: - break; // ok! + break; // ok! case HTTP_MOVED_PERMANENTLY: case HTTP_FOUND: case HTTP_SEE_OTHER: - case HTTP_TEMPORARY_REDIRECT: // moved! + case HTTP_TEMPORARY_REDIRECT: // moved! // recalculer adr et fil! - if (ident_url_absolute(loc,adr,fil)!=-1) { - tryagain=1; // retenter - retry++; // ..encore une fois + if (ident_url_absolute(loc, adr, fil) != -1) { + tryagain = 1; // retenter + retry++; // ..encore une fois } } - } while((tryagain) && (retry<5+3)); + } while((tryagain) && (retry < 5 + 3)); return retour; } - // teste si une URL (validité, header, taille) // retourne 200 ou le code d'erreur (404=NOT FOUND, etc) // en cas de moved xx, dans location // abandonne désormais au bout de 30 secondes (aurevoir les sites // qui nous font poireauter 5 heures..) -> -2=timeout -htsblk http_test(httrackp *opt,char* adr,char* fil,char* loc) { +htsblk http_test(httrackp * opt, char *adr, char *fil, char *loc) { T_SOC soc; htsblk retour; + //int rcvsize=-1; //char* rcv=NULL; // adresse de retour //int bufl=TAILLE_BUFFER; // 8Ko de buffer TStamp tl; - int timeout=30; // timeout pour un check (arbitraire) // ** + int timeout = 30; // timeout pour un check (arbitraire) // ** // pour abandonner un site trop lent - tl=time_local(); + tl = time_local(); - loc[0]='\0'; + loc[0] = '\0'; hts_init_htsblk(&retour); //memset(&retour, 0, sizeof(htsblk)); // effacer - retour.location=loc; // si non nul, contiendra l'adresse véritable en cas de moved xx + retour.location = loc; // si non nul, contiendra l'adresse véritable en cas de moved xx //soc=http_fopen(adr,fil,&retour,NULL); // ouvrir, + header // on ouvre en head, et on traite l'en tête - soc=http_xfopen(opt,1,0,1,NULL,adr,fil,&retour); // ouvrir HEAD, + envoi header - - if (soc!=INVALID_SOCKET) { - int e=0; + soc = http_xfopen(opt, 1, 0, 1, NULL, adr, fil, &retour); // ouvrir HEAD, + envoi header + + if (soc != INVALID_SOCKET) { + int e = 0; + // tant qu'on a des données, et qu'on ne recoit pas deux LF, et que le timeout n'arrie pas do { - if (http_xfread1(&retour,0) < 0) - e=1; + if (http_xfread1(&retour, 0) < 0) + e = 1; else { - if (retour.adr!=NULL) { - if ((retour.adr[retour.size-1]!=10) || (retour.adr[retour.size-2]!=10)) - e=1; + if (retour.adr != NULL) { + if ((retour.adr[retour.size - 1] != 10) + || (retour.adr[retour.size - 2] != 10)) + e = 1; } } - + if (!e) { - if ((time_local()-tl)>=timeout) { - e=-1; + if ((time_local() - tl) >= timeout) { + e = -1; } } - - } while (!e); - - if (e==1) { - if (adr!=NULL) { - int ptr=0; + + } while(!e); + + if (e == 1) { + if (adr != NULL) { + int ptr = 0; char rcvd[1100]; // note: en gros recopie du traitement de back_wait() // - // ---------------------------------------- // traiter en-tête! // status-line à récupérer - ptr+=binput(retour.adr+ptr,rcvd,1024); - if (strnotempty(rcvd)==0) - ptr+=binput(retour.adr+ptr,rcvd,1024); // "certains serveurs buggés envoient un \n au début" (RFC) - + ptr += binput(retour.adr + ptr, rcvd, 1024); + if (strnotempty(rcvd) == 0) + ptr += binput(retour.adr + ptr, rcvd, 1024); // "certains serveurs buggés envoient un \n au début" (RFC) + // traiter status-line - treatfirstline(&retour,rcvd); - + treatfirstline(&retour, rcvd); + #if HDEBUG - printf("(Buffer) Status-Code=%d\n",retour.statuscode); + printf("(Buffer) Status-Code=%d\n", retour.statuscode); #endif - + // en-tête - + // header // ** !attention! HTTP/0.9 non supporté do { - ptr+=binput(retour.adr+ptr,rcvd,1024); + ptr += binput(retour.adr + ptr, rcvd, 1024); #if HDEBUG - printf("(buffer)>%s\n",rcvd); + printf("(buffer)>%s\n", rcvd); #endif if (strnotempty(rcvd)) - treathead(NULL,NULL,NULL,&retour,rcvd); // traiter - + treathead(NULL, NULL, NULL, &retour, rcvd); // traiter + } while(strnotempty(rcvd)); // ---------------------------------------- - + // libérer mémoire - if (retour.adr!=NULL) { freet(retour.adr); retour.adr=NULL; } + if (retour.adr != NULL) { + freet(retour.adr); + retour.adr = NULL; + } } } else { - retour.statuscode=STATUSCODE_TIMEOUT; - strcpybuff(retour.msg,"Timeout While Testing"); + retour.statuscode = STATUSCODE_TIMEOUT; + strcpybuff(retour.msg, "Timeout While Testing"); } - - + #if HTS_DEBUG_CLOSESOCK DEBUG_W("http_test: deletehttp\n"); #endif deletehttp(&retour); - retour.soc=INVALID_SOCKET; + retour.soc = INVALID_SOCKET; } - return retour; + return retour; } // Crée un lien (http) vers une adresse internet iadr // retour: structure (adresse, taille, message si erreur (si !adr)) // peut ouvrir avec des connect() non bloquants: waitconnect=0/1 -T_SOC newhttp(httrackp *opt,const char* _iadr,htsblk* retour,int port,int waitconnect) { - t_fullhostent fullhostent_buffer; // buffer pour resolver - T_SOC soc; // descipteur de la socket - char* iadr; +T_SOC newhttp(httrackp * opt, const char *_iadr, htsblk * retour, int port, + int waitconnect) { + t_fullhostent fullhostent_buffer; // buffer pour resolver + T_SOC soc; // descipteur de la socket + char *iadr; + // unsigned short int port; - + // si iadr="#" alors c'est une fausse URL, mais un vrai fichier // local. // utile pour les tests! //## if (iadr[0]!=lOCAL_CHAR) { - if (strcmp(_iadr,"file://") != 0) { /* non fichier */ + if (strcmp(_iadr, "file://") != 0) { /* non fichier */ SOCaddr server; - int server_size=sizeof(server); - t_hostent* hp; + int server_size = sizeof(server); + t_hostent *hp; + // effacer structure memset(&server, 0, sizeof(server)); // tester un éventuel id:pass et virer id:pass@ si détecté iadr = jump_identification(_iadr); - + #if HDEBUG printf("gethostbyname\n"); #endif - + // tester un éventuel port - if (port==-1) { - char *a=jump_toport(iadr); + if (port == -1) { + char *a = jump_toport(iadr); + #if HTS_USEOPENSSL if (retour->ssl) - port=443; + port = 443; else - port=80; // port par défaut + port = 80; // port par défaut #else - port=80; // port par défaut + port = 80; // port par défaut #endif if (a) { - char BIGSTK iadr2[HTS_URLMAXSIZE*2]; - int i=-1; - iadr2[0]='\0'; - sscanf(a+1,"%d",&i); - if (i!=-1) { - port=(unsigned short int) i; + char BIGSTK iadr2[HTS_URLMAXSIZE * 2]; + int i = -1; + + iadr2[0] = '\0'; + sscanf(a + 1, "%d", &i); + if (i != -1) { + port = (unsigned short int) i; } - // adresse véritable (sans :xx) - strncatbuff(iadr2,iadr,(int) (a - iadr)); + strncatbuff(iadr2, iadr, (int) (a - iadr)); // adresse sans le :xx - hp = hts_gethostbyname(opt,iadr2, &fullhostent_buffer); - + hp = hts_gethostbyname(opt, iadr2, &fullhostent_buffer); + } else { // adresse normale (port par défaut par la suite) - hp = hts_gethostbyname(opt,iadr, &fullhostent_buffer); - + hp = hts_gethostbyname(opt, iadr, &fullhostent_buffer); + } - - } else // port défini - hp = hts_gethostbyname(opt,iadr, &fullhostent_buffer); - + } else // port défini + hp = hts_gethostbyname(opt, iadr, &fullhostent_buffer); + // Conversion iadr -> adresse // structure recevant le nom de l'hôte, etc - //struct hostent *hp; + //struct hostent *hp; if (hp == NULL) { #if DEBUG printf("erreur gethostbyname\n"); @@ -2149,283 +2298,304 @@ T_SOC newhttp(httrackp *opt,const char* _iadr,htsblk* retour,int port,int waitco if (retour && retour->msg) { #ifdef _WIN32 int last_errno = WSAGetLastError(); - sprintf(retour->msg,"Unable to get server's address: %s", strerror(last_errno)); + + sprintf(retour->msg, "Unable to get server's address: %s", + strerror(last_errno)); #else int last_errno = errno; - sprintf(retour->msg,"Unable to get server's address: %s", strerror(last_errno)); + + sprintf(retour->msg, "Unable to get server's address: %s", + strerror(last_errno)); #endif } return INVALID_SOCKET; - } + } // copie adresse SOCaddr_copyaddr(server, server_size, hp->h_addr_list[0], hp->h_length); // make a copy for external clients retour->address_size = sizeof(retour->address); - SOCaddr_copyaddr(retour->address, retour->address_size, hp->h_addr_list[0], hp->h_length); + SOCaddr_copyaddr(retour->address, retour->address_size, hp->h_addr_list[0], + hp->h_length); // memcpy(&SOCaddr_sinaddr(server), hp->h_addr_list[0], hp->h_length); - + // créer ("attachement") une socket (point d'accès) internet,en flot #if HDEBUG printf("socket\n"); #endif -#if HTS_WIDE_DEBUG +#if HTS_WIDE_DEBUG DEBUG_W("socket\n"); #endif soc = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0); if (retour != NULL) { retour->debugid = HTS_STAT.stat_sockid++; } -#if HTS_WIDE_DEBUG - DEBUG_W("socket()=%d\n" _ (int) soc); +#if HTS_WIDE_DEBUG + DEBUG_W("socket()=%d\n" _(int) soc); #endif - if (soc==INVALID_SOCKET) { + if (soc == INVALID_SOCKET) { if (retour && retour->msg) { #ifdef _WIN32 int last_errno = WSAGetLastError(); - sprintf(retour->msg,"Unable to create a socket: %s", strerror(last_errno)); + + sprintf(retour->msg, "Unable to create a socket: %s", + strerror(last_errno)); #else int last_errno = errno; - sprintf(retour->msg,"Unable to create a socket: %s", strerror(last_errno)); + + sprintf(retour->msg, "Unable to create a socket: %s", + strerror(last_errno)); #endif } - return INVALID_SOCKET; // erreur création socket impossible + return INVALID_SOCKET; // erreur création socket impossible } - // bind this address if (retour != NULL && retour->req.proxy.bindhost[0] != 0) { t_fullhostent bind_buffer; + hp = hts_gethostbyname(opt, retour->req.proxy.bindhost, &bind_buffer); - if (hp == NULL || - bind(soc, (struct sockaddr *)hp->h_addr_list[0], hp->h_length) != 0) { - if (retour && retour->msg) { + if (hp == NULL + || bind(soc, (struct sockaddr *) hp->h_addr_list[0], + hp->h_length) != 0) { + if (retour && retour->msg) { #ifdef _WIN32 - int last_errno = WSAGetLastError(); - sprintf(retour->msg,"Unable to bind the specificied server address: %s", strerror(last_errno)); + int last_errno = WSAGetLastError(); + + sprintf(retour->msg, + "Unable to bind the specificied server address: %s", + strerror(last_errno)); #else - int last_errno = errno; - sprintf(retour->msg,"Unable to bind the specificied server address: %s", strerror(last_errno)); + int last_errno = errno; + + sprintf(retour->msg, + "Unable to bind the specificied server address: %s", + strerror(last_errno)); #endif - } - deletesoc(soc); - return INVALID_SOCKET; + } + deletesoc(soc); + return INVALID_SOCKET; } } - // structure: connexion au domaine internet, port 80 (ou autre) SOCaddr_initport(server, port); #if HDEBUG - printf("==%d\n",soc); + printf("==%d\n", soc); #endif // connexion non bloquante? - if (!waitconnect ) { - unsigned long p=1; // non bloquant + if (!waitconnect) { + unsigned long p = 1; // non bloquant + #ifdef _WIN32 - ioctlsocket(soc,FIONBIO,&p); + ioctlsocket(soc, FIONBIO, &p); #else - ioctl(soc,FIONBIO,&p); + ioctl(soc, FIONBIO, &p); #endif } - // Connexion au serveur lui même #if HDEBUG printf("connect\n"); #endif HTS_STAT.last_connect = mtime_local(); - + #if HTS_WIDE_DEBUG DEBUG_W("connect\n"); #endif #ifdef _WIN32 - if (connect(soc, (const struct sockaddr FAR *)&server, server_size) != 0) { + if (connect(soc, (const struct sockaddr FAR *) &server, server_size) != 0) { #else - if (connect(soc, (struct sockaddr *)&server, server_size) == -1) { + if (connect(soc, (struct sockaddr *) &server, server_size) == -1) { #endif - // bloquant - if (waitconnect) { + // bloquant + if (waitconnect) { #if HDEBUG - printf("unable to connect!\n"); + printf("unable to connect!\n"); #endif - if (retour && retour->msg) { + if (retour && retour->msg) { #ifdef _WIN32 - int last_errno = WSAGetLastError(); - sprintf(retour->msg,"Unable to connect to the server: %s", strerror(last_errno)); + int last_errno = WSAGetLastError(); + + sprintf(retour->msg, "Unable to connect to the server: %s", + strerror(last_errno)); #else - int last_errno = errno; - sprintf(retour->msg,"Unable to connect to the server: %s", strerror(last_errno)); + int last_errno = errno; + + sprintf(retour->msg, "Unable to connect to the server: %s", + strerror(last_errno)); #endif - } - /* Close the socket and notify the error!!! */ - deletesoc(soc); - return INVALID_SOCKET; } + /* Close the socket and notify the error!!! */ + deletesoc(soc); + return INVALID_SOCKET; } -#if HTS_WIDE_DEBUG - DEBUG_W("connect done\n"); + } +#if HTS_WIDE_DEBUG + DEBUG_W("connect done\n"); #endif - + #if HDEBUG - printf("connexion établie\n"); + printf("connexion établie\n"); #endif - + // A partir de maintenant, on peut envoyer et recevoir des données // via le flot identifié par soc (socket): write(soc,adr,taille) et // read(soc,adr,taille) - } else { // on doit ouvrir un fichier local! + } else { // on doit ouvrir un fichier local! // il sera géré de la même manière qu'une socket (c'est idem!) - soc=LOCAL_SOCKET_ID; // pseudo-socket locale.. + soc = LOCAL_SOCKET_ID; // pseudo-socket locale.. // soc sera remplacé lors d'un http_fopen() par un handle véritable! - } // teste fichier local ou http - + } // teste fichier local ou http + return soc; } - - // couper http://www.truc.fr/pub/index.html -> www.truc.fr /pub/index.html // retour=-1 si erreur. // si file://... alors adresse=file:// (et coupe le ?query dans ce cas) -int ident_url_absolute(const char* url,char* adr,char* fil) { - int pos=0; - int scheme=0; +int ident_url_absolute(const char *url, char *adr, char *fil) { + int pos = 0; + int scheme = 0; // effacer adr et fil - adr[0]=fil[0]='\0'; - + adr[0] = fil[0] = '\0'; + #if HDEBUG - printf("protocol: %s\n",url); + printf("protocol: %s\n", url); #endif // Scheme? { - const char* a=url; - while (isalpha((unsigned char)*a)) + const char *a = url; + + while(isalpha((unsigned char) *a)) a++; if (*a == ':') - scheme=1; + scheme = 1; } // 1. optional scheme ":" - if ((pos=strfield(url,"file:"))) { // fichier local!! (pour les tests) + if ((pos = strfield(url, "file:"))) { // fichier local!! (pour les tests) //!!p+=3; - strcpybuff(adr,"file://"); - } else if ((pos=strfield(url,"http:"))) { // HTTP + strcpybuff(adr, "file://"); + } else if ((pos = strfield(url, "http:"))) { // HTTP //!!p+=3; - } else if ((pos=strfield(url,"ftp:"))) { // FTP - strcpybuff(adr,"ftp://"); // FTP!! + } else if ((pos = strfield(url, "ftp:"))) { // FTP + strcpybuff(adr, "ftp://"); // FTP!! //!!p+=3; #if HTS_USEOPENSSL - } else if (SSL_is_available && (pos=strfield(url,"https:"))) { // HTTPS - strcpybuff(adr,"https://"); + } else if (SSL_is_available && (pos = strfield(url, "https:"))) { // HTTPS + strcpybuff(adr, "https://"); #endif #if HTS_USEMMS - } else if ((pos = strfield(url,"mms:"))) { // mms - strcpybuff(adr,"mms://"); + } else if ((pos = strfield(url, "mms:"))) { // mms + strcpybuff(adr, "mms://"); #endif } else if (scheme) { - return -1; // erreur non reconnu + return -1; // erreur non reconnu } else - pos=0; + pos = 0; // 2. optional "//" authority - if (strncmp(url+pos,"//",2)==0) - pos+=2; + if (strncmp(url + pos, "//", 2) == 0) + pos += 2; // (url+pos) now points to the path (not net path) //## if (adr[0]!=lOCAL_CHAR) { // adresse normale http - if (!strfield(adr,"file:")) { // PAS file:// - const char *p,*q; - p=url+pos; + if (!strfield(adr, "file:")) { // PAS file:// + const char *p, *q; + + p = url + pos; // p pointe sur le début de l'adresse, ex: www.truc.fr/sommaire/index.html - q=strchr(jump_identification(p),'/'); - if (q==0) q=strchr(jump_identification(p),'?'); // http://www.foo.com?bar=1 - if (q==0) q=p+strlen(p); // pointe sur \0 + q = strchr(jump_identification(p), '/'); + if (q == 0) + q = strchr(jump_identification(p), '?'); // http://www.foo.com?bar=1 + if (q == 0) + q = p + strlen(p); // pointe sur \0 // q pointe sur le chemin, ex: index.html?query=recherche - + // chemin www... trop long!! - if ( ( ((int) (q - p)) ) > HTS_URLMAXSIZE) { + if ((((int) (q - p))) > HTS_URLMAXSIZE) { //strcpybuff(retour.msg,"Path too long"); - return -1; // erreur + return -1; // erreur } - // recopier adresse www.. - strncatbuff(adr,p, ((int) (q - p)) ); + strncatbuff(adr, p, ((int) (q - p))); // *( adr+( ((int) q) - ((int) p) ) )=0; // faut arrêter la fumette! // recopier chemin /pub/.. - if (q[0] != '/') // page par défaut (/) - strcatbuff(fil,"/"); - strcatbuff(fil,q); + if (q[0] != '/') // page par défaut (/) + strcatbuff(fil, "/"); + strcatbuff(fil, q); // SECURITE: // simplifier url pour les ../ fil_simplifie(fil); - } else { // localhost file:// + } else { // localhost file:// const char *p; int i; - char* a; - - p=url+pos; - if (*p == '/' || *p == '\\') { /* file:///.. */ - strcatbuff(fil,p); // fichier local ; adr="#" + char *a; + + p = url + pos; + if (*p == '/' || *p == '\\') { /* file:///.. */ + strcatbuff(fil, p); // fichier local ; adr="#" } else { if (p[1] != ':') { - strcatbuff(fil,"//"); /* file://server/foo */ - strcatbuff(fil,p); + strcatbuff(fil, "//"); /* file://server/foo */ + strcatbuff(fil, p); } else { - strcatbuff(fil,p); // file://C:\.. + strcatbuff(fil, p); // file://C:\.. } } - - a=strchr(fil,'?'); - if (a) - *a='\0'; /* couper query (inutile pour file:// lors de la requête) */ + + a = strchr(fil, '?'); + if (a) + *a = '\0'; /* couper query (inutile pour file:// lors de la requête) */ // filtrer les \\ -> / pour les fichiers DOS - for(i=0;i<(int) strlen(fil);i++) - if (fil[i]=='\\') - fil[i]='/'; + for(i = 0; i < (int) strlen(fil); i++) + if (fil[i] == '\\') + fil[i] = '/'; } // no hostname if (!strnotempty(adr)) - return -1; // erreur non reconnu + return -1; // erreur non reconnu // nommer au besoin.. (non utilisé normalement) if (!strnotempty(fil)) - strcpybuff(fil,"default-index.html"); + strcpybuff(fil, "default-index.html"); // case insensitive pour adresse { - char *a=jump_identification(adr); + char *a = jump_identification(adr); + while(*a) { - if ((*a>='A') && (*a<='Z')) - *a+='a'-'A'; + if ((*a >= 'A') && (*a <= 'Z')) + *a += 'a' - 'A'; a++; } } - + return 0; } /* simplify ../ and ./ */ -void fil_simplifie(char* f) { +void fil_simplifie(char *f) { char *a, *b; char *rollback[128]; int rollid = 0; char lc = '/'; int query = 0; int wasAbsolute = (*f == '/'); - for(a = b = f ; *a != '\0' ; ) { + + for(a = b = f; *a != '\0';) { if (*a == '?') query = 1; - if (query == 0 && lc == '/' && a[0] == '.' && a[1] == '/') { /* foo/./bar or ./foo */ + if (query == 0 && lc == '/' && a[0] == '.' && a[1] == '/') { /* foo/./bar or ./foo */ a += 2; - } - else if (query == 0 && lc == '/' && a[0] == '.' && a[1] == '.' && ( a[2] == '/' || a[2] == '\0' ) ) { /* foo/../bar or ../foo or .. */ + } else if (query == 0 && lc == '/' && a[0] == '.' && a[1] == '.' && (a[2] == '/' || a[2] == '\0')) { /* foo/../bar or ../foo or .. */ if (a[2] == '\0') a += 2; else @@ -2433,19 +2603,18 @@ void fil_simplifie(char* f) { if (rollid > 1) { rollid--; b = rollback[rollid - 1]; - } else { /* too many ../ */ + } else { /* too many ../ */ rollid = 0; b = f; if (wasAbsolute) - b++; /* after the / */ + b++; /* after the / */ } - } - else { + } else { *b++ = lc = *a; if (*a == '/') { rollback[rollid++] = b; if (rollid >= 127) { - *f = '\0'; /* ERROR */ + *f = '\0'; /* ERROR */ break; } } @@ -2466,34 +2635,34 @@ void fil_simplifie(char* f) { } // fermer liaison fichier ou socket -HTS_INLINE void deletehttp(htsblk* r) { +HTS_INLINE void deletehttp(htsblk * r) { #if HTS_DEBUG_CLOSESOCK - DEBUG_W("deletehttp: (htsblk*) 0x%p\n" _ (void*) r); + DEBUG_W("deletehttp: (htsblk*) 0x%p\n" _(void *)r); #endif #if HTS_USEOPENSSL - /* Free OpenSSL structures */ - if (SSL_is_available && r->ssl_con) { - SSL_shutdown(r->ssl_con); - SSL_free(r->ssl_con); - r->ssl_con=NULL; - } -#endif - if (r->soc!=INVALID_SOCKET) { + /* Free OpenSSL structures */ + if (SSL_is_available && r->ssl_con) { + SSL_shutdown(r->ssl_con); + SSL_free(r->ssl_con); + r->ssl_con = NULL; + } +#endif + if (r->soc != INVALID_SOCKET) { if (r->is_file) { if (r->fp) fclose(r->fp); - r->fp=NULL; + r->fp = NULL; } else { - if (r->soc!=LOCAL_SOCKET_ID) + if (r->soc != LOCAL_SOCKET_ID) deletesoc_r(r); } - r->soc=INVALID_SOCKET; + r->soc = INVALID_SOCKET; } } // free the addr buffer // always returns 1 -HTS_INLINE int deleteaddr(htsblk* r) { +HTS_INLINE int deleteaddr(htsblk * r) { if (r->adr != NULL) { freet(r->adr); r->adr = NULL; @@ -2507,40 +2676,42 @@ HTS_INLINE int deleteaddr(htsblk* r) { // fermer une socket HTS_INLINE void deletesoc(T_SOC soc) { - if (soc!=INVALID_SOCKET && soc!=LOCAL_SOCKET_ID) { -#if HTS_WIDE_DEBUG - DEBUG_W("close %d\n" _ (int) soc); + if (soc != INVALID_SOCKET && soc != LOCAL_SOCKET_ID) { +#if HTS_WIDE_DEBUG + DEBUG_W("close %d\n" _(int) soc); #endif #ifdef _WIN32 if (closesocket(soc) != 0) { int err = WSAGetLastError(); + fprintf(stderr, "* error closing socket %d: %s\n", soc, strerror(err)); } #else if (close(soc) != 0) { const int err = errno; + fprintf(stderr, "* error closing socket %d: %s\n", soc, strerror(err)); } #endif -#if HTS_WIDE_DEBUG +#if HTS_WIDE_DEBUG DEBUG_W(".. done\n"); #endif } } /* Will also clean other things */ -HTS_INLINE void deletesoc_r(htsblk* r) { +HTS_INLINE void deletesoc_r(htsblk * r) { #if HTS_USEOPENSSL if (SSL_is_available && r->ssl_con) { SSL_shutdown(r->ssl_con); // SSL_CTX_set_quiet_shutdown(r->ssl_con->ctx, 1); SSL_free(r->ssl_con); - r->ssl_con=NULL; + r->ssl_con = NULL; } #endif - if (r->soc!=INVALID_SOCKET) { + if (r->soc != INVALID_SOCKET) { deletesoc(r->soc); - r->soc=INVALID_SOCKET; + r->soc = INVALID_SOCKET; } } @@ -2553,210 +2724,230 @@ HTS_INLINE TStamp time_local(void) { HTSEXT_API HTS_INLINE TStamp mtime_local(void) { #ifndef HTS_DO_NOT_USE_FTIME struct timeb B; - ftime( &B ); - return (TStamp) ( ((TStamp) B.time * (TStamp) 1000) - + ((TStamp) B.millitm) ); + + ftime(&B); + return (TStamp) (((TStamp) B.time * (TStamp) 1000) + + ((TStamp) B.millitm)); #else // not precise.. - return (TStamp) ( ((TStamp) time_local() * (TStamp) 1000) - + ((TStamp) 0) ); + return (TStamp) (((TStamp) time_local() * (TStamp) 1000) + + ((TStamp) 0)); #endif } // convertit un nombre de secondes en temps (chaine) -void sec2str(char *st,TStamp t) { - int j,h,m,s; - - j=(int) (t/(3600*24)); - t-=((TStamp) j)*(3600*24); - h=(int) (t/(3600)); - t-=((TStamp) h)*3600; - m=(int) (t/60); - t-=((TStamp) m)*60; - s=(int) t; - - if (j>0) - sprintf(st,"%d days, %d hours %d minutes %d seconds",j,h,m,s); - else if (h>0) - sprintf(st,"%d hours %d minutes %d seconds",h,m,s); - else if (m>0) - sprintf(st,"%d minutes %d seconds",m,s); +void sec2str(char *st, TStamp t) { + int j, h, m, s; + + j = (int) (t / (3600 * 24)); + t -= ((TStamp) j) * (3600 * 24); + h = (int) (t / (3600)); + t -= ((TStamp) h) * 3600; + m = (int) (t / 60); + t -= ((TStamp) m) * 60; + s = (int) t; + + if (j > 0) + sprintf(st, "%d days, %d hours %d minutes %d seconds", j, h, m, s); + else if (h > 0) + sprintf(st, "%d hours %d minutes %d seconds", h, m, s); + else if (m > 0) + sprintf(st, "%d minutes %d seconds", m, s); else - sprintf(st,"%d seconds",s); + sprintf(st, "%d seconds", s); } // idem, plus court (chaine) -HTSEXT_API void qsec2str(char *st,TStamp t) { - int j,h,m,s; - - j=(int) (t/(3600*24)); - t-=((TStamp) j)*(3600*24); - h=(int) (t/(3600)); - t-=((TStamp) h)*3600; - m=(int) (t/60); - t-=((TStamp) m)*60; - s=(int) t; - - if (j>0) - sprintf(st,"%dd,%02dh,%02dmin%02ds",j,h,m,s); - else if (h>0) - sprintf(st,"%dh,%02dmin%02ds",h,m,s); - else if (m>0) - sprintf(st,"%dmin%02ds",m,s); +HTSEXT_API void qsec2str(char *st, TStamp t) { + int j, h, m, s; + + j = (int) (t / (3600 * 24)); + t -= ((TStamp) j) * (3600 * 24); + h = (int) (t / (3600)); + t -= ((TStamp) h) * 3600; + m = (int) (t / 60); + t -= ((TStamp) m) * 60; + s = (int) t; + + if (j > 0) + sprintf(st, "%dd,%02dh,%02dmin%02ds", j, h, m, s); + else if (h > 0) + sprintf(st, "%dh,%02dmin%02ds", h, m, s); + else if (m > 0) + sprintf(st, "%dmin%02ds", m, s); else - sprintf(st,"%ds",s); + sprintf(st, "%ds", s); } - // heure actuelle, GMT, format rfc (taille buffer 256o) -void time_gmt_rfc822(char* s) { +void time_gmt_rfc822(char *s) { time_t tt; - struct tm* A; - tt=time(NULL); - A=gmtime(&tt); - if (A==NULL) - A=localtime(&tt); - time_rfc822(s,A); + struct tm *A; + + tt = time(NULL); + A = gmtime(&tt); + if (A == NULL) + A = localtime(&tt); + time_rfc822(s, A); } // heure actuelle, format rfc (taille buffer 256o) -void time_local_rfc822(char* s) { +void time_local_rfc822(char *s) { time_t tt; - struct tm* A; - tt=time(NULL); - A=localtime(&tt); - time_rfc822_local(s,A); + struct tm *A; + + tt = time(NULL); + A = localtime(&tt); + time_rfc822_local(s, A); } /* convertir une chaine en temps */ -struct tm* convert_time_rfc822(struct tm *result, const char* s) { - char months[]="jan feb mar apr may jun jul aug sep oct nov dec"; +struct tm *convert_time_rfc822(struct tm *result, const char *s) { + char months[] = "jan feb mar apr may jun jul aug sep oct nov dec"; char str[256]; - char* a; + char *a; + /* */ - int result_mm=-1; - int result_dd=-1; - int result_n1=-1; - int result_n2=-1; - int result_n3=-1; - int result_n4=-1; + int result_mm = -1; + int result_dd = -1; + int result_n1 = -1; + int result_n2 = -1; + int result_n3 = -1; + int result_n4 = -1; + /* */ if ((int) strlen(s) > 200) return NULL; - strcpybuff(str,s); + strcpybuff(str, s); hts_lowcase(str); /* éliminer :,- */ - while( (a=strchr(str,'-')) ) *a=' '; - while( (a=strchr(str,':')) ) *a=' '; - while( (a=strchr(str,',')) ) *a=' '; + while((a = strchr(str, '-'))) + *a = ' '; + while((a = strchr(str, ':'))) + *a = ' '; + while((a = strchr(str, ','))) + *a = ' '; /* tokeniser */ - a=str; + a = str; while(*a) { - char *first,*last; + char *first, *last; char tok[256]; + /* découper mot */ - while(*a==' ') a++; /* sauter espaces */ - first=a; - while((*a) && (*a!=' ')) a++; - last=a; - tok[0]='\0'; - if (first!=last) { - char* pos; - strncatbuff(tok,first,(int) (last - first)); + while(*a == ' ') + a++; /* sauter espaces */ + first = a; + while((*a) && (*a != ' ')) + a++; + last = a; + tok[0] = '\0'; + if (first != last) { + char *pos; + + strncatbuff(tok, first, (int) (last - first)); /* analyser */ - if ( (pos=strstr(months,tok)) ) { /* month always in letters */ - result_mm=((int) (pos - months))/4; + if ((pos = strstr(months, tok))) { /* month always in letters */ + result_mm = ((int) (pos - months)) / 4; } else { int number; - if (sscanf(tok,"%d",&number) == 1) { /* number token */ - if (result_dd<0) /* day always first number */ - result_dd=number; - else if (result_n1<0) - result_n1=number; - else if (result_n2<0) - result_n2=number; - else if (result_n3<0) - result_n3=number; - else if (result_n4<0) - result_n4=number; - } /* sinon, bruit de fond(+1GMT for exampel) */ + + if (sscanf(tok, "%d", &number) == 1) { /* number token */ + if (result_dd < 0) /* day always first number */ + result_dd = number; + else if (result_n1 < 0) + result_n1 = number; + else if (result_n2 < 0) + result_n2 = number; + else if (result_n3 < 0) + result_n3 = number; + else if (result_n4 < 0) + result_n4 = number; + } /* sinon, bruit de fond(+1GMT for exampel) */ } } } - if ((result_n1>=0) && (result_mm>=0) && (result_dd>=0) && (result_n2>=0) && (result_n3>=0) && (result_n4>=0)) { - if (result_n4>=1000) { /* Sun Nov 6 08:49:37 1994 */ - result->tm_year=result_n4-1900; - result->tm_hour=result_n1; - result->tm_min=result_n2; - result->tm_sec=max(result_n3,0); - } else { /* Sun, 06 Nov 1994 08:49:37 GMT or Sunday, 06-Nov-94 08:49:37 GMT */ - result->tm_hour=result_n2; - result->tm_min=result_n3; - result->tm_sec=max(result_n4,0); - if (result_n1<=50) /* 00 means 2000 */ - result->tm_year=result_n1+100; - else if (result_n1<1000) /* 99 means 1999 */ - result->tm_year=result_n1; - else /* 2000 */ - result->tm_year=result_n1-1900; - } - result->tm_isdst=0; /* assume GMT */ - result->tm_yday=-1; /* don't know */ - result->tm_wday=-1; /* don't know */ - result->tm_mon=result_mm; - result->tm_mday=result_dd; + if ((result_n1 >= 0) && (result_mm >= 0) && (result_dd >= 0) + && (result_n2 >= 0) && (result_n3 >= 0) && (result_n4 >= 0)) { + if (result_n4 >= 1000) { /* Sun Nov 6 08:49:37 1994 */ + result->tm_year = result_n4 - 1900; + result->tm_hour = result_n1; + result->tm_min = result_n2; + result->tm_sec = max(result_n3, 0); + } else { /* Sun, 06 Nov 1994 08:49:37 GMT or Sunday, 06-Nov-94 08:49:37 GMT */ + result->tm_hour = result_n2; + result->tm_min = result_n3; + result->tm_sec = max(result_n4, 0); + if (result_n1 <= 50) /* 00 means 2000 */ + result->tm_year = result_n1 + 100; + else if (result_n1 < 1000) /* 99 means 1999 */ + result->tm_year = result_n1; + else /* 2000 */ + result->tm_year = result_n1 - 1900; + } + result->tm_isdst = 0; /* assume GMT */ + result->tm_yday = -1; /* don't know */ + result->tm_wday = -1; /* don't know */ + result->tm_mon = result_mm; + result->tm_mday = result_dd; return result; } return NULL; } -static time_t getGMT(struct tm *tm) { /* hey, time_t is local! */ - time_t t = mktime(tm); - if (t != (time_t) -1 && t != (time_t) 0) { +static time_t getGMT(struct tm *tm) { /* hey, time_t is local! */ + time_t t = mktime(tm); + + if (t != (time_t) - 1 && t != (time_t) 0) { /* BSD does not have static "timezone" declared */ #if (defined(BSD) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__FreeBSD_kernel__)) - time_t now = time(NULL); - time_t timezone = - localtime(&now)->tm_gmtoff; + time_t now = time(NULL); + time_t timezone = -localtime(&now)->tm_gmtoff; #endif return (time_t) (t - timezone); - } - return (time_t) -1; + } + return (time_t) - 1; } /* sets file time. -1 if error */ /* Note: utf-8 */ -int set_filetime(const char* file, struct tm* tm_time) { - time_t t = getGMT(tm_time); - if (t != (time_t) -1) { - STRUCT_UTIMBUF tim; - memset(&tim, 0, sizeof(tim)); - tim.actime = tim.modtime = t; - return UTIME(file, &tim); - } - return -1; +int set_filetime(const char *file, struct tm *tm_time) { + time_t t = getGMT(tm_time); + + if (t != (time_t) - 1) { + STRUCT_UTIMBUF tim; + + memset(&tim, 0, sizeof(tim)); + tim.actime = tim.modtime = t; + return UTIME(file, &tim); + } + return -1; } /* sets file time from RFC822 date+time, -1 if error*/ /* Note: utf-8 */ -int set_filetime_rfc822(const char* file, const char* date) { - struct tm buffer; - struct tm* tm_s = convert_time_rfc822(&buffer, date); +int set_filetime_rfc822(const char *file, const char *date) { + struct tm buffer; + struct tm *tm_s = convert_time_rfc822(&buffer, date); + if (tm_s) { - return set_filetime(file,tm_s); - } else return -1; + return set_filetime(file, tm_s); + } else + return -1; } /* Note: utf-8 */ -int get_filetime_rfc822(const char* file, char* date) { +int get_filetime_rfc822(const char *file, char *date) { STRUCT_STAT buf; + date[0] = '\0'; if (STAT(file, &buf) == 0) { - struct tm* A; + struct tm *A; time_t tt = buf.st_mtime; - A=gmtime(&tt); - if (A==NULL) - A=localtime(&tt); + + A = gmtime(&tt); + if (A == NULL) + A = localtime(&tt); if (A != NULL) { time_rfc822(date, A); return 1; @@ -2766,40 +2957,44 @@ int get_filetime_rfc822(const char* file, char* date) { } // heure au format rfc (taille buffer 256o) -HTS_INLINE void time_rfc822(char* s,struct tm * A) { +HTS_INLINE void time_rfc822(char *s, struct tm *A) { if (A == NULL) { - int localtime_returned_null=0; + int localtime_returned_null = 0; + assert(localtime_returned_null); } - strftime(s,256,"%a, %d %b %Y %H:%M:%S GMT",A); + strftime(s, 256, "%a, %d %b %Y %H:%M:%S GMT", A); } // heure locale au format rfc (taille buffer 256o) -HTS_INLINE void time_rfc822_local(char* s,struct tm * A) { +HTS_INLINE void time_rfc822_local(char *s, struct tm *A) { if (A == NULL) { - int localtime_returned_null=0; + int localtime_returned_null = 0; + assert(localtime_returned_null); } - strftime(s,256,"%a, %d %b %Y %H:%M:%S",A); + strftime(s, 256, "%a, %d %b %Y %H:%M:%S", A); } // conversion en b,Kb,Mb -HTSEXT_API char* int2bytes(strc_int2bytes2* strc, LLint n) { - char** a = int2bytes2(strc, n); +HTSEXT_API char *int2bytes(strc_int2bytes2 * strc, LLint n) { + char **a = int2bytes2(strc, n); + strcpybuff(strc->catbuff, a[0]); strcatbuff(strc->catbuff, a[1]); return strc->catbuff; } // conversion en b/s,Kb/s,Mb/s -HTSEXT_API char* int2bytessec(strc_int2bytes2* strc, long int n) { - char buff[256]; - char** a = int2bytes2(strc, n); +HTSEXT_API char *int2bytessec(strc_int2bytes2 * strc, long int n) { + char buff[256]; + char **a = int2bytes2(strc, n); + strcpybuff(buff, a[0]); strcatbuff(buff, a[1]); return concat(strc->catbuff, buff, "/s"); } -HTSEXT_API char* int2char(strc_int2bytes2* strc, int n) { +HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) { sprintf(strc->buff2, "%d", n); return strc->buff2; } @@ -2816,60 +3011,67 @@ HTSEXT_API char* int2char(strc_int2bytes2* strc, int n) { #define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB) #define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB) #endif -HTSEXT_API char** int2bytes2(strc_int2bytes2* strc, LLint n) { +HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) { if (n < ToLLintKiB) { - sprintf(strc->buff1,"%d",(int)(LLint)n); - strcpybuff(strc->buff2,"B"); + sprintf(strc->buff1, "%d", (int) (LLint) n); + strcpybuff(strc->buff2, "B"); } else if (n < ToLLintMiB) { - sprintf(strc->buff1,"%d,%02d",(int)((LLint)(n/ToLLintKiB)),(int)((LLint)((n%ToLLintKiB)*100)/ToLLintKiB)); - strcpybuff(strc->buff2,"KiB"); + sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)), + (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB)); + strcpybuff(strc->buff2, "KiB"); } #ifdef HTS_LONGLONG else if (n < ToLLintGiB) { - sprintf(strc->buff1,"%d,%02d",(int)((LLint)(n/(ToLLintMiB))),(int)((LLint)(((n%(ToLLintMiB))*100)/(ToLLintMiB)))); - strcpybuff(strc->buff2,"MiB"); + sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))), + (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB)))); + strcpybuff(strc->buff2, "MiB"); } else if (n < ToLLintTiB) { - sprintf(strc->buff1,"%d,%02d",(int)((LLint)(n/(ToLLintGiB))),(int)((LLint)(((n%(ToLLintGiB))*100)/(ToLLintGiB)))); - strcpybuff(strc->buff2,"GiB"); + sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintGiB))), + (int) ((LLint) (((n % (ToLLintGiB)) * 100) / (ToLLintGiB)))); + strcpybuff(strc->buff2, "GiB"); } else if (n < ToLLintPiB) { - sprintf(strc->buff1,"%d,%02d",(int)((LLint)(n/(ToLLintTiB))),(int)((LLint)(((n%(ToLLintTiB))*100)/(ToLLintTiB)))); - strcpybuff(strc->buff2,"TiB"); + sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintTiB))), + (int) ((LLint) (((n % (ToLLintTiB)) * 100) / (ToLLintTiB)))); + strcpybuff(strc->buff2, "TiB"); } else { - sprintf(strc->buff1,"%d,%02d",(int)((LLint)(n/(ToLLintPiB))),(int)((LLint)(((n%(ToLLintPiB))*100)/(ToLLintPiB)))); - strcpybuff(strc->buff2,"PiB"); + sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintPiB))), + (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB)))); + strcpybuff(strc->buff2, "PiB"); } #else else { - sprintf(strc->buff1,"%d,%02d",(int)((LLint)(n/(ToLLintMiB))),(int)((LLint)(((n%(ToLLintMiB))*100)/(ToLLintMiB)))); - strcpybuff(strc->buff2,"MiB"); + sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))), + (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB)))); + strcpybuff(strc->buff2, "MiB"); } #endif - strc->buffadr[0]=strc->buff1; - strc->buffadr[1]=strc->buff2; + strc->buffadr[0] = strc->buff1; + strc->buffadr[1] = strc->buff2; return strc->buffadr; } #ifdef _WIN32 #else // ignore sigpipe? -int sig_ignore_flag( int setflag ) { // flag ignore - static int flag=0; /* YES, this one is true static */ - if (setflag>=0) - flag=setflag; +int sig_ignore_flag(int setflag) { // flag ignore + static int flag = 0; /* YES, this one is true static */ + + if (setflag >= 0) + flag = setflag; return flag; } #endif // envoi de texte (en têtes généralement) sur la socket soc -HTS_INLINE int sendc(htsblk* r, const char* s) { - int n, ssz = (int)strlen(s); +HTS_INLINE int sendc(htsblk * r, const char *s) { + int n, ssz = (int) strlen(s); #ifdef _WIN32 #else sig_ignore_flag(1); #endif #if HDEBUG - write(0,s,ssz); + write(0, s, ssz); #endif #if HTS_USEOPENSSL @@ -2877,45 +3079,51 @@ HTS_INLINE int sendc(htsblk* r, const char* s) { n = SSL_write(r->ssl_con, s, ssz); } else #endif - n = send(r->soc,s,ssz,0); + n = send(r->soc, s, ssz, 0); #ifdef _WIN32 #else sig_ignore_flag(0); #endif - return ( n == ssz ) ? n : -1; + return (n == ssz) ? n : -1; } - // Remplace read -int finput(int fd,char* s,int max) { +int finput(int fd, char *s, int max) { char c; - int j=0; + int j = 0; + do { //c=fgetc(fp); - if (read(fd,&c,1)<=0) { - c=0; - } - if (c!=0) { - switch(c) { - case 10: c=0; break; - case 13: break; // sauter ces caractères - default: s[j++]=c; break; + if (read(fd, &c, 1) <= 0) { + c = 0; + } + if (c != 0) { + switch (c) { + case 10: + c = 0; + break; + case 13: + break; // sauter ces caractères + default: + s[j++] = c; + break; } } - } while((c!=0) && (j<max-1)); - s[j]='\0'; + } while((c != 0) && (j < max - 1)); + s[j] = '\0'; return j; -} +} // Like linput, but in memory (optimized) -int binput(char* buff, char* s, int max) { +int binput(char *buff, char *s, int max) { int count = 0; int destCount = 0; // Note: \0 will return 1 - while(destCount < max && buff != NULL && buff[count] != '\0' && buff[count] != '\n') { + while(destCount < max && buff != NULL && buff[count] != '\0' + && buff[count] != '\n') { if (buff[count] != '\r') { s[destCount++] = buff[count]; } @@ -2925,75 +3133,96 @@ int binput(char* buff, char* s, int max) { // then return the supplemental jump offset return count + 1; -} +} // Lecture d'une ligne (peut être unicode à priori) -int linput(FILE* fp,char* s,int max) { +int linput(FILE * fp, char *s, int max) { int c; - int j=0; + int j = 0; + do { - c=fgetc(fp); - if (c!=EOF) { - switch(c) { - case 13: break; // sauter CR - case 10: c=-1; break; - case 9: case 12: break; // sauter ces caractères - default: s[j++]=(char) c; break; + c = fgetc(fp); + if (c != EOF) { + switch (c) { + case 13: + break; // sauter CR + case 10: + c = -1; + break; + case 9: + case 12: + break; // sauter ces caractères + default: + s[j++] = (char) c; + break; } } - } while((c!=-1) && (c!=EOF) && (j<(max-1))); - s[j]='\0'; + } while((c != -1) && (c != EOF) && (j < (max - 1))); + s[j] = '\0'; return j; } -int linputsoc(T_SOC soc, char* s, int max) { +int linputsoc(T_SOC soc, char *s, int max) { int c; - int j=0; + int j = 0; + do { unsigned char ch; + if (recv(soc, &ch, 1, 0) == 1) { c = ch; } else { c = EOF; } - if (c!=EOF) { - switch(c) { - case 13: break; // sauter CR - case 10: c=-1; break; - case 9: case 12: break; // sauter ces caractères - default: s[j++]=(char) c; break; + if (c != EOF) { + switch (c) { + case 13: + break; // sauter CR + case 10: + c = -1; + break; + case 9: + case 12: + break; // sauter ces caractères + default: + s[j++] = (char) c; + break; } } - } while((c!=-1) && (c!=EOF) && (j<(max-1))); - s[j]='\0'; + } while((c != -1) && (c != EOF) && (j < (max - 1))); + s[j] = '\0'; return j; } -int linputsoc_t(T_SOC soc, char* s, int max, int timeout) { +int linputsoc_t(T_SOC soc, char *s, int max, int timeout) { if (check_readinput_t(soc, timeout)) { return linputsoc(soc, s, max); } return -1; } -int linput_trim(FILE* fp,char* s,int max) { - int rlen=0; - char* ls=(char*) malloct(max+2); - s[0]='\0'; +int linput_trim(FILE * fp, char *s, int max) { + int rlen = 0; + char *ls = (char *) malloct(max + 2); + + s[0] = '\0'; if (ls) { - char* a; + char *a; + // lire ligne - rlen=linput(fp,ls,max); + rlen = linput(fp, ls, max); if (rlen) { // sauter espaces et tabs en fin - while( (rlen>0) && ((ls[max(rlen-1,0)]==' ') || (ls[max(rlen-1,0)]=='\t')) ) - ls[--rlen]='\0'; + while((rlen > 0) + && ((ls[max(rlen - 1, 0)] == ' ') + || (ls[max(rlen - 1, 0)] == '\t'))) + ls[--rlen] = '\0'; // sauter espaces en début - a=ls; - while((rlen>0) && ((*a==' ') || (*a=='\t'))) { + a = ls; + while((rlen > 0) && ((*a == ' ') || (*a == '\t'))) { a++; rlen--; } - if (rlen>0) { - memcpy(s,a,rlen); // can copy \0 chars - s[rlen]='\0'; + if (rlen > 0) { + memcpy(s, a, rlen); // can copy \0 chars + s[rlen] = '\0'; } } // @@ -3001,47 +3230,56 @@ int linput_trim(FILE* fp,char* s,int max) { } return rlen; } -int linput_cpp(FILE* fp,char* s,int max) { - int rlen=0; - s[0]='\0'; +int linput_cpp(FILE * fp, char *s, int max) { + int rlen = 0; + + s[0] = '\0'; do { int ret; - if (rlen>0) - if (s[rlen-1]=='\\') - s[--rlen]='\0'; // couper \ final + + if (rlen > 0) + if (s[rlen - 1] == '\\') + s[--rlen] = '\0'; // couper \ final // lire ligne - ret=linput_trim(fp,s+rlen,max-rlen); - if (ret>0) - rlen+=ret; - } while((s[max(rlen-1,0)]=='\\') && (rlen<max)); + ret = linput_trim(fp, s + rlen, max - rlen); + if (ret > 0) + rlen += ret; + } while((s[max(rlen - 1, 0)] == '\\') && (rlen < max)); return rlen; } // idem avec les car spéciaux -void rawlinput(FILE* fp,char* s,int max) { +void rawlinput(FILE * fp, char *s, int max) { int c; - int j=0; + int j = 0; + do { - c=fgetc(fp); - if (c!=EOF) { - switch(c) { - case 13: break; // sauter CR - case 10: c=-1; break; - default: s[j++]=(char) c; break; + c = fgetc(fp); + if (c != EOF) { + switch (c) { + case 13: + break; // sauter CR + case 10: + c = -1; + break; + default: + s[j++] = (char) c; + break; } } - } while((c!=-1) && (c!=EOF) && (j<(max-1))); - s[j++]='\0'; + } while((c != -1) && (c != EOF) && (j < (max - 1))); + s[j++] = '\0'; } //cherche chaine, case insensitive -char* strstrcase(char *s,char *o) { - while((*s) && (strfield(s,o)==0)) s++; - if (*s=='\0') return NULL; - return s; +char *strstrcase(char *s, char *o) { + while((*s) && (strfield(s, o) == 0)) + s++; + if (*s == '\0') + return NULL; + return s; } - // Unicode detector // See http://www.unicode.org/unicode/reports/tr28/ // (sect Table 3.1B. Legal UTF-8 Byte Sequences) @@ -3070,45 +3308,52 @@ typedef struct { // 0 : no // 1 : yes // -1: don't know -int is_unicode_utf8(const char* buffer_, size_t size) { - const unsigned char* buffer = (const unsigned char*) buffer_; +int is_unicode_utf8(const char *buffer_, size_t size) { + const unsigned char *buffer = (const unsigned char *) buffer_; t_auto_seq seq; size_t i; int is_utf = -1; - seq.pos=0; - for(i = 0 ; i < size ; i++) { - unsigned int ok=0; - unsigned int inseq=0; - unsigned int err=0; - - seq.data[seq.pos]=buffer[i]; - /**/ if ( SEQBEG && BLK(0,00,7F) && IN_SEQ && SEQEND ) { } - else if ( SEQBEG && BLK(0,C2,DF) && IN_SEQ && BLK(1,80,BF) && SEQEND ) { } - else if ( SEQBEG && ELT(0,E0 ) && IN_SEQ && BLK(1,A0,BF) && BLK(2,80,BF) && SEQEND ) { } - else if ( SEQBEG && BLK(0,E1,EC) && IN_SEQ && BLK(1,80,BF) && BLK(2,80,BF) && SEQEND ) { } - else if ( SEQBEG && ELT(0,ED ) && IN_SEQ && BLK(1,80,9F) && BLK(2,80,BF) && SEQEND ) { } - else if ( SEQBEG && BLK(0,EE,EF) && IN_SEQ && BLK(1,80,BF) && BLK(2,80,BF) && SEQEND ) { } - else if ( SEQBEG && ELT(0,F0 ) && IN_SEQ && BLK(1,90,BF) && BLK(2,80,BF) && BLK(3,80,BF) && SEQEND ) { } - else if ( SEQBEG && BLK(0,F1,F3) && IN_SEQ && BLK(1,80,BF) && BLK(2,80,BF) && BLK(3,80,BF) && SEQEND ) { } - else if ( SEQBEG && ELT(0,F4 ) && IN_SEQ && BLK(1,80,8F) && BLK(2,80,BF) && BLK(3,80,BF) && SEQEND ) { } - else if ( NO_SEQ ) { // bad, unknown + seq.pos = 0; + for(i = 0; i < size; i++) { + unsigned int ok = 0; + unsigned int inseq = 0; + unsigned int err = 0; + + seq.data[seq.pos] = buffer[i]; + /**/ if (SEQBEG && BLK(0, 00, 7F) && IN_SEQ && SEQEND) { + } else if (SEQBEG && BLK(0, C2, DF) && IN_SEQ && BLK(1, 80, BF) && SEQEND) { + } else if (SEQBEG && ELT(0, E0) && IN_SEQ && BLK(1, A0, BF) + && BLK(2, 80, BF) && SEQEND) { + } else if (SEQBEG && BLK(0, E1, EC) && IN_SEQ && BLK(1, 80, BF) + && BLK(2, 80, BF) && SEQEND) { + } else if (SEQBEG && ELT(0, ED) && IN_SEQ && BLK(1, 80, 9F) + && BLK(2, 80, BF) && SEQEND) { + } else if (SEQBEG && BLK(0, EE, EF) && IN_SEQ && BLK(1, 80, BF) + && BLK(2, 80, BF) && SEQEND) { + } else if (SEQBEG && ELT(0, F0) && IN_SEQ && BLK(1, 90, BF) + && BLK(2, 80, BF) && BLK(3, 80, BF) && SEQEND) { + } else if (SEQBEG && BLK(0, F1, F3) && IN_SEQ && BLK(1, 80, BF) + && BLK(2, 80, BF) && BLK(3, 80, BF) && SEQEND) { + } else if (SEQBEG && ELT(0, F4) && IN_SEQ && BLK(1, 80, 8F) + && BLK(2, 80, BF) && BLK(3, 80, BF) && SEQEND) { + } else if (NO_SEQ) { // bad, unknown return 0; } /* */ - + /* Error */ - if ( BAD_SEQ ) { + if (BAD_SEQ) { return 0; } /* unicode character */ if (seq.pos > 0) - is_utf=1; + is_utf = 1; /* Next */ if (ok) - seq.pos=0; + seq.pos = 0; else seq.pos++; @@ -3121,25 +3366,26 @@ int is_unicode_utf8(const char* buffer_, size_t size) { return is_utf; } -void map_characters(unsigned char* buffer, unsigned int size, unsigned int* map) { +void map_characters(unsigned char *buffer, unsigned int size, unsigned int *map) { unsigned int i; + memset(map, 0, sizeof(unsigned int) * 256); - for(i = 0 ; i < size ; i++) { + for(i = 0; i < size; i++) { map[buffer[i]]++; } } - // le fichier est-il un fichier html? // 0 : non // 1 : oui // -1 : on sait pas // -2 : on sait pas, pas d'extension -int ishtml(httrackp *opt,const char* fil) { +int ishtml(httrackp * opt, const char *fil) { /* User-defined MIME types (overrides ishtml()) */ - char BIGSTK fil_noquery[HTS_URLMAXSIZE*2]; + char BIGSTK fil_noquery[HTS_URLMAXSIZE * 2]; char mime[256]; - char* a; + char *a; + strcpybuff(fil_noquery, fil); if ((a = strchr(fil_noquery, '?')) != NULL) { *a = '\0'; @@ -3157,133 +3403,148 @@ int ishtml(httrackp *opt,const char* fil) { } /* Search for known ext */ - for (a = fil_noquery + strlen(fil_noquery) - 1 ; *a != '.' && *a != '/' && a > fil_noquery ; a-- ); - if (*a == '.') { // a une extension - char BIGSTK fil_noquery[HTS_URLMAXSIZE*2]; - char* b; + for(a = fil_noquery + strlen(fil_noquery) - 1; + *a != '.' && *a != '/' && a > fil_noquery; a--) ; + if (*a == '.') { // a une extension + char BIGSTK fil_noquery[HTS_URLMAXSIZE * 2]; + char *b; int ret; - char* dotted = a; - fil_noquery[0]='\0'; - a++; // pointer sur extension - strncatbuff(fil_noquery,a,HTS_URLMAXSIZE); - b=strchr(fil_noquery,'?'); + char *dotted = a; + + fil_noquery[0] = '\0'; + a++; // pointer sur extension + strncatbuff(fil_noquery, a, HTS_URLMAXSIZE); + b = strchr(fil_noquery, '?'); if (b) - *b='\0'; - ret = ishtml_ext(fil_noquery); // retour + *b = '\0'; + ret = ishtml_ext(fil_noquery); // retour if (ret == -1) { - switch(is_knowntype(opt,dotted)) { + switch (is_knowntype(opt, dotted)) { case 1: - ret = 0; // connu, non html + ret = 0; // connu, non html break; case 2: - ret = 1; // connu, html + ret = 1; // connu, html break; default: - ret = -1; // inconnu.. + ret = -1; // inconnu.. break; } } return ret; - } else return -2; // indéterminé, par exemple /truc + } else + return -2; // indéterminé, par exemple /truc } // idem, mais pour uniquement l'extension -int ishtml_ext(const char* a) { - int html=0; +int ishtml_ext(const char *a) { + int html = 0; + // - if (strfield2(a,"html")) html = 1; - else if (strfield2(a,"htm")) html = 1; - else if (strfield2(a,"shtml")) html = 1; - else if (strfield2(a,"phtml")) html = 1; - else if (strfield2(a,"htmlx")) html = 1; - else if (strfield2(a,"shtm")) html = 1; - else if (strfield2(a,"phtm")) html = 1; - else if (strfield2(a,"htmx")) html = 1; + if (strfield2(a, "html")) + html = 1; + else if (strfield2(a, "htm")) + html = 1; + else if (strfield2(a, "shtml")) + html = 1; + else if (strfield2(a, "phtml")) + html = 1; + else if (strfield2(a, "htmlx")) + html = 1; + else if (strfield2(a, "shtm")) + html = 1; + else if (strfield2(a, "phtm")) + html = 1; + else if (strfield2(a, "htmx")) + html = 1; // // insuccès.. else { #if 1 - html = -1; // inconnu.. + html = -1; // inconnu.. #else // XXXXXX not suitable (ext) - switch(is_knownext(a)) { + switch (is_knownext(a)) { case 1: - html = 0; // connu, non html + html = 0; // connu, non html break; case 2: - html = 1; // connu, html + html = 1; // connu, html break; default: - html = -1; // inconnu.. + html = -1; // inconnu.. break; } #endif } - return html; + return html; } // error (404,500..) HTS_INLINE int ishttperror(int err) { - switch (err/100) { - case 4: case 5: return 1; - break; + switch (err / 100) { + case 4: + case 5: + return 1; + break; } return 0; } - // retourne le pointeur ou le pointeur + offset si il existe dans la chaine un @ signifiant // une identification -HTSEXT_API char* jump_identification(const char* source) { - const char *a,*trytofind; +HTSEXT_API char *jump_identification(const char *source) { + const char *a, *trytofind; + if (strcmp(source, "file://") == 0) - return (char*) source; + return (char *) source; // rechercher dernier @ (car parfois email transmise dans adresse!) // mais sauter ftp:// éventuel a = jump_protocol(source); - trytofind = strrchr_limit(a, '@', strchr(a,'/')); - return (char*) ( (trytofind != NULL) ? trytofind : a ); + trytofind = strrchr_limit(a, '@', strchr(a, '/')); + return (char *) ((trytofind != NULL) ? trytofind : a); } -HTSEXT_API char* jump_normalized(const char* source) { +HTSEXT_API char *jump_normalized(const char *source) { if (strcmp(source, "file://") == 0) - return (char*) source; - source = jump_identification(source); + return (char *) source; + source = jump_identification(source); if (strfield(source, "www") && source[3] != '\0') { - if (source[3] == '.') { // www.foo.com -> foo.com - source += 4; - } else { // www-4.foo.com -> foo.com - const char* a = source + 3; - while(*a && ( isdigit(*a) || *a == '-') ) a++; + if (source[3] == '.') { // www.foo.com -> foo.com + source += 4; + } else { // www-4.foo.com -> foo.com + const char *a = source + 3; + + while(*a && (isdigit(*a) || *a == '-')) + a++; if (*a == '.') { source = a + 1; } } } - return (char*) source; + return (char *) source; } -static int sortNormFnc(const void * a_, const void * b_) { - char** a = (char**) a_; - char** b = (char**) b_; - return strcmp(*a+1, *b+1); -} +static int sortNormFnc(const void *a_, const void *b_) { + char **a = (char **) a_; + char **b = (char **) b_; + return strcmp(*a + 1, *b + 1); +} -HTSEXT_API char* fil_normalized(const char* source, char* dest) { +HTSEXT_API char *fil_normalized(const char *source, char *dest) { char lastc = 0; - int gotquery=0; - int ampargs=0; - int i,j; - char* query=NULL; - for(i=j=0 ; source[i] != '\0'; i++) { + int gotquery = 0; + int ampargs = 0; + int i, j; + char *query = NULL; + + for(i = j = 0; source[i] != '\0'; i++) { if (!gotquery && source[i] == '?') - gotquery=ampargs=1; - if ( - (!gotquery && lastc == '/' && source[i] == '/') // foo//bar -> foo/bar + gotquery = ampargs = 1; + if ((!gotquery && lastc == '/' && source[i] == '/') // foo//bar -> foo/bar ) { - } - else { + } else { if (gotquery && source[i] == '&') { ampargs++; } @@ -3295,17 +3556,18 @@ HTSEXT_API char* fil_normalized(const char* source, char* dest) { /* Sort arguments (&foo=1&bar=2 == &bar=2&foo=1) */ if (ampargs > 1) { - char** amps = malloct(ampargs * sizeof(char*)); - char* copyBuff = NULL; - int qLen=0; + char **amps = malloct(ampargs * sizeof(char *)); + char *copyBuff = NULL; + int qLen = 0; + assertf(amps != NULL); gotquery = 0; - for(i=j=0 ; dest[i] != '\0'; i++) { - if ( (gotquery && dest[i] == '&') || ( !gotquery && dest[i] == '?') ) { + for(i = j = 0; dest[i] != '\0'; i++) { + if ((gotquery && dest[i] == '&') || (!gotquery && dest[i] == '?')) { if (!gotquery) { - gotquery=1; + gotquery = 1; query = &dest[i]; - qLen = (int)strlen(query); + qLen = (int) strlen(query); } assertf(j < ampargs); amps[j++] = &dest[i]; @@ -3315,391 +3577,408 @@ HTSEXT_API char* fil_normalized(const char* source, char* dest) { assertf(j == ampargs); /* Sort 'em all */ - qsort(amps, ampargs, sizeof(char*), sortNormFnc); + qsort(amps, ampargs, sizeof(char *), sortNormFnc); /* Replace query by sorted query */ copyBuff = malloct(qLen + 1); assertf(copyBuff != NULL); copyBuff[0] = '\0'; - for(i = 0 ; i < ampargs ; i++) { + for(i = 0; i < ampargs; i++) { if (i == 0) strcatbuff(copyBuff, "?"); else strcatbuff(copyBuff, "&"); strcatbuff(copyBuff, amps[i] + 1); } - assert((int)strlen(copyBuff) <= qLen); + assert((int) strlen(copyBuff) <= qLen); strcpybuff(query, copyBuff); /* Cleanup */ freet(amps); freet(copyBuff); } - + return dest; } #define endwith(a) ( (len >= (sizeof(a)-1)) ? ( strncmp(dest, a+len-(sizeof(a)-1), sizeof(a)-1) == 0 ) : 0 ); -HTSEXT_API char* adr_normalized(const char* source, char* dest) { +HTSEXT_API char *adr_normalized(const char *source, char *dest) { /* not yet too aggressive (no com<->net<->org checkings) */ strcpybuff(dest, jump_normalized(source)); return dest; } -#undef endwith +#undef endwith // find port (:80) or NULL if not found // can handle IPV6 addresses -HTSEXT_API char* jump_toport(const char* source) { - const char *a,*trytofind; +HTSEXT_API char *jump_toport(const char *source) { + const char *a, *trytofind; + a = jump_identification(source); - trytofind = strrchr_limit(a, ']', strchr(source, '/')); // find last ] (http://[3ffe:b80:1234::1]:80/foo.html) - a = strchr( (trytofind)?trytofind:a, ':'); - return (char*)a; + trytofind = strrchr_limit(a, ']', strchr(source, '/')); // find last ] (http://[3ffe:b80:1234::1]:80/foo.html) + a = strchr((trytofind) ? trytofind : a, ':'); + return (char *) a; } // strrchr, but not too far -char* strrchr_limit(const char* s, char c, const char* limit) { +char *strrchr_limit(const char *s, char c, const char *limit) { if (limit == NULL) { - const char* p = strrchr(s, c); - return (char*) ( p ? (p+1) : NULL ); + const char *p = strrchr(s, c); + + return (char *) (p ? (p + 1) : NULL); } else { const char *a = NULL, *p; + for(;;) { - p = strchr( (a) ? a : s, c); + p = strchr((a) ? a : s, c); if ((p >= limit) || (p == NULL)) - return (char*) a; - a=p+1; + return (char *) a; + a = p + 1; } } } // strrchr, but not too far -char* strstr_limit(const char* s, const char* sub, const char* limit) { +char *strstr_limit(const char *s, const char *sub, const char *limit) { if (limit == NULL) { return strstr(s, sub); } else { - const char* pos = strstr(s, sub); + const char *pos = strstr(s, sub); + if (pos != NULL) { - const char* farpos = strstr(s, limit); + const char *farpos = strstr(s, limit); + if (farpos == NULL || pos < farpos) - return (char*) pos; + return (char *) pos; } } return NULL; } // retourner adr sans ftp:// -HTS_INLINE char* jump_protocol(const char* source) { +HTS_INLINE char *jump_protocol(const char *source) { int p; + // scheme // "Comparisons of scheme names MUST be case-insensitive" (RFC2616) - if ((p=strfield(source,"http:"))) - source+=p; - else if ((p=strfield(source,"ftp:"))) - source+=p; - else if ((p=strfield(source,"https:"))) - source+=p; - else if ((p=strfield(source,"file:"))) - source+=p; + if ((p = strfield(source, "http:"))) + source += p; + else if ((p = strfield(source, "ftp:"))) + source += p; + else if ((p = strfield(source, "https:"))) + source += p; + else if ((p = strfield(source, "file:"))) + source += p; #if HTS_USEMMS - else if ((p=strfield(source,"mms:"))) - source+=p; + else if ((p = strfield(source, "mms:"))) + source += p; #endif // net_path - if (strncmp(source,"//",2)==0) - source+=2; - return (char*) source; + if (strncmp(source, "//", 2) == 0) + source += 2; + return (char *) source; } // codage base 64 a vers b -void code64(unsigned char* a,int size_a,unsigned char* b,int crlf) { - int i1=0,i2=0,i3=0,i4=0; - int loop=0; +void code64(unsigned char *a, int size_a, unsigned char *b, int crlf) { + int i1 = 0, i2 = 0, i3 = 0, i4 = 0; + int loop = 0; unsigned long int store; int n; - const char _hts_base64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - while(size_a-- > 0) { + const char _hts_base64[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + while(size_a-- > 0) { // 24 bits - n=1; + n = 1; store = *a++; - if (size_a-- > 0) { n=2; store <<= 8; store |= *a++; } - if (size_a-- > 0) { n=3; store <<= 8; store |= *a++; } - if (n==3) { - i4=store & 63; - i3=(store>>6) & 63; - i2=(store>>12) & 63; - i1=(store>>18) & 63; - } else if (n==2) { - store<<=2; - i3=store & 63; - i2=(store>>6) & 63; - i1=(store>>12) & 63; + if (size_a-- > 0) { + n = 2; + store <<= 8; + store |= *a++; + } + if (size_a-- > 0) { + n = 3; + store <<= 8; + store |= *a++; + } + if (n == 3) { + i4 = store & 63; + i3 = (store >> 6) & 63; + i2 = (store >> 12) & 63; + i1 = (store >> 18) & 63; + } else if (n == 2) { + store <<= 2; + i3 = store & 63; + i2 = (store >> 6) & 63; + i1 = (store >> 12) & 63; } else { - store<<=4; - i2=store & 63; - i1=(store>>6) & 63; + store <<= 4; + i2 = store & 63; + i1 = (store >> 6) & 63; } - + *b++ = _hts_base64[i1]; *b++ = _hts_base64[i2]; - if (n>=2) + if (n >= 2) *b++ = _hts_base64[i3]; else *b++ = '='; - if (n>=3) + if (n >= 3) *b++ = _hts_base64[i4]; else *b++ = '='; - if (crlf && ( ( loop += 3 ) % 60) == 0 ) { + if (crlf && ((loop += 3) % 60) == 0) { *b++ = '\r'; *b++ = '\n'; } } - *b++='\0'; + *b++ = '\0'; } // remplacer " par " etc.. // buffer MAX 1Ko #define strcmpbeg(a, b) strncmp(a, b, strlen(b)) -HTSEXT_API void unescape_amp(char* s) { +HTSEXT_API void unescape_amp(char *s) { while(*s) { - if (*s=='&') { - char* end=strchr(s,';'); - if ( end && (((int) (end - s)) <= 8) ) { - unsigned char c=0; - + if (*s == '&') { + char *end = strchr(s, ';'); + + if (end && (((int) (end - s)) <= 8)) { + unsigned char c = 0; + // http://www.w3.org/TR/xhtml-modularization/dtd_module_defs.html if (strcmpbeg(s, "&#") == 0) { - int num=0; - if ( (s[2] == 'x') || (s[2] == 'X')) { - if (sscanf(s+3, "%x", &num) == 1 && num <= 0xff) { - c=(unsigned char) num; + int num = 0; + + if ((s[2] == 'x') || (s[2] == 'X')) { + if (sscanf(s + 3, "%x", &num) == 1 && num <= 0xff) { + c = (unsigned char) num; } } else { - if (sscanf(s+2, "%d", &num) == 1 && num <= 0xff) { - c=(unsigned char) num; + if (sscanf(s + 2, "%d", &num) == 1 && num <= 0xff) { + c = (unsigned char) num; } } - } - else if (strcmpbeg(s, " ")==0) - c=32; // hack - c=160; - else if (strcmpbeg(s, "¡")==0) - c=161; - else if (strcmpbeg(s, "¢")==0) - c=162; - else if (strcmpbeg(s, "£")==0) - c=163; - else if (strcmpbeg(s, "¤")==0) - c=164; - else if (strcmpbeg(s, "¥")==0) - c=165; - else if (strcmpbeg(s, "¦")==0) - c=166; - else if (strcmpbeg(s, "§")==0) - c=167; - else if (strcmpbeg(s, "¨")==0) - c=168; - else if (strcmpbeg(s, "©")==0) - c=169; - else if (strcmpbeg(s, "ª")==0) - c=170; + } else if (strcmpbeg(s, " ") == 0) + c = 32; // hack - c=160; + else if (strcmpbeg(s, "¡") == 0) + c = 161; + else if (strcmpbeg(s, "¢") == 0) + c = 162; + else if (strcmpbeg(s, "£") == 0) + c = 163; + else if (strcmpbeg(s, "¤") == 0) + c = 164; + else if (strcmpbeg(s, "¥") == 0) + c = 165; + else if (strcmpbeg(s, "¦") == 0) + c = 166; + else if (strcmpbeg(s, "§") == 0) + c = 167; + else if (strcmpbeg(s, "¨") == 0) + c = 168; + else if (strcmpbeg(s, "©") == 0) + c = 169; + else if (strcmpbeg(s, "ª") == 0) + c = 170; //else if (strcmpbeg(s, "«")==0) // c=171; - else if (strcmpbeg(s, "¬")==0) - c=172; + else if (strcmpbeg(s, "¬") == 0) + c = 172; //else if (strcmpbeg(s, "­")==0) // c=173; - else if (strcmpbeg(s, "®")==0) - c=174; - else if (strcmpbeg(s, "¯")==0) - c=175; - else if (strcmpbeg(s, "°")==0) - c=176; - else if (strcmpbeg(s, "±")==0) - c=177; - else if (strcmpbeg(s, "²")==0) - c=178; - else if (strcmpbeg(s, "³")==0) - c=179; - else if (strcmpbeg(s, "´")==0) - c=180; - else if (strcmpbeg(s, "µ")==0) - c=181; - else if (strcmpbeg(s, "¶")==0) - c=182; - else if (strcmpbeg(s, "·")==0) - c=183; - else if (strcmpbeg(s, "¸")==0) - c=184; - else if (strcmpbeg(s, "¹")==0) - c=185; - else if (strcmpbeg(s, "º")==0) - c=186; + else if (strcmpbeg(s, "®") == 0) + c = 174; + else if (strcmpbeg(s, "¯") == 0) + c = 175; + else if (strcmpbeg(s, "°") == 0) + c = 176; + else if (strcmpbeg(s, "±") == 0) + c = 177; + else if (strcmpbeg(s, "²") == 0) + c = 178; + else if (strcmpbeg(s, "³") == 0) + c = 179; + else if (strcmpbeg(s, "´") == 0) + c = 180; + else if (strcmpbeg(s, "µ") == 0) + c = 181; + else if (strcmpbeg(s, "¶") == 0) + c = 182; + else if (strcmpbeg(s, "·") == 0) + c = 183; + else if (strcmpbeg(s, "¸") == 0) + c = 184; + else if (strcmpbeg(s, "¹") == 0) + c = 185; + else if (strcmpbeg(s, "º") == 0) + c = 186; //else if (strcmpbeg(s, "»")==0) // c=187; - else if (strcmpbeg(s, "¼")==0) - c=188; - else if (strcmpbeg(s, "½")==0) - c=189; - else if (strcmpbeg(s, "¾")==0) - c=190; - else if (strcmpbeg(s, "¿")==0) - c=191; - else if (strcmpbeg(s, "À")==0) - c=192; - else if (strcmpbeg(s, "Á")==0) - c=193; - else if (strcmpbeg(s, "Â")==0) - c=194; - else if (strcmpbeg(s, "Ã")==0) - c=195; - else if (strcmpbeg(s, "Ä")==0) - c=196; - else if (strcmpbeg(s, "Å")==0) - c=197; - else if (strcmpbeg(s, "Æ")==0) - c=198; - else if (strcmpbeg(s, "Ç")==0) - c=199; - else if (strcmpbeg(s, "È")==0) - c=200; - else if (strcmpbeg(s, "É")==0) - c=201; - else if (strcmpbeg(s, "Ê")==0) - c=202; - else if (strcmpbeg(s, "Ë")==0) - c=203; - else if (strcmpbeg(s, "Ì")==0) - c=204; - else if (strcmpbeg(s, "Í")==0) - c=205; - else if (strcmpbeg(s, "Î")==0) - c=206; - else if (strcmpbeg(s, "Ï")==0) - c=207; - else if (strcmpbeg(s, "Ð")==0) - c=208; - else if (strcmpbeg(s, "Ñ")==0) - c=209; - else if (strcmpbeg(s, "Ò")==0) - c=210; - else if (strcmpbeg(s, "Ó")==0) - c=211; - else if (strcmpbeg(s, "Ô")==0) - c=212; - else if (strcmpbeg(s, "Õ")==0) - c=213; - else if (strcmpbeg(s, "Ö")==0) - c=214; - else if (strcmpbeg(s, "×")==0) - c=215; - else if (strcmpbeg(s, "Ø")==0) - c=216; - else if (strcmpbeg(s, "Ù")==0) - c=217; - else if (strcmpbeg(s, "Ú")==0) - c=218; - else if (strcmpbeg(s, "Û")==0) - c=219; - else if (strcmpbeg(s, "Ü")==0) - c=220; - else if (strcmpbeg(s, "Ý")==0) - c=221; - else if (strcmpbeg(s, "Þ")==0) - c=222; - else if (strcmpbeg(s, "ß")==0) - c=223; - else if (strcmpbeg(s, "à")==0) - c=224; - else if (strcmpbeg(s, "á")==0) - c=225; - else if (strcmpbeg(s, "â")==0) - c=226; - else if (strcmpbeg(s, "ã")==0) - c=227; - else if (strcmpbeg(s, "ä")==0) - c=228; - else if (strcmpbeg(s, "å")==0) - c=229; - else if (strcmpbeg(s, "æ")==0) - c=230; - else if (strcmpbeg(s, "ç")==0) - c=231; - else if (strcmpbeg(s, "è")==0) - c=232; - else if (strcmpbeg(s, "é")==0) - c=233; - else if (strcmpbeg(s, "ê")==0) - c=234; - else if (strcmpbeg(s, "ë")==0) - c=235; - else if (strcmpbeg(s, "ì")==0) - c=236; - else if (strcmpbeg(s, "í")==0) - c=237; - else if (strcmpbeg(s, "î")==0) - c=238; - else if (strcmpbeg(s, "ï")==0) - c=239; - else if (strcmpbeg(s, "ð")==0) - c=240; - else if (strcmpbeg(s, "ñ")==0) - c=241; - else if (strcmpbeg(s, "ò")==0) - c=242; - else if (strcmpbeg(s, "ó")==0) - c=243; - else if (strcmpbeg(s, "ô")==0) - c=244; - else if (strcmpbeg(s, "õ")==0) - c=245; - else if (strcmpbeg(s, "ö")==0) - c=246; - else if (strcmpbeg(s, "÷")==0) - c=247; - else if (strcmpbeg(s, "ø")==0) - c=248; - else if (strcmpbeg(s, "ù")==0) - c=249; - else if (strcmpbeg(s, "ú")==0) - c=250; - else if (strcmpbeg(s, "û")==0) - c=251; - else if (strcmpbeg(s, "ü")==0) - c=252; - else if (strcmpbeg(s, "ý")==0) - c=253; - else if (strcmpbeg(s, "þ")==0) - c=254; - else if (strcmpbeg(s, "ÿ")==0) - c=255; + else if (strcmpbeg(s, "¼") == 0) + c = 188; + else if (strcmpbeg(s, "½") == 0) + c = 189; + else if (strcmpbeg(s, "¾") == 0) + c = 190; + else if (strcmpbeg(s, "¿") == 0) + c = 191; + else if (strcmpbeg(s, "À") == 0) + c = 192; + else if (strcmpbeg(s, "Á") == 0) + c = 193; + else if (strcmpbeg(s, "Â") == 0) + c = 194; + else if (strcmpbeg(s, "Ã") == 0) + c = 195; + else if (strcmpbeg(s, "Ä") == 0) + c = 196; + else if (strcmpbeg(s, "Å") == 0) + c = 197; + else if (strcmpbeg(s, "Æ") == 0) + c = 198; + else if (strcmpbeg(s, "Ç") == 0) + c = 199; + else if (strcmpbeg(s, "È") == 0) + c = 200; + else if (strcmpbeg(s, "É") == 0) + c = 201; + else if (strcmpbeg(s, "Ê") == 0) + c = 202; + else if (strcmpbeg(s, "Ë") == 0) + c = 203; + else if (strcmpbeg(s, "Ì") == 0) + c = 204; + else if (strcmpbeg(s, "Í") == 0) + c = 205; + else if (strcmpbeg(s, "Î") == 0) + c = 206; + else if (strcmpbeg(s, "Ï") == 0) + c = 207; + else if (strcmpbeg(s, "Ð") == 0) + c = 208; + else if (strcmpbeg(s, "Ñ") == 0) + c = 209; + else if (strcmpbeg(s, "Ò") == 0) + c = 210; + else if (strcmpbeg(s, "Ó") == 0) + c = 211; + else if (strcmpbeg(s, "Ô") == 0) + c = 212; + else if (strcmpbeg(s, "Õ") == 0) + c = 213; + else if (strcmpbeg(s, "Ö") == 0) + c = 214; + else if (strcmpbeg(s, "×") == 0) + c = 215; + else if (strcmpbeg(s, "Ø") == 0) + c = 216; + else if (strcmpbeg(s, "Ù") == 0) + c = 217; + else if (strcmpbeg(s, "Ú") == 0) + c = 218; + else if (strcmpbeg(s, "Û") == 0) + c = 219; + else if (strcmpbeg(s, "Ü") == 0) + c = 220; + else if (strcmpbeg(s, "Ý") == 0) + c = 221; + else if (strcmpbeg(s, "Þ") == 0) + c = 222; + else if (strcmpbeg(s, "ß") == 0) + c = 223; + else if (strcmpbeg(s, "à") == 0) + c = 224; + else if (strcmpbeg(s, "á") == 0) + c = 225; + else if (strcmpbeg(s, "â") == 0) + c = 226; + else if (strcmpbeg(s, "ã") == 0) + c = 227; + else if (strcmpbeg(s, "ä") == 0) + c = 228; + else if (strcmpbeg(s, "å") == 0) + c = 229; + else if (strcmpbeg(s, "æ") == 0) + c = 230; + else if (strcmpbeg(s, "ç") == 0) + c = 231; + else if (strcmpbeg(s, "è") == 0) + c = 232; + else if (strcmpbeg(s, "é") == 0) + c = 233; + else if (strcmpbeg(s, "ê") == 0) + c = 234; + else if (strcmpbeg(s, "ë") == 0) + c = 235; + else if (strcmpbeg(s, "ì") == 0) + c = 236; + else if (strcmpbeg(s, "í") == 0) + c = 237; + else if (strcmpbeg(s, "î") == 0) + c = 238; + else if (strcmpbeg(s, "ï") == 0) + c = 239; + else if (strcmpbeg(s, "ð") == 0) + c = 240; + else if (strcmpbeg(s, "ñ") == 0) + c = 241; + else if (strcmpbeg(s, "ò") == 0) + c = 242; + else if (strcmpbeg(s, "ó") == 0) + c = 243; + else if (strcmpbeg(s, "ô") == 0) + c = 244; + else if (strcmpbeg(s, "õ") == 0) + c = 245; + else if (strcmpbeg(s, "ö") == 0) + c = 246; + else if (strcmpbeg(s, "÷") == 0) + c = 247; + else if (strcmpbeg(s, "ø") == 0) + c = 248; + else if (strcmpbeg(s, "ù") == 0) + c = 249; + else if (strcmpbeg(s, "ú") == 0) + c = 250; + else if (strcmpbeg(s, "û") == 0) + c = 251; + else if (strcmpbeg(s, "ü") == 0) + c = 252; + else if (strcmpbeg(s, "ý") == 0) + c = 253; + else if (strcmpbeg(s, "þ") == 0) + c = 254; + else if (strcmpbeg(s, "ÿ") == 0) + c = 255; // - else if (strcmpbeg(s,"&")==0) - c='&'; - else if (strcmpbeg(s,">")==0) - c='>'; - else if (strcmpbeg(s,"«")==0) - c='\"'; - else if (strcmpbeg(s,"<")==0) - c='<'; - else if (strcmpbeg(s," ")==0) - c=' '; - else if (strcmpbeg(s,""")==0) - c='\"'; - else if (strcmpbeg(s,"»")==0) - c='\"'; - else if (strcmpbeg(s,"­")==0) - c='-'; - else if (strcmpbeg(s,"˜")==0) - c='~'; + else if (strcmpbeg(s, "&") == 0) + c = '&'; + else if (strcmpbeg(s, ">") == 0) + c = '>'; + else if (strcmpbeg(s, "«") == 0) + c = '\"'; + else if (strcmpbeg(s, "<") == 0) + c = '<'; + else if (strcmpbeg(s, " ") == 0) + c = ' '; + else if (strcmpbeg(s, """) == 0) + c = '\"'; + else if (strcmpbeg(s, "»") == 0) + c = '\"'; + else if (strcmpbeg(s, "­") == 0) + c = '-'; + else if (strcmpbeg(s, "˜") == 0) + c = '~'; // remplacer? if (c) { - char BIGSTK buff[HTS_URLMAXSIZE*2]; - buff[0]=(char) c; - strcpybuff(buff+1,end+1); - strcpybuff(s,buff); + char BIGSTK buff[HTS_URLMAXSIZE * 2]; + + buff[0] = (char) c; + strcpybuff(buff + 1, end + 1); + strcpybuff(s, buff); } } } @@ -3708,183 +3987,189 @@ HTSEXT_API void unescape_amp(char* s) { } static int ehexh(char c) { - if ((c>='0') && (c<='9')) return c-'0'; - if ((c>='a') && (c<='f')) c-=('a'-'A'); - if ((c>='A') && (c<='F')) return (c-'A'+10); + if ((c >= '0') && (c <= '9')) + return c - '0'; + if ((c >= 'a') && (c <= 'f')) + c -= ('a' - 'A'); + if ((c >= 'A') && (c <= 'F')) + return (c - 'A' + 10); return 0; } -static int ehex(const char* s) { - return 16*ehexh(*s)+ehexh(*(s+1)); +static int ehex(const char *s) { + return 16 * ehexh(*s) + ehexh(*(s + 1)); } // remplacer %20 par ' ', | par : etc.. // buffer MAX 1Ko -HTSEXT_API char* unescape_http(char *catbuff, const char* s) { - int i,j=0; - for (i=0;i<(int) strlen(s);i++) { - if (s[i]=='%') { +HTSEXT_API char *unescape_http(char *catbuff, const char *s) { + int i, j = 0; + + for(i = 0; i < (int) strlen(s); i++) { + if (s[i] == '%') { i++; - catbuff[j++]=(char) ehex(s+i); - i++; // sauter 2 caractères finalement + catbuff[j++] = (char) ehex(s + i); + i++; // sauter 2 caractères finalement } /* - NON a cause de trucs comme /home/0,1837,1|7|1173|Content,00.html - else if (s[i]=='|') { // exemple: file:///C|Program%20Files... - tempo[j++]=':'; - } - */ + NON a cause de trucs comme /home/0,1837,1|7|1173|Content,00.html + else if (s[i]=='|') { // exemple: file:///C|Program%20Files... + tempo[j++]=':'; + } + */ else - catbuff[j++]=s[i]; + catbuff[j++] = s[i]; } - catbuff[j++]='\0'; + catbuff[j++] = '\0'; return catbuff; } // unescape in URL/URI ONLY what has to be escaped, to form a standard URL/URI // DOES NOT DECODE %25 (part of CHAR_DELIM) -HTSEXT_API char* unescape_http_unharm(char *catbuff, const char* s, int no_high) { - int i,j=0; - for (i=0;i<(int) strlen(s);i++) { - if (s[i]=='%') { - int nchar=(char) ehex(s+i+1); - - int test = ( ( CHAR_RESERVED(nchar) && nchar != '+' ) /* %2B => + (not in query!) */ - || CHAR_DELIM(nchar) - || CHAR_UNWISE(nchar) - || CHAR_LOW(nchar) /* CHAR_SPECIAL */ - || CHAR_XXAVOID(nchar) - || ( - (no_high) - && - CHAR_HIG(nchar) - ) - ); +HTSEXT_API char *unescape_http_unharm(char *catbuff, const char *s, int no_high) { + int i, j = 0; + + for(i = 0; i < (int) strlen(s); i++) { + if (s[i] == '%') { + int nchar = (char) ehex(s + i + 1); + + int test = ((CHAR_RESERVED(nchar) && nchar != '+') /* %2B => + (not in query!) */ + ||CHAR_DELIM(nchar) + || CHAR_UNWISE(nchar) + || CHAR_LOW(nchar) /* CHAR_SPECIAL */ + ||CHAR_XXAVOID(nchar) + || ((no_high) + && CHAR_HIG(nchar) + ) + ); if (!test) { - catbuff[j++]=(char) ehex(s+i+1); - i+=2; + catbuff[j++] = (char) ehex(s + i + 1); + i += 2; } else { - catbuff[j++]='%'; + catbuff[j++] = '%'; } } /* - NON a cause de trucs comme /home/0,1837,1|7|1173|Content,00.html - else if (s[i]=='|') { // exemple: file:///C|Program%20Files... - tempo[j++]=':'; - } - */ + NON a cause de trucs comme /home/0,1837,1|7|1173|Content,00.html + else if (s[i]=='|') { // exemple: file:///C|Program%20Files... + tempo[j++]=':'; + } + */ else - catbuff[j++]=s[i]; + catbuff[j++] = s[i]; } - catbuff[j++]='\0'; + catbuff[j++] = '\0'; return catbuff; } // remplacer " par %xx etc.. // buffer MAX 1Ko -HTSEXT_API void escape_spc_url(char* s) { - x_escape_http(s,2); +HTSEXT_API void escape_spc_url(char *s) { + x_escape_http(s, 2); } + // smith / john -> smith%20%2f%20john -HTSEXT_API void escape_in_url(char* s) { - x_escape_http(s,1); +HTSEXT_API void escape_in_url(char *s) { + x_escape_http(s, 1); } + // smith / john -> smith%20/%20john -HTSEXT_API void escape_uri(char* s) { - x_escape_http(s,3); +HTSEXT_API void escape_uri(char *s) { + x_escape_http(s, 3); } -HTSEXT_API void escape_uri_utf(char* s) { - x_escape_http(s,30); +HTSEXT_API void escape_uri_utf(char *s) { + x_escape_http(s, 30); } -HTSEXT_API void escape_check_url(char* s) { - x_escape_http(s,0); +HTSEXT_API void escape_check_url(char *s) { + x_escape_http(s, 0); } + // same as escape_check_url, but returns char* -HTSEXT_API char* escape_check_url_addr(char *catbuff, const char* s) { - char* adr; +HTSEXT_API char *escape_check_url_addr(char *catbuff, const char *s) { + char *adr; + escape_check_url(adr = concat(catbuff, s, "")); return adr; } // strip all control characters -HTSEXT_API void escape_remove_control(char* s) { - unsigned char* ss = (unsigned char*) s; +HTSEXT_API void escape_remove_control(char *s) { + unsigned char *ss = (unsigned char *) s; + while(*ss) { - if (*ss < 32) { /* CONTROL characters go away! */ - char BIGSTK tmp[HTS_URLMAXSIZE*2]; - strcpybuff(tmp, (char*) ss+1); - strcpybuff((char*) ss, tmp); + if (*ss < 32) { /* CONTROL characters go away! */ + char BIGSTK tmp[HTS_URLMAXSIZE * 2]; + + strcpybuff(tmp, (char *) ss + 1); + strcpybuff((char *) ss, tmp); } else { ss++; } } } -HTSEXT_API void x_escape_html(char* s) { +HTSEXT_API void x_escape_html(char *s) { while(*s) { - int test=0; - test = ( - CHAR_HIG(*s) - || CHAR_XXAVOID(*s) ); + int test = 0; + + test = (CHAR_HIG(*s) + || CHAR_XXAVOID(*s)); if (test) { - char BIGSTK buffer[HTS_URLMAXSIZE*3]; + char BIGSTK buffer[HTS_URLMAXSIZE * 3]; int n; - n = (int)(unsigned char) *s; - strcpybuff(buffer, s+1); - sprintf(s,"&#x%02x;", n); + + n = (int) (unsigned char) *s; + strcpybuff(buffer, s + 1); + sprintf(s, "&#x%02x;", n); strcatbuff(s, buffer); } s++; } } - -HTSEXT_API void x_escape_http(char* s,int mode) { +HTSEXT_API void x_escape_http(char *s, int mode) { while(*s) { - int test=0; + int test = 0; + if (mode == 0) - test=(strchr("\" ",*s)!=0 || CHAR_SPECIAL(*s)); - else if (mode==1) { - test = ( CHAR_RESERVED(*s) - || CHAR_DELIM(*s) - || CHAR_UNWISE(*s) - || CHAR_SPECIAL(*s) - || CHAR_XXAVOID(*s) - || CHAR_MARK(*s)); - } - else if (mode==2) - test=(*s == ' '); // n'escaper que espace - else if (mode==3) { // échapper que ce qui est nécessaire - test = ( - CHAR_SPECIAL(*s) - || CHAR_XXAVOID(*s) ); - } - else if (mode==30) { // échapper que ce qui est nécessaire - test = - ( *s != '/' && CHAR_RESERVED(*s) ) + test = (strchr("\" ", *s) != 0 || CHAR_SPECIAL(*s)); + else if (mode == 1) { + test = (CHAR_RESERVED(*s) + || CHAR_DELIM(*s) + || CHAR_UNWISE(*s) + || CHAR_SPECIAL(*s) + || CHAR_XXAVOID(*s) + || CHAR_MARK(*s)); + } else if (mode == 2) + test = (*s == ' '); // n'escaper que espace + else if (mode == 3) { // échapper que ce qui est nécessaire + test = (CHAR_SPECIAL(*s) + || CHAR_XXAVOID(*s)); + } else if (mode == 30) { // échapper que ce qui est nécessaire + test = (*s != '/' && CHAR_RESERVED(*s)) || CHAR_DELIM(*s) || CHAR_UNWISE(*s) || CHAR_SPECIAL(*s) - || CHAR_XXAVOID(*s) - ; + || CHAR_XXAVOID(*s); } if (test) { - char BIGSTK buffer[HTS_URLMAXSIZE*3]; + char BIGSTK buffer[HTS_URLMAXSIZE * 3]; int n; - n=(int)(unsigned char) *s; - strcpybuff(buffer,s+1); - sprintf(s,"%%%02x",n); - strcatbuff(s,buffer); + + n = (int) (unsigned char) *s; + strcpybuff(buffer, s + 1); + sprintf(s, "%%%02x", n); + strcatbuff(s, buffer); } s++; } } -HTSEXT_API void escape_for_html_print(char* s, char* d) { - for( ; *s ; s++) { +HTSEXT_API void escape_for_html_print(char *s, char *d) { + for(; *s; s++) { if (*s == '&') { strcpybuff(d, "&"); d += strlen(d); @@ -3895,8 +4180,8 @@ HTSEXT_API void escape_for_html_print(char* s, char* d) { *d = '\0'; } -HTSEXT_API void escape_for_html_print_full(char* s, char* d) { - for( ; *s ; s++) { +HTSEXT_API void escape_for_html_print_full(char *s, char *d) { + for(; *s; s++) { if (*s == '&') { strcpybuff(d, "&"); d += strlen(d); @@ -3913,71 +4198,75 @@ HTSEXT_API void escape_for_html_print_full(char* s, char* d) { // concat, concatène deux chaines et renvoi le résultat // permet d'alléger grandement le code // il faut savoir qu'on ne peut mettre plus de 16 concat() dans une expression -HTSEXT_API char* concat(char *catbuff,const char* a,const char* b) { - if (a != NULL && a[0] != '\0') { - strcpybuff(catbuff, a); - } else { - catbuff[0] = '\0'; - } - if (b != NULL && b[0] != '\0') { - strcatbuff(catbuff, b); - } +HTSEXT_API char *concat(char *catbuff, const char *a, const char *b) { + if (a != NULL && a[0] != '\0') { + strcpybuff(catbuff, a); + } else { + catbuff[0] = '\0'; + } + if (b != NULL && b[0] != '\0') { + strcatbuff(catbuff, b); + } return catbuff; } + // conversion fichier / -> antislash -static char* __fconv(char* a) { +static char *__fconv(char *a) { #if HTS_DOSNAME int i; - for(i = 0 ; a[i] != 0 ; i++) - if (a[i] == '/') // Unix-to-DOS style + + for(i = 0; a[i] != 0; i++) + if (a[i] == '/') // Unix-to-DOS style a[i] = '\\'; #endif return a; } -HTSEXT_API char* fconcat(char *catbuff, const char* a, const char* b) { - return __fconv(concat(catbuff,a,b)); +HTSEXT_API char *fconcat(char *catbuff, const char *a, const char *b) { + return __fconv(concat(catbuff, a, b)); } -HTSEXT_API char* fconv(char *catbuff, const char* a) { - return __fconv(concat(catbuff,a,"")); +HTSEXT_API char *fconv(char *catbuff, const char *a) { + return __fconv(concat(catbuff, a, "")); } /* / et \\ en / */ -static char* __fslash(char* a) { +static char *__fslash(char *a) { int i; - for(i = 0 ; a[i] != 0 ; i++) - if (a[i] == '\\') // convertir + + for(i = 0; a[i] != 0; i++) + if (a[i] == '\\') // convertir a[i] = '/'; return a; } -char* fslash(char *catbuff, const char* a) { - return __fslash(concat(catbuff,a,NULL)); +char *fslash(char *catbuff, const char *a) { + return __fslash(concat(catbuff, a, NULL)); } // conversion minuscules, avec buffer -char* convtolower(char *catbuff, const char* a) { - strcpybuff(catbuff,a); - hts_lowcase(catbuff); // lower case +char *convtolower(char *catbuff, const char *a) { + strcpybuff(catbuff, a); + hts_lowcase(catbuff); // lower case return catbuff; } // conversion en minuscules -void hts_lowcase(char* s) { +void hts_lowcase(char *s) { int i; - for(i=0;i<(int) strlen(s);i++) - if ((s[i]>='A') && (s[i]<='Z')) - s[i]+=('a'-'A'); + + for(i = 0; i < (int) strlen(s); i++) + if ((s[i] >= 'A') && (s[i] <= 'Z')) + s[i] += ('a' - 'A'); } // remplacer un caractère d'une chaîne dans une autre -HTS_INLINE void hts_replace(char *s,char from,char to) { - char* a; - while ((a=strchr(s,from))!=NULL) { - *a=to; +HTS_INLINE void hts_replace(char *s, char from, char to) { + char *a; + + while((a = strchr(s, from)) != NULL) { + *a = to; } } - // caractère espace, guillemets, CR, LF etc.. /* SECTION OPTIMISEE: #define is_space(c) (strchr(" \"\x0d\x0a\x09'",c)!=NULL) @@ -4007,47 +4296,47 @@ HTS_INLINE int is_realspace(char c) { } */ - - - - // deviner type d'un fichier local.. // ex: fil="toto.gif" -> s="image/gif" -void guess_httptype(httrackp *opt,char *s,const char *fil) { - get_httptype(opt,s, fil, 1); +void guess_httptype(httrackp * opt, char *s, const char *fil) { + get_httptype(opt, s, fil, 1); } + // idem // flag: 1 si toujours renvoyer un type -HTSEXT_API void get_httptype(httrackp *opt,char *s,const char *fil,int flag) { +HTSEXT_API void get_httptype(httrackp * opt, char *s, const char *fil, int flag) { // userdef overrides get_httptype if (get_userhttptype(opt, s, fil)) { - return ; + return; } // regular tests - if (ishtml(opt,fil) == 1) { - strcpybuff(s,"text/html"); + if (ishtml(opt, fil) == 1) { + strcpybuff(s, "text/html"); } else { /* Check html -> text/html */ - const char* a = fil + strlen(fil) - 1; - while ( (*a!='.') && (*a!='/') && (a>fil)) a--; - if (*a=='.' && strlen(a) < 32) { - int j=0; + const char *a = fil + strlen(fil) - 1; + + while((*a != '.') && (*a != '/') && (a > fil)) + a--; + if (*a == '.' && strlen(a) < 32) { + int j = 0; + a++; while(strnotempty(hts_mime[j][1])) { - if (strfield2(hts_mime[j][1],a)) { - if (hts_mime[j][0][0]!='*') { // Une correspondance existe - strcpybuff(s,hts_mime[j][0]); + if (strfield2(hts_mime[j][1], a)) { + if (hts_mime[j][0][0] != '*') { // Une correspondance existe + strcpybuff(s, hts_mime[j][0]); return; } } j++; } - + if (flag) - sprintf(s,"application/%s",a); + sprintf(s, "application/%s", a); } else { if (flag) - strcpybuff(s,"application/octet-stream"); + strcpybuff(s, "application/octet-stream"); } } } @@ -4055,10 +4344,10 @@ HTSEXT_API void get_httptype(httrackp *opt,char *s,const char *fil,int flag) { // get type of fil (php) // s: buffer (text/html) or NULL // return: 1 if known by user -int get_userhttptype(httrackp *opt, char *s, const char *fil) { +int get_userhttptype(httrackp * opt, char *s, const char *fil) { if (s != NULL) { if (s) - s[0]='\0'; + s[0] = '\0'; if (fil == NULL || *fil == '\0') return 0; #if 1 @@ -4068,47 +4357,55 @@ int get_userhttptype(httrackp *opt, char *s, const char *fil) { /* also: --assume baz,bar,foooo/foo/bar.cgi=text/html */ /* start from path beginning */ do { - const char* next; - const char* mimedefs = StringBuff(opt->mimedefs); /* loop through mime definitions : \nfoo=bar\nzoo=baz\n.. */ + const char *next; + const char *mimedefs = StringBuff(opt->mimedefs); /* loop through mime definitions : \nfoo=bar\nzoo=baz\n.. */ + while(*mimedefs != '\0') { - const char* segment = fil + 1; + const char *segment = fil + 1; + if (*mimedefs == '\n') { mimedefs++; } /* compare current segment with user's definition */ do { int i; + /* check current item */ - for(i = 0 ; - mimedefs[i] != '\0' /* end of all defs */ - && mimedefs[i] != ' ' /* next item in left list */ - && mimedefs[i] != '=' /* end of left list */ - && mimedefs[i] != '\n' /* end of this def (?) */ - && mimedefs[i] == segment[i] /* same item */ - ; i++); + for(i = 0; mimedefs[i] != '\0' /* end of all defs */ + && mimedefs[i] != ' ' /* next item in left list */ + && mimedefs[i] != '=' /* end of left list */ + && mimedefs[i] != '\n' /* end of this def (?) */ + && mimedefs[i] == segment[i] /* same item */ + ; i++) ; /* success */ - if ( ( mimedefs[i] == '=' || mimedefs[i] == ' ' ) && segment[i] == '\0') { + if ((mimedefs[i] == '=' || mimedefs[i] == ' ') + && segment[i] == '\0') { int i2; - while(mimedefs[i] != 0 && mimedefs[i] != '\n' && mimedefs[i] != '=') i++; + + while(mimedefs[i] != 0 && mimedefs[i] != '\n' + && mimedefs[i] != '=') + i++; if (mimedefs[i] == '=') { i++; - for(i2 = 0 ; mimedefs[i + i2] != '\n' && mimedefs[i + i2] != '\0' ; i2++) { + for(i2 = 0; + mimedefs[i + i2] != '\n' && mimedefs[i + i2] != '\0'; + i2++) { s[i2] = mimedefs[i + i2]; } s[i2] = '\0'; - return 1; /* SUCCESS! */ + return 1; /* SUCCESS! */ } } /* next item in list */ - for(mimedefs += i ; - *mimedefs != '\0' && *mimedefs != '\n' && *mimedefs != '=' && *mimedefs != ' ' ; - mimedefs++); + for(mimedefs += i; + *mimedefs != '\0' && *mimedefs != '\n' && *mimedefs != '=' + && *mimedefs != ' '; mimedefs++) ; if (*mimedefs == ' ') { mimedefs++; } } while(*mimedefs != '\0' && *mimedefs != '\n' && *mimedefs != '='); /* next user-def */ - for( ; *mimedefs != '\0' && *mimedefs != '\n' ; mimedefs++); + for(; *mimedefs != '\0' && *mimedefs != '\n'; mimedefs++) ; } /* shorten segment */ next = strchr(fil + 1, '/'); @@ -4122,24 +4419,24 @@ int get_userhttptype(httrackp *opt, char *s, const char *fil) { #else if (*buffer) { char BIGSTK search[1024]; - char* detect; + char *detect; - - sprintf(search,"\n%s=",ext); // php=text/html - detect=strstr(*buffer,search); + sprintf(search, "\n%s=", ext); // php=text/html + detect = strstr(*buffer, search); if (!detect) { - sprintf(search,"\n%s\n",ext); // php\ncgi=text/html - detect=strstr(*buffer,search); + sprintf(search, "\n%s\n", ext); // php\ncgi=text/html + detect = strstr(*buffer, search); } if (detect) { - detect=strchr(detect,'='); + detect = strchr(detect, '='); if (detect) { detect++; if (s) { - char* a; - a=strchr(detect,'\n'); + char *a; + + a = strchr(detect, '\n'); if (a) { - strncatbuff(s,detect,(int) (a - detect)); + strncatbuff(s, detect, (int) (a - detect)); } } return 1; @@ -4150,17 +4447,19 @@ int get_userhttptype(httrackp *opt, char *s, const char *fil) { } return 0; } + // renvoyer extesion d'un type mime.. // ex: "image/gif" -> gif -void give_mimext(char *s,const char *st) { - int ok=0; - int j=0; - s[0]='\0'; - while( (!ok) && (strnotempty(hts_mime[j][1])) ) { - if (strfield2(hts_mime[j][0],st)) { - if (hts_mime[j][1][0]!='*') { // Une correspondance existe - strcpybuff(s,hts_mime[j][1]); - ok=1; +void give_mimext(char *s, const char *st) { + int ok = 0; + int j = 0; + + s[0] = '\0'; + while((!ok) && (strnotempty(hts_mime[j][1]))) { + if (strfield2(hts_mime[j][0], st)) { + if (hts_mime[j][1][0] != '*') { // Une correspondance existe + strcpybuff(s, hts_mime[j][1]); + ok = 1; } } j++; @@ -4171,29 +4470,32 @@ void give_mimext(char *s,const char *st) { // application/mp3 if (!ok) { int p; - const char* a=NULL; - if ((p=strfield(st,"application/x-"))) - a=st+p; - else if ((p=strfield(st,"application/"))) - a=st+p; + const char *a = NULL; + + if ((p = strfield(st, "application/x-"))) + a = st + p; + else if ((p = strfield(st, "application/"))) + a = st + p; if (a) { - if ((int)strlen(a) >= 1) { - if ((int)strlen(a) <= 4) { - strcpybuff(s,a); - ok=1; + if ((int) strlen(a) >= 1) { + if ((int) strlen(a) <= 4) { + strcpybuff(s, a); + ok = 1; } } } } } + // extension connue?.. // 0 : non // 1 : oui // 2 : html -HTSEXT_API int is_knowntype(httrackp *opt,const char *fil) { - char catbuff[CATBUFF_SIZE]; +HTSEXT_API int is_knowntype(httrackp * opt, const char *fil) { + char catbuff[CATBUFF_SIZE]; const char *ext; - int j=0; + int j = 0; + if (!fil) return 0; ext = get_ext(catbuff, fil); @@ -4208,40 +4510,44 @@ HTSEXT_API int is_knowntype(httrackp *opt,const char *fil) { } // Known by user? - return (is_userknowntype(opt,fil)); + return (is_userknowntype(opt, fil)); } + // extension : html,gif.. -HTSEXT_API char* get_ext(char *catbuff, const char *fil) { - const char *a=fil+strlen(fil)-1; - - while ( (*a!='.') && (*a!='/') && (a>fil)) a--; - if (*a=='.') { - char fil_noquery[HTS_URLMAXSIZE*2]; - char* b; - fil_noquery[0]='\0'; - a++; // pointer sur extension - strncatbuff(fil_noquery,a,HTS_URLMAXSIZE); - b=strchr(fil_noquery,'?'); +HTSEXT_API char *get_ext(char *catbuff, const char *fil) { + const char *a = fil + strlen(fil) - 1; + + while((*a != '.') && (*a != '/') && (a > fil)) + a--; + if (*a == '.') { + char fil_noquery[HTS_URLMAXSIZE * 2]; + char *b; + + fil_noquery[0] = '\0'; + a++; // pointer sur extension + strncatbuff(fil_noquery, a, HTS_URLMAXSIZE); + b = strchr(fil_noquery, '?'); if (b) - *b='\0'; - return concat(catbuff,fil_noquery,""); - } - else + *b = '\0'; + return concat(catbuff, fil_noquery, ""); + } else return ""; } + // known type?.. // 0 : no // 1 : yes // 2 : html // setdefs : set mime buffer: // file=(char*) "asp=text/html\nphp=text/html\n" -HTSEXT_API int is_userknowntype(httrackp *opt,const char *fil) { +HTSEXT_API int is_userknowntype(httrackp * opt, const char *fil) { char BIGSTK mime[1024]; + if (!fil) return 0; if (!strnotempty(fil)) return 0; - mime[0]='\0'; + mime[0] = '\0'; get_userhttptype(opt, mime, fil); if (!strnotempty(mime)) return 0; @@ -4254,13 +4560,14 @@ HTSEXT_API int is_userknowntype(httrackp *opt,const char *fil) { // page dynamique? // is_dyntype(get_ext("foo.asp")) HTSEXT_API int is_dyntype(const char *fil) { - int j=0; + int j = 0; + if (!fil) return 0; if (!strnotempty(fil)) return 0; while(strnotempty(hts_ext_dynamic[j])) { - if (strfield2(hts_ext_dynamic[j],fil)) { + if (strfield2(hts_ext_dynamic[j], fil)) { return 1; } j++; @@ -4270,53 +4577,61 @@ HTSEXT_API int is_dyntype(const char *fil) { // types critiques qui ne doivent pas être changés car renvoyés par des serveurs qui ne // connaissent pas le type -int may_unknown(httrackp *opt,const char* st) { - int j=0; +int may_unknown(httrackp * opt, const char *st) { + int j = 0; + // types média - if (may_be_hypertext_mime(opt,st, "")) { + if (may_be_hypertext_mime(opt, st, "")) { return 1; } while(strnotempty(hts_mime_keep[j])) { - if (strfield2(hts_mime_keep[j],st)) { // trouvé + if (strfield2(hts_mime_keep[j], st)) { // trouvé return 1; } j++; - } + } return 0; } /* returns 1 if the mime/filename seems to be bogus because of badly recognized multiple extension ; such as "application/x-wais-source" for "httrack-3.42-1.el5.src.rpm" reported by Hippy Dave 08/2008 (3.43) */ -int may_bogus_multiple(httrackp *opt, const char* mime, const char *filename) { +int may_bogus_multiple(httrackp * opt, const char *mime, const char *filename) { int j; - for(j = 0 ; strnotempty(hts_mime_bogus_multiple[j]) ; j++) { - if (strfield2(hts_mime_bogus_multiple[j], mime)) { /* found mime type in suspicious list */ + + for(j = 0; strnotempty(hts_mime_bogus_multiple[j]); j++) { + if (strfield2(hts_mime_bogus_multiple[j], mime)) { /* found mime type in suspicious list */ char ext[64]; + ext[0] = '\0'; give_mimext(ext, mime); - if (ext[0] != 0) { /* we have an extension for that */ + if (ext[0] != 0) { /* we have an extension for that */ const size_t ext_size = strlen(ext); - const char *file = strrchr(filename, '/'); /* fetch terminal filename */ + const char *file = strrchr(filename, '/'); /* fetch terminal filename */ + if (file != NULL) { int i; - for(i = 0 ; file[i] != 0 ; i++) { - if (i > 0 && file[i - 1] == '.' && strncasecmp(&file[i], ext, ext_size) == 0 - && ( file[i + ext_size] == 0 || file[i + ext_size] == '.' || file[i + ext_size] == '?' ) ) { - return 1; /* is ambiguous */ + + for(i = 0; file[i] != 0; i++) { + if (i > 0 && file[i - 1] == '.' + && strncasecmp(&file[i], ext, ext_size) == 0 + && (file[i + ext_size] == 0 || file[i + ext_size] == '.' + || file[i + ext_size] == '?')) { + return 1; /* is ambiguous */ } } } } return 0; } - } + } return 0; } /* filename extension should not be changed because potentially bogus ; replaces may_unknown() (3.43) */ -int may_unknown2(httrackp *opt, const char* mime, const char *filename) { +int may_unknown2(httrackp * opt, const char *mime, const char *filename) { int ret = may_unknown(opt, mime); + if (ret == 0) { ret = may_bogus_multiple(opt, mime, filename); } @@ -4326,20 +4641,22 @@ int may_unknown2(httrackp *opt, const char* mime, const char *filename) { // -- Utils fichiers // pretty print for i/o -void fprintfio(FILE* fp,char* buff,char* prefix) { - char nl=1; +void fprintfio(FILE * fp, char *buff, char *prefix) { + char nl = 1; + while(*buff) { - switch(*buff) { - case 13: break; + switch (*buff) { + case 13: + break; case 10: - fprintf(fp,"\r\n"); - nl=1; - break; + fprintf(fp, "\r\n"); + nl = 1; + break; default: if (nl) - fprintf(fp,"%s", prefix); - nl=0; - fputc(*buff,fp); + fprintf(fp, "%s", prefix); + nl = 0; + fputc(*buff, fp); } buff++; } @@ -4348,12 +4665,13 @@ void fprintfio(FILE* fp,char* buff,char* prefix) { /* Le fichier existe-t-il? (ou est-il accessible?) */ /* Note: NOT utf-8 */ /* Note: preserve errno */ -int fexist(const char* s) { - char catbuff[CATBUFF_SIZE]; +int fexist(const char *s) { + char catbuff[CATBUFF_SIZE]; const int err = errno; struct stat st; + memset(&st, 0, sizeof(st)); - if (stat(fconv(catbuff,s), &st) == 0) { + if (stat(fconv(catbuff, s), &st) == 0) { if (S_ISREG(st.st_mode)) { return 1; } else { @@ -4362,17 +4680,18 @@ int fexist(const char* s) { } errno = err; return 0; -} +} /* Le fichier existe-t-il? (ou est-il accessible?) */ /* Note: utf-8 */ /* Note: preserve errno */ -int fexist_utf8(const char* s) { - char catbuff[CATBUFF_SIZE]; +int fexist_utf8(const char *s) { + char catbuff[CATBUFF_SIZE]; const int err = errno; STRUCT_STAT st; + memset(&st, 0, sizeof(st)); - if (STAT(fconv(catbuff,s), &st) == 0) { + if (STAT(fconv(catbuff, s), &st) == 0) { if (S_ISREG(st.st_mode)) { return 1; } else { @@ -4381,13 +4700,14 @@ int fexist_utf8(const char* s) { } errno = err; return 0; -} +} /* Taille d'un fichier, -1 si n'existe pas */ /* Note: NOT utf-8 */ -off_t fsize(const char* s) { +off_t fsize(const char *s) { struct stat st; - if (!strnotempty(s)) // nom vide: erreur + + if (!strnotempty(s)) // nom vide: erreur return -1; if (stat(s, &st) == 0) { return st.st_size; @@ -4398,9 +4718,10 @@ off_t fsize(const char* s) { /* Taille d'un fichier, -1 si n'existe pas */ /* Note: utf-8 */ -off_t fsize_utf8(const char* s) { +off_t fsize_utf8(const char *s) { STRUCT_STAT st; - if (!strnotempty(s)) // nom vide: erreur + + if (!strnotempty(s)) // nom vide: erreur return -1; if (STAT(s, &st) == 0) { return st.st_size; @@ -4409,51 +4730,54 @@ off_t fsize_utf8(const char* s) { } } -off_t fpsize(FILE* fp) { - off_t oldpos,size; +off_t fpsize(FILE * fp) { + off_t oldpos, size; + if (!fp) return -1; #ifdef HTS_FSEEKO - oldpos=ftello(fp); + oldpos = ftello(fp); #else - oldpos=ftell(fp); + oldpos = ftell(fp); #endif - fseek(fp,0,SEEK_END); + fseek(fp, 0, SEEK_END); #ifdef HTS_FSEEKO - size=ftello(fp); - fseeko(fp,oldpos,SEEK_SET); + size = ftello(fp); + fseeko(fp, oldpos, SEEK_SET); #else - size=ftell(fp); - fseek(fp,oldpos,SEEK_SET); + size = ftell(fp); + fseek(fp, oldpos, SEEK_SET); #endif return size; } /* root dir, with ending / */ typedef struct { - char path[1024+4]; + char path[1024 + 4]; int init; } hts_rootdir_strc; -HTSEXT_API char* hts_rootdir(char* file) { - static hts_rootdir_strc strc = {"", 0}; +HTSEXT_API char *hts_rootdir(char *file) { + static hts_rootdir_strc strc = { "", 0 }; if (file) { if (!strc.init) { - strc.path[0]='\0'; - strc.init=1; + strc.path[0] = '\0'; + strc.init = 1; if (strnotempty(file)) { - char* a; - strcpybuff(strc.path,file); - while((a=strrchr(strc.path,'\\'))) *a='/'; - if ((a=strrchr(strc.path,'/'))) { - *(a+1)='\0'; + char *a; + + strcpybuff(strc.path, file); + while((a = strrchr(strc.path, '\\'))) + *a = '/'; + if ((a = strrchr(strc.path, '/'))) { + *(a + 1) = '\0'; } else - strc.path[0]='\0'; + strc.path[0] = '\0'; } if (!strnotempty(strc.path)) { - if( getcwd( strc.path, 1024 ) == NULL ) - strc.path[0]='\0'; + if (getcwd(strc.path, 1024) == NULL) + strc.path[0] = '\0'; else - strcatbuff(strc.path,"/"); + strcatbuff(strc.path, "/"); } } return NULL; @@ -4463,16 +4787,15 @@ HTSEXT_API char* hts_rootdir(char* file) { return ""; } - - HTSEXT_API hts_stat_struct HTS_STAT; + // // return number of downloadable bytes, depending on rate limiter // see engine_stats() routine, too // this routine works quite well for big files and regular ones, but apparently the rate limiter has // some problems with very small files (rate too high) LLint check_downloadable_bytes(int rate) { - if (rate>0) { + if (rate > 0) { TStamp time_now; TStamp elapsed_useconds; LLint bytes_transfered_during_period; @@ -4481,15 +4804,16 @@ LLint check_downloadable_bytes(int rate) { // get the older timer int id_timer = (HTS_STAT.istat_idlasttimer + 1) % 2; - time_now=mtime_local(); + time_now = mtime_local(); elapsed_useconds = time_now - HTS_STAT.istat_timestart[id_timer]; // NO totally stupid - elapsed_useconds+=1000; // for the next second, too - bytes_transfered_during_period = (HTS_STAT.HTS_TOTAL_RECV-HTS_STAT.istat_bytes[id_timer]); - - left = ((rate * elapsed_useconds)/1000) - bytes_transfered_during_period; + bytes_transfered_during_period = + (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[id_timer]); + + left = ((rate * elapsed_useconds) / 1000) - bytes_transfered_during_period; if (left <= 0) left = 0; - + return left; } else return TAILLE_BUFFER; @@ -4502,21 +4826,21 @@ LLint check_downloadable_bytes(int rate) { int HTS_TOTAL_RECV_CHECK(int var) { if (HTS_STAT.HTS_TOTAL_RECV_STATE) return 1; - /* - { - if (HTS_STAT.HTS_TOTAL_RECV_STATE==3) { - var = min(var,32); - Sleep(250); - } else if (HTS_STAT.HTS_TOTAL_RECV_STATE==2) { - var = min(var,256); - Sleep(100); - } else { - var/=2; - if (var<=0) var=1; - Sleep(50); - } - } - */ + /* + { + if (HTS_STAT.HTS_TOTAL_RECV_STATE==3) { + var = min(var,32); + Sleep(250); + } else if (HTS_STAT.HTS_TOTAL_RECV_STATE==2) { + var = min(var,256); + Sleep(100); + } else { + var/=2; + if (var<=0) var=1; + Sleep(50); + } + } + */ return 0; } #endif @@ -4526,21 +4850,23 @@ int HTS_TOTAL_RECV_CHECK(int var) { // >0 : data received // == 0 : not yet data // <0: error or no data: READ_ERROR, READ_EOF or READ_TIMEOUT -int hts_read(htsblk* r,char* buff,int size) { +int hts_read(htsblk * r, char *buff, int size) { int retour; + // return read(soc,buff,size); if (r->is_file) { -#if HTS_WIDE_DEBUG - DEBUG_W("read(%p, %d, %d)\n" _ (void*) buff _ (int) size _ (int) r->fp); +#if HTS_WIDE_DEBUG + DEBUG_W("read(%p, %d, %d)\n" _(void *)buff _(int) size _(int) r->fp); #endif if (r->fp) - retour = (int)fread(buff,1,size,r->fp); + retour = (int) fread(buff, 1, size, r->fp); else retour = READ_ERROR; } else { -#if HTS_WIDE_DEBUG - DEBUG_W("recv(%d, %p, %d)\n" _ (int) r->soc _ (void*) buff _ (int) size); - if (r->soc==INVALID_SOCKET) +#if HTS_WIDE_DEBUG + DEBUG_W("recv(%d, %p, %d)\n" _(int) r->soc _(void *)buff _(int) size); + + if (r->soc == INVALID_SOCKET) printf("!!WIDE_DEBUG ERROR, soc==INVALID hts_read\n"); #endif //HTS_TOTAL_RECV_CHECK(size); // Diminuer au besoin si trop de données reçues @@ -4549,54 +4875,52 @@ int hts_read(htsblk* r,char* buff,int size) { retour = SSL_read(r->ssl_con, buff, size); if (retour <= 0) { int err_code = SSL_get_error(r->ssl_con, retour); - if ( - (err_code == SSL_ERROR_WANT_READ) - || - (err_code == SSL_ERROR_WANT_WRITE) - ) - { - retour = 0; /* no data yet (ssl cache) */ + + if ((err_code == SSL_ERROR_WANT_READ) + || (err_code == SSL_ERROR_WANT_WRITE) + ) { + retour = 0; /* no data yet (ssl cache) */ } else if (err_code == SSL_ERROR_ZERO_RETURN) { - retour = READ_EOF; /* completed */ + retour = READ_EOF; /* completed */ } else { - retour = READ_ERROR; /* eof or error */ + retour = READ_ERROR; /* eof or error */ } } } else { #endif - retour=recv(r->soc,buff,size,0); - if (retour == 0) { - retour = READ_EOF; - } else if (retour < 0) { - retour = READ_ERROR; + retour = recv(r->soc, buff, size, 0); + if (retour == 0) { + retour = READ_EOF; + } else if (retour < 0) { + retour = READ_ERROR; + } } - } - if (retour > 0) // compter flux entrant - HTS_STAT.HTS_TOTAL_RECV+=retour; + if (retour > 0) // compter flux entrant + HTS_STAT.HTS_TOTAL_RECV += retour; #if HTS_USEOPENSSL } #endif -#if HTS_WIDE_DEBUG - DEBUG_W("recv/read done (%d bytes)\n" _ (int) retour); +#if HTS_WIDE_DEBUG + DEBUG_W("recv/read done (%d bytes)\n" _(int) retour); #endif return retour; } - // -- Gestion cache DNS -- // 'RX98 #if HTS_DNSCACHE // 'capsule' contenant uniquement le cache -t_dnscache* _hts_cache(httrackp *opt) { - if (opt->state.dns_cache == NULL) { - opt->state.dns_cache = (t_dnscache*)malloct(sizeof(t_dnscache)); - memset(opt->state.dns_cache, 0, sizeof(t_dnscache)); - } +t_dnscache *_hts_cache(httrackp * opt) { + if (opt->state.dns_cache == NULL) { + opt->state.dns_cache = (t_dnscache *) malloct(sizeof(t_dnscache)); + memset(opt->state.dns_cache, 0, sizeof(t_dnscache)); + } return opt->state.dns_cache; } + // free the cache -static void hts_cache_free_(t_dnscache* cache) { +static void hts_cache_free_(t_dnscache * cache) { if (cache != NULL) { if (cache->n != NULL) { hts_cache_free_(cache->n); @@ -4604,11 +4928,11 @@ static void hts_cache_free_(t_dnscache* cache) { freet(cache); } } -void hts_cache_free(t_dnscache* cache) { - if (cache != NULL && cache->n != NULL) { - hts_cache_free_(cache->n); - cache->n = NULL; - } +void hts_cache_free(t_dnscache * cache) { + if (cache != NULL && cache->n != NULL) { + hts_cache_free_(cache->n); + cache->n = NULL; + } } // lock le cache dns pour tout opération d'ajout @@ -4623,33 +4947,34 @@ htsmutex dns_lock = HTSMUTEX_INIT; // routine pour le cache - retour optionnel à donner à chaque fois // NULL: nom non encore testé dans le cache // si h_length==0 alors le nom n'existe pas dans le dns -t_hostent* _hts_ghbn(t_dnscache* cache,const char* iadr,t_hostent* retour) { - t_hostent* ret = NULL; +t_hostent *_hts_ghbn(t_dnscache * cache, const char *iadr, t_hostent * retour) { + t_hostent *ret = NULL; + hts_mutexlock(&dns_lock); for(;;) { - if (strcmp(cache->iadr,iadr) == 0) { // ok trouvé - if (cache->host_length > 0) { // entrée valide + if (strcmp(cache->iadr, iadr) == 0) { // ok trouvé + if (cache->host_length > 0) { // entrée valide if (retour->h_addr_list[0]) memcpy(retour->h_addr_list[0], cache->host_addr, cache->host_length); - retour->h_length=cache->host_length; - } else if (cache->host_length == 0) { // en cours + retour->h_length = cache->host_length; + } else if (cache->host_length == 0) { // en cours ret = NULL; break; - } else { // erreur dans le dns, déja vérifié + } else { // erreur dans le dns, déja vérifié if (retour->h_addr_list[0]) - retour->h_addr_list[0][0]='\0'; - retour->h_length=0; // erreur, n'existe pas + retour->h_addr_list[0][0] = '\0'; + retour->h_length = 0; // erreur, n'existe pas } ret = retour; break; - } else { // on a pas encore trouvé - if (cache->n!=NULL) { // chercher encore - cache = cache->n; // suivant! + } else { // on a pas encore trouvé + if (cache->n != NULL) { // chercher encore + cache = cache->n; // suivant! } else { ret = NULL; break; } - } + } } hts_mutexrelease(&dns_lock); return ret; @@ -4659,48 +4984,49 @@ t_hostent* _hts_ghbn(t_dnscache* cache,const char* iadr,t_hostent* retour) { // 0 non encore // 1 ok // 2 non présent -int hts_dnstest(httrackp *opt, const char* _iadr) { +int hts_dnstest(httrackp * opt, const char *_iadr) { int ret = 0; - t_dnscache* cache=_hts_cache(opt); // adresse du cache - char iadr[HTS_URLMAXSIZE*2]; + t_dnscache *cache = _hts_cache(opt); // adresse du cache + char iadr[HTS_URLMAXSIZE * 2]; // sauter user:pass@ éventuel strcpybuff(iadr, jump_identification(_iadr)); // couper éventuel : { char *a; - if ( (a = jump_toport(iadr)) ) - *a='\0'; + + if ((a = jump_toport(iadr))) + *a = '\0'; } #ifdef _WIN32 - if (inet_addr(iadr)!=INADDR_NONE) // numérique + if (inet_addr(iadr) != INADDR_NONE) // numérique #else - if (inet_addr(iadr)!=(in_addr_t) -1 ) // numérique + if (inet_addr(iadr) != (in_addr_t) - 1) // numérique #endif return 1; hts_mutexlock(&dns_lock); for(;;) { - if (strcmp(cache->iadr, iadr)==0) { // ok trouvé + if (strcmp(cache->iadr, iadr) == 0) { // ok trouvé ret = 1; break; - } else { // on a pas encore trouvé - if (cache->n!=NULL) { // chercher encore - cache=cache->n; // suivant! + } else { // on a pas encore trouvé + if (cache->n != NULL) { // chercher encore + cache = cache->n; // suivant! } else { - ret = 2; // non présent - break ; + ret = 2; // non présent + break; } - } + } } hts_mutexrelease(&dns_lock); return ret; } +HTSEXT_API t_hostent *vxgethostbyname(char *hostname, void *v_buffer) { + t_fullhostent *buffer = (t_fullhostent *) v_buffer; -HTSEXT_API t_hostent* vxgethostbyname(char* hostname, void* v_buffer) { - t_fullhostent* buffer = (t_fullhostent*) v_buffer; /* Clear */ fullhostent_init(buffer); @@ -4710,24 +5036,27 @@ HTSEXT_API t_hostent* vxgethostbyname(char* hostname, void* v_buffer) { } /* - Strip [] if any : [3ffe:b80:1234:1::1] - The resolver doesn't seem to handle IP6 addresses in brackets - */ - if ((hostname[0] == '[') && (hostname[strlen(hostname)-1] == ']')) { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - tempo[0]='\0'; - strncatbuff(tempo, hostname+1, strlen(hostname)-2); + Strip [] if any : [3ffe:b80:1234:1::1] + The resolver doesn't seem to handle IP6 addresses in brackets + */ + if ((hostname[0] == '[') && (hostname[strlen(hostname) - 1] == ']')) { + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + + tempo[0] = '\0'; + strncatbuff(tempo, hostname + 1, strlen(hostname) - 2); strcpybuff(hostname, tempo); } { #if HTS_INET6==0 - /* - ipV4 resolver - */ - t_hostent* hp=gethostbyname(hostname); - if (hp!=NULL) { - if ( (hp->h_length) && ( ((unsigned int) hp->h_length) <= buffer->addr_maxlen) ) { + /* + ipV4 resolver + */ + t_hostent *hp = gethostbyname(hostname); + + if (hp != NULL) { + if ((hp->h_length) + && (((unsigned int) hp->h_length) <= buffer->addr_maxlen)) { memcpy(buffer->hp.h_addr_list[0], hp->h_addr_list[0], hp->h_length); buffer->hp.h_length = hp->h_length; return &(buffer->hp); @@ -4735,27 +5064,29 @@ HTSEXT_API t_hostent* vxgethostbyname(char* hostname, void* v_buffer) { } #else /* - ipV6 resolver - */ + ipV6 resolver + */ /* - int error_num=0; - t_hostent* hp=getipnodebyname(hostname, AF_INET6, AI_DEFAULT, &error_num); - oops, deprecated :( - */ - struct addrinfo* res = NULL; + int error_num=0; + t_hostent* hp=getipnodebyname(hostname, AF_INET6, AI_DEFAULT, &error_num); + oops, deprecated :( + */ + struct addrinfo *res = NULL; struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); - if (IPV6_resolver == 1) // V4 only (for bogus V6 entries) + if (IPV6_resolver == 1) // V4 only (for bogus V6 entries) hints.ai_family = PF_INET; - else if (IPV6_resolver == 2) // V6 only (for testing V6 only) + else if (IPV6_resolver == 2) // V6 only (for testing V6 only) hints.ai_family = PF_INET6; - else // V4 + V6 + else // V4 + V6 hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = IPPROTO_TCP; if (getaddrinfo(hostname, NULL, &hints, &res) == 0) { if (res) { - if ( (res->ai_addr) && (res->ai_addrlen) && (res->ai_addrlen <= buffer->addr_maxlen) ) { + if ((res->ai_addr) && (res->ai_addrlen) + && (res->ai_addrlen <= buffer->addr_maxlen)) { memcpy(buffer->hp.h_addr_list[0], res->ai_addr, res->ai_addrlen); buffer->hp.h_length = (short) res->ai_addrlen; freeaddrinfo(res); @@ -4766,129 +5097,135 @@ HTSEXT_API t_hostent* vxgethostbyname(char* hostname, void* v_buffer) { if (res) { freeaddrinfo(res); } - #endif } return NULL; } // cache dns interne à HTS // ** FREE A FAIRE sur la chaine -t_hostent* hts_gethostbyname(httrackp *opt,const char* _iadr, void* v_buffer) { - char BIGSTK iadr[HTS_URLMAXSIZE*2]; - t_fullhostent* buffer = (t_fullhostent*) v_buffer; - t_dnscache* cache=_hts_cache(opt); // adresse du cache - t_hostent* hp; +t_hostent *hts_gethostbyname(httrackp * opt, const char *_iadr, void *v_buffer) { + char BIGSTK iadr[HTS_URLMAXSIZE * 2]; + t_fullhostent *buffer = (t_fullhostent *) v_buffer; + t_dnscache *cache = _hts_cache(opt); // adresse du cache + t_hostent *hp; /* Clear */ fullhostent_init(buffer); - strcpybuff(iadr,jump_identification(_iadr)); + strcpybuff(iadr, jump_identification(_iadr)); // couper éventuel : { char *a; - if ( (a=jump_toport(iadr)) ) - *a='\0'; + + if ((a = jump_toport(iadr))) + *a = '\0'; } // effacer structure de retour, créer nouvelle /* - memset(&host, 0, sizeof(t_hostent)); - host.h_addr_list=he; - he[0]=NULL; - he[1]=NULL; - host.h_length=0; - */ - cache->iadr[0]='*'; - cache->iadr[1]='\0'; - + memset(&host, 0, sizeof(t_hostent)); + host.h_addr_list=he; + he[0]=NULL; + he[1]=NULL; + host.h_length=0; + */ + cache->iadr[0] = '*'; + cache->iadr[1] = '\0'; + /* get IP from the dns cache */ hp = _hts_ghbn(cache, iadr, &buffer->hp); if (hp) { - if (hp->h_length>0) + if (hp->h_length > 0) return hp; else - return NULL; // entrée erronée (erreur DNS) dans le DNS - } else { // non présent dans le cache dns, tester - t_dnscache* c=cache; - while(c->n) c=c->n; // calculer queue - -#if HTS_WIDE_DEBUG + return NULL; // entrée erronée (erreur DNS) dans le DNS + } else { // non présent dans le cache dns, tester + t_dnscache *c = cache; + + while(c->n) + c = c->n; // calculer queue + +#if HTS_WIDE_DEBUG DEBUG_W("gethostbyname\n"); -#endif +#endif #if HDEBUG printf("gethostbyname (not in cache)\n"); #endif { unsigned long inetaddr; + #ifdef _WIN32 - if ((inetaddr=inet_addr(iadr))==INADDR_NONE) { + if ((inetaddr = inet_addr(iadr)) == INADDR_NONE) { #else - if ((inetaddr=inet_addr(iadr))==(in_addr_t) -1 ) { -#endif -#if DEBUGDNS - printf("resolving (not cached) %s\n",iadr); + if ((inetaddr = inet_addr(iadr)) == (in_addr_t) - 1) { +#endif +#if DEBUGDNS + printf("resolving (not cached) %s\n", iadr); #endif - hp=vxgethostbyname(iadr, buffer); // calculer IP host - } else { // numérique, convertir sans passer par le dns - buffer->hp.h_addr_list[0]=(char*) &inetaddr; - buffer->hp.h_length=4; - hp=&buffer->hp; + hp = vxgethostbyname(iadr, buffer); // calculer IP host + } else { // numérique, convertir sans passer par le dns + buffer->hp.h_addr_list[0] = (char *) &inetaddr; + buffer->hp.h_length = 4; + hp = &buffer->hp; } } -#if HTS_WIDE_DEBUG +#if HTS_WIDE_DEBUG DEBUG_W("gethostbyname done\n"); #endif - cache->n=(t_dnscache*) calloct(1,sizeof(t_dnscache)); - if (cache->n!=NULL) { - strcpybuff(cache->n->iadr,iadr); - if (hp!=NULL) { + cache->n = (t_dnscache *) calloct(1, sizeof(t_dnscache)); + if (cache->n != NULL) { + strcpybuff(cache->n->iadr, iadr); + if (hp != NULL) { memcpy(cache->n->host_addr, hp->h_addr_list[0], hp->h_length); - cache->n->host_length=hp->h_length; + cache->n->host_length = hp->h_length; } else { - cache->n->host_addr[0]='\0'; - cache->n->host_length=0; // non existant dans le dns + cache->n->host_addr[0] = '\0'; + cache->n->host_length = 0; // non existant dans le dns } - cache->n->n=NULL; + cache->n->n = NULL; return hp; - } else { // on peut pas noter, mais on peut renvoyer le résultat + } else { // on peut pas noter, mais on peut renvoyer le résultat return hp; - } - } // retour hp du cache + } + } // retour hp du cache } #else -HTS_INLINE t_hostent* hts_gethostbyname(httrackp *opt,char* iadr, t_fullhostent* buffer) { - t_hostent* retour; -#if HTS_WIDE_DEBUG +HTS_INLINE t_hostent *hts_gethostbyname(httrackp * opt, char *iadr, + t_fullhostent * buffer) { + t_hostent *retour; + +#if HTS_WIDE_DEBUG DEBUG_W("gethostbyname (2)\n"); #endif -#if DEBUGDNS - printf("blocking method gethostbyname() in progress for %s\n",iadr); +#if DEBUGDNS + printf("blocking method gethostbyname() in progress for %s\n", iadr); #endif - retour=vxgethostbyname(jump_identification(iadr), ); -#if HTS_WIDE_DEBUG + retour = vxgethostbyname(jump_identification(iadr),); +#if HTS_WIDE_DEBUG DEBUG_W("gethostbyname (2) done\n"); #endif return retour; } #endif - // --- Tracage des mallocs() --- #ifdef HTS_TRACE_MALLOC //#define htsLocker(A, N) htsLocker(A, N) #define htsLocker(A, N) do {} while(0) -static mlink trmalloc = {NULL,0,0,NULL}; -static int trmalloc_id=0; -static htsmutex* mallocMutex = NULL; +static mlink trmalloc = { NULL, 0, 0, NULL }; + +static int trmalloc_id = 0; +static htsmutex *mallocMutex = NULL; static void hts_meminit(void) { //if (mallocMutex == NULL) { // mallocMutex = calloc(sizeof(*mallocMutex), 1); // htsLocker(mallocMutex, -999); //} } -void* hts_malloc(size_t len) { - void* adr; +void *hts_malloc(size_t len) { + void *adr; + hts_meminit(); htsLocker(mallocMutex, 1); fassert(len > 0); @@ -4896,8 +5233,9 @@ void* hts_malloc(size_t len) { htsLocker(mallocMutex, 0); return adr; } -void* hts_calloc(size_t len,size_t len2) { - void* adr; +void *hts_calloc(size_t len, size_t len2) { + void *adr; + hts_meminit(); fassert(len > 0); fassert(len2 > 0); @@ -4907,33 +5245,36 @@ void* hts_calloc(size_t len,size_t len2) { memset(adr, 0, len * len2); return adr; } -void* hts_strdup(char* str) { +void *hts_strdup(char *str) { size_t size = str ? strlen(str) : 0; - char* adr = (char*) hts_malloc(size + 1); + char *adr = (char *) hts_malloc(size + 1); + fassert(adr != NULL); strcpy(adr, str ? str : ""); return adr; } -void* hts_xmalloc(size_t len,size_t len2) { - mlink* lnk = (mlink*) calloc(1,sizeof(mlink)); +void *hts_xmalloc(size_t len, size_t len2) { + mlink *lnk = (mlink *) calloc(1, sizeof(mlink)); + fassert(lnk != NULL); fassert(len > 0); fassert(len2 >= 0); if (lnk) { - void* r = NULL; + void *r = NULL; int size, bsize = sizeof(t_htsboundary); + if (len2) size = len * len2; else size = len; - size += ((bsize - (size % bsize)) % bsize); /* check alignement */ - r = malloc(size + bsize*2); + size += ((bsize - (size % bsize)) % bsize); /* check alignement */ + r = malloc(size + bsize * 2); fassert(r != NULL); if (r) { - * ( (t_htsboundary*) ((char*) r ) ) - = * ( (t_htsboundary*) ( (char*) r + size + bsize ) ) + *((t_htsboundary *) ((char *) r)) + = *((t_htsboundary *) ((char *) r + size + bsize)) = htsboundary; - ((char*) r) += bsize; /* boundary */ + ((char *) r) += bsize; /* boundary */ lnk->adr = r; lnk->len = size; lnk->id = trmalloc_id++; @@ -4946,9 +5287,10 @@ void* hts_xmalloc(size_t len,size_t len2) { } return NULL; } -void hts_free(void* adr) { - mlink* lnk = &trmalloc; +void hts_free(void *adr) { + mlink *lnk = &trmalloc; int bsize = sizeof(t_htsboundary); + fassert(adr != NULL); if (!adr) { return; @@ -4956,14 +5298,16 @@ void hts_free(void* adr) { htsLocker(mallocMutex, 1); while(lnk->next != NULL) { if (lnk->next->adr == adr) { - mlink* blk_free=lnk->next; + mlink *blk_free = lnk->next; + fassert(blk_free->id != -1); - fassert( * ( (t_htsboundary*) ( (char*) adr - bsize ) ) == htsboundary ); - fassert( * ( (t_htsboundary*) ( (char*) adr + blk_free->len ) ) == htsboundary ); - lnk->next=lnk->next->next; - free((void*) blk_free); + fassert(*((t_htsboundary *) ((char *) adr - bsize)) == htsboundary); + fassert(*((t_htsboundary *) ((char *) adr + blk_free->len)) == + htsboundary); + lnk->next = lnk->next->next; + free((void *) blk_free); //blk_free->id=-1; - free((char*) adr - bsize); + free((char *) adr - bsize); htsLocker(mallocMutex, 0); return; } @@ -4973,29 +5317,33 @@ void hts_free(void* adr) { free(adr); htsLocker(mallocMutex, 0); } -void* hts_realloc(void* adr,size_t len) { +void *hts_realloc(void *adr, size_t len) { int bsize = sizeof(t_htsboundary); - len += ((bsize - (len % bsize)) % bsize); /* check alignement */ + + len += ((bsize - (len % bsize)) % bsize); /* check alignement */ if (adr != NULL) { - mlink* lnk = &trmalloc; + mlink *lnk = &trmalloc; + htsLocker(mallocMutex, 1); - while(lnk->next != NULL) { - if (lnk->next->adr==adr) { + while(lnk->next != NULL) { + if (lnk->next->adr == adr) { { - mlink* blk_free=lnk->next; + mlink *blk_free = lnk->next; + fassert(blk_free->id != -1); - fassert( * ( (t_htsboundary*) ( (char*) adr - bsize ) ) == htsboundary ); - fassert( * ( (t_htsboundary*) ( (char*) adr + blk_free->len ) ) == htsboundary ); + fassert(*((t_htsboundary *) ((char *) adr - bsize)) == htsboundary); + fassert(*((t_htsboundary *) ((char *) adr + blk_free->len)) == + htsboundary); } - adr = realloc((char*) adr - bsize, len + bsize * 2); + adr = realloc((char *) adr - bsize, len + bsize * 2); fassert(adr != NULL); - lnk->next->adr = (char*) adr + bsize; + lnk->next->adr = (char *) adr + bsize; lnk->next->len = len; - * ( (t_htsboundary*) ( (char*) adr ) ) - = * ( (t_htsboundary*) ( (char*) adr + len + bsize) ) + *((t_htsboundary *) ((char *) adr)) + = *((t_htsboundary *) ((char *) adr + len + bsize)) = htsboundary; htsLocker(mallocMutex, 0); - return (char*) adr + bsize; + return (char *) adr + bsize; } lnk = lnk->next; fassert(lnk->next != NULL); @@ -5004,17 +5352,18 @@ void* hts_realloc(void* adr,size_t len) { } return hts_malloc(len); } -mlink* hts_find(char* adr) { - char* stkframe = (char*) &stkframe; - mlink* lnk = &trmalloc; +mlink *hts_find(char *adr) { + char *stkframe = (char *) &stkframe; + mlink *lnk = &trmalloc; int bsize = sizeof(t_htsboundary); + fassert(adr != NULL); if (!adr) { return NULL; } htsLocker(mallocMutex, 1); while(lnk->next != NULL) { - if (adr >= lnk->next->adr && adr <= lnk->next->adr + lnk->next->len) { /* found */ + if (adr >= lnk->next->adr && adr <= lnk->next->adr + lnk->next->len) { /* found */ htsLocker(mallocMutex, 0); return lnk->next; } @@ -5023,49 +5372,56 @@ mlink* hts_find(char* adr) { htsLocker(mallocMutex, 0); { int depl = (int) (adr - stkframe); - if (depl < 0) depl = -depl; + + if (depl < 0) + depl = -depl; //fassert(depl < 512000); /* near the stack frame.. doesn't look like malloc but stack variable */ return NULL; } } + // check the malloct() and calloct() trace stack -void hts_freeall(void) { +void hts_freeall(void) { int bsize = sizeof(t_htsboundary); + while(trmalloc.next) { #if MEMDEBUG - printf("* block %d\t not released: at %d\t (%d\t bytes)\n",trmalloc.next->id,trmalloc.next->adr,trmalloc.next->len); + printf("* block %d\t not released: at %d\t (%d\t bytes)\n", + trmalloc.next->id, trmalloc.next->adr, trmalloc.next->len); #endif if (trmalloc.next->id != -1) { - free((char*) trmalloc.next->adr - bsize); + free((char *) trmalloc.next->adr - bsize); } } } #endif - // -- divers // // cut path and project name // patch also initial path -void cut_path(char* fullpath,char* path,char* pname) { - path[0]=pname[0]='\0'; +void cut_path(char *fullpath, char *path, char *pname) { + path[0] = pname[0] = '\0'; if (strnotempty(fullpath)) { - if ((fullpath[strlen(fullpath)-1]=='/') || (fullpath[strlen(fullpath)-1]=='\\')) - fullpath[strlen(fullpath)-1]='\0'; - if (strlen(fullpath)>1) { - char* a; - while( (a=strchr(fullpath,'\\')) ) *a='/'; // remplacer par / - a=fullpath+strlen(fullpath)-2; - while( (*a!='/') && ( a > fullpath)) a--; - if (*a=='/') a++; - strcpybuff(pname,a); - strncatbuff(path,fullpath,(int) (a - fullpath)); + if ((fullpath[strlen(fullpath) - 1] == '/') + || (fullpath[strlen(fullpath) - 1] == '\\')) + fullpath[strlen(fullpath) - 1] = '\0'; + if (strlen(fullpath) > 1) { + char *a; + + while((a = strchr(fullpath, '\\'))) + *a = '/'; // remplacer par / + a = fullpath + strlen(fullpath) - 2; + while((*a != '/') && (a > fullpath)) + a--; + if (*a == '/') + a++; + strcpybuff(pname, a); + strncatbuff(path, fullpath, (int) (a - fullpath)); } } } - - // -- Gestion protocole ftp -- #ifdef _WIN32 @@ -5074,7 +5430,7 @@ int ftp_available(void) { } #else int ftp_available(void) { - return 1; // ok! + return 1; // ok! //return 0; // SOUS UNIX, PROBLEMESs } #endif @@ -5082,7 +5438,7 @@ int ftp_available(void) { static void hts_debug_log_print(const char *format, ...); static int hts_dgb_init = 0; -static FILE* hts_dgb_init_fp = NULL; +static FILE *hts_dgb_init_fp = NULL; HTSEXT_API void hts_debug(int level) { hts_dgb_init = level; if (hts_dgb_init > 0) { @@ -5113,6 +5469,7 @@ static void hts_debug_log_print(const char *format, ...) { const int error = errno; FILE *const fp = hts_dgb_(); va_list args; + assertf(format != NULL); va_start(args, format); (void) vfprintf(fp, format, args); @@ -5126,6 +5483,7 @@ static void hts_debug_log_print(const char *format, ...) { static int hts_init_ok = 0; HTSEXT_API int hts_init(void) { const char *dbg_env; + /* */ if (hts_init_ok) return 1; @@ -5135,12 +5493,13 @@ HTSEXT_API int hts_init(void) { dbg_env = getenv("HTS_LOG"); if (dbg_env != NULL && *dbg_env != 0) { int level = 0; + if (sscanf(dbg_env, "%d", &level) == 1) { hts_debug(level); } } - hts_debug_log_print("entering hts_init()"); /* debug */ + hts_debug_log_print("entering hts_init()"); /* debug */ #ifdef _WIN32_WCE #ifndef HTS_CECOMPAT @@ -5152,28 +5511,31 @@ HTSEXT_API int hts_init(void) { htsthread_init(); /* Ensure external modules are loaded */ - hts_debug_log_print("calling htspe_init()"); /* debug */ - htspe_init(); /* module load (lazy) */ + hts_debug_log_print("calling htspe_init()"); /* debug */ + htspe_init(); /* module load (lazy) */ /* MD5 Auto-test */ { char digest[32 + 2]; - const char* atest = "MD5 Checksum Autotest"; + const char *atest = "MD5 Checksum Autotest"; + digest[0] = '\0'; - domd5mem(atest, strlen(atest), digest, 1); /* a42ec44369da07ace5ec1d660ba4a69a */ + domd5mem(atest, strlen(atest), digest, 1); /* a42ec44369da07ace5ec1d660ba4a69a */ if (strcmp(digest, "a42ec44369da07ace5ec1d660ba4a69a") != 0) { int fatal_broken_md5 = 0; + assertf(fatal_broken_md5); } } - hts_debug_log_print("initializing SSL"); /* debug */ + hts_debug_log_print("initializing SSL"); /* debug */ #if HTS_USEOPENSSL /* - Initialize the OpensSSL library - */ + Initialize the OpensSSL library + */ if (!openssl_ctx && SSL_is_available) { - if (SSL_load_error_strings) SSL_load_error_strings(); + if (SSL_load_error_strings) + SSL_load_error_strings(); SSL_library_init(); ///if (SSL_load_error_strings) SSL_load_error_strings(); //if (ERR_load_crypto_strings) ERR_load_crypto_strings(); @@ -5181,21 +5543,22 @@ HTSEXT_API int hts_init(void) { // OpenSSL_add_all_algorithms(); openssl_ctx = SSL_CTX_new(SSLv23_client_method()); if (!openssl_ctx) { - fprintf(stderr, "fatal: unable to initialize TLS: SSL_CTX_new(SSLv23_client_method)\n"); + fprintf(stderr, + "fatal: unable to initialize TLS: SSL_CTX_new(SSLv23_client_method)\n"); abortLog("unable to initialize TLS: SSL_CTX_new(SSLv23_client_method)"); assertf("unable to initialize TLS" == NULL); } } #endif - - hts_debug_log_print("ending hts_init()"); /* debug */ + + hts_debug_log_print("ending hts_init()"); /* debug */ return 1; } /* will not free thread env. */ HTSEXT_API int hts_uninit(void) { /* hts_init() is a lazy initializer, with limited a allocation (one or two mutexes) ; - we won't free anything here as the .h semantic was never being very clear */ + we won't free anything here as the .h semantic was never being very clear */ return 1; } @@ -5209,40 +5572,41 @@ HTSEXT_API int hts_uninit_module(void) { } // legacy. do not use -HTSEXT_API int hts_log(httrackp *opt, const char* prefix, const char *msg) { +HTSEXT_API int hts_log(httrackp * opt, const char *prefix, const char *msg) { if (opt->log != NULL) { fspc(opt, opt->log, prefix); - fprintf(opt->log, "%s"LF, msg); + fprintf(opt->log, "%s" LF, msg); return 0; } - return 1; /* Error */ + return 1; /* Error */ } -HTSEXT_API void hts_log_print(httrackp *opt, int type, const char *format, ...) { +HTSEXT_API void hts_log_print(httrackp * opt, int type, const char *format, ...) { assertf(format != NULL); if (opt != NULL && opt->log != NULL) { va_list args; const int save_errno = errno; const char *s_type = "unknown"; - switch(type & 0xff) { + + switch (type & 0xff) { case LOG_DEBUG: // check verbosity if (opt->debug < 2) { - return ; + return; } s_type = "debug"; break; case LOG_INFO: // check verbosity if (opt->debug < 1) { - return ; + return; } s_type = "info"; break; case LOG_NOTICE: // check verbosity if (opt->debug < 1) { - return ; + return; } /* nobreak; */ case LOG_WARNING: @@ -5270,50 +5634,68 @@ HTSEXT_API void hts_log_print(httrackp *opt, int type, const char *format, ...) } } -HTSEXT_API void set_wrappers(httrackp *opt) { // LEGACY +HTSEXT_API void set_wrappers(httrackp * opt) { // LEGACY } -HTSEXT_API int plug_wrapper(httrackp *opt, const char *moduleName, const char* argv) { - void* handle = openFunctionLib(moduleName); +HTSEXT_API int plug_wrapper(httrackp * opt, const char *moduleName, + const char *argv) { + void *handle = openFunctionLib(moduleName); + if (handle != NULL) { t_hts_plug plug = (t_hts_plug) getFunctionPtr(handle, "hts_plug"); t_hts_unplug unplug = (t_hts_unplug) getFunctionPtr(handle, "hts_unplug"); + if (plug != NULL) { int ret = plug(opt, argv); + if (hts_dgb_init > 0 && opt->log != NULL) { - hts_debug_log_print("plugged module '%s' (return code=%d)", moduleName, ret); + hts_debug_log_print("plugged module '%s' (return code=%d)", moduleName, + ret); } - if (ret == 1) { /* Success! */ - opt->libHandles.handles = (htslibhandle*) realloct(opt->libHandles.handles, ( opt->libHandles.count + 1 )*sizeof(htslibhandle)); + if (ret == 1) { /* Success! */ + opt->libHandles.handles = + (htslibhandle *) realloct(opt->libHandles.handles, + (opt->libHandles.count + + 1) * sizeof(htslibhandle)); opt->libHandles.handles[opt->libHandles.count].handle = handle; - opt->libHandles.handles[opt->libHandles.count].moduleName = strdupt(moduleName); + opt->libHandles.handles[opt->libHandles.count].moduleName = + strdupt(moduleName); opt->libHandles.count++; return 1; } else { - hts_debug_log_print("* note: error while running entry point 'hts_plug' in %s", moduleName); + hts_debug_log_print + ("* note: error while running entry point 'hts_plug' in %s", + moduleName); if (unplug) unplug(opt); } } else { int last_errno = errno; - hts_debug_log_print("* note: can't find entry point 'hts_plug' in %s: %s", moduleName, strerror(last_errno)); + + hts_debug_log_print("* note: can't find entry point 'hts_plug' in %s: %s", + moduleName, strerror(last_errno)); } closeFunctionLib(handle); return 0; } else { int last_errno = errno; - hts_debug_log_print("* note: can't load %s: %s", moduleName, strerror(last_errno)); + + hts_debug_log_print("* note: can't load %s: %s", moduleName, + strerror(last_errno)); } return -1; } -static void unplug_wrappers(httrackp *opt) { +static void unplug_wrappers(httrackp * opt) { if (opt->libHandles.handles != NULL) { int i; - for(i = 0 ; i < opt->libHandles.count ; i++) { + + for(i = 0; i < opt->libHandles.count; i++) { if (opt->libHandles.handles[i].handle != NULL) { /* hts_unplug(), the dll exit point (finalizer) */ - t_hts_unplug unplug = (t_hts_unplug) getFunctionPtr(opt->libHandles.handles[i].handle, "hts_unplug"); + t_hts_unplug unplug = + (t_hts_unplug) getFunctionPtr(opt->libHandles.handles[i].handle, + "hts_unplug"); if (unplug != NULL) unplug(opt); closeFunctionLib(opt->libHandles.handles[i].handle); @@ -5333,16 +5715,17 @@ static void unplug_wrappers(httrackp *opt) { int multipleStringMatch(const char *s, const char *match) { int ret = 0; String name = STRING_EMPTY; - if (match == NULL || s == NULL || *s == 0) + + if (match == NULL || s == NULL || *s == 0) return 0; - for( ; *match != 0 ; match++) { + for(; *match != 0; match++) { StringClear(name); - for( ; *match != 0 && *match != '\n' ; match++) { + for(; *match != 0 && *match != '\n'; match++) { StringAddchar(name, *match); } if (StringLength(name) > 0 && strstr(s, StringBuff(name)) != NULL) { ret = 1; - break ; + break; } } StringFree(name); @@ -5351,12 +5734,12 @@ int multipleStringMatch(const char *s, const char *match) { HTSEXT_API httrackp *hts_create_opt(void) { #ifdef _WIN32 - static const char *defaultModules[] = { - "htsswf", "htsjava", "httrack-plugin", NULL + static const char *defaultModules[] = { + "htsswf", "htsjava", "httrack-plugin", NULL }; #else - static const char *defaultModules[] = { - "libhtsswf.so.1", "libhtsjava.so.2", "httrack-plugin", NULL + static const char *defaultModules[] = { + "libhtsswf.so.1", "libhtsjava.so.2", "httrack-plugin", NULL }; #endif httrackp *opt = malloc(sizeof(httrackp)); @@ -5368,148 +5751,159 @@ HTSEXT_API httrackp *hts_create_opt(void) { /* mutexes */ hts_mutexinit(&opt->state.lock); - /* custom wrappers */ + /* custom wrappers */ opt->libHandles.count = 0; - /* default settings */ + /* default settings */ - opt->wizard=2; // wizard automatique - opt->quiet=0; // questions + opt->wizard = 2; // wizard automatique + opt->quiet = 0; // questions // - opt->travel=0; // même adresse - opt->depth=9999; // mirror total par défaut - opt->extdepth=0; // mais pas à l'extérieur - opt->seeker=1; // down - opt->urlmode=2; // relatif par défaut - opt->debug=0; // pas de débug en plus - opt->getmode=3; // linear scan - opt->maxsite=-1; // taille max site (aucune) - opt->maxfile_nonhtml=-1; // taille max fichier non html - opt->maxfile_html=-1; // idem pour html - opt->maxsoc=4; // nbre socket max - opt->fragment=-1; // pas de fragmentation - opt->nearlink=0; // ne pas prendre les liens non-html "adjacents" - opt->makeindex=1; // faire un index - opt->kindex=0; // index 'keyword' - opt->delete_old=1; // effacer anciens fichiers - opt->background_on_suspend=1; // Background the process if Control Z calls signal suspend. - opt->makestat=0; // pas de fichier de stats - opt->maketrack=0; // ni de tracking - opt->timeout=120; // timeout par défaut (2 minutes) - opt->cache=1; // cache prioritaire - opt->shell=0; // pas de shell par defaut - opt->proxy.active=0; // pas de proxy - opt->user_agent_send=1; // envoyer un user-agent - StringCopy(opt->user_agent, "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"); + opt->travel = 0; // même adresse + opt->depth = 9999; // mirror total par défaut + opt->extdepth = 0; // mais pas à l'extérieur + opt->seeker = 1; // down + opt->urlmode = 2; // relatif par défaut + opt->debug = 0; // pas de débug en plus + opt->getmode = 3; // linear scan + opt->maxsite = -1; // taille max site (aucune) + opt->maxfile_nonhtml = -1; // taille max fichier non html + opt->maxfile_html = -1; // idem pour html + opt->maxsoc = 4; // nbre socket max + opt->fragment = -1; // pas de fragmentation + opt->nearlink = 0; // ne pas prendre les liens non-html "adjacents" + opt->makeindex = 1; // faire un index + opt->kindex = 0; // index 'keyword' + opt->delete_old = 1; // effacer anciens fichiers + opt->background_on_suspend = 1; // Background the process if Control Z calls signal suspend. + opt->makestat = 0; // pas de fichier de stats + opt->maketrack = 0; // ni de tracking + opt->timeout = 120; // timeout par défaut (2 minutes) + opt->cache = 1; // cache prioritaire + opt->shell = 0; // pas de shell par defaut + opt->proxy.active = 0; // pas de proxy + opt->user_agent_send = 1; // envoyer un user-agent + StringCopy(opt->user_agent, + "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"); StringCopy(opt->referer, ""); StringCopy(opt->from, ""); - opt->savename_83=0; // noms longs par défaut - opt->savename_type=0; // avec structure originale - opt->savename_delayed=2;// hard delayed type (default) - opt->delayed_cached=1; // cached delayed type (default) - opt->mimehtml=0; // pas MIME-html - opt->parsejava=HTSPARSE_DEFAULT; // parser classes - opt->hostcontrol=0; // PAS de control host pour timeout et traffic jammer - opt->retry=2; // 2 retry par défaut - opt->errpage=1; // copier ou générer une page d'erreur en cas d'erreur (404 etc.) - opt->check_type=1; // vérifier type si inconnu (cgi,asp..) SAUF / considéré comme html - opt->all_in_cache=0; // ne pas tout stocker en cache - opt->robots=2; // traiter les robots.txt - opt->external=0; // liens externes normaux - opt->passprivacy=0; // mots de passe dans les fichiers - opt->includequery=1; // include query-string par défaut - opt->mirror_first_page=0; // pas mode mirror links - opt->accept_cookie=1; // gérer les cookies - opt->cookie=NULL; - opt->http10=0; // laisser http/1.1 - opt->nokeepalive = 0; // pas keep-alive - opt->nocompression=0; // pas de compression - opt->tolerant=0; // ne pas accepter content-length incorrect - opt->parseall=1; // tout parser (tags inconnus, par exemple) - opt->parsedebug=0; // pas de mode débuggage - opt->norecatch=0; // ne pas reprendre les fichiers effacés par l'utilisateur - opt->verbosedisplay=0; // pas d'animation texte - opt->sizehack=0; // size hack - opt->urlhack=1; // url hack (normalizer) - StringCopy(opt->footer,HTS_DEFAULT_FOOTER); - opt->ftp_proxy=1; // proxy http pour ftp - opt->convert_utf8 = 1; // convert html to UTF-8 - StringCopy(opt->filelist,""); - StringCopy(opt->lang_iso,"en, *"); - StringCopy(opt->mimedefs,"\n"); // aucun filtre mime (\n IMPORTANT) + opt->savename_83 = 0; // noms longs par défaut + opt->savename_type = 0; // avec structure originale + opt->savename_delayed = 2; // hard delayed type (default) + opt->delayed_cached = 1; // cached delayed type (default) + opt->mimehtml = 0; // pas MIME-html + opt->parsejava = HTSPARSE_DEFAULT; // parser classes + opt->hostcontrol = 0; // PAS de control host pour timeout et traffic jammer + opt->retry = 2; // 2 retry par défaut + opt->errpage = 1; // copier ou générer une page d'erreur en cas d'erreur (404 etc.) + opt->check_type = 1; // vérifier type si inconnu (cgi,asp..) SAUF / considéré comme html + opt->all_in_cache = 0; // ne pas tout stocker en cache + opt->robots = 2; // traiter les robots.txt + opt->external = 0; // liens externes normaux + opt->passprivacy = 0; // mots de passe dans les fichiers + opt->includequery = 1; // include query-string par défaut + opt->mirror_first_page = 0; // pas mode mirror links + opt->accept_cookie = 1; // gérer les cookies + opt->cookie = NULL; + opt->http10 = 0; // laisser http/1.1 + opt->nokeepalive = 0; // pas keep-alive + opt->nocompression = 0; // pas de compression + opt->tolerant = 0; // ne pas accepter content-length incorrect + opt->parseall = 1; // tout parser (tags inconnus, par exemple) + opt->parsedebug = 0; // pas de mode débuggage + opt->norecatch = 0; // ne pas reprendre les fichiers effacés par l'utilisateur + opt->verbosedisplay = 0; // pas d'animation texte + opt->sizehack = 0; // size hack + opt->urlhack = 1; // url hack (normalizer) + StringCopy(opt->footer, HTS_DEFAULT_FOOTER); + opt->ftp_proxy = 1; // proxy http pour ftp + opt->convert_utf8 = 1; // convert html to UTF-8 + StringCopy(opt->filelist, ""); + StringCopy(opt->lang_iso, "en, *"); + StringCopy(opt->mimedefs, "\n"); // aucun filtre mime (\n IMPORTANT) StringClear(opt->mod_blacklist); // opt->log = stdout; opt->errlog = stderr; - opt->flush = 1; // flush sur les fichiers log + opt->flush = 1; // flush sur les fichiers log //opt->aff_progress=0; - opt->keyboard=0; + opt->keyboard = 0; // - StringCopy(opt->path_html,""); - StringCopy(opt->path_html_utf8,""); - StringCopy(opt->path_log,""); - StringCopy(opt->path_bin,""); + StringCopy(opt->path_html, ""); + StringCopy(opt->path_html_utf8, ""); + StringCopy(opt->path_log, ""); + StringCopy(opt->path_bin, ""); // #if HTS_SPARE_MEMORY==0 - opt->maxlink=100000; // 100,000 liens max par défaut (400Kb) - opt->maxfilter=200; // 200 filtres max par défaut + opt->maxlink = 100000; // 100,000 liens max par défaut (400Kb) + opt->maxfilter = 200; // 200 filtres max par défaut #else - opt->maxlink=10000; // 10,000 liens max par défaut (40Kb) - opt->maxfilter=50; // 50 filtres max par défaut + opt->maxlink = 10000; // 10,000 liens max par défaut (40Kb) + opt->maxfilter = 50; // 50 filtres max par défaut #endif - opt->maxcache=1048576*32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT -- + opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT -- //opt->maxcache_anticipate=256; // maximum de liens à anticiper - opt->maxtime=-1; // temps max en secondes + opt->maxtime = -1; // temps max en secondes #if HTS_USEMMS - opt->mms_maxtime = 60*3600; // max time for mms streams (one hour) + opt->mms_maxtime = 60 * 3600; // max time for mms streams (one hour) #endif - opt->maxrate=25000; // taux maxi - opt->maxconn=5.0; // nombre connexions/s - opt->waittime=-1; // wait until.. hh*3600+mm*60+ss + opt->maxrate = 25000; // taux maxi + opt->maxconn = 5.0; // nombre connexions/s + opt->waittime = -1; // wait until.. hh*3600+mm*60+ss // - opt->exec=""; - opt->is_update=0; // not an update (yet) - opt->dir_topindex=0; // do not built top index (yet) + opt->exec = ""; + opt->is_update = 0; // not an update (yet) + opt->dir_topindex = 0; // do not built top index (yet) // - opt->bypass_limits=0; // enforce limits by default - opt->state.stop=0; // stopper - opt->state.exit_xh=0; // abort + opt->bypass_limits = 0; // enforce limits by default + opt->state.stop = 0; // stopper + opt->state.exit_xh = 0; // abort /* Alocated buffers */ - opt->callbacks_fun = (t_hts_htmlcheck_callbacks*) malloct(sizeof(t_hts_htmlcheck_callbacks)); - memset(opt->callbacks_fun, 0, sizeof(t_hts_htmlcheck_callbacks)); + opt->callbacks_fun = + (t_hts_htmlcheck_callbacks *) malloct(sizeof(t_hts_htmlcheck_callbacks)); + memset(opt->callbacks_fun, 0, sizeof(t_hts_htmlcheck_callbacks)); /* Preload callbacks : java and flash parser, and the automatic user-defined callback */ { int i; - for(i = 0 ; defaultModules[i] != NULL ; i++) { + + for(i = 0; defaultModules[i] != NULL; i++) { int ret = plug_wrapper(opt, defaultModules[i], defaultModules[i]); - if (ret == 0) { /* Module aborted initialization */ + + if (ret == 0) { /* Module aborted initialization */ /* Ignored. */ } } } - return opt; + return opt; } -HTSEXT_API void hts_free_opt(httrackp *opt) { +HTSEXT_API void hts_free_opt(httrackp * opt) { if (opt != NULL) { /* Alocated callbacks */ if (opt->callbacks_fun != NULL) { int i; - t_hts_htmlcheck_callbacks_item *items = (t_hts_htmlcheck_callbacks_item*) opt->callbacks_fun; - const int size = (int) sizeof(t_hts_htmlcheck_callbacks) / sizeof(t_hts_htmlcheck_callbacks_item); - assertf(sizeof(t_hts_htmlcheck_callbacks_item)*size == sizeof(t_hts_htmlcheck_callbacks)); + t_hts_htmlcheck_callbacks_item *items = + (t_hts_htmlcheck_callbacks_item *) opt->callbacks_fun; + const int size = + (int) sizeof(t_hts_htmlcheck_callbacks) / + sizeof(t_hts_htmlcheck_callbacks_item); + assertf(sizeof(t_hts_htmlcheck_callbacks_item) * size == + sizeof(t_hts_htmlcheck_callbacks)); /* Free all linked lists */ - for(i = 0 ; i < size ; i++) { + for(i = 0; i < size; i++) { t_hts_callbackarg *carg, *next_carg; - for(carg = items[i].carg ; carg != NULL && (next_carg = carg->prev.carg, carg != NULL) ; carg = next_carg ) { + + for(carg = items[i].carg; + carg != NULL && (next_carg = carg->prev.carg, carg != NULL); + carg = next_carg) { hts_free(carg); } } @@ -5530,8 +5924,10 @@ HTSEXT_API void hts_free_opt(httrackp *opt) { /* Cancel chain */ if (opt->state.cancel != NULL) { htsoptstatecancel *cancel; - for(cancel = opt->state.cancel ; cancel != NULL ; ) { + + for(cancel = opt->state.cancel; cancel != NULL;) { htsoptstatecancel *next = cancel->next; + if (cancel->url != NULL) { freet(cancel->url); } @@ -5572,109 +5968,153 @@ HTSEXT_API void hts_free_opt(httrackp *opt) { } // defaut wrappers -static void __cdecl htsdefault_init(t_hts_callbackarg *carg) { +static void __cdecl htsdefault_init(t_hts_callbackarg * carg) { } -static void __cdecl htsdefault_uninit(t_hts_callbackarg *carg) { +static void __cdecl htsdefault_uninit(t_hts_callbackarg * carg) { // hts_freevar(); } -static int __cdecl htsdefault_start(t_hts_callbackarg *carg, httrackp* opt) { - return 1; +static int __cdecl htsdefault_start(t_hts_callbackarg * carg, httrackp * opt) { + return 1; } -static int __cdecl htsdefault_chopt(t_hts_callbackarg *carg, httrackp* opt) { +static int __cdecl htsdefault_chopt(t_hts_callbackarg * carg, httrackp * opt) { return 1; } -static int __cdecl htsdefault_end(t_hts_callbackarg *carg, httrackp* opt) { - return 1; +static int __cdecl htsdefault_end(t_hts_callbackarg * carg, httrackp * opt) { + return 1; } -static int __cdecl htsdefault_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_adresse,const char* url_fichier) { +static int __cdecl htsdefault_preprocesshtml(t_hts_callbackarg * carg, + httrackp * opt, char **html, + int *len, const char *url_adresse, + const char *url_fichier) { return 1; } -static int __cdecl htsdefault_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_adresse,const char* url_fichier) { +static int __cdecl htsdefault_postprocesshtml(t_hts_callbackarg * carg, + httrackp * opt, char **html, + int *len, const char *url_adresse, + const char *url_fichier) { return 1; } -static int __cdecl htsdefault_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_adresse,const char* url_fichier) { +static int __cdecl htsdefault_checkhtml(t_hts_callbackarg * carg, + httrackp * opt, char *html, int len, + const char *url_adresse, + const char *url_fichier) { return 1; } -static int __cdecl htsdefault_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time,hts_stat_struct* stats) { // appelé à chaque boucle de HTTrack +static int __cdecl htsdefault_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * back, int back_max, int back_index, int lien_n, int lien_tot, int stat_time, hts_stat_struct * stats) { // appelé à chaque boucle de HTTrack return 1; } -static const char* __cdecl htsdefault_query(t_hts_callbackarg *carg, httrackp *opt, const char* question) { +static const char *__cdecl htsdefault_query(t_hts_callbackarg * carg, + httrackp * opt, + const char *question) { return ""; } -static const char* __cdecl htsdefault_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question) { +static const char *__cdecl htsdefault_query2(t_hts_callbackarg * carg, + httrackp * opt, + const char *question) { return ""; } -static const char* __cdecl htsdefault_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question) { +static const char *__cdecl htsdefault_query3(t_hts_callbackarg * carg, + httrackp * opt, + const char *question) { return ""; } -static int __cdecl htsdefault_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status) { +static int __cdecl htsdefault_check(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + int status) { return -1; } -static int __cdecl htsdefault_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status) { +static int __cdecl htsdefault_check_mime(t_hts_callbackarg * carg, + httrackp * opt, const char *adr, + const char *fil, const char *mime, + int status) { return -1; } -static void __cdecl htsdefault_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile) { - while (fexist(lockfile)) { +static void __cdecl htsdefault_pause(t_hts_callbackarg * carg, httrackp * opt, + const char *lockfile) { + while(fexist(lockfile)) { Sleep(1000); } } -static void __cdecl htsdefault_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file) { +static void __cdecl htsdefault_filesave(t_hts_callbackarg * carg, + httrackp * opt, const char *file) { } -static void __cdecl htsdefault_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* file, const char* sav, int is_new, int is_modified, int not_updated) { +static void __cdecl htsdefault_filesave2(t_hts_callbackarg * carg, + httrackp * opt, const char *adr, + const char *file, const char *sav, + int is_new, int is_modified, + int not_updated) { } -static int __cdecl htsdefault_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link) { +static int __cdecl htsdefault_linkdetected(t_hts_callbackarg * carg, + httrackp * opt, char *link) { return 1; } -static int __cdecl htsdefault_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag) { +static int __cdecl htsdefault_linkdetected2(t_hts_callbackarg * carg, + httrackp * opt, char *link, + const char *start_tag) { return 1; } -static int __cdecl htsdefault_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, lien_back* back) { +static int __cdecl htsdefault_xfrstatus(t_hts_callbackarg * carg, + httrackp * opt, lien_back * back) { return 1; } -static int __cdecl htsdefault_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save) { +static int __cdecl htsdefault_savename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, + const char *fil_complete, + const char *referer_adr, + const char *referer_fil, char *save) { return 1; } -static int __cdecl htsdefault_sendhead(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing) { +static int __cdecl htsdefault_sendhead(t_hts_callbackarg * carg, httrackp * opt, + char *buff, const char *adr, + const char *fil, const char *referer_adr, + const char *referer_fil, + htsblk * outgoing) { return 1; } -static int __cdecl htsdefault_receivehead(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming) { +static int __cdecl htsdefault_receivehead(t_hts_callbackarg * carg, + httrackp * opt, char *buff, + const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, + htsblk * incoming) { return 1; } -static int __cdecl htsdefault_detect(t_hts_callbackarg *carg, httrackp *opt, htsmoduleStruct* str) { +static int __cdecl htsdefault_detect(t_hts_callbackarg * carg, httrackp * opt, + htsmoduleStruct * str) { return 0; } -static int __cdecl htsdefault_parse(t_hts_callbackarg *carg, httrackp *opt, htsmoduleStruct* str) { +static int __cdecl htsdefault_parse(t_hts_callbackarg * carg, httrackp * opt, + htsmoduleStruct * str) { return 0; } - /* Default internal dummy callbacks */ const t_hts_htmlcheck_callbacks default_callbacks = { - { htsdefault_init, NULL }, - { htsdefault_uninit, NULL }, - { htsdefault_start, NULL }, - { htsdefault_end, NULL }, - { htsdefault_chopt, NULL }, - { htsdefault_preprocesshtml, NULL }, - { htsdefault_postprocesshtml, NULL }, - { htsdefault_checkhtml, NULL }, - { htsdefault_query, NULL }, - { htsdefault_query2, NULL }, - { htsdefault_query3, NULL }, - { htsdefault_loop, NULL }, - { htsdefault_check, NULL }, - { htsdefault_check_mime, NULL }, - { htsdefault_pause, NULL }, - { htsdefault_filesave, NULL }, - { htsdefault_filesave2, NULL }, - { htsdefault_linkdetected, NULL }, - { htsdefault_linkdetected2, NULL }, - { htsdefault_xfrstatus, NULL }, - { htsdefault_savename, NULL }, - { htsdefault_sendhead, NULL }, - { htsdefault_receivehead, NULL }, - { htsdefault_detect, NULL }, - { htsdefault_parse, NULL } + {htsdefault_init, NULL}, + {htsdefault_uninit, NULL}, + {htsdefault_start, NULL}, + {htsdefault_end, NULL}, + {htsdefault_chopt, NULL}, + {htsdefault_preprocesshtml, NULL}, + {htsdefault_postprocesshtml, NULL}, + {htsdefault_checkhtml, NULL}, + {htsdefault_query, NULL}, + {htsdefault_query2, NULL}, + {htsdefault_query3, NULL}, + {htsdefault_loop, NULL}, + {htsdefault_check, NULL}, + {htsdefault_check_mime, NULL}, + {htsdefault_pause, NULL}, + {htsdefault_filesave, NULL}, + {htsdefault_filesave2, NULL}, + {htsdefault_linkdetected, NULL}, + {htsdefault_linkdetected2, NULL}, + {htsdefault_xfrstatus, NULL}, + {htsdefault_savename, NULL}, + {htsdefault_sendhead, NULL}, + {htsdefault_receivehead, NULL}, + {htsdefault_detect, NULL}, + {htsdefault_parse, NULL} }; #define CHARCAST(A) ( (char*) (A) ) @@ -5684,82 +6124,87 @@ const t_hts_htmlcheck_callbacks default_callbacks = { #define MEMBER_OF(STRUCT, OFFSET, TYPE) ( * ((TYPE*)((char*)(STRUCT) + (OFFSET))) ) const t_hts_callback_ref default_callbacks_ref[] = { - CALLBACK_REF("init", init), - CALLBACK_REF("free", uninit), - CALLBACK_REF("start", start), - CALLBACK_REF("end", end), - CALLBACK_REF("change-options", chopt), - CALLBACK_REF("preprocess-html", preprocess), - CALLBACK_REF("postprocess-html", postprocess), - CALLBACK_REF("check-html", check_html), - CALLBACK_REF("query", query), - CALLBACK_REF("query2", query2), - CALLBACK_REF("query3", query3), - CALLBACK_REF("loop", loop), - CALLBACK_REF("check-link", check_link), - CALLBACK_REF("check-mime", check_mime), - CALLBACK_REF("pause", pause), - CALLBACK_REF("save-file", filesave), - CALLBACK_REF("save-file2", filesave2), - CALLBACK_REF("link-detected", linkdetected), - CALLBACK_REF("link-detected2", linkdetected2), - CALLBACK_REF("transfer-status", xfrstatus), - CALLBACK_REF("save-name", savename), - CALLBACK_REF("send-header", sendhead), - CALLBACK_REF("receive-header", receivehead), - { NULL, 0 } + CALLBACK_REF("init", init), + CALLBACK_REF("free", uninit), + CALLBACK_REF("start", start), + CALLBACK_REF("end", end), + CALLBACK_REF("change-options", chopt), + CALLBACK_REF("preprocess-html", preprocess), + CALLBACK_REF("postprocess-html", postprocess), + CALLBACK_REF("check-html", check_html), + CALLBACK_REF("query", query), + CALLBACK_REF("query2", query2), + CALLBACK_REF("query3", query3), + CALLBACK_REF("loop", loop), + CALLBACK_REF("check-link", check_link), + CALLBACK_REF("check-mime", check_mime), + CALLBACK_REF("pause", pause), + CALLBACK_REF("save-file", filesave), + CALLBACK_REF("save-file2", filesave2), + CALLBACK_REF("link-detected", linkdetected), + CALLBACK_REF("link-detected2", linkdetected2), + CALLBACK_REF("transfer-status", xfrstatus), + CALLBACK_REF("save-name", savename), + CALLBACK_REF("send-header", sendhead), + CALLBACK_REF("receive-header", receivehead), + {NULL, 0} }; size_t hts_get_callback_offs(const char *name) { - const t_hts_callback_ref *ref; - for(ref = &default_callbacks_ref[0] ; ref->name != NULL ; ref++) { - if (strcmp(name, ref->name) == 0) { - return ref->offset; - } - } - return (size_t)(-1); + const t_hts_callback_ref *ref; + + for(ref = &default_callbacks_ref[0]; ref->name != NULL; ref++) { + if (strcmp(name, ref->name) == 0) { + return ref->offset; + } + } + return (size_t) (-1); } -int hts_set_callback(t_hts_htmlcheck_callbacks *callbacks, const char *name, void *function) { - size_t offs = hts_get_callback_offs(name); - if (offs != (size_t) -1) { - MEMBER_OF(callbacks, offs, void*) = function; - return 0; - } - return 1; +int hts_set_callback(t_hts_htmlcheck_callbacks * callbacks, const char *name, + void *function) { + size_t offs = hts_get_callback_offs(name); + + if (offs != (size_t) - 1) { + MEMBER_OF(callbacks, offs, void *) = function; + + return 0; + } + return 1; } -void *hts_get_callback(t_hts_htmlcheck_callbacks *callbacks, const char *name) { - size_t offs = hts_get_callback_offs(name); - if (offs != (size_t) -1) { - return MEMBER_OF(callbacks, offs, void*); - } - return NULL; +void *hts_get_callback(t_hts_htmlcheck_callbacks * callbacks, const char *name) { + size_t offs = hts_get_callback_offs(name); + + if (offs != (size_t) - 1) { + return MEMBER_OF(callbacks, offs, void *); + } + return NULL; } // end defaut wrappers /* libc stubs */ -HTSEXT_API char* hts_strdup(const char* str) { +HTSEXT_API char *hts_strdup(const char *str) { return strdup(str); } -HTSEXT_API void* hts_malloc(size_t size) { +HTSEXT_API void *hts_malloc(size_t size) { return malloc(size); } -HTSEXT_API void* hts_realloc(void* data, size_t size) { +HTSEXT_API void *hts_realloc(void *data, size_t size) { return realloc(data, size); } -HTSEXT_API void hts_free(void* data) { +HTSEXT_API void hts_free(void *data) { free(data); } /* Dummy functions */ HTSEXT_API int hts_resetvar(void) { - return 0; + return 0; } #ifdef _WIN32 @@ -5770,12 +6215,13 @@ DIR *opendir(const char *name) { DIR *dir; size_t len; int i; + if (name == NULL || *name == '\0') { errno = ENOENT; return NULL; } if (!GetFileAttributesEx(name, GetFileExInfoStandard, &st) - || ( st.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) == 0) { + || (st.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) { errno = ENOENT; return NULL; } @@ -5788,7 +6234,7 @@ DIR *opendir(const char *name) { dir->h = INVALID_HANDLE_VALUE; dir->name = malloc(len + 2 + 1); strcpy(dir->name, name); - for(i = 0 ; dir->name[i] != '\0' ; i++) { + for(i = 0; dir->name[i] != '\0'; i++) { if (dir->name[i] == '/') { dir->name[i] = '\\'; } @@ -5797,8 +6243,9 @@ DIR *opendir(const char *name) { return dir; } -struct dirent *readdir(DIR *dir) { +struct dirent *readdir(DIR * dir) { WIN32_FIND_DATAA find; + if (dir->h == INVALID_HANDLE_VALUE) { dir->h = FindFirstFileA(dir->name, &find); } else { @@ -5816,7 +6263,7 @@ struct dirent *readdir(DIR *dir) { return NULL; } -int closedir(DIR *dir) { +int closedir(DIR * dir) { if (dir != NULL) { if (dir->h != INVALID_HANDLE_VALUE) { CloseHandle(dir->h); @@ -5835,19 +6282,22 @@ int closedir(DIR *dir) { static void copyWchar(LPWSTR dest, const char *src) { int i; - for(i = 0 ; src[i] ; i++) { + + for(i = 0; src[i]; i++) { dest[i] = src[i]; } dest[i] = '\0'; } -FILE* hts_fopen_utf8(const char *path, const char *mode) { +FILE *hts_fopen_utf8(const char *path, const char *mode) { WCHAR wmode[32]; LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL); + assertf(strlen(mode) < sizeof(wmode) / sizeof(WCHAR)); copyWchar(wmode, mode); if (wpath != NULL) { FILE *const fp = _wfopen(wpath, wmode); + free(wpath); return fp; } else { @@ -5856,10 +6306,12 @@ FILE* hts_fopen_utf8(const char *path, const char *mode) { } } -int hts_stat_utf8(const char *path, STRUCT_STAT *buf) { +int hts_stat_utf8(const char *path, STRUCT_STAT * buf) { LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL); + if (wpath != NULL) { const int result = _wstat(wpath, buf); + free(wpath); return result; } else { @@ -5870,8 +6322,10 @@ int hts_stat_utf8(const char *path, STRUCT_STAT *buf) { int hts_unlink_utf8(const char *path) { LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL); + if (wpath != NULL) { const int result = _wunlink(wpath); + free(wpath); return result; } else { @@ -5881,10 +6335,13 @@ int hts_unlink_utf8(const char *path) { } int hts_rename_utf8(const char *oldpath, const char *newpath) { - LPWSTR woldpath = hts_convertUTF8StringToUCS2(oldpath, (int) strlen(oldpath), NULL); - LPWSTR wnewpath = hts_convertUTF8StringToUCS2(newpath, (int) strlen(newpath), NULL); + LPWSTR woldpath = + hts_convertUTF8StringToUCS2(oldpath, (int) strlen(oldpath), NULL); + LPWSTR wnewpath = + hts_convertUTF8StringToUCS2(newpath, (int) strlen(newpath), NULL); if (woldpath != NULL && wnewpath != NULL) { const int result = _wrename(woldpath, wnewpath); + free(woldpath); free(wnewpath); return result; @@ -5900,8 +6357,10 @@ int hts_rename_utf8(const char *oldpath, const char *newpath) { int hts_mkdir_utf8(const char *path) { LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL); + if (wpath != NULL) { const int result = _wmkdir(wpath); + free(wpath); return result; } else { @@ -5910,11 +6369,13 @@ int hts_mkdir_utf8(const char *path) { } } -HTSEXT_API int hts_utime_utf8(const char *path, const STRUCT_UTIMBUF *times) { +HTSEXT_API int hts_utime_utf8(const char *path, const STRUCT_UTIMBUF * times) { STRUCT_UTIMBUF mtimes = *times; LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL); + if (wpath != NULL) { const int result = _wutime(wpath, &mtimes); + free(wpath); return result; } else { @@ -5926,4 +6387,3 @@ HTSEXT_API int hts_utime_utf8(const char *path, const STRUCT_UTIMBUF *times) { #endif // Fin - diff --git a/src/htslib.h b/src/htslib.h index 3121004..b622137 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Subroutines .h */ /* Author: Xavier Roche */ @@ -37,7 +34,7 @@ Please visit our Website: http://www.httrack.com // Fichier librairie .h #ifndef HTS_DEFH -#define HTS_DEFH +#define HTS_DEFH /* Forward definitions */ #ifndef HTS_DEF_FWSTRUCT_htsrequest @@ -90,10 +87,11 @@ typedef struct t_dnscache t_dnscache; #else #define MSVC2003INLINEBUG #endif -MSVC2003INLINEBUG HTS_STATIC char* getHtsOptBuff_(httrackp *opt) { - opt->state.concat.index = ( opt->state.concat.index + 1 ) % 16; - return opt->state.concat.buff[opt->state.concat.index]; +MSVC2003INLINEBUG HTS_STATIC char *getHtsOptBuff_(httrackp * opt) { + opt->state.concat.index = (opt->state.concat.index + 1) % 16; + return opt->state.concat.buff[opt->state.concat.index]; } + #undef MSVC2003INLINEBUG #define OPT_GET_BUFF(OPT) ( getHtsOptBuff_(OPT) ) @@ -106,76 +104,75 @@ struct htsrequest_proxy { int active; char name[1024]; int port; - char bindhost[256]; // bind this host -}; + char bindhost[256]; // bind this host +}; + #ifndef HTS_DEF_FWSTRUCT_htsrequest #define HTS_DEF_FWSTRUCT_htsrequest typedef struct htsrequest htsrequest; #endif struct htsrequest { - short int user_agent_send; // user agent (ex: httrack/1.0 [sun]) - short int http11; // l'en tête peut (doit) être signé HTTP/1.1 et non HTTP/1.0 - short int nokeepalive; // pas de keep-alive - short int range_used; // Range utilisé - short int nocompression; // Pas de compression - short int flush_garbage; // recycled + short int user_agent_send; // user agent (ex: httrack/1.0 [sun]) + short int http11; // l'en tête peut (doit) être signé HTTP/1.1 et non HTTP/1.0 + short int nokeepalive; // pas de keep-alive + short int range_used; // Range utilisé + short int nocompression; // Pas de compression + short int flush_garbage; // recycled char user_agent[128]; char referer[256]; char from[256]; char lang_iso[64]; - htsrequest_proxy proxy; // proxy + htsrequest_proxy proxy; // proxy }; - // structure pour retour d'une connexion/prise d'en tête #ifndef HTS_DEF_FWSTRUCT_htsblk #define HTS_DEF_FWSTRUCT_htsblk typedef struct htsblk htsblk; #endif struct htsblk { - int statuscode; // status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945) - short int notmodified; // page ou fichier NON modifié (transféré) - short int is_write; // sortie sur disque (out) ou en mémoire (adr) - short int is_chunk; // mode chunk - short int compressed; // compressé? - short int empty; // vide? - short int keep_alive; // Keep-Alive? - short int keep_alive_trailers; // ..with trailers extension - int keep_alive_t; // KA timeout - int keep_alive_max; // KA number of requests - char* adr; // adresse du bloc de mémoire, NULL=vide - char* headers; // adresse des en têtes si présents - FILE* out; // écriture directe sur disque (si is_write=1) - LLint size; // taille fichier - char msg[80]; // message éventuel si échec ("\0"=non précisé) - char contenttype[64]; // content-type ("text/html" par exemple) - char charset[64]; // charset ("iso-8859-1" par exemple) - char contentencoding[64]; // content-encoding ("gzip" par exemple) - char* location; // on copie dedans éventuellement la véritable 'location' - LLint totalsize; // taille totale à télécharger (-1=inconnue) - short int is_file; // ce n'est pas une socket mais un descripteur de fichier si 1 - T_SOC soc; // ID socket - SOCaddr address; // IP address - int address_size; // IP address structure length - FILE* fp; // fichier pour file:// + int statuscode; // status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945) + short int notmodified; // page ou fichier NON modifié (transféré) + short int is_write; // sortie sur disque (out) ou en mémoire (adr) + short int is_chunk; // mode chunk + short int compressed; // compressé? + short int empty; // vide? + short int keep_alive; // Keep-Alive? + short int keep_alive_trailers; // ..with trailers extension + int keep_alive_t; // KA timeout + int keep_alive_max; // KA number of requests + char *adr; // adresse du bloc de mémoire, NULL=vide + char *headers; // adresse des en têtes si présents + FILE *out; // écriture directe sur disque (si is_write=1) + LLint size; // taille fichier + char msg[80]; // message éventuel si échec ("\0"=non précisé) + char contenttype[64]; // content-type ("text/html" par exemple) + char charset[64]; // charset ("iso-8859-1" par exemple) + char contentencoding[64]; // content-encoding ("gzip" par exemple) + char *location; // on copie dedans éventuellement la véritable 'location' + LLint totalsize; // taille totale à télécharger (-1=inconnue) + short int is_file; // ce n'est pas une socket mais un descripteur de fichier si 1 + T_SOC soc; // ID socket + SOCaddr address; // IP address + int address_size; // IP address structure length + FILE *fp; // fichier pour file:// #if HTS_USEOPENSSL - short int ssl; // is this connection a SSL one? (https) + short int ssl; // is this connection a SSL one? (https) // BIO* ssl_soc; // SSL structure - SSL * ssl_con; // connection structure -#endif - char lastmodified[64]; // Last-Modified - char etag[64]; // Etag - char cdispo[256]; // Content-Disposition coupé - LLint crange; // Content-Range - LLint crange_start; // Content-Range - LLint crange_end; // Content-Range - int debugid; // debug connection + SSL *ssl_con; // connection structure +#endif + char lastmodified[64]; // Last-Modified + char etag[64]; // Etag + char cdispo[256]; // Content-Disposition coupé + LLint crange; // Content-Range + LLint crange_start; // Content-Range + LLint crange_end; // Content-Range + int debugid; // debug connection /* */ - htsrequest req; // paramètres pour la requête - /*char digest[32+2]; // digest md5 généré par le moteur ("" si non généré)*/ + htsrequest req; // paramètres pour la requête + /*char digest[32+2]; // digest md5 généré par le moteur ("" si non généré) */ }; - /* ANCIENNE STURCTURE pour cache 1.0 */ #ifndef HTS_DEF_FWSTRUCT_OLD_t_proxy #define HTS_DEF_FWSTRUCT_OLD_t_proxy @@ -185,30 +182,32 @@ struct OLD_t_proxy { int active; char name[1024]; int port; -}; +}; + #ifndef HTS_DEF_FWSTRUCT_OLD_htsblk #define HTS_DEF_FWSTRUCT_OLD_htsblk typedef struct OLD_htsblk OLD_htsblk; #endif struct OLD_htsblk { - int statuscode; // ANCIENNE STURCTURE - status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945) - int notmodified; // ANCIENNE STURCTURE - page ou fichier NON modifié (transféré) - int is_write; // ANCIENNE STURCTURE - sortie sur disque (out) ou en mémoire (adr) - char* adr; // ANCIENNE STURCTURE - adresse du bloc de mémoire, NULL=vide - FILE* out; // ANCIENNE STURCTURE - écriture directe sur disque (si is_write=1) - int size; // ANCIENNE STURCTURE - taille fichier - char msg[80]; // ANCIENNE STURCTURE - message éventuel si échec ("\0"=non précisé) - char contenttype[64]; // ANCIENNE STURCTURE - content-type ("text/html" par exemple) - char* location; // ANCIENNE STURCTURE - on copie dedans éventuellement la véritable 'location' - int totalsize; // ANCIENNE STURCTURE - taille totale à télécharger (-1=inconnue) - int is_file; // ANCIENNE STURCTURE - ce n'est pas une socket mais un descripteur de fichier si 1 - T_SOC soc; // ANCIENNE STURCTURE - ID socket - FILE* fp; // ANCIENNE STURCTURE - fichier pour file:// - OLD_t_proxy proxy; // ANCIENNE STURCTURE - proxy - int user_agent_send; // ANCIENNE STURCTURE - user agent (ex: httrack/1.0 [sun]) + int statuscode; // ANCIENNE STURCTURE - status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945) + int notmodified; // ANCIENNE STURCTURE - page ou fichier NON modifié (transféré) + int is_write; // ANCIENNE STURCTURE - sortie sur disque (out) ou en mémoire (adr) + char *adr; // ANCIENNE STURCTURE - adresse du bloc de mémoire, NULL=vide + FILE *out; // ANCIENNE STURCTURE - écriture directe sur disque (si is_write=1) + int size; // ANCIENNE STURCTURE - taille fichier + char msg[80]; // ANCIENNE STURCTURE - message éventuel si échec ("\0"=non précisé) + char contenttype[64]; // ANCIENNE STURCTURE - content-type ("text/html" par exemple) + char *location; // ANCIENNE STURCTURE - on copie dedans éventuellement la véritable 'location' + int totalsize; // ANCIENNE STURCTURE - taille totale à télécharger (-1=inconnue) + int is_file; // ANCIENNE STURCTURE - ce n'est pas une socket mais un descripteur de fichier si 1 + T_SOC soc; // ANCIENNE STURCTURE - ID socket + FILE *fp; // ANCIENNE STURCTURE - fichier pour file:// + OLD_t_proxy proxy; // ANCIENNE STURCTURE - proxy + int user_agent_send; // ANCIENNE STURCTURE - user agent (ex: httrack/1.0 [sun]) char user_agent[64]; - int http11; // ANCIENNE STURCTURE - l'en tête doit être signé HTTP/1.1 et non HTTP/1.0 + int http11; // ANCIENNE STURCTURE - l'en tête doit être signé HTTP/1.1 et non HTTP/1.0 }; + /* fin ANCIENNE STURCTURE pour cache 1.0 */ // cache pour le dns, pour éviter de faire des gethostbyname sans arrêt @@ -218,24 +217,24 @@ typedef struct t_dnscache t_dnscache; #endif struct t_dnscache { char iadr[1024]; - struct t_dnscache* n; - char host_addr[HTS_MAXADDRLEN]; // 4 octets (v4), ou 16 octets (v6) - int host_length; // 4 normalement - ==0 alors en cours de résolution - // ou >16 si sockaddr - // ==-1 alors erreur (host n'éxiste pas) + struct t_dnscache *n; + char host_addr[HTS_MAXADDRLEN]; // 4 octets (v4), ou 16 octets (v6) + int host_length; // 4 normalement - ==0 alors en cours de résolution + // ou >16 si sockaddr + // ==-1 alors erreur (host n'éxiste pas) }; - /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE extern htsmutex dns_lock; // initialize an htsblk structure -void hts_init_htsblk(htsblk* r); +void hts_init_htsblk(htsblk * r); // fonctions unix/winsock -int hts_read(htsblk* r,char* buff,int size); +int hts_read(htsblk * r, char *buff, int size); + //int HTS_TOTAL_RECV_CHECK(int var); LLint check_downloadable_bytes(int rate); @@ -243,207 +242,233 @@ LLint check_downloadable_bytes(int rate); HTSEXT_API int hts_init(void); HTSEXT_API int hts_uninit(void); HTSEXT_API int hts_uninit_module(void); -HTSEXT_API int hts_resetvar(void); /* dummy */ +HTSEXT_API int hts_resetvar(void); /* dummy */ HTSEXT_API void hts_debug(int level); -HTSEXT_API httrackp* hts_create_opt(void); -HTSEXT_API void hts_free_opt(httrackp *opt); -HTSEXT_API void set_wrappers(httrackp *opt); /* LEGACY */ -HTSEXT_API int plug_wrapper(httrackp *opt, const char *moduleName, const char* argv); +HTSEXT_API httrackp *hts_create_opt(void); +HTSEXT_API void hts_free_opt(httrackp * opt); +HTSEXT_API void set_wrappers(httrackp * opt); /* LEGACY */ +HTSEXT_API int plug_wrapper(httrackp * opt, const char *moduleName, + const char *argv); -HTSEXT_API char* hts_strdup(const char* string); -HTSEXT_API void* hts_malloc(size_t size); -HTSEXT_API void* hts_realloc(void* data, size_t size); -HTSEXT_API void hts_free(void* data); +HTSEXT_API char *hts_strdup(const char *string); +HTSEXT_API void *hts_malloc(size_t size); +HTSEXT_API void *hts_realloc(void *data, size_t size); +HTSEXT_API void hts_free(void *data); #endif // fonctions principales -T_SOC http_fopen(httrackp *opt,char* adr,char* fil,htsblk* retour); -T_SOC http_xfopen(httrackp *opt,int mode,int treat,int waitconnect,char* xsend,char* adr,char* fil,htsblk* retour); -int http_sendhead(httrackp *opt,t_cookie* cookie,int mode,char* xsend,char* adr,char* fil,char* referer_adr,char* referer_fil,htsblk* retour); -htsblk httpget(httrackp *opt,char* url); +T_SOC http_fopen(httrackp * opt, char *adr, char *fil, htsblk * retour); +T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect, + char *xsend, char *adr, char *fil, htsblk * retour); +int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, char *xsend, + char *adr, char *fil, char *referer_adr, char *referer_fil, + htsblk * retour); +htsblk httpget(httrackp * opt, char *url); + //int newhttp(char* iadr,char* err=NULL); -T_SOC newhttp(httrackp *opt,const char* iadr,htsblk* retour,int port,int waitconnect); -HTS_INLINE void deletehttp(htsblk* r); -HTS_INLINE int deleteaddr(htsblk* r); +T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port, + int waitconnect); +HTS_INLINE void deletehttp(htsblk * r); +HTS_INLINE int deleteaddr(htsblk * r); HTS_INLINE void deletesoc(T_SOC soc); -HTS_INLINE void deletesoc_r(htsblk* r); -htsblk http_location(httrackp *opt,char* adr,char* fil,char* loc); -htsblk http_test(httrackp *opt,char* adr,char* fil,char* loc); -int check_readinput(htsblk* r); +HTS_INLINE void deletesoc_r(htsblk * r); +htsblk http_location(httrackp * opt, char *adr, char *fil, char *loc); +htsblk http_test(httrackp * opt, char *adr, char *fil, char *loc); +int check_readinput(htsblk * r); int check_readinput_t(T_SOC soc, int timeout); -void http_fread(T_SOC soc,htsblk* retour); -LLint http_fread1(htsblk* r); -void treathead(t_cookie* cookie,char* adr,char* fil,htsblk* retour,char* rcvd); -void treatfirstline(htsblk* retour,char* rcvd); +void http_fread(T_SOC soc, htsblk * retour); +LLint http_fread1(htsblk * r); +void treathead(t_cookie * cookie, char *adr, char *fil, htsblk * retour, + char *rcvd); +void treatfirstline(htsblk * retour, char *rcvd); + #ifndef HTTRACK_DEFLIB -HTSEXT_API void infostatuscode(char* msg,int statuscode); +HTSEXT_API void infostatuscode(char *msg, int statuscode); #endif // sous-fonctions -htsblk xhttpget(httrackp *opt,char* adr,char* fil); -htsblk http_gethead(httrackp *opt,char* adr,char* fil); -LLint http_xfread1(htsblk* r,int bufl); -HTS_INLINE t_hostent* hts_gethostbyname(httrackp *opt,const char* iadr, void* v_buffer); +htsblk xhttpget(httrackp * opt, char *adr, char *fil); +htsblk http_gethead(httrackp * opt, char *adr, char *fil); +LLint http_xfread1(htsblk * r, int bufl); +HTS_INLINE t_hostent *hts_gethostbyname(httrackp * opt, const char *iadr, + void *v_buffer); #ifndef HTTRACK_DEFLIB -HTSEXT_API t_hostent* vxgethostbyname(char* hostname, void* v_buffer); +HTSEXT_API t_hostent *vxgethostbyname(char *hostname, void *v_buffer); #endif -t_hostent* _hts_ghbn(t_dnscache* cache,const char* iadr,t_hostent* retour); +t_hostent *_hts_ghbn(t_dnscache * cache, const char *iadr, t_hostent * retour); int ftp_available(void); + #if HTS_DNSCACHE -void hts_cache_free(t_dnscache* cache); -int hts_dnstest(httrackp *opt, const char* _iadr); -t_dnscache* _hts_cache(httrackp *opt); +void hts_cache_free(t_dnscache * cache); +int hts_dnstest(httrackp * opt, const char *_iadr); +t_dnscache *_hts_cache(httrackp * opt); #endif // outils divers HTS_INLINE TStamp time_local(void); + #ifndef HTTRACK_DEFLIB HTSEXT_API HTS_INLINE TStamp mtime_local(void); #endif -void sec2str(char *s,TStamp t); +void sec2str(char *s, TStamp t); + #ifndef HTTRACK_DEFLIB -HTSEXT_API void qsec2str(char *st,TStamp t); -#endif -void time_gmt_rfc822(char* s); -void time_local_rfc822(char* s); -struct tm* convert_time_rfc822(struct tm* buffer, const char* s); -int set_filetime(const char* file,struct tm* tm_time); -int set_filetime_rfc822(const char* file,const char* date); -int get_filetime_rfc822(const char* file,char* date); -HTS_INLINE void time_rfc822(char* s,struct tm * A); -HTS_INLINE void time_rfc822_local(char* s,struct tm * A); +HTSEXT_API void qsec2str(char *st, TStamp t); +#endif +void time_gmt_rfc822(char *s); +void time_local_rfc822(char *s); +struct tm *convert_time_rfc822(struct tm *buffer, const char *s); +int set_filetime(const char *file, struct tm *tm_time); +int set_filetime_rfc822(const char *file, const char *date); +int get_filetime_rfc822(const char *file, char *date); +HTS_INLINE void time_rfc822(char *s, struct tm *A); +HTS_INLINE void time_rfc822_local(char *s, struct tm *A); + #ifndef HTTRACK_DEFLIB -HTSEXT_API char* int2char(strc_int2bytes2* strc, int n); -HTSEXT_API char* int2bytes(strc_int2bytes2* strc, LLint n); -HTSEXT_API char* int2bytessec(strc_int2bytes2* strc, long int n); -HTSEXT_API char** int2bytes2(strc_int2bytes2* strc, LLint n); -#endif -HTS_INLINE int sendc(htsblk* r, const char* s); -int finput(int fd,char* s,int max); -int binput(char* buff,char* s,int max); -int linput(FILE* fp,char* s,int max); -int linputsoc(T_SOC soc, char* s, int max); -int linputsoc_t(T_SOC soc, char* s, int max, int timeout); -int linput_trim(FILE* fp,char* s,int max); -int linput_cpp(FILE* fp,char* s,int max); -void rawlinput(FILE* fp,char* s,int max); -char* strstrcase(char *s,char *o); -int ident_url_absolute(const char* url,char* adr,char* fil); -void fil_simplifie(char* f); -int is_unicode_utf8(const char* buffer, size_t size); -void map_characters(unsigned char* buffer, unsigned int size, unsigned int* map); -int ishtml(httrackp *opt,const char* urlfil); -int ishtml_ext(const char* a); +HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n); +HTSEXT_API char *int2bytes(strc_int2bytes2 * strc, LLint n); +HTSEXT_API char *int2bytessec(strc_int2bytes2 * strc, long int n); +HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n); +#endif +HTS_INLINE int sendc(htsblk * r, const char *s); +int finput(int fd, char *s, int max); +int binput(char *buff, char *s, int max); +int linput(FILE * fp, char *s, int max); +int linputsoc(T_SOC soc, char *s, int max); +int linputsoc_t(T_SOC soc, char *s, int max, int timeout); +int linput_trim(FILE * fp, char *s, int max); +int linput_cpp(FILE * fp, char *s, int max); +void rawlinput(FILE * fp, char *s, int max); +char *strstrcase(char *s, char *o); +int ident_url_absolute(const char *url, char *adr, char *fil); +void fil_simplifie(char *f); +int is_unicode_utf8(const char *buffer, size_t size); +void map_characters(unsigned char *buffer, unsigned int size, + unsigned int *map); +int ishtml(httrackp * opt, const char *urlfil); +int ishtml_ext(const char *a); int ishttperror(int err); -void guess_httptype(httrackp *opt,char *s,const char *fil); +void guess_httptype(httrackp * opt, char *s, const char *fil); + #ifndef HTTRACK_DEFLIB -HTSEXT_API void get_httptype(httrackp *opt,char *s,const char *fil,int flag); +HTSEXT_API void get_httptype(httrackp * opt, char *s, const char *fil, + int flag); #endif -int get_userhttptype(httrackp *opt,char *s,const char *fil); -void give_mimext(char *s,const char *st); +int get_userhttptype(httrackp * opt, char *s, const char *fil); +void give_mimext(char *s, const char *st); + #ifndef HTTRACK_DEFLIB -HTSEXT_API int is_knowntype(httrackp *opt,const char *fil); -HTSEXT_API int is_userknowntype(httrackp *opt,const char *fil); +HTSEXT_API int is_knowntype(httrackp * opt, const char *fil); +HTSEXT_API int is_userknowntype(httrackp * opt, const char *fil); HTSEXT_API int is_dyntype(const char *fil); -HTSEXT_API char* get_ext(char *catbuff, const char *fil); +HTSEXT_API char *get_ext(char *catbuff, const char *fil); #endif -int may_unknown(httrackp *opt,const char* st); -int may_bogus_multiple(httrackp *opt, const char* mime, const char *filename); -int may_unknown2(httrackp *opt,const char* mime, const char *filename); +int may_unknown(httrackp * opt, const char *st); +int may_bogus_multiple(httrackp * opt, const char *mime, const char *filename); +int may_unknown2(httrackp * opt, const char *mime, const char *filename); + #ifndef HTTRACK_DEFLIB -HTSEXT_API char* jump_identification(const char*); -HTSEXT_API char* jump_normalized(const char*); -HTSEXT_API char* jump_toport(const char*); -HTSEXT_API char* fil_normalized(const char* source, char* dest); -HTSEXT_API char* adr_normalized(const char* source, char* dest); -#endif -char* strrchr_limit(const char* s, char c, const char* limit); -char* strstr_limit(const char* s, const char* sub, const char* limit); -HTS_INLINE char* jump_protocol(const char* source); -void code64(unsigned char* a,int size_a,unsigned char* b,int crlf); +HTSEXT_API char *jump_identification(const char *); +HTSEXT_API char *jump_normalized(const char *); +HTSEXT_API char *jump_toport(const char *); +HTSEXT_API char *fil_normalized(const char *source, char *dest); +HTSEXT_API char *adr_normalized(const char *source, char *dest); +#endif +char *strrchr_limit(const char *s, char c, const char *limit); +char *strstr_limit(const char *s, const char *sub, const char *limit); +HTS_INLINE char *jump_protocol(const char *source); +void code64(unsigned char *a, int size_a, unsigned char *b, int crlf); + #ifndef HTTRACK_DEFLIB -HTSEXT_API void unescape_amp(char* s); -HTSEXT_API void escape_spc_url(char* s); -HTSEXT_API void escape_in_url(char* s); -HTSEXT_API void escape_uri(char* s); -HTSEXT_API void escape_uri_utf(char* s); -HTSEXT_API void escape_check_url(char* s); -HTSEXT_API char* escape_check_url_addr(char *catbuff, const char* s); -HTSEXT_API void x_escape_http(char* s,int mode); -HTSEXT_API void x_escape_html(char* s); -HTSEXT_API void escape_remove_control(char* s); -HTSEXT_API void escape_for_html_print(char* s, char* d); -HTSEXT_API void escape_for_html_print_full(char* s, char* d); +HTSEXT_API void unescape_amp(char *s); +HTSEXT_API void escape_spc_url(char *s); +HTSEXT_API void escape_in_url(char *s); +HTSEXT_API void escape_uri(char *s); +HTSEXT_API void escape_uri_utf(char *s); +HTSEXT_API void escape_check_url(char *s); +HTSEXT_API char *escape_check_url_addr(char *catbuff, const char *s); +HTSEXT_API void x_escape_http(char *s, int mode); +HTSEXT_API void x_escape_html(char *s); +HTSEXT_API void escape_remove_control(char *s); +HTSEXT_API void escape_for_html_print(char *s, char *d); +HTSEXT_API void escape_for_html_print_full(char *s, char *d); #endif #ifndef HTTRACK_DEFLIB -HTSEXT_API char* unescape_http(char *catbuff, const char* s); -HTSEXT_API char* unescape_http_unharm(char *catbuff, const char* s, int no_high); -HTSEXT_API char* antislash_unescaped(char *catbuff, const char* s); -HTSEXT_API char* concat(char *catbuff,const char* a,const char* b); -HTSEXT_API char* fconcat(char *catbuff, const char* a, const char* b); -HTSEXT_API char* fconv(char *catbuff, const char* a); +HTSEXT_API char *unescape_http(char *catbuff, const char *s); +HTSEXT_API char *unescape_http_unharm(char *catbuff, const char *s, + int no_high); +HTSEXT_API char *antislash_unescaped(char *catbuff, const char *s); +HTSEXT_API char *concat(char *catbuff, const char *a, const char *b); +HTSEXT_API char *fconcat(char *catbuff, const char *a, const char *b); +HTSEXT_API char *fconv(char *catbuff, const char *a); #endif #define copychar(catbuff,a) concat(catbuff,(a),NULL) -char* fslash(char *catbuff, const char* a); +char *fslash(char *catbuff, const char *a); + #ifndef HTTRACK_DEFLIB -HTSEXT_API int hts_log(httrackp *opt, const char* prefix, const char *msg); -HTSEXT_API void hts_log_print(httrackp *opt, int type, const char *format, ...) HTS_PRINTF_FUN(3, 4); +HTSEXT_API int hts_log(httrackp * opt, const char *prefix, const char *msg); +HTSEXT_API void hts_log_print(httrackp * opt, int type, const char *format, + ...) HTS_PRINTF_FUN(3, 4); #endif -char* convtolower(char *catbuff, const char* a); -void hts_lowcase(char* s); -void hts_replace(char *s,char from,char to); -int multipleStringMatch(const char *s, const char *match); + char *convtolower(char *catbuff, const char *a); + void hts_lowcase(char *s); + void hts_replace(char *s, char from, char to); + int multipleStringMatch(const char *s, const char *match); - -void fprintfio(FILE* fp,char* buff,char* prefix); + void fprintfio(FILE * fp, char *buff, char *prefix); #ifdef _WIN32 #else -int sig_ignore_flag( int setflag ); // flag ignore + int sig_ignore_flag(int setflag); // flag ignore #endif -void cut_path(char* fullpath,char* path,char* pname); -int fexist(const char* s); -int fexist_utf8(const char* s); + void cut_path(char *fullpath, char *path, char *pname); + int fexist(const char *s); + int fexist_utf8(const char *s); + /*LLint fsize(const char* s); */ -off_t fpsize(FILE* fp); -off_t fsize(const char* s); -off_t fsize_utf8(const char* s); + off_t fpsize(FILE * fp); + off_t fsize(const char *s); + off_t fsize_utf8(const char *s); + /* root dir */ #ifndef HTTRACK_DEFLIB -HTSEXT_API char* hts_rootdir(char* file); + HTSEXT_API char *hts_rootdir(char *file); #endif // Threads -typedef void* ( *beginthread_type )( void * ); + typedef void *(*beginthread_type) (void *); + /*unsigned long _beginthread( beginthread_type start_address, unsigned stack_size, void *arglist );*/ /* variables globales */ -extern HTSEXT_API hts_stat_struct HTS_STAT; -extern int _DEBUG_HEAD; -extern FILE* ioinfo; + extern HTSEXT_API hts_stat_struct HTS_STAT; + extern int _DEBUG_HEAD; + extern FILE *ioinfo; /* constantes */ -extern const char* hts_mime_keep[]; -extern const char* hts_mime[][2]; -extern const char* hts_main_mime[]; -extern const char* hts_detect[]; -extern const char* hts_detectbeg[]; -extern const char* hts_nodetect[]; -extern const char* hts_detectURL[]; -extern const char* hts_detectandleave[]; -extern const char* hts_detect_js[]; + extern const char *hts_mime_keep[]; + extern const char *hts_mime[][2]; + extern const char *hts_main_mime[]; + extern const char *hts_detect[]; + extern const char *hts_detectbeg[]; + extern const char *hts_nodetect[]; + extern const char *hts_detectURL[]; + extern const char *hts_detectandleave[]; + extern const char *hts_detect_js[]; // htsmodule.c definitions -extern void* openFunctionLib(const char* file_); -extern void* getFunctionPtr(void* handle, const char* fncname); -extern void closeFunctionLib(void* handle); + extern void *openFunctionLib(const char *file_); + extern void *getFunctionPtr(void *handle, const char *fncname); + extern void closeFunctionLib(void *handle); -extern void clearCallbacks(htscallbacks* chain); -extern size_t hts_get_callback_offs(const char *name); -int hts_set_callback(t_hts_htmlcheck_callbacks *callbacks, const char *name, void *function); -void *hts_get_callback(t_hts_htmlcheck_callbacks *callbacks, const char *name); + extern void clearCallbacks(htscallbacks * chain); + extern size_t hts_get_callback_offs(const char *name); + int hts_set_callback(t_hts_htmlcheck_callbacks * callbacks, + const char *name, void *function); + void *hts_get_callback(t_hts_htmlcheck_callbacks * callbacks, + const char *name); #define CBSTRUCT(OPT) ((t_hts_htmlcheck_callbacks*) ((OPT)->callbacks_fun)) #define GET_USERCALLBACK(OPT, NAME) ( CBSTRUCT(OPT)-> NAME .fun ) @@ -488,34 +513,42 @@ void *hts_get_callback(t_hts_htmlcheck_callbacks *callbacks, const char *name); #ifndef HTS_DEF_FILEAPI #ifdef _WIN32 #define FOPEN hts_fopen_utf8 -HTSEXT_API FILE* hts_fopen_utf8(const char *path, const char *mode); + HTSEXT_API FILE *hts_fopen_utf8(const char *path, const char *mode); + #define STAT hts_stat_utf8 -typedef struct _stat STRUCT_STAT; -HTSEXT_API int hts_stat_utf8(const char *path, STRUCT_STAT *buf); + typedef struct _stat STRUCT_STAT; + HTSEXT_API int hts_stat_utf8(const char *path, STRUCT_STAT * buf); + #define UNLINK hts_unlink_utf8 -HTSEXT_API int hts_unlink_utf8(const char *pathname); + HTSEXT_API int hts_unlink_utf8(const char *pathname); + #define RENAME hts_rename_utf8 -HTSEXT_API int hts_rename_utf8(const char *oldpath, const char *newpath); + HTSEXT_API int hts_rename_utf8(const char *oldpath, const char *newpath); + #define MKDIR(F) hts_mkdir_utf8(F) -HTSEXT_API int hts_mkdir_utf8(const char *pathname); + HTSEXT_API int hts_mkdir_utf8(const char *pathname); + #define UTIME(A,B) hts_utime_utf8(A,B) -typedef struct _utimbuf STRUCT_UTIMBUF; -HTSEXT_API int hts_utime_utf8(const char *filename, const STRUCT_UTIMBUF *times); + typedef struct _utimbuf STRUCT_UTIMBUF; + HTSEXT_API int hts_utime_utf8(const char *filename, + const STRUCT_UTIMBUF * times); #else /* The underlying filesystem charset is supposed to be UTF-8 */ #define FOPEN fopen #define STAT stat -typedef struct stat STRUCT_STAT; + typedef struct stat STRUCT_STAT; + #define UNLINK unlink #define RENAME rename #define MKDIR(F) mkdir(F, HTS_ACCESS_FOLDER) -typedef struct utimbuf STRUCT_UTIMBUF; + typedef struct utimbuf STRUCT_UTIMBUF; + #define UTIME(A,B) utime(A,B) #endif #define HTS_DEF_FILEAPI #endif -#endif // internals +#endif // internals #undef PATH_SEPARATOR #ifdef _WIN32 @@ -547,17 +580,23 @@ typedef struct utimbuf STRUCT_UTIMBUF; // compare le début de f avec s et retourne la position de la fin // 'A=a' (case insensitive) -HTS_STATIC int strfield(const char* f,const char* s) { - int r=0; - while (streql(*f,*s) && ((*f)!=0) && ((*s)!=0)) { f++; s++; r++; } - if (*s==0) + HTS_STATIC int strfield(const char *f, const char *s) { + int r = 0; + + while(streql(*f, *s) && ((*f) != 0) && ((*s) != 0)) { + f++; + s++; + r++; + } + if (*s == 0) return r; else return 0; -} -HTS_STATIC int strcmpnocase(char* a,char* b) { + } +HTS_STATIC int strcmpnocase(char *a, char *b) { while(*a) { int cmp = hichar(*a) - hichar(*b); + if (cmp != 0) return cmp; a++; @@ -609,39 +648,45 @@ HTS_STATIC int strcmpnocase(char* a,char* b) { #ifdef HTS_INTERNAL_BYTECODE // check if (mime, file) is hypertext -HTS_STATIC int is_hypertext_mime(httrackp *opt,const char* mime, const char* file) { +HTS_STATIC int is_hypertext_mime(httrackp * opt, const char *mime, + const char *file) { if (is_hypertext_mime__(mime)) return 1; - if (may_unknown(opt,mime)) { + if (may_unknown(opt, mime)) { char guessed[256]; + guessed[0] = '\0'; - guess_httptype(opt,guessed, file); + guess_httptype(opt, guessed, file); return is_hypertext_mime__(guessed); } return 0; } // check if (mime, file) might be "false" hypertext -HTS_STATIC int may_be_hypertext_mime(httrackp *opt,const char* mime, const char* file) { +HTS_STATIC int may_be_hypertext_mime(httrackp * opt, const char *mime, + const char *file) { if (may_be_hypertext_mime__(mime)) return 1; - if (file != NULL && file[0] != '\0' && may_unknown(opt,mime)) { + if (file != NULL && file[0] != '\0' && may_unknown(opt, mime)) { char guessed[256]; + guessed[0] = '\0'; - guess_httptype(opt,guessed, file); + guess_httptype(opt, guessed, file); return may_be_hypertext_mime__(guessed); } return 0; } // compare (mime, file) with reference -HTS_STATIC int compare_mime(httrackp *opt,const char* mime, const char* file, const char* reference) { +HTS_STATIC int compare_mime(httrackp * opt, const char *mime, const char *file, + const char *reference) { if (is_hypertext_mime__(mime) || may_be_hypertext_mime__(mime)) return strfield2(mime, reference); - if (file != NULL && file[0] != '\0' && may_unknown(opt,mime)) { + if (file != NULL && file[0] != '\0' && may_unknown(opt, mime)) { char guessed[256]; + guessed[0] = '\0'; - guess_httptype(opt,guessed, file); + guess_httptype(opt, guessed, file); return strfield2(guessed, reference); } return 0; @@ -650,26 +695,26 @@ HTS_STATIC int compare_mime(httrackp *opt,const char* mime, const char* file, co #endif #ifdef _WIN32_WCE_XXC -extern char cwd[MAX_PATH+1]; +extern char cwd[MAX_PATH + 1]; HTS_STATIC char *getcwd_ce(char *buffer, int maxlen) { - TCHAR fileUnc[MAX_PATH+1]; - char* plast; - - if(cwd[0] == 0) - { - GetModuleFileName(NULL, fileUnc, MAX_PATH); - WideCharToMultiByte(CP_ACP, 0, fileUnc, -1, cwd, MAX_PATH, NULL, NULL); - plast = strrchr(cwd, '\\'); - if(plast) - *plast = 0; - /* Special trick to keep start menu clean... */ - if(_stricmp(cwd, "\\windows\\start menu") == 0) - strcpy(cwd, "\\Apps"); - } - if(buffer) - strncpy(buffer, cwd, maxlen); - return cwd; + TCHAR fileUnc[MAX_PATH + 1]; + char *plast; + + if (cwd[0] == 0) { + GetModuleFileName(NULL, fileUnc, MAX_PATH); + WideCharToMultiByte(CP_ACP, 0, fileUnc, -1, cwd, MAX_PATH, NULL, NULL); + plast = strrchr(cwd, '\\'); + if (plast) + *plast = 0; + /* Special trick to keep start menu clean... */ + if (_stricmp(cwd, "\\windows\\start menu") == 0) + strcpy(cwd, "\\Apps"); + } + if (buffer) + strncpy(buffer, cwd, maxlen); + return cwd; } + #undef getcwd #define getcwd getcwd_ce #endif @@ -678,11 +723,11 @@ HTS_STATIC char *getcwd_ce(char *buffer, int maxlen) { #ifdef _WIN32 #define HTS_DIRENT_SIZE 256 struct dirent { - ino_t d_ino; /* ignored */ - off_t d_off; /* ignored */ - unsigned short d_reclen; /* ignored */ - unsigned char d_type; /* ignored */ - char d_name[HTS_DIRENT_SIZE]; /* filename */ + ino_t d_ino; /* ignored */ + off_t d_off; /* ignored */ + unsigned short d_reclen; /* ignored */ + unsigned char d_type; /* ignored */ + char d_name[HTS_DIRENT_SIZE]; /* filename */ }; typedef struct DIR DIR; struct DIR { @@ -691,8 +736,8 @@ struct DIR { char *name; }; DIR *opendir(const char *name); -struct dirent *readdir(DIR *dir); -int closedir(DIR *dir); +struct dirent *readdir(DIR * dir); +int closedir(DIR * dir); #endif #endif diff --git a/src/htsmd5.c b/src/htsmd5.c index c26a11c..e045df9 100644 --- a/src/htsmd5.c +++ b/src/htsmd5.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsmd5.c subroutines: */ /* generate a md5 hash */ @@ -48,34 +45,36 @@ Please visit our Website: http://www.httrack.com #include "htsmd5.h" #include "md5.h" -int domd5mem(const char * buf, size_t len, char * digest, int asAscii) { +int domd5mem(const char *buf, size_t len, char *digest, int asAscii) { int endian = 1; unsigned char bindigest[16]; MD5_CTX ctx; - MD5Init(&ctx, * ( (char*) &endian)); - MD5Update(&ctx, (const unsigned char*) buf, (unsigned int) len); + MD5Init(&ctx, *((char *) &endian)); + MD5Update(&ctx, (const unsigned char *) buf, (unsigned int) len); MD5Final(bindigest, &ctx); if (!asAscii) { memcpy(digest, bindigest, 16); } else { - sprintf(digest, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x" - "%02x%02x%02x%02x%02x", - bindigest[0], bindigest[1], bindigest[2], bindigest[3], - bindigest[4], bindigest[5], bindigest[6], bindigest[7], - bindigest[8], bindigest[9], bindigest[10], bindigest[11], - bindigest[12], bindigest[13], bindigest[14], bindigest[15]); + sprintf(digest, + "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x" + "%02x%02x%02x%02x%02x", bindigest[0], bindigest[1], bindigest[2], + bindigest[3], bindigest[4], bindigest[5], bindigest[6], + bindigest[7], bindigest[8], bindigest[9], bindigest[10], + bindigest[11], bindigest[12], bindigest[13], bindigest[14], + bindigest[15]); } - + return 0; } -unsigned long int md5sum32(const char* buff) { +unsigned long int md5sum32(const char *buff) { union { char md5digest[16]; unsigned long int hash; } u; + domd5mem(buff, strlen(buff), u.md5digest, 0); return u.hash; } diff --git a/src/htsmd5.h b/src/htsmd5.h index 872699d..965f39f 100644 --- a/src/htsmd5.h +++ b/src/htsmd5.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsmd5.c subroutines: */ /* generate a md5 hash */ @@ -40,15 +37,12 @@ Please visit our Website: http://www.httrack.com /* ------------------------------------------------------------ */ #ifndef HTSMD5_DEFH -#define HTSMD5_DEFH +#define HTSMD5_DEFH /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -int domd5mem(const char * buf, size_t len, char * digest, int asAscii); -unsigned long int md5sum32(const char* buff); +int domd5mem(const char *buf, size_t len, char *digest, int asAscii); +unsigned long int md5sum32(const char *buff); #endif #endif - - - diff --git a/src/htsmms.c b/src/htsmms.c index 588141f..b54caa8 100644 --- a/src/htsmms.c +++ b/src/htsmms.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: basic mms protocol manager */ /* Author: Xavier Roche */ @@ -54,179 +51,185 @@ Please visit our Website: http://www.httrack.com #define FTP_STATUS_READY 1001 -static int run_launch_mms(MMSDownloadStruct* back); -static void back_launch_mms( void* pP ) { - MMSDownloadStruct *pStruct = (MMSDownloadStruct*)pP; +static int run_launch_mms(MMSDownloadStruct * back); +static void back_launch_mms(void *pP) { + MMSDownloadStruct *pStruct = (MMSDownloadStruct *) pP; + if (pStruct == NULL) - return ; + return; + + /* Initialize */ + hts_init(); - /* Initialize */ - hts_init(); + /* Run */ + run_launch_mms(pStruct); - /* Run */ - run_launch_mms(pStruct); + /* Set as ready */ + { + lien_back *back = pStruct->pBack; - /* Set as ready */ - { - lien_back* back = pStruct->pBack; - back->status=FTP_STATUS_READY; - } + back->status = FTP_STATUS_READY; + } - /* Delete structure */ - free(pP); + /* Delete structure */ + free(pP); - /* Uninitialize */ + /* Uninitialize */ hts_uninit(); - return ; + return; } /* download cancelled */ -static int stop_mms(lien_back* back) { - if (back->stop_ftp) { - strcpybuff(back->r.msg, "Cancelled by User"); - back->r.statuscode = STATUSCODE_INVALID; - return 1; - } - return 0; +static int stop_mms(lien_back * back) { + if (back->stop_ftp) { + strcpybuff(back->r.msg, "Cancelled by User"); + back->r.statuscode = STATUSCODE_INVALID; + return 1; + } + return 0; } /* Background launch */ -void launch_mms(const MMSDownloadStruct* pStruct) { - MMSDownloadStruct *pCopy = calloc(sizeof(MMSDownloadStruct), 1); - memcpy(pCopy, pStruct, sizeof(*pCopy)); - hts_newthread(back_launch_mms, (void*) pCopy); +void launch_mms(const MMSDownloadStruct * pStruct) { + MMSDownloadStruct *pCopy = calloc(sizeof(MMSDownloadStruct), 1); + + memcpy(pCopy, pStruct, sizeof(*pCopy)); + hts_newthread(back_launch_mms, (void *) pCopy); } /* Code mainly written by Nicolas BENOIT */ -static int run_launch_mms(MMSDownloadStruct* pStruct) { - lien_back* back = pStruct->pBack; - httrackp* opt = pStruct->pOpt; - /* */ - char url[HTS_URLMAXSIZE*2]; - char catbuff[CATBUFF_SIZE]; - char catbuff2[CATBUFF_SIZE]; +static int run_launch_mms(MMSDownloadStruct * pStruct) { + lien_back *back = pStruct->pBack; + httrackp *opt = pStruct->pOpt; + + /* */ + char url[HTS_URLMAXSIZE * 2]; + char catbuff[CATBUFF_SIZE]; + char catbuff2[CATBUFF_SIZE]; MMS *mms; FILE *f; ssize_t len_written; - uint64_t total_len_written; - int delay = opt->mms_maxtime; - time_t end = time(NULL) + delay; - short checkPending = 0; - ssize_t existingSize = fsize(back->url_sav); - - // effacer - strcpybuff(back->r.msg,""); - back->status=STATUS_FTP_TRANSFER; - back->r.statuscode=HTTP_OK; - back->r.size=0; - - /* Create file */ - if (existingSize > 0) { - /* back->r.out = fileappend(back->url_sav); - */ - (void) unlink(fconcat(catbuff,back->url_sav, ".old")); - if (rename(fconcat(catbuff,back->url_sav, ""), fconcat(catbuff2,back->url_sav, ".old")) == 0) { - checkPending = 1; - } - back->r.out = filecreate(&pStruct->pOpt->state.strc, back->url_sav); - } else { - back->r.out = filecreate(&pStruct->pOpt->state.strc, back->url_sav); - } - if ((f = back->r.out) != NULL) { - // create mms resource - strcpybuff(url, back->url_adr); /* mms:// */ - strcatbuff(url, back->url_fil); - if ( ( mms = mms_create( url, f, NULL, 0, 1 ) ) != NULL ) { - if ( mms_connect ( mms ) == 0 ) { - if ( mms_handshake ( mms ) == 0 ) { - if ( ( len_written = mms_write_stream_header ( mms ) ) != -1 ) { - total_len_written = len_written; - HTS_STAT.HTS_TOTAL_RECV += len_written; - - /* not modified */ - if (checkPending) { - if (mms->is_live != MMS_LIVE - && mms->expected_file_size == existingSize + 50 /* Why 50 additional bytes declared ?? */ - ) // abort download - { - fclose(back->r.out); - f = back->r.out = NULL; - if (unlink(fconcat(catbuff, back->url_sav, "")) == 0 - && rename(fconcat(catbuff, back->url_sav, ".old"), fconcat(catbuff2, back->url_sav, "")) == 0) - { - back->r.notmodified = 1; - back->r.statuscode = HTTP_OK; - strcpybuff(back->r.msg, "Not modified"); - } else { - back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; - strcpybuff(back->r.msg, "Unable to rename previous file (not updated)"); - } - } else { - (void) unlink(fconcat(catbuff, back->url_sav, ".old")); - } - } - - /* begin rip */ - if ( f != NULL && mms_begin_rip ( mms ) == 0 ) { - if ( mms->is_live != MMS_LIVE ) { - back->r.totalsize = mms->expected_file_size; - back->r.totalsize -= 50; /* Why 50 additional bytes declared ?? */ - } else - back->r.totalsize = -1; - - /* Start download */ - while ( !stop_mms(back) ) { - len_written = mms_write_stream_data ( mms ); - if ( len_written == 0 ) { - break; - } else if ( len_written == -1 ) { - back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; - strcpybuff(back->r.msg, "Unable to write stream data"); - break; - } - - total_len_written += len_written; - back->r.size = total_len_written; - HTS_STAT.HTS_TOTAL_RECV += len_written; - - fflush ( f ); - - if ( delay != 0 && end <= time(NULL) ) { - delay = -1; - back->r.statuscode = HTTP_OK; - strcpybuff(back->r.msg, "Download interrupted"); - break; - } - } // while - - back->r.statuscode = HTTP_OK; /* Finished */ - } else if (f != NULL) { - back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; - strcpybuff(back->r.msg, "Can not begin ripping"); - } - } else { - back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; - strcpybuff(back->r.msg, "Can not write stream header"); - } - } else { - back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; - strcpybuff(back->r.msg, "Can not handshake"); - } - mms_disconnect ( mms ); - } else { - back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; - strcpybuff(back->r.msg, "Can not connect"); - } - mms_destroy ( mms ); - } else { - back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; - strcpybuff(back->r.msg, "Can not create mms resource"); - } - } else { - back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; - strcpybuff(back->r.msg, "Unable to open local output file"); - } - return 0; + uint64_t total_len_written; + int delay = opt->mms_maxtime; + time_t end = time(NULL) + delay; + short checkPending = 0; + ssize_t existingSize = fsize(back->url_sav); + + // effacer + strcpybuff(back->r.msg, ""); + back->status = STATUS_FTP_TRANSFER; + back->r.statuscode = HTTP_OK; + back->r.size = 0; + + /* Create file */ + if (existingSize > 0) { + /* back->r.out = fileappend(back->url_sav); + */ + (void) unlink(fconcat(catbuff, back->url_sav, ".old")); + if (rename + (fconcat(catbuff, back->url_sav, ""), + fconcat(catbuff2, back->url_sav, ".old")) == 0) { + checkPending = 1; + } + back->r.out = filecreate(&pStruct->pOpt->state.strc, back->url_sav); + } else { + back->r.out = filecreate(&pStruct->pOpt->state.strc, back->url_sav); + } + if ((f = back->r.out) != NULL) { + // create mms resource + strcpybuff(url, back->url_adr); /* mms:// */ + strcatbuff(url, back->url_fil); + if ((mms = mms_create(url, f, NULL, 0, 1)) != NULL) { + if (mms_connect(mms) == 0) { + if (mms_handshake(mms) == 0) { + if ((len_written = mms_write_stream_header(mms)) != -1) { + total_len_written = len_written; + HTS_STAT.HTS_TOTAL_RECV += len_written; + + /* not modified */ + if (checkPending) { + if (mms->is_live != MMS_LIVE && mms->expected_file_size == existingSize + 50 /* Why 50 additional bytes declared ?? */ + ) // abort download + { + fclose(back->r.out); + f = back->r.out = NULL; + if (unlink(fconcat(catbuff, back->url_sav, "")) == 0 + && rename(fconcat(catbuff, back->url_sav, ".old"), + fconcat(catbuff2, back->url_sav, "")) == 0) { + back->r.notmodified = 1; + back->r.statuscode = HTTP_OK; + strcpybuff(back->r.msg, "Not modified"); + } else { + back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; + strcpybuff(back->r.msg, + "Unable to rename previous file (not updated)"); + } + } else { + (void) unlink(fconcat(catbuff, back->url_sav, ".old")); + } + } + + /* begin rip */ + if (f != NULL && mms_begin_rip(mms) == 0) { + if (mms->is_live != MMS_LIVE) { + back->r.totalsize = mms->expected_file_size; + back->r.totalsize -= 50; /* Why 50 additional bytes declared ?? */ + } else + back->r.totalsize = -1; + + /* Start download */ + while(!stop_mms(back)) { + len_written = mms_write_stream_data(mms); + if (len_written == 0) { + break; + } else if (len_written == -1) { + back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; + strcpybuff(back->r.msg, "Unable to write stream data"); + break; + } + + total_len_written += len_written; + back->r.size = total_len_written; + HTS_STAT.HTS_TOTAL_RECV += len_written; + + fflush(f); + + if (delay != 0 && end <= time(NULL)) { + delay = -1; + back->r.statuscode = HTTP_OK; + strcpybuff(back->r.msg, "Download interrupted"); + break; + } + } // while + + back->r.statuscode = HTTP_OK; /* Finished */ + } else if (f != NULL) { + back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; + strcpybuff(back->r.msg, "Can not begin ripping"); + } + } else { + back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; + strcpybuff(back->r.msg, "Can not write stream header"); + } + } else { + back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; + strcpybuff(back->r.msg, "Can not handshake"); + } + mms_disconnect(mms); + } else { + back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; + strcpybuff(back->r.msg, "Can not connect"); + } + mms_destroy(mms); + } else { + back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; + strcpybuff(back->r.msg, "Can not create mms resource"); + } + } else { + back->r.statuscode = HTTP_INTERNAL_SERVER_ERROR; + strcpybuff(back->r.msg, "Unable to open local output file"); + } + return 0; } #endif diff --git a/src/htsmms.h b/src/htsmms.h index e5e18c2..77d7287 100644 --- a/src/htsmms.h +++ b/src/htsmms.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: basic mms protocol manager .h */ /* Author: Xavier Roche */ @@ -38,9 +35,8 @@ Please visit our Website: http://www.httrack.com /* Thanks to all of them! */ /* ------------------------------------------------------------ */ - #ifndef HTSMMS_DEFH -#define HTSMMS_DEFH +#define HTSMMS_DEFH #if HTS_USEMMS @@ -59,11 +55,11 @@ typedef struct httrackp httrackp; typedef struct MMSDownloadStruct MMSDownloadStruct; #endif struct MMSDownloadStruct { - lien_back *pBack; - httrackp *pOpt; + lien_back *pBack; + httrackp *pOpt; }; -void launch_mms(const MMSDownloadStruct* pStruct); +void launch_mms(const MMSDownloadStruct * pStruct); #endif #endif diff --git a/src/htsmodules.c b/src/htsmodules.c index 4c37047..e1d2ec6 100644 --- a/src/htsmodules.c +++ b/src/htsmodules.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsmodules.c subroutines: */ /* external modules (parsers) */ @@ -44,7 +41,7 @@ Please visit our Website: http://www.httrack.com #include "htsbasenet.h" #include "htslib.h" -extern int fspc(httrackp *opt,FILE* fp,const char* type); +extern int fspc(httrackp * opt, FILE * fp, const char *type); #ifndef _WIN32 #if HTS_DLOPEN @@ -60,6 +57,7 @@ extern int fspc(httrackp *opt,FILE* fp,const char* type); /* >>> Put all modules variables here */ int gz_is_available = 0; + #if 0 t_gzopen gzopen = NULL; t_gzread gzread = NULL; @@ -88,18 +86,22 @@ t_SSL_load_error_strings SSL_load_error_strings = NULL; int V6_is_available = HTS_INET6; -static char WHAT_is_available[64]=""; +static char WHAT_is_available[64] = ""; + /* <<< */ -HTSEXT_API const char* hts_get_version_info(httrackp *opt) { +HTSEXT_API const char *hts_get_version_info(httrackp * opt) { size_t size; int i; + strcpy(opt->state.HTbuff, WHAT_is_available); size = strlen(opt->state.HTbuff); - for(i = 0 ; i < opt->libHandles.count ; i++) { + for(i = 0; i < opt->libHandles.count; i++) { const char *name = opt->libHandles.handles[i].moduleName; + if (name != NULL) { size_t nsize = strlen(name) + sizeof("+"); + size += nsize; if (size + 1 >= sizeof(opt->state.HTbuff)) break; @@ -113,24 +115,30 @@ HTSEXT_API const char* hts_get_version_info(httrackp *opt) { /* memory checks */ HTSEXT_API htsErrorCallback htsCallbackErr = NULL; HTSEXT_API int htsMemoryFastXfr = 1; /* fast xfr by default */ -void abortLog__fnc(char* msg, char* file, int line); -void abortLog__fnc(char* msg, char* file, int line) { - FILE* fp = fopen("CRASH.TXT", "wb"); - if (!fp) fp = fopen("/tmp/CRASH.TXT", "wb"); - if (!fp) fp = fopen("C:\\CRASH.TXT", "wb"); - if (!fp) fp = fopen("CRASH.TXT", "wb"); +void abortLog__fnc(char *msg, char *file, int line); +void abortLog__fnc(char *msg, char *file, int line) { + FILE *fp = fopen("CRASH.TXT", "wb"); + + if (!fp) + fp = fopen("/tmp/CRASH.TXT", "wb"); + if (!fp) + fp = fopen("C:\\CRASH.TXT", "wb"); + if (!fp) + fp = fopen("CRASH.TXT", "wb"); if (fp) { - fprintf(fp, "HTTrack " HTTRACK_VERSIONID " closed at '%s', line %d\r\n", file, line); + fprintf(fp, "HTTrack " HTTRACK_VERSIONID " closed at '%s', line %d\r\n", + file, line); fprintf(fp, "Reason:\r\n%s\r\n", msg); fflush(fp); fclose(fp); } } -HTSEXT_API t_abortLog abortLog__ = abortLog__fnc; /* avoid VC++ inlining */ -static void htspe_log(htsmoduleStruct* str, const char* msg); +HTSEXT_API t_abortLog abortLog__ = abortLog__fnc; /* avoid VC++ inlining */ + +static void htspe_log(htsmoduleStruct * str, const char *msg); -int hts_parse_externals(htsmoduleStruct* str) { +int hts_parse_externals(htsmoduleStruct * str) { str->wrapper_name = "wrapper-lib"; /* External callback */ @@ -138,7 +146,8 @@ int hts_parse_externals(htsmoduleStruct* str) { if (str->wrapper_name == NULL) str->wrapper_name = "wrapper-lib"; /* Blacklisted */ - if (multipleStringMatch(str->wrapper_name, StringBuff(str->opt->mod_blacklist))) { + if (multipleStringMatch + (str->wrapper_name, StringBuff(str->opt->mod_blacklist))) { return -1; } else { htspe_log(str, str->wrapper_name); @@ -162,13 +171,14 @@ int hts_parse_externals(htsmoduleStruct* str) { // chain->moduleHandle = moduleHandle; //} -void clearCallbacks(htscallbacks* chain_); -void clearCallbacks(htscallbacks* chain_) { - htscallbacks* chain; +void clearCallbacks(htscallbacks * chain_); +void clearCallbacks(htscallbacks * chain_) { + htscallbacks *chain; + chain = chain_; while(chain != NULL) { if (chain->exitFnc != NULL) { - (void) chain->exitFnc(); /* result ignored */ + (void) chain->exitFnc(); /* result ignored */ chain->exitFnc = NULL; } chain = chain->next; @@ -184,18 +194,20 @@ void clearCallbacks(htscallbacks* chain_) { } chain = chain->next; } - chain = chain_->next; // Don't free the block #0 + chain = chain_->next; // Don't free the block #0 while(chain != NULL) { - htscallbacks* nextchain = chain->next; + htscallbacks *nextchain = chain->next; + freet(chain); chain = nextchain; } - chain_->next = NULL; // Empty + chain_->next = NULL; // Empty } -void* openFunctionLib(const char* file_) { - void* handle; +void *openFunctionLib(const char *file_) { + void *handle; char *file = malloct(strlen(file_) + 32); + strcpy(file, file_); #ifdef _WIN32 handle = LoadLibraryA(file); @@ -214,7 +226,7 @@ void* openFunctionLib(const char* file_) { return handle; } -void closeFunctionLib(void* handle) { +void closeFunctionLib(void *handle) { #ifdef _WIN32 FreeLibrary(handle); #else @@ -222,19 +234,20 @@ void closeFunctionLib(void* handle) { #endif } -void* getFunctionPtr(void* handle, const char* fncname_) { +void *getFunctionPtr(void *handle, const char *fncname_) { if (handle) { - void* userfunction = NULL; + void *userfunction = NULL; char *fncname = strdupt(fncname_); /* Strip optional comma */ char *comma; - if ((comma = strchr(fncname, ',')) != NULL) { /* empty arg */ + + if ((comma = strchr(fncname, ',')) != NULL) { /* empty arg */ *comma++ = '\0'; } - + /* the function itself */ - userfunction = (void*) DynamicGet(handle, (char*)fncname); + userfunction = (void *) DynamicGet(handle, (char *) fncname); freet(fncname); @@ -243,34 +256,37 @@ void* getFunctionPtr(void* handle, const char* fncname_) { return NULL; } -void* ssl_handle = NULL; +void *ssl_handle = NULL; + #ifdef _WIN32 -void* ssl_handle_2 = NULL; +void *ssl_handle_2 = NULL; #endif void htspe_init(void) { static int initOk = 0; + if (!initOk) { initOk = 1; /* See CVE-2010-5252 */ #if (defined(_WIN32) && (!defined(_DEBUG))) /* See KB 2389418 - "If this parameter is an empty string (""), the call removes the - current directory from the default DLL search order" */ + "If this parameter is an empty string (""), the call removes the + current directory from the default DLL search order" */ if (!SetDllDirectory("")) { - assertf(! "SetDllDirectory failed"); + assertf(!"SetDllDirectory failed"); } #endif - + /* Zlib is now statically linked */ gz_is_available = 1; /* OpenSSL */ #if HTS_DLOPEN { - void* handle; + void *handle; + #ifdef _WIN32 - handle = LoadLibraryA((char*)"ssleay32"); + handle = LoadLibraryA((char *) "ssleay32"); #else /* We are compatible with 0.9.6/7/8/8b and potentially above */ static const char *const libs[] = { @@ -288,62 +304,82 @@ void htspe_init(void) { "libssl.so.0.9.8n", "libssl.so.0.9.8m", "libssl.so.0.9.8l", - "libssl.so.0.9.8k", /* (Debarshi Ray) */ - "libssl.so.0.9.8j", /* (Debarshi Ray) */ - "libssl.so.0.9.8g", /* Added 8g release too (Debarshi Ray) */ + "libssl.so.0.9.8k", /* (Debarshi Ray) */ + "libssl.so.0.9.8j", /* (Debarshi Ray) */ + "libssl.so.0.9.8g", /* Added 8g release too (Debarshi Ray) */ "libssl.so.0.9.8b", "libssl.so.0.9.8", "libssl.so.0.9.7", "libssl.so.0.9.6", - "libssl.so", /* Try harder with devel link */ + "libssl.so", /* Try harder with devel link */ NULL }; int i; - for(i = 0, handle = NULL ; handle == NULL && libs[i] != NULL ; i++) { + + for(i = 0, handle = NULL; handle == NULL && libs[i] != NULL; i++) { handle = dlopen(libs[i], RTLD_LAZY); } #endif ssl_handle = handle; if (handle != NULL) { - SSL_shutdown = (t_SSL_shutdown) DynamicGet(handle, (char*)"SSL_shutdown"); - SSL_free = (t_SSL_free) DynamicGet(handle, (char*)"SSL_free"); - SSL_new = (t_SSL_new) DynamicGet(handle, (char*)"SSL_new"); - SSL_clear = (t_SSL_clear) DynamicGet(handle, (char*)"SSL_clear"); - SSL_set_fd = (t_SSL_set_fd) DynamicGet(handle, (char*)"SSL_set_fd"); - SSL_set_connect_state = (t_SSL_set_connect_state) DynamicGet(handle, (char*)"SSL_set_connect_state"); - SSL_connect = (t_SSL_connect) DynamicGet(handle, (char*)"SSL_connect"); - SSL_get_error = (t_SSL_get_error) DynamicGet(handle, (char*)"SSL_get_error"); - SSL_write = (t_SSL_write) DynamicGet(handle, (char*)"SSL_write"); - SSL_read = (t_SSL_read) DynamicGet(handle, (char*)"SSL_read"); - SSL_library_init = (t_SSL_library_init) DynamicGet(handle, (char*)"SSL_library_init"); - ERR_load_SSL_strings = (t_ERR_load_SSL_strings) DynamicGet(handle, (char*)"ERR_load_SSL_strings"); - SSLv23_client_method = (t_SSLv23_client_method) DynamicGet(handle, (char*)"SSLv23_client_method"); - SSL_CTX_new = (t_SSL_CTX_new) DynamicGet(handle, (char*)"SSL_CTX_new"); - SSL_load_error_strings = (t_SSL_load_error_strings) DynamicGet(handle, (char*)"SSL_load_error_strings"); - SSL_CTX_ctrl = (t_SSL_CTX_ctrl) DynamicGet(handle, (char*)"SSL_CTX_ctrl"); + SSL_shutdown = + (t_SSL_shutdown) DynamicGet(handle, (char *) "SSL_shutdown"); + SSL_free = (t_SSL_free) DynamicGet(handle, (char *) "SSL_free"); + SSL_new = (t_SSL_new) DynamicGet(handle, (char *) "SSL_new"); + SSL_clear = (t_SSL_clear) DynamicGet(handle, (char *) "SSL_clear"); + SSL_set_fd = (t_SSL_set_fd) DynamicGet(handle, (char *) "SSL_set_fd"); + SSL_set_connect_state = + (t_SSL_set_connect_state) DynamicGet(handle, + (char *) + "SSL_set_connect_state"); + SSL_connect = + (t_SSL_connect) DynamicGet(handle, (char *) "SSL_connect"); + SSL_get_error = + (t_SSL_get_error) DynamicGet(handle, (char *) "SSL_get_error"); + SSL_write = (t_SSL_write) DynamicGet(handle, (char *) "SSL_write"); + SSL_read = (t_SSL_read) DynamicGet(handle, (char *) "SSL_read"); + SSL_library_init = + (t_SSL_library_init) DynamicGet(handle, (char *) "SSL_library_init"); + ERR_load_SSL_strings = + (t_ERR_load_SSL_strings) DynamicGet(handle, + (char *) "ERR_load_SSL_strings"); + SSLv23_client_method = + (t_SSLv23_client_method) DynamicGet(handle, + (char *) "SSLv23_client_method"); + SSL_CTX_new = + (t_SSL_CTX_new) DynamicGet(handle, (char *) "SSL_CTX_new"); + SSL_load_error_strings = + (t_SSL_load_error_strings) DynamicGet(handle, + (char *) + "SSL_load_error_strings"); + SSL_CTX_ctrl = + (t_SSL_CTX_ctrl) DynamicGet(handle, (char *) "SSL_CTX_ctrl"); #ifdef _WIN32 - handle = LoadLibraryA((char*)"libeay32"); + handle = LoadLibraryA((char *) "libeay32"); ssl_handle_2 = handle; #endif - ERR_load_crypto_strings = (t_ERR_load_crypto_strings) DynamicGet(handle, (char*)"ERR_load_crypto_strings"); - ERR_error_string = (t_ERR_error_string) DynamicGet(handle, (char*)"ERR_error_string"); - - if (SSL_shutdown && SSL_free && SSL_CTX_ctrl && SSL_new && SSL_clear && - SSL_set_fd && SSL_set_connect_state && SSL_connect && SSL_get_error && SSL_write - && SSL_read && SSL_library_init && SSLv23_client_method && SSL_CTX_new - && SSL_load_error_strings && ERR_error_string) { + ERR_load_crypto_strings = + (t_ERR_load_crypto_strings) DynamicGet(handle, + (char *) + "ERR_load_crypto_strings"); + ERR_error_string = + (t_ERR_error_string) DynamicGet(handle, (char *) "ERR_error_string"); + + if (SSL_shutdown && SSL_free && SSL_CTX_ctrl && SSL_new && SSL_clear + && SSL_set_fd && SSL_set_connect_state && SSL_connect + && SSL_get_error && SSL_write && SSL_read && SSL_library_init + && SSLv23_client_method && SSL_CTX_new && SSL_load_error_strings + && ERR_error_string) { SSL_is_available = 1; } } } #endif /* */ - + /* Options availability */ - sprintf(WHAT_is_available, "%s%s%s", - V6_is_available ? "" : "-noV6", - gz_is_available ? "" : "-nozip", - SSL_is_available ? "" : "-nossl"); + sprintf(WHAT_is_available, "%s%s%s", V6_is_available ? "" : "-noV6", + gz_is_available ? "" : "-nozip", SSL_is_available ? "" : "-nossl"); } } @@ -359,13 +395,14 @@ void htspe_uninit(void) { #endif } -static void htspe_log(htsmoduleStruct* str, const char* msg) { - const char* savename = str->filename; - httrackp* opt = (httrackp*) str->opt; - hts_log_print(opt, LOG_DEBUG, "(External module): parsing %s using module %s", - savename, msg); +static void htspe_log(htsmoduleStruct * str, const char *msg) { + const char *savename = str->filename; + httrackp *opt = (httrackp *) str->opt; + + hts_log_print(opt, LOG_DEBUG, "(External module): parsing %s using module %s", + savename, msg); } -HTSEXT_API const char* hts_is_available(void) { +HTSEXT_API const char *hts_is_available(void) { return WHAT_is_available; } diff --git a/src/htsmodules.h b/src/htsmodules.h index e03354e..b982241 100644 --- a/src/htsmodules.h +++ b/src/htsmodules.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsmodules.h subroutines: */ /* external modules (parsers) */ @@ -69,56 +66,56 @@ typedef struct hash_struct hash_struct; #define HTS_DEF_FWSTRUCT_htsmoduleStruct typedef struct htsmoduleStruct htsmoduleStruct; #endif -typedef int (* t_htsAddLink)(htsmoduleStruct* str, char* link); +typedef int (*t_htsAddLink) (htsmoduleStruct * str, char *link); /* Structure passed to the module */ struct htsmoduleStruct { /* Read-only elements */ - const char* filename; /* filename (C:\My Web Sites\...) */ - int size; /* size of filename (should be > 0) */ - const char* mime; /* MIME type of the object */ - const char* url_host; /* incoming hostname (www.foo.com) */ - const char* url_file; /* incoming filename (/bar/bar.gny) */ - + const char *filename; /* filename (C:\My Web Sites\...) */ + int size; /* size of filename (should be > 0) */ + const char *mime; /* MIME type of the object */ + const char *url_host; /* incoming hostname (www.foo.com) */ + const char *url_file; /* incoming filename (/bar/bar.gny) */ + /* Write-only */ - const char* wrapper_name; /* name of wrapper (static string) */ - char* err_msg; /* if an error occured, the error message (max. 1KB) */ - + const char *wrapper_name; /* name of wrapper (static string) */ + char *err_msg; /* if an error occured, the error message (max. 1KB) */ + /* Read/Write */ - int relativeToHtmlLink; /* set this to 1 if all urls you pass to addLink - are in fact relative to the html file where your - module was originally */ - + int relativeToHtmlLink; /* set this to 1 if all urls you pass to addLink + are in fact relative to the html file where your + module was originally */ + /* Callbacks */ - t_htsAddLink addLink; /* call this function when links are - being detected. it if not your responsability to decide - if the engine will keep them, or not. */ + t_htsAddLink addLink; /* call this function when links are + being detected. it if not your responsability to decide + if the engine will keep them, or not. */ /* Optional */ - char* localLink; /* if non null, the engine will write there the local - relative filename of the link added by addLink(), or - the absolute path if the link was refused by the wizard */ - int localLinkSize; /* size of the optionnal buffer */ - + char *localLink; /* if non null, the engine will write there the local + relative filename of the link added by addLink(), or + the absolute path if the link was refused by the wizard */ + int localLinkSize; /* size of the optionnal buffer */ + /* User-defined */ - void* userdef; /* can be used by callback routines - */ + void *userdef; /* can be used by callback routines + */ /* The parser httrackp structure (may be used) */ - httrackp* opt; + httrackp *opt; /* Internal use - please don't touch */ - lien_url** liens; - struct_back* sback; - cache_back* cache; - hash_struct* hashptr; + lien_url **liens; + struct_back *sback; + cache_back *cache; + hash_struct *hashptr; int numero_passe; int add_tab_alloc; /* */ - int* lien_tot_; - int* ptr_; - size_t* lien_size_; - char** lien_buffer_; + int *lien_tot_; + int *ptr_; + size_t *lien_size_; + char **lien_buffer_; const char *page_charset_; /* Internal use - please don't touch */ @@ -126,19 +123,20 @@ struct htsmoduleStruct { /* Used to wrap module initialization */ /* return 1 if init was ok */ -typedef int (*t_htsWrapperInit)(char *fn, char *args); -typedef int (*t_htsWrapperExit)(void); -typedef int (*t_htsWrapperPlugInit)(char *args); +typedef int (*t_htsWrapperInit) (char *fn, char *args); +typedef int (*t_htsWrapperExit) (void); +typedef int (*t_htsWrapperPlugInit) (char *args); /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -HTSEXT_API const char* hts_get_version_info(httrackp *opt); -HTSEXT_API const char* hts_is_available(void); +HTSEXT_API const char *hts_get_version_info(httrackp * opt); +HTSEXT_API const char *hts_is_available(void); extern void htspe_init(void); extern void htspe_uninit(void); -extern int hts_parse_externals(htsmoduleStruct* str); +extern int hts_parse_externals(htsmoduleStruct * str); extern int gz_is_available; + /*extern int swf_is_available;*/ extern int SSL_is_available; extern int V6_is_available; diff --git a/src/htsname.c b/src/htsname.c index 18a3dc3..779c27d 100644 --- a/src/htsname.c +++ b/src/htsname.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* savename routine (compute output filename) */ @@ -61,20 +58,17 @@ Please visit our Website: http://www.httrack.com url_savename_addstr(save,buff);\ } - /* Avoid stupid DOS system folders/file such as 'nul' */ /* Based on linux/fs/umsdos/mangle.c */ -static const char *hts_tbdev[] = -{ - "/prn", "/con", "/aux", "/nul", - "/lpt1", "/lpt2", "/lpt3", "/lpt4", - "/com1", "/com2", "/com3", "/com4", - "/clock$", - "/emmxxxx0", "/xmsxxxx0", "/setverxx", - "" +static const char *hts_tbdev[] = { + "/prn", "/con", "/aux", "/nul", + "/lpt1", "/lpt2", "/lpt3", "/lpt4", + "/com1", "/com2", "/com3", "/com4", + "/clock$", + "/emmxxxx0", "/xmsxxxx0", "/setverxx", + "" }; - #define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET() do { \ int prev = opt->state._hts_in_html_parsing; \ while(back_pluggable_sockets_strict(sback, opt) <= 0) { \ @@ -103,7 +97,8 @@ static const char *hts_tbdev[] = /* Strip all // */ static void cleanDoubleSlash(char *s) { int i, j; - for(i = 0, j = 0 ; s[i] != '\0' ; i++) { + + for(i = 0, j = 0; s[i] != '\0'; i++) { if (s[i] == '/' && i != 0 && s[i - 1] == '/') { continue; } @@ -118,67 +113,75 @@ static void cleanDoubleSlash(char *s) { } // legacy version, without page charset -int url_savename(char* adr_complete, char* fil_complete, char* save, - char* former_adr, char* former_fil, - char* referer_adr, char* referer_fil, - httrackp* opt, - lien_url** liens, int lien_tot, - struct_back* sback, cache_back* cache, hash_struct* hash, - int ptr, int numero_passe, const lien_back* headers) { - return url_savename2(adr_complete, fil_complete, save, former_adr, former_fil, - referer_adr, referer_fil, opt, - liens, lien_tot, sback, cache, hash, ptr, numero_passe, headers, /* unknown */ NULL); +int url_savename(char *adr_complete, char *fil_complete, char *save, + char *former_adr, char *former_fil, char *referer_adr, + char *referer_fil, httrackp * opt, lien_url ** liens, + int lien_tot, struct_back * sback, cache_back * cache, + hash_struct * hash, int ptr, int numero_passe, + const lien_back * headers) { + return url_savename2(adr_complete, fil_complete, save, former_adr, former_fil, + referer_adr, referer_fil, opt, liens, lien_tot, sback, + cache, hash, ptr, numero_passe, headers, /* unknown */ + NULL); } // forme le nom du fichier à sauver (save) à partir de fil et adr // système intelligent, qui renomme en cas de besoin (exemple: deux INDEX.HTML et index.html) -int url_savename2(char* adr_complete, char* fil_complete, char* save, - char* former_adr, char* former_fil, - char* referer_adr, char* referer_fil, - httrackp* opt, - lien_url** liens, int lien_tot, - struct_back* sback, cache_back* cache, hash_struct* hash, - int ptr, int numero_passe, const lien_back* headers, const char *charset) { - char catbuff[CATBUFF_SIZE]; - const char* mime_type = ( headers && !HTTP_IS_REDIRECT(headers->r.statuscode) ) ? headers->r.contenttype : NULL; - /*const char* mime_type = ( headers && HTTP_IS_OK(headers->r.statuscode) ) ? headers->r.contenttype : NULL;*/ - lien_back* const back = sback->lnk; +int url_savename2(char *adr_complete, char *fil_complete, char *save, + char *former_adr, char *former_fil, char *referer_adr, + char *referer_fil, httrackp * opt, lien_url ** liens, + int lien_tot, struct_back * sback, cache_back * cache, + hash_struct * hash, int ptr, int numero_passe, + const lien_back * headers, const char *charset) { + char catbuff[CATBUFF_SIZE]; + const char *mime_type = (headers + && !HTTP_IS_REDIRECT(headers->r. + statuscode)) ? headers->r. + contenttype : NULL; + /*const char* mime_type = ( headers && HTTP_IS_OK(headers->r.statuscode) ) ? headers->r.contenttype : NULL; */ + lien_back *const back = sback->lnk; + /* */ - char BIGSTK newfil[HTS_URLMAXSIZE*2]; /* ="" */ - /*char BIGSTK normadr_[HTS_URLMAXSIZE*2];*/ - char BIGSTK normadr_[HTS_URLMAXSIZE*2], normfil_[HTS_URLMAXSIZE*2]; - enum { PROTOCOL_HTTP, PROTOCOL_HTTPS, PROTOCOL_FTP, PROTOCOL_FILE, PROTOCOL_MMS, PROTOCOL_UNKNOWN }; - static const char* protocol_str[] = {"http", "https", "ftp", "file", "mms", "unknown"}; + char BIGSTK newfil[HTS_URLMAXSIZE * 2]; /* ="" */ + + /*char BIGSTK normadr_[HTS_URLMAXSIZE*2]; */ + char BIGSTK normadr_[HTS_URLMAXSIZE * 2], normfil_[HTS_URLMAXSIZE * 2]; + enum { PROTOCOL_HTTP, PROTOCOL_HTTPS, PROTOCOL_FTP, PROTOCOL_FILE, + PROTOCOL_MMS, PROTOCOL_UNKNOWN }; + static const char *protocol_str[] = + { "http", "https", "ftp", "file", "mms", "unknown" }; int protocol = PROTOCOL_HTTP; - char* normadr; - char* normfil; - char* fil; - char* adr; - char* print_adr; - char *start_pos=NULL,*nom_pos=NULL,*dot_pos=NULL; // Position nom et point + char *normadr; + char *normfil; + char *fil; + char *adr; + char *print_adr; + char *start_pos = NULL, *nom_pos = NULL, *dot_pos = NULL; // Position nom et point + // pour changement d'extension ou de nom (content-disposition) - int ext_chg=0, ext_chg_delayed=0; - int is_html=0; + int ext_chg = 0, ext_chg_delayed = 0; + int is_html = 0; char ext[256]; - int max_char=0; + int max_char = 0; + //CLEAR - newfil[0]=ext[0]='\0'; + newfil[0] = ext[0] = '\0'; /* 8-3 ? */ - switch(opt->savename_83) { - case 1: // 8-3 - max_char=8; + switch (opt->savename_83) { + case 1: // 8-3 + max_char = 8; break; - case 2: // Level 2 File names may be up to 31 characters. - max_char=31; + case 2: // Level 2 File names may be up to 31 characters. + max_char = 31; break; default: - max_char=8; + max_char = 8; break; } // effacer save - save[0]='\0'; + save[0] = '\0'; // fil fil = fil_complete; // copy of fil, used for lookups (see urlhack) @@ -197,167 +200,157 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, // foo.com/bar//foobar -> foo.com/bar/foobar if (opt->urlhack) { // copy of adr (without protocol), used for lookups (see urlhack) - normadr=adr_normalized(adr, normadr_); - normfil=fil_normalized(fil,normfil_); + normadr = adr_normalized(adr, normadr_); + normfil = fil_normalized(fil, normfil_); } else { if (link_has_authority(adr_complete)) { // https or other protocols : in "http/" subfolder - char* pos = strchr(adr_complete, ':'); + char *pos = strchr(adr_complete, ':'); + if (pos != NULL) { normadr_[0] = '\0'; - strncatbuff(normadr_, adr_complete, (int)(pos - adr_complete)); + strncatbuff(normadr_, adr_complete, (int) (pos - adr_complete)); strcatbuff(normadr_, "://"); strcatbuff(normadr_, normadr); - normadr=normadr_; + normadr = normadr_; } } } // à afficher sans ftp:// - print_adr=jump_protocol(adr); + print_adr = jump_protocol(adr); if (strfield(adr_complete, "https:")) { - protocol = PROTOCOL_HTTPS; + protocol = PROTOCOL_HTTPS; } else if (strfield(adr_complete, "ftp:")) { - protocol = PROTOCOL_FTP; + protocol = PROTOCOL_FTP; } else if (strfield(adr_complete, "file:")) { - protocol = PROTOCOL_FILE; + protocol = PROTOCOL_FILE; } else if (strfield(adr_complete, "mms:")) { - protocol = PROTOCOL_MMS; + protocol = PROTOCOL_MMS; } else { - protocol = PROTOCOL_HTTP; + protocol = PROTOCOL_HTTP; } // court-circuit pour lien primaire - if (strnotempty(adr)==0) { - if (strcmp(fil,"primary")==0) { - strcatbuff(save,"primary.html"); + if (strnotempty(adr) == 0) { + if (strcmp(fil, "primary") == 0) { + strcatbuff(save, "primary.html"); return 0; } } - // vérifier que le nom n'a pas déja été calculé (si oui le renvoyer tel que) // vérifier que le nom n'est pas déja pris... // NOTE: si on cherche /toto/ et que /toto est trouvé on le prend (et réciproquqment) ** // ** - if (liens!=NULL) { + if (liens != NULL) { int i; - i=hash_read(hash,normadr,normfil,1,opt->urlhack); // recherche table 1 (adr+fil) - if (i>=0) { // ok, trouvé - strcpybuff(save,liens[i]->sav); + i = hash_read(hash, normadr, normfil, 1, opt->urlhack); // recherche table 1 (adr+fil) + if (i >= 0) { // ok, trouvé + strcpybuff(save, liens[i]->sav); return 0; } - i=hash_read(hash,normadr,normfil,2,opt->urlhack); // recherche table 2 (former_adr+former_fil) - if (i>=0) { // ok, trouvé + i = hash_read(hash, normadr, normfil, 2, opt->urlhack); // recherche table 2 (former_adr+former_fil) + if (i >= 0) { // ok, trouvé // copier location moved! - strcpybuff(adr_complete,liens[i]->adr); - strcpybuff(fil_complete,liens[i]->fil); + strcpybuff(adr_complete, liens[i]->adr); + strcpybuff(fil_complete, liens[i]->fil); // et save - strcpybuff(save,liens[i]->sav); // copier (formé à partir du nouveau lien!) + strcpybuff(save, liens[i]->sav); // copier (formé à partir du nouveau lien!) return 0; } - // chercher sans / ou avec / dans former { - char BIGSTK fil_complete_patche[HTS_URLMAXSIZE*2]; - strcpybuff(fil_complete_patche,normfil); + char BIGSTK fil_complete_patche[HTS_URLMAXSIZE * 2]; + + strcpybuff(fil_complete_patche, normfil); // Version avec ou sans / - if (fil_complete_patche[strlen(fil_complete_patche)-1]=='/') - fil_complete_patche[strlen(fil_complete_patche)-1]='\0'; + if (fil_complete_patche[strlen(fil_complete_patche) - 1] == '/') + fil_complete_patche[strlen(fil_complete_patche) - 1] = '\0'; else - strcatbuff(fil_complete_patche,"/"); - i=hash_read(hash,normadr,fil_complete_patche,2,opt->urlhack); // recherche table 2 (former_adr+former_fil) - if (i>=0) { + strcatbuff(fil_complete_patche, "/"); + i = hash_read(hash, normadr, fil_complete_patche, 2, opt->urlhack); // recherche table 2 (former_adr+former_fil) + if (i >= 0) { // écraser fil et adr (pas former_fil?????) - strcpybuff(adr_complete,liens[i]->adr); - strcpybuff(fil_complete,liens[i]->fil); + strcpybuff(adr_complete, liens[i]->adr); + strcpybuff(fil_complete, liens[i]->fil); // écrire save - strcpybuff(save,liens[i]->sav); + strcpybuff(save, liens[i]->sav); return 0; } } } - // vérifier la non présence de paramètres dans le nom de fichier // si il y en a, les supprimer (ex: truc.cgi?subj=aspirateur) // néanmoins, gardé pour vérifier la non duplication (voir après) { - char* a; - a=strchr(fil,'?'); - if (a!=NULL) { - strncatbuff(newfil,fil,(int) (a - fil)); + char *a; + + a = strchr(fil, '?'); + if (a != NULL) { + strncatbuff(newfil, fil, (int) (a - fil)); } else { - strcpybuff(newfil,fil); + strcpybuff(newfil, fil); } - fil=newfil; + fil = newfil; } // Decode remaining % - strcpybuff(fil,unescape_http(catbuff,fil)); - // , BUT do not decode high chars + strcpybuff(fil, unescape_http(catbuff, fil)); + // , BUT do not decode high chars //strcpybuff(fil,unescape_http_unharm(fil, 1)); - // YES (not server side, but fs/client side) + // YES (not server side, but fs/client side) #if HTS_USEMMS - /* .asx hack */ - if (headers != NULL && headers->r.cdispo[0] != 0 - && strfield(headers->r.contenttype, "video/") - && strfield2(get_ext(OPT_GET_BUFF(opt),headers->r.cdispo), "asx") == 0) - { - ext_chg = 1; - strcpybuff(ext, "asx"); - } - else if (headers != NULL && headers->r.contenttype[0] != 0 - && strfield2(headers->r.contenttype, "video/x-ms-asf")) - { - char *exts = get_ext(OPT_GET_BUFF(opt),headers->url_fil); - if (strfield2(exts, "wmv") == 0) - { - ext_chg = 1; - strcpybuff(ext, "wmv"); - } - else if (strfield2(exts, "asf") == 0) - { - ext_chg = 1; - strcpybuff(ext, "asf"); - } - else if (strfield2(exts, "avi") == 0) - { - ext_chg = 1; - strcpybuff(ext, "avi"); - } - else if (strfield2(exts, "asx") == 0) - { - ext_chg = 1; - strcpybuff(ext, "asx"); - } - } + /* .asx hack */ + if (headers != NULL && headers->r.cdispo[0] != 0 + && strfield(headers->r.contenttype, "video/") + && strfield2(get_ext(OPT_GET_BUFF(opt), headers->r.cdispo), "asx") == 0) { + ext_chg = 1; + strcpybuff(ext, "asx"); + } else if (headers != NULL && headers->r.contenttype[0] != 0 + && strfield2(headers->r.contenttype, "video/x-ms-asf")) { + char *exts = get_ext(OPT_GET_BUFF(opt), headers->url_fil); + + if (strfield2(exts, "wmv") == 0) { + ext_chg = 1; + strcpybuff(ext, "wmv"); + } else if (strfield2(exts, "asf") == 0) { + ext_chg = 1; + strcpybuff(ext, "asf"); + } else if (strfield2(exts, "avi") == 0) { + ext_chg = 1; + strcpybuff(ext, "avi"); + } else if (strfield2(exts, "asx") == 0) { + ext_chg = 1; + strcpybuff(ext, "asx"); + } + } #endif /* replace shtml to html.. */ if (opt->savename_delayed == 2) - is_html = -1; /* ALWAYS delay type */ + is_html = -1; /* ALWAYS delay type */ else - is_html = ishtml(opt,fil); - switch ( is_html ) { /* .html,.shtml,.. */ + is_html = ishtml(opt, fil); + switch (is_html) { /* .html,.shtml,.. */ case 1: - if ( - (strfield2(get_ext(OPT_GET_BUFF(opt),fil),"html") == 0) - && (strfield2(get_ext(OPT_GET_BUFF(opt),fil),"htm") == 0) - ) - { - strcpybuff(ext,"html"); - ext_chg=1; + if ((strfield2(get_ext(OPT_GET_BUFF(opt), fil), "html") == 0) + && (strfield2(get_ext(OPT_GET_BUFF(opt), fil), "htm") == 0) + ) { + strcpybuff(ext, "html"); + ext_chg = 1; } break; case 0: if (!strnotempty(ext)) { - if (is_userknowntype(opt,fil)) { // mime known by user + if (is_userknowntype(opt, fil)) { // mime known by user char BIGSTK mime[1024]; - mime[0]=ext[0]='\0'; - get_userhttptype(opt,mime,fil); + + mime[0] = ext[0] = '\0'; + get_userhttptype(opt, mime, fil); if (strnotempty(mime)) { - give_mimext(ext,mime); + give_mimext(ext, mime); if (strnotempty(ext)) { - ext_chg=1; + ext_chg = 1; } } } @@ -367,651 +360,707 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, // si option check_type activée if (is_html < 0 && opt->check_type && !ext_chg) { - int ishtest=0; - if ( (!strfield(adr_complete,"file://")) - && (!strfield(adr_complete,"ftp://")) + int ishtest = 0; + + if ((!strfield(adr_complete, "file://")) + && (!strfield(adr_complete, "ftp://")) #if HTS_USEMMS - && (!strfield(adr_complete,"mms://")) + && (!strfield(adr_complete, "mms://")) #endif ) { // tester type avec requète HEAD si on ne connait pas le type du fichier - if (!( (opt->check_type==1) && (fil[strlen(fil)-1]=='/') )) // slash doit être html? - if ( opt->savename_delayed == 2 || (ishtest=ishtml(opt,fil)) < 0) { // on ne sait pas si c'est un html ou un fichier.. - // lire dans le cache - htsblk r = cache_read_including_broken(opt,cache,adr,fil); // test uniquement - if (r.statuscode != -1) { // pas d'erreur de lecture cache - char s[32]; s[0]='\0'; - hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s",adr_complete,fil_complete); - if (strnotempty(r.cdispo)) { /* filename given */ - ext_chg=2; /* change filename */ - strcpybuff(ext,r.cdispo); - } - else if (!may_unknown2(opt,r.contenttype,fil)) { // on peut patcher à priori? - give_mimext(s,r.contenttype); // obtenir extension - if (strnotempty(s)>0) { // on a reconnu l'extension - ext_chg=1; - strcpybuff(ext,s); + if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/'))) // slash doit être html? + if (opt->savename_delayed == 2 || (ishtest = ishtml(opt, fil)) < 0) { // on ne sait pas si c'est un html ou un fichier.. + // lire dans le cache + htsblk r = cache_read_including_broken(opt, cache, adr, fil); // test uniquement + + if (r.statuscode != -1) { // pas d'erreur de lecture cache + char s[32]; + + s[0] = '\0'; + hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s", + adr_complete, fil_complete); + if (strnotempty(r.cdispo)) { /* filename given */ + ext_chg = 2; /* change filename */ + strcpybuff(ext, r.cdispo); + } else if (!may_unknown2(opt, r.contenttype, fil)) { // on peut patcher à priori? + give_mimext(s, r.contenttype); // obtenir extension + if (strnotempty(s) > 0) { // on a reconnu l'extension + ext_chg = 1; + strcpybuff(ext, s); + } } - } #ifdef DEFAULT_BIN_EXT - // no extension and potentially bogus - else if (ishtest == -2) { - ext_chg=1; - strcpybuff(ext, DEFAULT_BIN_EXT + 1); - } + // no extension and potentially bogus + else if (ishtest == -2) { + ext_chg = 1; + strcpybuff(ext, DEFAULT_BIN_EXT + 1); + } #endif - // - } else if ( opt->savename_delayed != 2 && is_userknowntype(opt,fil)) { /* PATCH BY BRIAN SCHRÖDER. - Lookup mimetype not only by extension, - but also by filename */ - /* Note: "foo.cgi => text/html" means that foo.cgi shall have the text/html MIME file type, - that is, ".html" */ - char BIGSTK mime[1024]; - mime[0]=ext[0]='\0'; - get_userhttptype(opt, mime, fil); - if (strnotempty(mime)) { - give_mimext(ext, mime); - if (strnotempty(ext)) { - ext_chg=1; + // + } else if (opt->savename_delayed != 2 && is_userknowntype(opt, fil)) { /* PATCH BY BRIAN SCHRÖDER. + Lookup mimetype not only by extension, + but also by filename */ + /* Note: "foo.cgi => text/html" means that foo.cgi shall have the text/html MIME file type, + that is, ".html" */ + char BIGSTK mime[1024]; + + mime[0] = ext[0] = '\0'; + get_userhttptype(opt, mime, fil); + if (strnotempty(mime)) { + give_mimext(ext, mime); + if (strnotempty(ext)) { + ext_chg = 1; + } } } - } - // note: if savename_delayed is enabled, the naming will be temporary (and slightly invalid!) - // note: if we are about to stop (opt->state.stop), back_add() will fail later - else if (opt->savename_delayed != 0 && !opt->state.stop) { - if (mime_type != NULL) { - ext[0] = '\0'; - if (*mime_type) { - give_mimext(ext, mime_type); - } - if (strnotempty(ext)) { - char mime_from_file[128]; - mime_from_file[0] = 0; - get_httptype(opt, mime_from_file, fil, 1); - if (!strnotempty(mime_from_file) || strcasecmp(mime_type, mime_from_file) != 0) { /* different mime for this type */ - /* type change not forbidden (or no extension at all) */ - if (!may_unknown2(opt, mime_type, fil)) { - ext_chg = 1; - } + // note: if savename_delayed is enabled, the naming will be temporary (and slightly invalid!) + // note: if we are about to stop (opt->state.stop), back_add() will fail later + else if (opt->savename_delayed != 0 && !opt->state.stop) { + if (mime_type != NULL) { + ext[0] = '\0'; + if (*mime_type) { + give_mimext(ext, mime_type); + } + if (strnotempty(ext)) { + char mime_from_file[128]; + + mime_from_file[0] = 0; + get_httptype(opt, mime_from_file, fil, 1); + if (!strnotempty(mime_from_file) || strcasecmp(mime_type, mime_from_file) != 0) { /* different mime for this type */ + /* type change not forbidden (or no extension at all) */ + if (!may_unknown2(opt, mime_type, fil)) { + ext_chg = 1; + } #ifdef DEFAULT_BIN_EXT - // no extension and potentially bogus - else if (ishtml(opt,fil) == -2) { - ext_chg = 1; - strcpybuff(ext, DEFAULT_BIN_EXT + 1); - } + // no extension and potentially bogus + else if (ishtml(opt, fil) == -2) { + ext_chg = 1; + strcpybuff(ext, DEFAULT_BIN_EXT + 1); + } #endif - } else { - ext_chg = 0; + } else { + ext_chg = 0; + } } + } else { + /* Avoid collisions (no collisionning detection) */ + sprintf(ext, "%x.%s", opt->state.delayedId++, DELAYED_EXT); + ext_chg = 1; + ext_chg_delayed = 1; /* due to naming system */ } - } else { - /* Avoid collisions (no collisionning detection) */ - sprintf(ext, "%x.%s", opt->state.delayedId++, DELAYED_EXT); - ext_chg = 1; - ext_chg_delayed = 1; /* due to naming system */ } - } - // test imposible dans le cache, faire une requête - else { - // - int hihp = opt->state._hts_in_html_parsing; - int has_been_moved=0; - char BIGSTK curr_adr[HTS_URLMAXSIZE*2],curr_fil[HTS_URLMAXSIZE*2]; - - /* Ensure we don't use too many sockets by using a "testing" one - If we have only 1 simultaneous connection authorized, wait for pending download - Wait for an available slot - */ - URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET(); - - /* Rock'in */ - curr_adr[0]=curr_fil[0]='\0'; - opt->state._hts_in_html_parsing=2; // test - hts_log_print(opt, LOG_DEBUG, "Testing link type %s%s",adr_complete,fil_complete); - strcpybuff(curr_adr,adr_complete); - strcpybuff(curr_fil,fil_complete); - // ajouter dans le backing le fichier en mode test - // savename: rien car en mode test - if (back_add(sback,opt,cache,curr_adr,curr_fil,BACK_ADD_TEST,referer_adr,referer_fil,1)!=-1) { - int b; - b=back_index(opt,sback,curr_adr,curr_fil,BACK_ADD_TEST); - if (b>=0) { - int stop_looping=0; - int petits_tours=0; - int get_test_request=0; // en cas de bouclage sur soi même avec HEAD, tester avec GET.. parfois c'est la cause des problèmes - do { - // temps à attendre, et remplir autant que l'on peut le cache (backing) - if (back[b].status>0) { - back_wait(sback,opt,cache,0); - } - if (ptr>=0) { - back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot); - } - - // on est obligé d'appeler le shell pour le refresh.. - // Transfer rate - engine_stats(); - - // Refresh various stats - HTS_STAT.stat_nsocket=back_nsoc(sback); - HTS_STAT.stat_errors=fspc(opt,NULL,"error"); - HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); - HTS_STAT.stat_infos=fspc(opt,NULL,"info"); - HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr); - HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback); - - if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { - return -1; - } else if (opt->state._hts_cancel || !back_checkmirror(opt)) { // cancel 2 ou 1 (cancel parsing) - back_delete(opt,cache,sback,b); // cancel test - stop_looping = 1; - } - - // traitement des 304,303.. - if (back[b].status<=0) { - if (HTTP_IS_REDIRECT(back[b].r.statuscode)) { // agh moved.. un tit tour de plus - if ((petits_tours<5) && (former_adr) && (former_fil)) { // on va pas tourner en rond non plus! - if ((int) strnotempty(back[b].r.location)) { // location existe! - char BIGSTK mov_url[HTS_URLMAXSIZE*2],mov_adr[HTS_URLMAXSIZE*2],mov_fil[HTS_URLMAXSIZE*2]; - mov_url[0]=mov_adr[0]=mov_fil[0]='\0'; - // - strcpybuff(mov_url,back[b].r.location); // copier URL - if (ident_url_relatif(mov_url,curr_adr,curr_fil,mov_adr,mov_fil)>=0) { - // si non bouclage sur soi même, ou si test avec GET non testé - if ((strcmp(mov_adr,curr_adr)) || (strcmp(mov_fil,curr_fil)) || (get_test_request==0)) { - // bouclage? - if ((!strcmp(mov_adr,curr_adr)) && (!strcmp(mov_fil,curr_fil))) - get_test_request=1; // faire requète avec GET - - // recopier former_adr/fil? - if ((former_adr) && (former_fil)) { - if (strnotempty(former_adr)==0) { // Pas déja noté - strcpybuff(former_adr,curr_adr); - strcpybuff(former_fil,curr_fil); + // test imposible dans le cache, faire une requête + else { + // + int hihp = opt->state._hts_in_html_parsing; + int has_been_moved = 0; + char BIGSTK curr_adr[HTS_URLMAXSIZE * 2], + curr_fil[HTS_URLMAXSIZE * 2]; + + /* Ensure we don't use too many sockets by using a "testing" one + If we have only 1 simultaneous connection authorized, wait for pending download + Wait for an available slot + */ + URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET(); + + /* Rock'in */ + curr_adr[0] = curr_fil[0] = '\0'; + opt->state._hts_in_html_parsing = 2; // test + hts_log_print(opt, LOG_DEBUG, "Testing link type %s%s", + adr_complete, fil_complete); + strcpybuff(curr_adr, adr_complete); + strcpybuff(curr_fil, fil_complete); + // ajouter dans le backing le fichier en mode test + // savename: rien car en mode test + if (back_add + (sback, opt, cache, curr_adr, curr_fil, BACK_ADD_TEST, + referer_adr, referer_fil, 1) != -1) { + int b; + + b = back_index(opt, sback, curr_adr, curr_fil, BACK_ADD_TEST); + if (b >= 0) { + int stop_looping = 0; + int petits_tours = 0; + int get_test_request = 0; // en cas de bouclage sur soi même avec HEAD, tester avec GET.. parfois c'est la cause des problèmes + + do { + // temps à attendre, et remplir autant que l'on peut le cache (backing) + if (back[b].status > 0) { + back_wait(sback, opt, cache, 0); + } + if (ptr >= 0) { + back_fillmax(sback, opt, cache, liens, ptr, numero_passe, + lien_tot); + } + // on est obligé d'appeler le shell pour le refresh.. + // Transfer rate + engine_stats(); + + // Refresh various stats + HTS_STAT.stat_nsocket = back_nsoc(sback); + HTS_STAT.stat_errors = fspc(opt, NULL, "error"); + HTS_STAT.stat_warnings = fspc(opt, NULL, "warning"); + HTS_STAT.stat_infos = fspc(opt, NULL, "info"); + HTS_STAT.nbk = backlinks_done(sback, liens, lien_tot, ptr); + HTS_STAT.nb = back_transfered(HTS_STAT.stat_bytes, sback); + + if (!RUN_CALLBACK7 + (opt, loop, sback->lnk, sback->count, b, ptr, lien_tot, + (int) (time_local() - HTS_STAT.stat_timestart), + &HTS_STAT)) { + return -1; + } else if (opt->state._hts_cancel || !back_checkmirror(opt)) { // cancel 2 ou 1 (cancel parsing) + back_delete(opt, cache, sback, b); // cancel test + stop_looping = 1; + } + // traitement des 304,303.. + if (back[b].status <= 0) { + if (HTTP_IS_REDIRECT(back[b].r.statuscode)) { // agh moved.. un tit tour de plus + if ((petits_tours < 5) && (former_adr) && (former_fil)) { // on va pas tourner en rond non plus! + if ((int) strnotempty(back[b].r.location)) { // location existe! + char BIGSTK mov_url[HTS_URLMAXSIZE * 2], + mov_adr[HTS_URLMAXSIZE * 2], + mov_fil[HTS_URLMAXSIZE * 2]; + mov_url[0] = mov_adr[0] = mov_fil[0] = '\0'; + // + strcpybuff(mov_url, back[b].r.location); // copier URL + if (ident_url_relatif + (mov_url, curr_adr, curr_fil, mov_adr, + mov_fil) >= 0) { + // si non bouclage sur soi même, ou si test avec GET non testé + if ((strcmp(mov_adr, curr_adr)) + || (strcmp(mov_fil, curr_fil)) + || (get_test_request == 0)) { + // bouclage? + if ((!strcmp(mov_adr, curr_adr)) + && (!strcmp(mov_fil, curr_fil))) + get_test_request = 1; // faire requète avec GET + + // recopier former_adr/fil? + if ((former_adr) && (former_fil)) { + if (strnotempty(former_adr) == 0) { // Pas déja noté + strcpybuff(former_adr, curr_adr); + strcpybuff(former_fil, curr_fil); + } } - } - - // check explicit forbidden - don't follow 3xx in this case - { - int set_prio_to=0; - if (hts_acceptlink(opt,ptr,lien_tot,liens, - mov_adr,mov_fil, - NULL, NULL, - &set_prio_to, - NULL) == 1) - { /* forbidden */ - has_been_moved = 1; - back_maydelete(opt,cache,sback,b); // ok - strcpybuff(curr_adr,mov_adr); - strcpybuff(curr_fil,mov_fil); - mov_url[0]='\0'; - stop_looping = 1; + // check explicit forbidden - don't follow 3xx in this case + { + int set_prio_to = 0; + + if (hts_acceptlink(opt, ptr, lien_tot, liens, mov_adr, mov_fil, NULL, NULL, &set_prio_to, NULL) == 1) { /* forbidden */ + has_been_moved = 1; + back_maydelete(opt, cache, sback, b); // ok + strcpybuff(curr_adr, mov_adr); + strcpybuff(curr_fil, mov_fil); + mov_url[0] = '\0'; + stop_looping = 1; + } } - } - - // ftp: stop! - if (strfield(mov_url,"ftp://") + + // ftp: stop! + if (strfield(mov_url, "ftp://") #if HTS_USEMMS - || strfield(mov_url,"mms://") + || strfield(mov_url, "mms://") #endif - ) - { // ftp, ok on arrête - has_been_moved = 1; - back_maydelete(opt,cache,sback,b); // ok - strcpybuff(curr_adr,mov_adr); - strcpybuff(curr_fil,mov_fil); - stop_looping = 1; - } else if (*mov_url) { - char* methode; - if (!get_test_request) - methode=BACK_ADD_TEST; // tester avec HEAD - else { - methode=BACK_ADD_TEST2; // tester avec GET - hts_log_print(opt, LOG_WARNING, "Loop with HEAD request (during prefetch) at %s%s",curr_adr,curr_fil); - } - // Ajouter - URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET(); - if (back_add(sback,opt,cache,mov_adr,mov_fil,methode,referer_adr,referer_fil,1)!=-1) { // OK - hts_log_print(opt, LOG_DEBUG, "(during prefetch) %s (%d) to link %s at %s%s",back[b].r.msg,back[b].r.statuscode,back[b].r.location,curr_adr,curr_fil); - - // libérer emplacement backing actuel et attendre le prochain - back_maydelete(opt,cache,sback,b); - strcpybuff(curr_adr,mov_adr); - strcpybuff(curr_fil,mov_fil); - b=back_index(opt,sback,curr_adr,curr_fil,methode); + ) { // ftp, ok on arrête + has_been_moved = 1; + back_maydelete(opt, cache, sback, b); // ok + strcpybuff(curr_adr, mov_adr); + strcpybuff(curr_fil, mov_fil); + stop_looping = 1; + } else if (*mov_url) { + char *methode; + if (!get_test_request) - has_been_moved = 1; // sinon ne pas forcer has_been_moved car non déplacé - petits_tours++; - // - } else {// sinon on fait rien et on s'en va.. (ftp etc) - hts_log_print(opt, LOG_DEBUG, "Warning: Savename redirect backing error at %s%s",mov_adr,mov_fil); + methode = BACK_ADD_TEST; // tester avec HEAD + else { + methode = BACK_ADD_TEST2; // tester avec GET + hts_log_print(opt, LOG_WARNING, + "Loop with HEAD request (during prefetch) at %s%s", + curr_adr, curr_fil); + } + // Ajouter + URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET(); + if (back_add(sback, opt, cache, mov_adr, mov_fil, methode, referer_adr, referer_fil, 1) != -1) { // OK + hts_log_print(opt, LOG_DEBUG, + "(during prefetch) %s (%d) to link %s at %s%s", + back[b].r.msg, + back[b].r.statuscode, + back[b].r.location, curr_adr, + curr_fil); + + // libérer emplacement backing actuel et attendre le prochain + back_maydelete(opt, cache, sback, b); + strcpybuff(curr_adr, mov_adr); + strcpybuff(curr_fil, mov_fil); + b = + back_index(opt, sback, curr_adr, curr_fil, + methode); + if (!get_test_request) + has_been_moved = 1; // sinon ne pas forcer has_been_moved car non déplacé + petits_tours++; + // + } else { // sinon on fait rien et on s'en va.. (ftp etc) + hts_log_print(opt, LOG_DEBUG, + "Warning: Savename redirect backing error at %s%s", + mov_adr, mov_fil); + } } + } else { + hts_log_print(opt, LOG_WARNING, + "Unable to test %s%s (loop to same filename)", + adr_complete, fil_complete); } - } else { - hts_log_print(opt, LOG_WARNING, "Unable to test %s%s (loop to same filename)",adr_complete,fil_complete); + } - } + } else { // arrêter les frais + hts_log_print(opt, LOG_WARNING, + "Unable to test %s%s (loop)", + adr_complete, fil_complete); } - } else{ // arrêter les frais - hts_log_print(opt, LOG_WARNING, "Unable to test %s%s (loop)",adr_complete,fil_complete); - } - } // ok, leaving - } - } while(!stop_looping && back[b].status > 0 && back[b].status < 1000); - - // Si non déplacé, forcer type? - if (!has_been_moved) { - if (back[b].r.statuscode!=-10) { // erreur - if (strnotempty(back[b].r.contenttype)==0) - strcpybuff(back[b].r.contenttype,"text/html"); // message d'erreur en html - // Finalement on, renvoie un erreur, pour ne toucher à rien dans le code - // libérer emplacement backing - } - - { // pas d'erreur, changer type? - char s[16]; - s[0]='\0'; - if (strnotempty(back[b].r.cdispo)) { /* filename given */ - ext_chg=2; /* change filename */ - strcpybuff(ext,back[b].r.cdispo); + } // ok, leaving } - else if (!may_unknown2(opt, back[b].r.contenttype, back[b].url_fil)) { // on peut patcher à priori? (pas interdit ou pas de type) - give_mimext(s,back[b].r.contenttype); // obtenir extension - if (strnotempty(s)>0) { // on a reconnu l'extension - ext_chg=1; - strcpybuff(ext,s); - } + } while(!stop_looping && back[b].status > 0 + && back[b].status < 1000); + + // Si non déplacé, forcer type? + if (!has_been_moved) { + if (back[b].r.statuscode != -10) { // erreur + if (strnotempty(back[b].r.contenttype) == 0) + strcpybuff(back[b].r.contenttype, "text/html"); // message d'erreur en html + // Finalement on, renvoie un erreur, pour ne toucher à rien dans le code + // libérer emplacement backing } + + { // pas d'erreur, changer type? + char s[16]; + + s[0] = '\0'; + if (strnotempty(back[b].r.cdispo)) { /* filename given */ + ext_chg = 2; /* change filename */ + strcpybuff(ext, back[b].r.cdispo); + } else if (!may_unknown2(opt, back[b].r.contenttype, back[b].url_fil)) { // on peut patcher à priori? (pas interdit ou pas de type) + give_mimext(s, back[b].r.contenttype); // obtenir extension + if (strnotempty(s) > 0) { // on a reconnu l'extension + ext_chg = 1; + strcpybuff(ext, s); + } + } #ifdef DEFAULT_BIN_EXT - // no extension and potentially bogus - else if (ishtest == -2) { - ext_chg=1; - strcpybuff(ext, DEFAULT_BIN_EXT + 1); - } + // no extension and potentially bogus + else if (ishtest == -2) { + ext_chg = 1; + strcpybuff(ext, DEFAULT_BIN_EXT + 1); + } #endif + } } + // FIN Si non déplacé, forcer type? + + // libérer emplacement backing + back_maydelete(opt, cache, sback, b); + + // --- --- --- + // oops, a été déplacé.. on recalcule en récursif (osons!) + if (has_been_moved) { + // copier adr, fil (optionnel, mais sinon marche pas pour le rip) + strcpybuff(adr_complete, curr_adr); + strcpybuff(fil_complete, curr_fil); + // copier adr, fil + + return url_savename(curr_adr, curr_fil, save, NULL, NULL, + referer_adr, referer_fil, opt, liens, + lien_tot, sback, cache, hash, ptr, + numero_passe, NULL); + } + // --- --- --- + } - // FIN Si non déplacé, forcer type? - - // libérer emplacement backing - back_maydelete(opt,cache,sback,b); - - // --- --- --- - // oops, a été déplacé.. on recalcule en récursif (osons!) - if (has_been_moved) { - // copier adr, fil (optionnel, mais sinon marche pas pour le rip) - strcpybuff(adr_complete,curr_adr); - strcpybuff(fil_complete,curr_fil); - // copier adr, fil - - return url_savename(curr_adr,curr_fil,save,NULL,NULL,referer_adr,referer_fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL); - } - // --- --- --- - - } - - } else { - printf("PANIC! : Savename Crash adding error, unexpected error found.. [%d]\n",__LINE__); + + } else { + printf + ("PANIC! : Savename Crash adding error, unexpected error found.. [%d]\n", + __LINE__); #if BDEBUG==1 - printf("error while savename crash adding\n"); + printf("error while savename crash adding\n"); #endif - hts_log_print(opt, LOG_ERROR, "Unexpected savename backing error at %s%s",adr,fil_complete); - - } - // restaurer - opt->state._hts_in_html_parsing=hihp; - } // caché? - } + hts_log_print(opt, LOG_ERROR, + "Unexpected savename backing error at %s%s", adr, + fil_complete); + + } + // restaurer + opt->state._hts_in_html_parsing = hihp; + } // caché? + } } } - - // - - - DEBUT NOMMAGE - - - // Donner nom par défaut? - if (fil[strlen(fil)-1]=='/') { - if (!strfield(adr_complete,"ftp://") + if (fil[strlen(fil) - 1] == '/') { + if (!strfield(adr_complete, "ftp://") #if HTS_USEMMS - && !strfield(adr_complete,"mms://") + && !strfield(adr_complete, "mms://") #endif - ) - { - strcatbuff(fil,DEFAULT_HTML); // nommer page par défaut!! + ) { + strcatbuff(fil, DEFAULT_HTML); // nommer page par défaut!! #if HTS_USEMMS - } else if (strfield(adr_complete,"mms://")) { - strcatbuff(fil,DEFAULT_MMS); + } else if (strfield(adr_complete, "mms://")) { + strcatbuff(fil, DEFAULT_MMS); #endif - } else { + } else { if (!opt->proxy.active) - strcatbuff(fil,DEFAULT_FTP); // nommer page par défaut (texte) + strcatbuff(fil, DEFAULT_FTP); // nommer page par défaut (texte) else - strcatbuff(fil,DEFAULT_HTML); // nommer page par défaut (à priori ici html depuis un proxy http) + strcatbuff(fil, DEFAULT_HTML); // nommer page par défaut (à priori ici html depuis un proxy http) } } // Changer extension? // par exemple, php3 sera sauvé en html, cgi en html ou gif, xbm etc.. selon les cas - if (ext_chg) { // changer ext - char* a=fil+strlen(fil)-1; - if ( (opt->debug>1) && (opt->log!=NULL) ) { - if (ext_chg==1) - hts_log_print(opt, LOG_DEBUG, "Changing link extension %s%s to .%s",adr_complete,fil_complete,ext); + if (ext_chg) { // changer ext + char *a = fil + strlen(fil) - 1; + + if ((opt->debug > 1) && (opt->log != NULL)) { + if (ext_chg == 1) + hts_log_print(opt, LOG_DEBUG, "Changing link extension %s%s to .%s", + adr_complete, fil_complete, ext); else - hts_log_print(opt, LOG_DEBUG, "Changing link name %s%s to %s",adr_complete,fil_complete,ext); + hts_log_print(opt, LOG_DEBUG, "Changing link name %s%s to %s", + adr_complete, fil_complete, ext); } - if (ext_chg==1) { - while((a > fil) && (*a!='.') && (*a!='/')) a--; - if (*a=='.') *a='\0'; // couper - strcatbuff(fil,"."); // recopier point + if (ext_chg == 1) { + while((a > fil) && (*a != '.') && (*a != '/')) + a--; + if (*a == '.') + *a = '\0'; // couper + strcatbuff(fil, "."); // recopier point } else { - while(( a > fil) && (*a!='/')) a--; - if (*a=='/') a++; - *a='\0'; - } - strcatbuff(fil,ext); // copier ext/nom + while((a > fil) && (*a != '/')) + a--; + if (*a == '/') + a++; + *a = '\0'; + } + strcatbuff(fil, ext); // copier ext/nom } - // Rechercher premier / et dernier . - { - char* a=fil+strlen(fil)-1; + { + char *a = fil + strlen(fil) - 1; // passer structures - start_pos=fil; - while(( a > fil) && (*a != '/') && (*a != '\\')) { - if (*a == '.') // point? noter position + start_pos = fil; + while((a > fil) && (*a != '/') && (*a != '\\')) { + if (*a == '.') // point? noter position if (!dot_pos) - dot_pos=a; - a--; + dot_pos = a; + a--; } - if ((*a=='/') || (*a=='\\')) a++; + if ((*a == '/') || (*a == '\\')) + a++; nom_pos = a; } - // un nom de fichier est généré // s'il existe déja, alors on le mofifie légèrement // ajouter nom du site éventuellement en premier - if (opt->savename_type == -1) { // utiliser savename_userdef! (%h%p/%n%q.%t) - const char* a = StringBuff(opt->savename_userdef); - char* b = save; + if (opt->savename_type == -1) { // utiliser savename_userdef! (%h%p/%n%q.%t) + const char *a = StringBuff(opt->savename_userdef); + char *b = save; + /*char *nom_pos=NULL,*dot_pos=NULL; // Position nom et point */ char tok; /* - { // Rechercher premier / - char* a=fil+strlen(fil)-1; - // passer structures - while(((int) a>(int) fil) && (*a != '/') && (*a != '\\')) { - if (*a == '.') // point? noter position - if (!dot_pos) - dot_pos=a; - a--; - } - if ((*a=='/') || (*a=='\\')) a++; - nom_pos = a; - } - */ + { // Rechercher premier / + char* a=fil+strlen(fil)-1; + // passer structures + while(((int) a>(int) fil) && (*a != '/') && (*a != '\\')) { + if (*a == '.') // point? noter position + if (!dot_pos) + dot_pos=a; + a--; + } + if ((*a=='/') || (*a=='\\')) a++; + nom_pos = a; + } + */ // Construire nom - while ((*a) && (((int) (b - save)) < HTS_URLMAXSIZE ) ) { // parser, et pas trop long.. + while((*a) && (((int) (b - save)) < HTS_URLMAXSIZE)) { // parser, et pas trop long.. if (*a == '%') { - int short_ver=0; + int short_ver = 0; + a++; if (*a == 's') { - short_ver=1; + short_ver = 1; a++; } - *b='\0'; - switch(tok=*a++) { - case '[': // %[param:prefix_if_not_empty:suffix_if_not_empty:empty_replacement:notfound_replacement] - if (strchr(a,']')) { - int pos=0; - char name[5][256]; - char* c=name[0]; - for(pos = 0 ; pos < 5 ; pos++) { - name[pos][0]='\0'; - } - pos=0; - while(*a!=']') { - if (pos < 5) { - if (*a == ':') { // next token - c=name[++pos]; - a++; - } else { - *c++=*a++; - *c='\0'; - } + *b = '\0'; + switch (tok = *a++) { + case '[': // %[param:prefix_if_not_empty:suffix_if_not_empty:empty_replacement:notfound_replacement] + if (strchr(a, ']')) { + int pos = 0; + char name[5][256]; + char *c = name[0]; + + for(pos = 0; pos < 5; pos++) { + name[pos][0] = '\0'; + } + pos = 0; + while(*a != ']') { + if (pos < 5) { + if (*a == ':') { // next token + c = name[++pos]; + a++; + } else { + *c++ = *a++; + *c = '\0'; } } - a++; - strcatbuff(name[0],"="); /* param=.. */ - c=strchr(fil_complete,'?'); - /* parameters exists */ - if (c) { - char* cp; - while((cp = strstr(c+1, name[0])) && *(cp-1) != '?' && *(cp-1) != '&') { /* finds [?&]param= */ - c = cp; - } - if (cp) { - c = cp + strlen(name[0]); /* jumps "param=" */ - strcpybuff(b, name[1]); /* prefix */ - b += strlen(b); - if (*c != '\0' && *c != '&') { - char* d = name[0]; - /* */ - while(*c != '\0' && *c != '&') { - *d++ = *c++; - } - *d = '\0'; - d = unescape_http(catbuff,name[0]); - if (d && *d) { - strcpybuff(b, d); /* value */ - b += strlen(b); - } else { - strcpybuff(b, name[3]); /* empty replacement if any */ - b += strlen(b); - } + } + a++; + strcatbuff(name[0], "="); /* param=.. */ + c = strchr(fil_complete, '?'); + /* parameters exists */ + if (c) { + char *cp; + + while((cp = strstr(c + 1, name[0])) && *(cp - 1) != '?' && *(cp - 1) != '&') { /* finds [?&]param= */ + c = cp; + } + if (cp) { + c = cp + strlen(name[0]); /* jumps "param=" */ + strcpybuff(b, name[1]); /* prefix */ + b += strlen(b); + if (*c != '\0' && *c != '&') { + char *d = name[0]; + + /* */ + while(*c != '\0' && *c != '&') { + *d++ = *c++; + } + *d = '\0'; + d = unescape_http(catbuff, name[0]); + if (d && *d) { + strcpybuff(b, d); /* value */ + b += strlen(b); } else { - strcpybuff(b, name[3]); /* empty replacement if any */ + strcpybuff(b, name[3]); /* empty replacement if any */ b += strlen(b); } - strcpybuff(b, name[2]); /* suffix */ - b += strlen(b); } else { - strcpybuff(b, name[4]); /* not found replacement if any */ + strcpybuff(b, name[3]); /* empty replacement if any */ b += strlen(b); } - } else { - strcpybuff(b, name[4]); /* not found replacement if any */ - b += strlen(b); - } - } - break; - case '%': *b++='%'; break; - case 'n': // nom sans ext - *b='\0'; - if (dot_pos) { - if (!short_ver) // Noms longs - strncatbuff(b,nom_pos,(int) (dot_pos - nom_pos)); - else - strncatbuff(b,nom_pos,min((int) (dot_pos - nom_pos),8)); - } else { - if (!short_ver) // Noms longs - strcpybuff(b,nom_pos); - else - strncatbuff(b,nom_pos,8); - } - b+=strlen(b); // pointer à la fin - break; - case 'N': // nom avec ext - // RECOPIE NOM + EXT - *b='\0'; - if (dot_pos) { - if (!short_ver) // Noms longs - strncatbuff(b,nom_pos,(int) (dot_pos - nom_pos)); - else - strncatbuff(b,nom_pos,min((int) (dot_pos - nom_pos),8)); - } else { - if (!short_ver) // Noms longs - strcpybuff(b,nom_pos); - else - strncatbuff(b,nom_pos,8); - } - b+=strlen(b); // pointer à la fin - // RECOPIE NOM + EXT - *b='\0'; - if (dot_pos) { - if (!short_ver) // Noms longs - strcpybuff(b,dot_pos+1); - else - strncatbuff(b,dot_pos+1,3); - } else { - if (!short_ver) // Noms longs - strcpybuff(b,DEFAULT_EXT + 1); // pas de.. - else - strcpybuff(b,DEFAULT_EXT_SHORT + 1); // pas de.. - } - b+=strlen(b); // pointer à la fin - // - break; - case 't': // ext - *b='\0'; - if (dot_pos) { - if (!short_ver) // Noms longs - strcpybuff(b,dot_pos+1); - else - strncatbuff(b,dot_pos+1,3); - } else { - if (!short_ver) // Noms longs - strcpybuff(b,DEFAULT_EXT + 1); // pas de.. - else - strcpybuff(b,DEFAULT_EXT_SHORT + 1); // pas de.. - } - b+=strlen(b); // pointer à la fin - break; - case 'p': // path sans dernier / - *b='\0'; - if (nom_pos != fil + 1) { // pas: /index.html (chemin nul) - if (!short_ver) { // Noms longs - strncatbuff(b,fil,(int) (nom_pos - fil) - 1); + strcpybuff(b, name[2]); /* suffix */ + b += strlen(b); } else { - char BIGSTK pth[HTS_URLMAXSIZE*2],n83[HTS_URLMAXSIZE*2]; - pth[0]=n83[0]='\0'; - // - strncatbuff(pth,fil,(int) (nom_pos - fil) - 1); - long_to_83(opt->savename_83,n83,pth); - strcpybuff(b,n83); + strcpybuff(b, name[4]); /* not found replacement if any */ + b += strlen(b); } - } - b+=strlen(b); // pointer à la fin - break; - case 'h': // host - *b='\0'; - if (strcmp(adr_complete,"file://")==0) { - if (!short_ver) // Noms longs - strcpybuff(b,"localhost"); - else - strcpybuff(b,"local"); } else { - if (!short_ver) // Noms longs - strcpybuff(b,print_adr); - else - strncatbuff(b,print_adr,8); + strcpybuff(b, name[4]); /* not found replacement if any */ + b += strlen(b); } - b+=strlen(b); // pointer à la fin - break; - case 'M': /* host/address?query MD5 (128-bits) */ - *b='\0'; - { - char digest[32+2]; - char BIGSTK buff[HTS_URLMAXSIZE*2]; - digest[0]=buff[0]='\0'; - strcpybuff(buff,adr); - strcatbuff(buff,fil_complete); - domd5mem(buff,strlen(buff),digest,1); - strcpybuff(b,digest); + } + break; + case '%': + *b++ = '%'; + break; + case 'n': // nom sans ext + *b = '\0'; + if (dot_pos) { + if (!short_ver) // Noms longs + strncatbuff(b, nom_pos, (int) (dot_pos - nom_pos)); + else + strncatbuff(b, nom_pos, min((int) (dot_pos - nom_pos), 8)); + } else { + if (!short_ver) // Noms longs + strcpybuff(b, nom_pos); + else + strncatbuff(b, nom_pos, 8); + } + b += strlen(b); // pointer à la fin + break; + case 'N': // nom avec ext + // RECOPIE NOM + EXT + *b = '\0'; + if (dot_pos) { + if (!short_ver) // Noms longs + strncatbuff(b, nom_pos, (int) (dot_pos - nom_pos)); + else + strncatbuff(b, nom_pos, min((int) (dot_pos - nom_pos), 8)); + } else { + if (!short_ver) // Noms longs + strcpybuff(b, nom_pos); + else + strncatbuff(b, nom_pos, 8); + } + b += strlen(b); // pointer à la fin + // RECOPIE NOM + EXT + *b = '\0'; + if (dot_pos) { + if (!short_ver) // Noms longs + strcpybuff(b, dot_pos + 1); + else + strncatbuff(b, dot_pos + 1, 3); + } else { + if (!short_ver) // Noms longs + strcpybuff(b, DEFAULT_EXT + 1); // pas de.. + else + strcpybuff(b, DEFAULT_EXT_SHORT + 1); // pas de.. + } + b += strlen(b); // pointer à la fin + // + break; + case 't': // ext + *b = '\0'; + if (dot_pos) { + if (!short_ver) // Noms longs + strcpybuff(b, dot_pos + 1); + else + strncatbuff(b, dot_pos + 1, 3); + } else { + if (!short_ver) // Noms longs + strcpybuff(b, DEFAULT_EXT + 1); // pas de.. + else + strcpybuff(b, DEFAULT_EXT_SHORT + 1); // pas de.. + } + b += strlen(b); // pointer à la fin + break; + case 'p': // path sans dernier / + *b = '\0'; + if (nom_pos != fil + 1) { // pas: /index.html (chemin nul) + if (!short_ver) { // Noms longs + strncatbuff(b, fil, (int) (nom_pos - fil) - 1); + } else { + char BIGSTK pth[HTS_URLMAXSIZE * 2], n83[HTS_URLMAXSIZE * 2]; + + pth[0] = n83[0] = '\0'; + // + strncatbuff(pth, fil, (int) (nom_pos - fil) - 1); + long_to_83(opt->savename_83, n83, pth); + strcpybuff(b, n83); } - b+=strlen(b); // pointer à la fin - break; - case 'Q': case 'q': /* query MD5 (128-bits/16-bits) - GENERATED ONLY IF query string exists! */ - { - char md5[32 + 2]; - *b='\0'; - strncatbuff(b,url_md5(md5, fil_complete),(tok == 'Q')?32:4); - b+=strlen(b); // pointer à la fin - } - break; - case 'r': case 'R': // protocol - *b='\0'; - strcatbuff(b, protocol_str[protocol]); - b+=strlen(b); // pointer à la fin - break; - - /* Patch by Juan Fco Rodriguez to get the full query string */ - case 'k': - { - char *d = strchr(fil_complete,'?'); - if( d != NULL ) + } + b += strlen(b); // pointer à la fin + break; + case 'h': // host + *b = '\0'; + if (strcmp(adr_complete, "file://") == 0) { + if (!short_ver) // Noms longs + strcpybuff(b, "localhost"); + else + strcpybuff(b, "local"); + } else { + if (!short_ver) // Noms longs + strcpybuff(b, print_adr); + else + strncatbuff(b, print_adr, 8); + } + b += strlen(b); // pointer à la fin + break; + case 'M': /* host/address?query MD5 (128-bits) */ + *b = '\0'; { - strcatbuff(b, d); - b+=strlen(b); + char digest[32 + 2]; + char BIGSTK buff[HTS_URLMAXSIZE * 2]; + + digest[0] = buff[0] = '\0'; + strcpybuff(buff, adr); + strcatbuff(buff, fil_complete); + domd5mem(buff, strlen(buff), digest, 1); + strcpybuff(b, digest); } - } - break; + b += strlen(b); // pointer à la fin + break; + case 'Q': + case 'q': /* query MD5 (128-bits/16-bits) + GENERATED ONLY IF query string exists! */ + { + char md5[32 + 2]; + + *b = '\0'; + strncatbuff(b, url_md5(md5, fil_complete), (tok == 'Q') ? 32 : 4); + b += strlen(b); // pointer à la fin + } + break; + case 'r': + case 'R': // protocol + *b = '\0'; + strcatbuff(b, protocol_str[protocol]); + b += strlen(b); // pointer à la fin + break; + + /* Patch by Juan Fco Rodriguez to get the full query string */ + case 'k': + { + char *d = strchr(fil_complete, '?'); + + if (d != NULL) { + strcatbuff(b, d); + b += strlen(b); + } + } + break; } } else - *b++=*a++; + *b++ = *a++; } - *b++='\0'; + *b++ = '\0'; // // Types prédéfinis // - } + } // // Structure originale - else if (opt->savename_type%100==0) { + else if (opt->savename_type % 100 == 0) { /* recopier www.. */ - if (opt->savename_type!=100) { - if (((opt->savename_type/1000)%2)==0) { // >1000 signifie "pas de www/" - if (strcmp(adr_complete,"file://")==0) { + if (opt->savename_type != 100) { + if (((opt->savename_type / 1000) % 2) == 0) { // >1000 signifie "pas de www/" + if (strcmp(adr_complete, "file://") == 0) { //## if (*adr==lOCAL_CHAR) { - if (opt->savename_83 != 1) // noms longs - strcatbuff(save,"localhost"); + if (opt->savename_83 != 1) // noms longs + strcatbuff(save, "localhost"); else - strcatbuff(save,"local"); + strcatbuff(save, "local"); } else { // adresse url - if (!opt->savename_83) { // noms longs (et pas de .) - strcatbuff(save,print_adr); - } else { // noms 8-3 - if (strlen(print_adr)>4) { - if (strfield(print_adr,"www.")) - strncatbuff(save,print_adr+4,max_char); + if (!opt->savename_83) { // noms longs (et pas de .) + strcatbuff(save, print_adr); + } else { // noms 8-3 + if (strlen(print_adr) > 4) { + if (strfield(print_adr, "www.")) + strncatbuff(save, print_adr + 4, max_char); else - strncatbuff(save,print_adr,8); - } else strncatbuff(save,print_adr,max_char); + strncatbuff(save, print_adr, 8); + } else + strncatbuff(save, print_adr, max_char); } } - if (*fil!='/') strcatbuff(save,"/"); + if (*fil != '/') + strcatbuff(save, "/"); } } - + hts_lowcase(save); - + /* - // ne sert à rien car a déja été filtré normalement - if ((*fil=='.') && (*(fil+1)=='/')) // ./index.html ** // - url_savename_addstr(save,fil+2); - else // index.html ou /index.html - url_savename_addstr(save,fil); - if (save[strlen(save)-1]=='/') - strcatbuff(save,DEFAULT_HTML); // nommer page par défaut!! -*/ + // ne sert à rien car a déja été filtré normalement + if ((*fil=='.') && (*(fil+1)=='/')) // ./index.html ** // + url_savename_addstr(save,fil+2); + else // index.html ou /index.html + url_savename_addstr(save,fil); + if (save[strlen(save)-1]=='/') + strcatbuff(save,DEFAULT_HTML); // nommer page par défaut!! + */ /* add name */ ADD_STANDARD_PATH; @@ -1020,150 +1069,162 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, } // // Structure html/image - else { + else { // dossier "web" ou "www.xxx" ? - if (((opt->savename_type/1000)%2)==0) { // >1000 signifie "pas de www/" - if ((opt->savename_type/100)%2) { - if (strcmp(adr_complete,"file://")==0) { - //## if (*adr==lOCAL_CHAR) { - if (opt->savename_83 != 1) // noms longs - strcatbuff(save,"localhost/"); + if (((opt->savename_type / 1000) % 2) == 0) { // >1000 signifie "pas de www/" + if ((opt->savename_type / 100) % 2) { + if (strcmp(adr_complete, "file://") == 0) { + //## if (*adr==lOCAL_CHAR) { + if (opt->savename_83 != 1) // noms longs + strcatbuff(save, "localhost/"); else - strcatbuff(save,"local/"); + strcatbuff(save, "local/"); } else { // adresse url - if (!opt->savename_83) { // noms longs - strcatbuff(save,print_adr); strcatbuff(save,"/"); - } else { // noms 8-3 - if (strlen(print_adr)>4) { - if (strfield(print_adr,"www.")) - strncatbuff(save,print_adr+4,max_char); + if (!opt->savename_83) { // noms longs + strcatbuff(save, print_adr); + strcatbuff(save, "/"); + } else { // noms 8-3 + if (strlen(print_adr) > 4) { + if (strfield(print_adr, "www.")) + strncatbuff(save, print_adr + 4, max_char); else - strncatbuff(save,print_adr,max_char); - strcatbuff(save,"/"); - } else { - strncatbuff(save,print_adr,max_char); strcatbuff(save,"/"); + strncatbuff(save, print_adr, max_char); + strcatbuff(save, "/"); + } else { + strncatbuff(save, print_adr, max_char); + strcatbuff(save, "/"); } } } } else { - strcatbuff(save,"web/"); // répertoire général + strcatbuff(save, "web/"); // répertoire général } - } - + } // si un html à coup sûr - if ( (ext_chg!=0) ? (ishtml_ext(ext) == 1) : (ishtml(opt,fil)==1) ) { - if (opt->savename_type%100==2) { // html/ + if ((ext_chg != 0) ? (ishtml_ext(ext) == 1) : (ishtml(opt, fil) == 1)) { + if (opt->savename_type % 100 == 2) { // html/ strcatbuff(save, "html/"); } } else { - if ((opt->savename_type%100==1) || (opt->savename_type%100==2)) { // html & images + if ((opt->savename_type % 100 == 1) || (opt->savename_type % 100 == 2)) { // html & images strcatbuff(save, "images/"); } } - - switch (opt->savename_type%100) { - case 4: case 5: { // séparer par types - char* a=fil+strlen(fil)-1; - // passer structures - while(( a > fil) && (*a != '/') && (*a != '\\')) a--; - if ((*a=='/') || (*a=='\\')) a++; - - // html? - if ( (ext_chg!=0) ? (ishtml_ext(ext)==1) : (ishtml(opt,fil)==1) ) { - if (opt->savename_type%100==5) - strcatbuff(save,"html/"); - } else { - char* a=fil+strlen(fil)-1; - while(( a> fil) && (*a != '/') && (*a != '.')) a--; - if (*a!='.') - strcatbuff(save,"other"); - else - strcatbuff(save,a+1); - strcatbuff(save,"/"); + + switch (opt->savename_type % 100) { + case 4: + case 5:{ // séparer par types + char *a = fil + strlen(fil) - 1; + + // passer structures + while((a > fil) && (*a != '/') && (*a != '\\')) + a--; + if ((*a == '/') || (*a == '\\')) + a++; + + // html? + if ((ext_chg != 0) ? (ishtml_ext(ext) == 1) : (ishtml(opt, fil) == 1)) { + if (opt->savename_type % 100 == 5) + strcatbuff(save, "html/"); + } else { + char *a = fil + strlen(fil) - 1; + + while((a > fil) && (*a != '/') && (*a != '.')) + a--; + if (*a != '.') + strcatbuff(save, "other"); + else + strcatbuff(save, a + 1); + strcatbuff(save, "/"); + } + /*strcatbuff(save,a); */ + /* add name */ + ADD_STANDARD_NAME(0); } - /*strcatbuff(save,a);*/ - /* add name */ - ADD_STANDARD_NAME(0); - } break; - case 99: { // 'codé' .. c'est un gadget - int i; - int j; - char* a; - char C[]="ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-"; - int L; - // pseudo-CRC sur fil et adr pour initialiser générateur aléatoire.. - unsigned int s=0; - L = (int) strlen(C); - for(i=0;i<(int) strlen(fil_complete);i++) { - s+=(unsigned int) fil_complete[i]; - } - for(i=0;i<(int) strlen(adr_complete);i++) { - s+=(unsigned int) adr_complete[i]; - } - srand(s); - - j = (int) strlen(save); - for(i=0;i<8;i++) { - char c=C[(rand()%L)]; - save[i+j]=c; - } - save[i+j]='\0'; - // ajouter extension - a = fil + strlen(fil) - 1; - while(( a > fil) && (*a != '/') && (*a != '.')) a--; - if (*a=='.') { - strcatbuff(save,a); // ajouter + case 99:{ // 'codé' .. c'est un gadget + int i; + int j; + char *a; + char C[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-"; + int L; + + // pseudo-CRC sur fil et adr pour initialiser générateur aléatoire.. + unsigned int s = 0; + + L = (int) strlen(C); + for(i = 0; i < (int) strlen(fil_complete); i++) { + s += (unsigned int) fil_complete[i]; + } + for(i = 0; i < (int) strlen(adr_complete); i++) { + s += (unsigned int) adr_complete[i]; + } + srand(s); + + j = (int) strlen(save); + for(i = 0; i < 8; i++) { + char c = C[(rand() % L)]; + + save[i + j] = c; + } + save[i + j] = '\0'; + // ajouter extension + a = fil + strlen(fil) - 1; + while((a > fil) && (*a != '/') && (*a != '.')) + a--; + if (*a == '.') { + strcatbuff(save, a); // ajouter + } } - } break; - default: { // noms sans les noms des répertoires - // ne garder que le nom, pas la structure - /* - char* a=fil+strlen(fil)-1; - while(((int) a>(int) fil) && (*a != '/') && (*a != '\\')) a--; - if ((*a=='/') || (*a=='\\')) a++; - strcatbuff(save,a); - */ - - /* add name */ - ADD_STANDARD_NAME(0); - } + default:{ // noms sans les noms des répertoires + // ne garder que le nom, pas la structure + /* + char* a=fil+strlen(fil)-1; + while(((int) a>(int) fil) && (*a != '/') && (*a != '\\')) a--; + if ((*a=='/') || (*a=='\\')) a++; + strcatbuff(save,a); + */ + + /* add name */ + ADD_STANDARD_NAME(0); + } break; } hts_lowcase(save); - if (save[strlen(save)-1]=='/') - strcatbuff(save,DEFAULT_HTML); // nommer page par défaut!! + if (save[strlen(save) - 1] == '/') + strcatbuff(save, DEFAULT_HTML); // nommer page par défaut!! } - // vérifier qu'on ne doit pas forcer l'extension // par exemple, asp sera sauvé en html, cgi en html ou gif, xbm etc.. selon les cas /*if (ext_chg) { - char* a=save+strlen(save)-1; - while(((int) a>(int) save) && (*a!='.') && (*a!='/')) a--; - if (*a=='.') *a='\0'; // couper - // recopier extension - strcatbuff(save,"."); - strcatbuff(save,ext); // copier ext - }*/ + char* a=save+strlen(save)-1; + while(((int) a>(int) save) && (*a!='.') && (*a!='/')) a--; + if (*a=='.') *a='\0'; // couper + // recopier extension + strcatbuff(save,"."); + strcatbuff(save,ext); // copier ext + } */ // Not used anymore unless non-delayed types. // de même en cas de manque d'extension on en place une de manière forcée.. // cela évite les /chez/toto et les /chez/toto/index.html incompatibles if (opt->savename_type != -1 && opt->savename_delayed != 2) { - char* a=save+strlen(save)-1; - while(( a > save) && (*a!='.') && (*a!='/')) a--; - if (*a!='.') { // agh pas de point + char *a = save + strlen(save) - 1; + + while((a > save) && (*a != '.') && (*a != '/')) + a--; + if (*a != '.') { // agh pas de point //strcatbuff(save,".none"); // a éviter - strcatbuff(save,".html"); // préférable! - hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",adr_complete,fil_complete,save); + strcatbuff(save, ".html"); // préférable! + hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s", + adr_complete, fil_complete, save); } } - // effacer pass au besoin pour les autentifications // (plus la peine : masqué au début) /* @@ -1186,66 +1247,70 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, */ // éviter les / au début (cause: N100) - if (save[0]=='/') { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - strcpybuff(tempo,save+1); - strcpybuff(save,tempo); - } + if (save[0] == '/') { + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + strcpybuff(tempo, save + 1); + strcpybuff(save, tempo); + } // changer les ~,:,",*,? en _ pour sauver sur disque - hts_replace(save,'~','_'); // interdit sous unix (~foo) + hts_replace(save, '~', '_'); // interdit sous unix (~foo) // - hts_replace(save,'\\','_'); - hts_replace(save,':','_'); // interdit sous windows - hts_replace(save,'*','_'); // interdit sous windows - hts_replace(save,'?','_'); // doit pas arriver!! - hts_replace(save,'\"','_'); // interdit sous windows - hts_replace(save,'<','_'); // interdit sous windows - hts_replace(save,'>','_'); // interdit sous windows - hts_replace(save,'|','_'); // interdit sous windows + hts_replace(save, '\\', '_'); + hts_replace(save, ':', '_'); // interdit sous windows + hts_replace(save, '*', '_'); // interdit sous windows + hts_replace(save, '?', '_'); // doit pas arriver!! + hts_replace(save, '\"', '_'); // interdit sous windows + hts_replace(save, '<', '_'); // interdit sous windows + hts_replace(save, '>', '_'); // interdit sous windows + hts_replace(save, '|', '_'); // interdit sous windows // - hts_replace(save,'@','_'); - if (opt->savename_83 == 2) { // CDROM + hts_replace(save, '@', '_'); + if (opt->savename_83 == 2) { // CDROM // maybe other ones? - hts_replace(save,'-','_'); - hts_replace(save,'=','_'); - hts_replace(save,'+','_'); + hts_replace(save, '-', '_'); + hts_replace(save, '=', '_'); + hts_replace(save, '+', '_'); } // - { // éliminer les // (comme ftp://) - char* a; + { // éliminer les // (comme ftp://) + char *a; + //while( (a=strstr(save,"//")) ) *a='_'; cleanDoubleSlash(save); // Eliminer chars spéciaux - a=save -1 ; + a = save - 1; while(*(++a)) - if ( ((unsigned char)(*a) <= 31) - || ((unsigned char)(*a) == 127) ) - *a='_'; + if (((unsigned char) (*a) <= 31) + || ((unsigned char) (*a) == 127)) + *a = '_'; } - #if HTS_OVERRIDE_DOS_FOLDERS /* Replace /foo/nul/bar by /foo/nul_/bar */ { - int i=0; + int i = 0; + while(hts_tbdev[i][0]) { - char* a=save; - while((a=strstrcase(a,(char*)hts_tbdev[i]))) { - switch ( (int) a[strlen(hts_tbdev[i])] ) { + char *a = save; + + while((a = strstrcase(a, (char *) hts_tbdev[i]))) { + switch ((int) a[strlen(hts_tbdev[i])]) { case '\0': case '/': - case '.': - { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; tempo[0]='\0'; - strncatbuff(tempo,save,(int) (a - save) + strlen(hts_tbdev[i])); - strcatbuff(tempo,"_"); - strcatbuff(tempo,a+strlen(hts_tbdev[i])); - strcpybuff(save,tempo); - } + case '.': + { + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + + tempo[0] = '\0'; + strncatbuff(tempo, save, (int) (a - save) + strlen(hts_tbdev[i])); + strcatbuff(tempo, "_"); + strcatbuff(tempo, a + strlen(hts_tbdev[i])); + strcpybuff(save, tempo); + } break; } - a+=strlen(hts_tbdev[i]); + a += strlen(hts_tbdev[i]); } i++; } @@ -1253,16 +1318,17 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, /* Strip ending . or ' ' forbidden on windoz */ { int len; - char* a=save; - while((a=strstr(a,"./"))) { + char *a = save; + + while((a = strstr(a, "./"))) { *a = '_'; } - a=save; - while((a=strstr(a," /"))) { + a = save; + while((a = strstr(a, " /"))) { *a = '_'; } len = (int) strlen(save); - if (len > 0 && ( save[len - 1] == '.' || save[len - 1] == ' ') ) { + if (len > 0 && (save[len - 1] == '.' || save[len - 1] == ' ')) { save[len - 1] = '_'; } } @@ -1270,11 +1336,11 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, // conversion 8-3 .. y compris pour les répertoires if (opt->savename_83) { - char BIGSTK n83[HTS_URLMAXSIZE*2]; - long_to_83(opt->savename_83,n83,save); - strcpybuff(save,n83); - } + char BIGSTK n83[HTS_URLMAXSIZE * 2]; + long_to_83(opt->savename_83, n83, save); + strcpybuff(save, n83); + } // enforce stricter ISO9660 compliance (bug reported by Steffo Carlsson) // Level 1 File names are restricted to 8 characters with a 3 character extension, // upper case letters, numbers and underscore; maximum depth of directories is 8. @@ -1283,20 +1349,23 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, // A-Z,0-9,_ if (opt->savename_83 > 0) { char *a, *last; - for(last = save + strlen(save) - 1 ; last != save && *last != '/' && *last != '\\' && *last != '.' ; last--); + + for(last = save + strlen(save) - 1; + last != save && *last != '/' && *last != '\\' && *last != '.'; last--) ; if (*last != '.') { last = NULL; } - for(a = save ; *a != '\0' ; a++) { + for(a = save; *a != '\0'; a++) { if (*a >= 'a' && *a <= 'z') { *a -= 'a' - 'A'; - } - else if (*a == '.') { + } else if (*a == '.') { if (a != last) { *a = '_'; } - } - else if ( ! ( (*a >= 'A' && *a <= 'Z') || (*a >= '0' && *a <= '9') || *a == '_' || *a == '/' || *a == '\\') ) { + } else + if (! + ((*a >= 'A' && *a <= 'Z') || (*a >= '0' && *a <= '9') || *a == '_' + || *a == '/' || *a == '\\')) { *a = '_'; } } @@ -1308,53 +1377,64 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, /* convert name to UTF-8 ? */ if (charset != NULL && charset[0] != '\0') { char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset); + if (s != NULL) { - hts_log_print(opt, LOG_DEBUG, "engine: save-name: charset conversion from '%s' to '%s'", save, s); + hts_log_print(opt, LOG_DEBUG, + "engine: save-name: charset conversion from '%s' to '%s'", + save, s); strcpybuff(save, s); free(s); } } - /* callback */ - RUN_CALLBACK5(opt, savename, adr_complete,fil_complete,referer_adr,referer_fil,save); - - hts_log_print(opt, LOG_DEBUG, "engine: save-name: local name: %s%s -> %s",adr,fil,save); - - /* Ensure that the MANDATORY "temporary" extension is set */ - if (ext_chg_delayed) { - char *ptr; - char *lastDot = NULL; - for(ptr = save ; *ptr != 0 ; ptr++) { - if (*ptr == '.') { - lastDot = ptr; - } else if (*ptr == '/' || *ptr == '\\') { - lastDot = NULL; - } - } - if (lastDot == NULL) { - strcatbuff(save, "." DELAYED_EXT); - } else if (!IS_DELAYED_EXT(save)) { - strcatbuff(lastDot, "." DELAYED_EXT); - } - } + /* callback */ + RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr, + referer_fil, save); + + hts_log_print(opt, LOG_DEBUG, "engine: save-name: local name: %s%s -> %s", + adr, fil, save); + /* Ensure that the MANDATORY "temporary" extension is set */ + if (ext_chg_delayed) { + char *ptr; + char *lastDot = NULL; + + for(ptr = save; *ptr != 0; ptr++) { + if (*ptr == '.') { + lastDot = ptr; + } else if (*ptr == '/' || *ptr == '\\') { + lastDot = NULL; + } + } + if (lastDot == NULL) { + strcatbuff(save, "." DELAYED_EXT); + } else if (!IS_DELAYED_EXT(save)) { + strcatbuff(lastDot, "." DELAYED_EXT); + } + } // enforce 256-character path limit before inserting destination path #define HTS_MAX_PATH_LEN 250 #define MIN_LAST_SEG_RESERVE 30 - if (hts_stringLengthUTF8(save) + hts_stringLengthUTF8(StringBuff(opt->path_html_utf8)) >= HTS_MAX_PATH_LEN) { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - const size_t parentLen = hts_stringLengthUTF8(StringBuff(opt->path_html_utf8)); + if (hts_stringLengthUTF8(save) + + hts_stringLengthUTF8(StringBuff(opt->path_html_utf8)) >= + HTS_MAX_PATH_LEN) { + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + const size_t parentLen = + hts_stringLengthUTF8(StringBuff(opt->path_html_utf8)); // parent path length is not insane (otherwise, ignore and pick 200 as suffix length) - const size_t maxLen = parentLen < HTS_MAX_PATH_LEN/2 ? HTS_MAX_PATH_LEN - parentLen : HTS_MAX_PATH_LEN; + const size_t maxLen = + parentLen < + HTS_MAX_PATH_LEN / 2 ? HTS_MAX_PATH_LEN - parentLen : HTS_MAX_PATH_LEN; size_t i, j, lastSeg, sofar; + // pick up last segment - for(i = 0, lastSeg = 0 ; save[i] != '\0' ; i++) { + for(i = 0, lastSeg = 0; save[i] != '\0'; i++) { if (save[i] == '/') { lastSeg = i + 1; } } // add as much pathes as we can - for(i = 0, sofar = 0 ; i < lastSeg && i + MIN_LAST_SEG_RESERVE < maxLen ; i++) { + for(i = 0, sofar = 0; i < lastSeg && i + MIN_LAST_SEG_RESERVE < maxLen; i++) { tempo[i] = save[i]; if (save[i] == '/') { // validate segment so far @@ -1363,9 +1443,10 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, } // last segment #define MAX_UTF8_SEQ_CHARS 4 - for(j = 0 ; save[j + i] != '\0' ; j++) { + for(j = 0; save[j + i] != '\0'; j++) { // Stop here before next sequence - if (sofar + j + MAX_UTF8_SEQ_CHARS >= maxLen && HTS_IS_LEADING_UTF8(save[i + j])) { + if (sofar + j + MAX_UTF8_SEQ_CHARS >= maxLen + && HTS_IS_LEADING_UTF8(save[i + j])) { break; } // Stop is overflowing @@ -1377,7 +1458,8 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, // terminating \0 save[sofar + j] = '\0'; // log in debug - hts_log_print(opt, LOG_DEBUG, "Too long filename shortened: %s%s => %s", adr_complete, fil_complete, save); + hts_log_print(opt, LOG_DEBUG, "Too long filename shortened: %s%s => %s", + adr_complete, fil_complete, save); } #undef MAX_UTF8_SEQ_CHARS #undef MIN_LAST_SEG_RESERVE @@ -1385,206 +1467,218 @@ int url_savename2(char* adr_complete, char* fil_complete, char* save, // chemin primaire éventuel A METTRE AVANT if (strnotempty(StringBuff(opt->path_html_utf8))) { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - strcpybuff(tempo,StringBuff(opt->path_html_utf8)); - strcatbuff(tempo,save); - strcpybuff(save,tempo); - } + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + strcpybuff(tempo, StringBuff(opt->path_html_utf8)); + strcatbuff(tempo, save); + strcpybuff(save, tempo); + } // vérifier que le nom n'est pas déja pris... - if (liens!=NULL) { + if (liens != NULL) { int nom_ok; + do { int i; + // - nom_ok=1; // à priori bon + nom_ok = 1; // à priori bon // on part de la fin pour optimiser, plus les opti de taille pour aller encore plus vite.. #if DEBUG_SAVENAME -printf("\nStart search\n"); + printf("\nStart search\n"); #endif - i=hash_read(hash,save,"",0,0); // lecture type 0 (sav) - if (i>=0) - { - int sameAdr = ( strfield2(liens[i]->adr, normadr) != 0 ); - int sameFil; - // NO - URL hack is only for stripping // and www. - //if (opt->urlhack != 0) - // sameFil = ( strfield2(liens[i]->fil, normfil) != 0); - //else - sameFil = ( strcmp(liens[i]->fil, normfil) == 0); - if (sameAdr && sameFil) - { // ok c'est le même lien, adresse déja définie - /* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */ - if (strcmp(liens[i]->sav, save) != 0) { - strcpybuff(save, liens[i]->sav); - } - i=0; + i = hash_read(hash, save, "", 0, 0); // lecture type 0 (sav) + if (i >= 0) { + int sameAdr = (strfield2(liens[i]->adr, normadr) != 0); + int sameFil; + + // NO - URL hack is only for stripping // and www. + //if (opt->urlhack != 0) + // sameFil = ( strfield2(liens[i]->fil, normfil) != 0); + //else + sameFil = (strcmp(liens[i]->fil, normfil) == 0); + if (sameAdr && sameFil) { // ok c'est le même lien, adresse déja définie + /* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */ + if (strcmp(liens[i]->sav, save) != 0) { + strcpybuff(save, liens[i]->sav); + } + i = 0; #if DEBUG_SAVENAME -printf("\nOK ALREADY DEFINED\n",13,i); + printf("\nOK ALREADY DEFINED\n", 13, i); #endif - } else { // utilisé par un AUTRE, changer de nom - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - char* a=save+strlen(save)-1; - char* b; - int n=2; - char collisionSeparator = ( (opt->savename_83 != 2) ? '-' : '_'); - tempo[0]='\0'; + } else { // utilisé par un AUTRE, changer de nom + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + char *a = save + strlen(save) - 1; + char *b; + int n = 2; + char collisionSeparator = ((opt->savename_83 != 2) ? '-' : '_'); + + tempo[0] = '\0'; #if DEBUG_SAVENAME -printf("\nWRONG CASE UNMATCH : \n%s\n%s, REDEFINE\n",liens[i]->fil,fil_complete); + printf("\nWRONG CASE UNMATCH : \n%s\n%s, REDEFINE\n", liens[i]->fil, + fil_complete); #endif - nom_ok=0; - i=0; - - while(( a > save) && (*a!='.') && (*a!='\\') && (*a!='/')) a--; - if (*a=='.') - strncatbuff(tempo,save,(int) (a - save)); - else - strcatbuff(tempo,save); - - // tester la présence d'un -xx (ex: index-2.html -> index-3.html) - b=tempo+strlen(tempo)-1; - while (isdigit((unsigned char)*b)) b--; - if (*b == collisionSeparator) { - sscanf(b+1,"%d",&n); - *b='\0'; // couper - n++; // plus un - } - - // en plus il faut gérer le 8-3 .. pas facile le client - if (opt->savename_83) { - int max; - char* a=tempo+strlen(tempo)-1; - while(( a > tempo) && (*a!='/')) a--; - if (*a=='/') a++; - max=max_char-1-nombre_digit(n); - if ((int) strlen(a)>max) - *(a+max)='\0'; // couper sinon il n'y aura pas la place! - } - - // ajouter -xx (ex: index.html -> index-2.html) - sprintf(tempo+strlen(tempo),"%c%d", collisionSeparator, n); - - // ajouter extension - if (*a=='.') - strcatbuff(tempo,a); - - strcpybuff(save,tempo); - - //printf("switched: %s\n",save); - - } // if + nom_ok = 0; + i = 0; + + while((a > save) && (*a != '.') && (*a != '\\') && (*a != '/')) + a--; + if (*a == '.') + strncatbuff(tempo, save, (int) (a - save)); + else + strcatbuff(tempo, save); + + // tester la présence d'un -xx (ex: index-2.html -> index-3.html) + b = tempo + strlen(tempo) - 1; + while(isdigit((unsigned char) *b)) + b--; + if (*b == collisionSeparator) { + sscanf(b + 1, "%d", &n); + *b = '\0'; // couper + n++; // plus un } + // en plus il faut gérer le 8-3 .. pas facile le client + if (opt->savename_83) { + int max; + char *a = tempo + strlen(tempo) - 1; + + while((a > tempo) && (*a != '/')) + a--; + if (*a == '/') + a++; + max = max_char - 1 - nombre_digit(n); + if ((int) strlen(a) > max) + *(a + max) = '\0'; // couper sinon il n'y aura pas la place! + } + // ajouter -xx (ex: index.html -> index-2.html) + sprintf(tempo + strlen(tempo), "%c%d", collisionSeparator, n); + + // ajouter extension + if (*a == '.') + strcatbuff(tempo, a); + + strcpybuff(save, tempo); + + //printf("switched: %s\n",save); + + } // if + } #if DEBUG_SAVENAME -printf("\nEnd search, %s\n",fil_complete); + printf("\nEnd search, %s\n", fil_complete); #endif } while(!nom_ok); - + } - //printf("'%s' %s %s\n",save,adr,fil); - + return 0; } /* nom avec md5 urilisé partout */ -void standard_name(char* b,char* dot_pos,char* nom_pos,char* fil_complete,int short_ver) { - char md5[32 + 2]; +void standard_name(char *b, char *dot_pos, char *nom_pos, char *fil_complete, + int short_ver) { + char md5[32 + 2]; - b[0]='\0'; + b[0] = '\0'; /* Nom */ if (dot_pos) { - if (!short_ver) // Noms longs - strncatbuff(b,nom_pos,(int) (dot_pos - nom_pos)); + if (!short_ver) // Noms longs + strncatbuff(b, nom_pos, (int) (dot_pos - nom_pos)); else - strncatbuff(b,nom_pos,min((int) (dot_pos - nom_pos),8)); + strncatbuff(b, nom_pos, min((int) (dot_pos - nom_pos), 8)); } else { - if (!short_ver) // Noms longs - strcatbuff(b,nom_pos); + if (!short_ver) // Noms longs + strcatbuff(b, nom_pos); else - strncatbuff(b,nom_pos,8); + strncatbuff(b, nom_pos, 8); } /* MD5 - 16 bits */ - strncatbuff(b,url_md5(md5,fil_complete),4); + strncatbuff(b, url_md5(md5, fil_complete), 4); /* Ext */ if (dot_pos) { - strcatbuff(b,"."); - if (!short_ver) // Noms longs - strcatbuff(b,dot_pos+1); + strcatbuff(b, "."); + if (!short_ver) // Noms longs + strcatbuff(b, dot_pos + 1); else - strncatbuff(b,dot_pos+1,3); + strncatbuff(b, dot_pos + 1, 3); } // Allow extensionless #ifdef DO_NOT_ALLOW_EXTENSIONLESS else { - if (!short_ver) // Noms longs - strcatbuff(b,DEFAULT_EXT); + if (!short_ver) // Noms longs + strcatbuff(b, DEFAULT_EXT); else - strcatbuff(b,DEFAULT_EXT_SHORT); + strcatbuff(b, DEFAULT_EXT_SHORT); } #endif } - /* Petit md5 */ -char* url_md5(char* digest, char* fil_complete) { - char* a; - digest[0]='\0'; - a=strchr(fil_complete,'?'); +char *url_md5(char *digest, char *fil_complete) { + char *a; + + digest[0] = '\0'; + a = strchr(fil_complete, '?'); if (a) { if (strlen(a)) { - char BIGSTK buff[HTS_URLMAXSIZE*2]; + char BIGSTK buff[HTS_URLMAXSIZE * 2]; + a++; - digest[0]=buff[0]='\0'; - strcatbuff(buff,a); /* query string MD5 */ - domd5mem(buff,strlen(buff),digest,1); + digest[0] = buff[0] = '\0'; + strcatbuff(buff, a); /* query string MD5 */ + domd5mem(buff, strlen(buff), digest, 1); } } return digest; } // interne à url_savename: ajoute une chaîne à une autre avec \ -> / -void url_savename_addstr(char* d,char* s) { +void url_savename_addstr(char *d, char *s) { int i = (int) strlen(d); + while(*s) { - if (*s=='\\') // remplacer \ par des / - d[i++]='/'; + if (*s == '\\') // remplacer \ par des / + d[i++] = '/'; else - d[i++]=*s; + d[i++] = *s; s++; } - d[i]='\0'; + d[i] = '\0'; } /* "filename" should be at least 64 bytes. */ void url_savename_refname(const char *adr, const char *fil, char *filename) { unsigned char bindigest[16]; MD5_CTX ctx; + MD5Init(&ctx, 0); - MD5Update(&ctx, (const unsigned char*) adr, strlen(adr)); - MD5Update(&ctx, (const unsigned char*) ",", 1); - MD5Update(&ctx, (const unsigned char*) fil, strlen(fil)); + MD5Update(&ctx, (const unsigned char *) adr, strlen(adr)); + MD5Update(&ctx, (const unsigned char *) ",", 1); + MD5Update(&ctx, (const unsigned char *) fil, strlen(fil)); MD5Final(bindigest, &ctx); - sprintf(filename, CACHE_REFNAME "/" - "%02x%02x%02x%02x%02x%02x%02x%02x" - "%02x%02x%02x%02x%02x%02x%02x%02x" - ".ref", - bindigest[0], bindigest[1], bindigest[2], bindigest[3], - bindigest[4], bindigest[5], bindigest[6], bindigest[7], - bindigest[8], bindigest[9], bindigest[10], bindigest[11], - bindigest[12], bindigest[13], bindigest[14], bindigest[15]); + sprintf(filename, + CACHE_REFNAME "/" "%02x%02x%02x%02x%02x%02x%02x%02x" + "%02x%02x%02x%02x%02x%02x%02x%02x" ".ref", bindigest[0], bindigest[1], + bindigest[2], bindigest[3], bindigest[4], bindigest[5], bindigest[6], + bindigest[7], bindigest[8], bindigest[9], bindigest[10], + bindigest[11], bindigest[12], bindigest[13], bindigest[14], + bindigest[15]); } /* note: return a local filename */ -char *url_savename_refname_fullpath(httrackp* opt, const char *adr, const char *fil) { +char *url_savename_refname_fullpath(httrackp * opt, const char *adr, + const char *fil) { char digest_filename[64]; + url_savename_refname(adr, fil, digest_filename); return fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), digest_filename); } /* remove refname if any */ -void url_savename_refname_remove(httrackp* opt, const char *adr, const char *fil) { +void url_savename_refname_remove(httrackp * opt, const char *adr, + const char *fil) { char *filename = url_savename_refname_fullpath(opt, adr, fil); + (void) UNLINK(filename); } diff --git a/src/htsname.h b/src/htsname.h index 7cb7dda..433c75e 100644 --- a/src/htsname.h +++ b/src/htsname.h @@ -17,43 +17,40 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* savename routine (compute output filename) */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - - #ifndef HTSNAME_DEFH -#define HTSNAME_DEFH +#define HTSNAME_DEFH #include "htsglobal.h" #define DELAYED_EXT "delayed" #define IS_DELAYED_EXT(a) ( ((a) != NULL) && ((a)[0] != 0) && strendwith_(a, "." DELAYED_EXT) ) -HTS_STATIC int strendwith_(const char* a, const char* b) { +HTS_STATIC int strendwith_(const char *a, const char *b) { int i, j; - for(i = 0 ; a[i] != 0 ; i++); - for(j = 0 ; b[j] != 0 ; j++); + + for(i = 0; a[i] != 0; i++) ; + for(j = 0; b[j] != 0; j++) ; while(i >= 0 && j >= 0 && a[i] == b[j]) { i--; j--; } return (j == -1); } + #define CACHE_REFNAME "hts-cache/ref" /* Library internal definictions */ @@ -86,33 +83,27 @@ typedef struct lien_back lien_back; #endif // note: 'headers' can either be null, or incomplete (only r member filled) -int url_savename(char* adr_complete, char* fil_complete, char* save, - char* former_adr, char* former_fil, - char* referer_adr, char* referer_fil, - httrackp* opt, - lien_url** liens, int lien_tot, - struct_back* sback, - cache_back* cache, - hash_struct* hash, - int ptr, int numero_passe, - const lien_back* headers); -int url_savename2(char* adr_complete, char* fil_complete, char* save, - char* former_adr, char* former_fil, - char* referer_adr, char* referer_fil, - httrackp* opt, - lien_url** liens, int lien_tot, - struct_back* sback, - cache_back* cache, - hash_struct* hash, - int ptr, int numero_passe, - const lien_back* headers, - const char *charset); -void standard_name(char* b,char* dot_pos,char* nom_pos,char* fil_complete,int short_ver); -void url_savename_addstr(char* d,char* s); -char* url_md5(char* digest_buffer, char* fil_complete); +int url_savename(char *adr_complete, char *fil_complete, char *save, + char *former_adr, char *former_fil, char *referer_adr, + char *referer_fil, httrackp * opt, lien_url ** liens, + int lien_tot, struct_back * sback, cache_back * cache, + hash_struct * hash, int ptr, int numero_passe, + const lien_back * headers); +int url_savename2(char *adr_complete, char *fil_complete, char *save, + char *former_adr, char *former_fil, char *referer_adr, + char *referer_fil, httrackp * opt, lien_url ** liens, + int lien_tot, struct_back * sback, cache_back * cache, + hash_struct * hash, int ptr, int numero_passe, + const lien_back * headers, const char *charset); +void standard_name(char *b, char *dot_pos, char *nom_pos, char *fil_complete, + int short_ver); +void url_savename_addstr(char *d, char *s); +char *url_md5(char *digest_buffer, char *fil_complete); void url_savename_refname(const char *adr, const char *fil, char *filename); -char *url_savename_refname_fullpath(httrackp* opt, const char *adr, const char *fil); -void url_savename_refname_remove(httrackp* opt, const char *adr, const char *fil); +char *url_savename_refname_fullpath(httrackp * opt, const char *adr, + const char *fil); +void url_savename_refname_remove(httrackp * opt, const char *adr, + const char *fil); #endif #endif diff --git a/src/htsnet.h b/src/htsnet.h index 2243611..69ff9b1 100644 --- a/src/htsnet.h +++ b/src/htsnet.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Net definitions */ /* Used in .c files that needs connect() functions and so */ @@ -47,32 +44,32 @@ Please visit our Website: http://www.httrack.com #ifdef _WIN32 // pour read #ifndef _WIN32_WCE - #include <io.h> +#include <io.h> #endif // pour FindFirstFile - #include <winbase.h> +#include <winbase.h> #else //typedef int T_SOC; - #define INVALID_SOCKET -1 - #include <netdb.h> - #include <sys/types.h> - #include <sys/socket.h> - #include <netinet/in.h> - #include <sys/time.h> +#define INVALID_SOCKET -1 +#include <netdb.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <sys/time.h> /* Force for sun env. */ - #ifndef BSD_COMP - #define BSD_COMP - #endif - #include <sys/ioctl.h> +#ifndef BSD_COMP +#define BSD_COMP +#endif +#include <sys/ioctl.h> /* gethostname & co */ #ifndef _WIN32 #include <unistd.h> #endif /* inet_addr */ - #include <arpa/inet.h> +#include <arpa/inet.h> // pas la peine normalement.. #ifndef HTS_DO_NOT_REDEFINE_in_addr_t - typedef unsigned long in_addr_t; +typedef unsigned long in_addr_t; #endif #endif @@ -83,6 +80,7 @@ Please visit our Website: http://www.httrack.com /* Ipv4 structures */ typedef struct in_addr INaddr; + /* This should handle all cases */ #ifndef HTS_DEF_FWSTRUCT_SOCaddr #define HTS_DEF_FWSTRUCT_SOCaddr @@ -117,7 +115,6 @@ struct SOCaddr { server_len=sizeof(struct sockaddr_in); \ } while(0) - /* Copy sockaddr to another one */ #define SOCaddr_copyaddr(server, server_len, hpaddr, hpsize) do { \ if (hpsize == sizeof(struct sockaddr_in)) { \ @@ -157,6 +154,7 @@ strcpy(namebuf, dot); \ /* Ipv4 structures */ typedef struct in6_addr INaddr; + /* This should handle all cases */ #ifndef HTS_DEF_FWSTRUCT_SOCaddr #define HTS_DEF_FWSTRUCT_SOCaddr @@ -257,7 +255,7 @@ typedef struct t_fullhostent t_fullhostent; #endif struct t_fullhostent { t_hostent hp; - char* list[2]; + char *list[2]; char addr[HTS_MAXADDRLEN]; /* various struct sockaddr structures */ unsigned int addr_maxlen; }; @@ -271,7 +269,4 @@ memset((h), 0, sizeof(t_fullhostent)); \ (h)->addr_maxlen = HTS_MAXADDRLEN; \ } while(0) - #endif - - diff --git a/src/htsopt.h b/src/htsopt.h index eb6e84d..e76d8f2 100644 --- a/src/htsopt.h +++ b/src/htsopt.h @@ -17,25 +17,21 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: HTTrack parameters block */ /* Called by httrack.h and some other files */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - #ifndef HTTRACK_DEFOPT #define HTTRACK_DEFOPT @@ -72,7 +68,7 @@ typedef struct String String; #ifndef HTS_DEF_STRUCT_String #define HTS_DEF_STRUCT_String struct String { - char* buffer_; + char *buffer_; size_t length_; size_t capacity_; }; @@ -91,8 +87,8 @@ struct t_proxy { int active; String name; int port; - String bindhost; // bind this host -}; + String bindhost; // bind this host +}; /* Structure utile pour copier en bloc les paramètres */ #ifndef HTS_DEF_FWSTRUCT_htsfilters @@ -100,18 +96,18 @@ struct t_proxy { typedef struct htsfilters htsfilters; #endif struct htsfilters { - char*** filters; - int* filptr; + char ***filters; + int *filptr; //int* filter_max; }; /* User callbacks chain */ -typedef int (*htscallbacksfncptr)(void); +typedef int (*htscallbacksfncptr) (void); typedef struct htscallbacks htscallbacks; struct htscallbacks { - void* moduleHandle; + void *moduleHandle; htscallbacksfncptr exitFnc; - htscallbacks * next; + htscallbacks *next; }; /* filenote() internal file structure */ @@ -120,8 +116,8 @@ struct htscallbacks { typedef struct filenote_strc filenote_strc; #endif struct filenote_strc { - FILE* lst; - char path[STRING_SIZE*2]; + FILE *lst; + char path[STRING_SIZE * 2]; }; /* concat() functions */ @@ -131,7 +127,7 @@ typedef struct concat_strc concat_strc; #endif struct concat_strc { int index; - char buff[16][STRING_SIZE*2*2]; + char buff[16][STRING_SIZE * 2 * 2]; }; /* int2 functions */ @@ -140,10 +136,10 @@ struct concat_strc { typedef struct strc_int2bytes2 strc_int2bytes2; #endif struct strc_int2bytes2 { - char catbuff[CATBUFF_SIZE]; + char catbuff[CATBUFF_SIZE]; char buff1[256]; char buff2[32]; - char* buffadr[2]; + char *buffadr[2]; }; /* cmd callback */ @@ -170,13 +166,13 @@ struct fspc_strc { #ifndef HTS_DEF_DEFSTRUCT_hts_log_type #define HTS_DEF_DEFSTRUCT_hts_log_type typedef enum hts_log_type { - LOG_DEBUG, - LOG_INFO, - LOG_NOTICE, - LOG_WARNING, - LOG_ERROR, - LOG_PANIC, - LOG_ERRNO = 1 << 8 + LOG_DEBUG, + LOG_INFO, + LOG_NOTICE, + LOG_WARNING, + LOG_ERROR, + LOG_PANIC, + LOG_ERRNO = 1 << 8 } hts_log_type; #endif @@ -208,7 +204,7 @@ typedef struct struct_inthash struct_inthash, *inthash; typedef struct htsoptstate htsoptstate; #endif struct htsoptstate { - htsmutex lock; /* 3.41 */ + htsmutex lock; /* 3.41 */ /* */ int stop; int exit_xh; @@ -216,32 +212,32 @@ struct htsoptstate { /* */ int mimehtml_created; String mimemid; - FILE* mimefp; + FILE *mimefp; int delayedId; /* */ - filenote_strc strc; - /* Functions context (avoir thread variables!) */ + filenote_strc strc; + /* Functions context (avoir thread variables!) */ htscallbacks callbacks; - concat_strc concat; - usercommand_strc usercmd; - fspc_strc fspc; - char *userhttptype; - int verif_backblue_done; - int verif_external_status; - t_dnscache *dns_cache; - /* HTML parsing state */ - char _hts_errmsg[HTS_CDLMAXSIZE + 256]; - int _hts_in_html_parsing; - int _hts_in_html_done; - int _hts_in_html_poll; - int _hts_setpause; - int _hts_in_mirror; - char** _hts_addurl; + concat_strc concat; + usercommand_strc usercmd; + fspc_strc fspc; + char *userhttptype; + int verif_backblue_done; + int verif_external_status; + t_dnscache *dns_cache; + /* HTML parsing state */ + char _hts_errmsg[HTS_CDLMAXSIZE + 256]; + int _hts_in_html_parsing; + int _hts_in_html_done; + int _hts_in_html_poll; + int _hts_setpause; + int _hts_in_mirror; + char **_hts_addurl; int _hts_cancel; - htsoptstatecancel *cancel; /* 3.41 */ - char HTbuff[2048]; + htsoptstatecancel *cancel; /* 3.41 */ + char HTbuff[2048]; unsigned int debug_state; - unsigned int tmpnameid; /* 3.41 */ + unsigned int tmpnameid; /* 3.41 */ }; /* Library handles */ @@ -264,11 +260,11 @@ struct htslibhandles { /* Javascript parser flags */ typedef enum htsparsejava_flags { - HTSPARSE_NONE = 0, // don't parse - HTSPARSE_DEFAULT = 1, // parse default (all) - HTSPARSE_NO_CLASS = 2, // don't parse .java - HTSPARSE_NO_JAVASCRIPT = 4, // don't parse .js - HTSPARSE_NO_AGGRESSIVE = 8 // don't aggressively parse .js or .java + HTSPARSE_NONE = 0, // don't parse + HTSPARSE_DEFAULT = 1, // parse default (all) + HTSPARSE_NO_CLASS = 2, // don't parse .java + HTSPARSE_NO_JAVASCRIPT = 4, // don't parse .js + HTSPARSE_NO_AGGRESSIVE = 8 // don't aggressively parse .js or .java } htsparsejava_flags; // paramètres httrack (options) @@ -277,113 +273,113 @@ typedef enum htsparsejava_flags { typedef struct httrackp httrackp; #endif struct httrackp { - size_t size_httrackp; // size of this structure + size_t size_httrackp; // size of this structure /* */ - int wizard; // wizard aucun/grand/petit - int flush; // fflush sur les fichiers log - int travel; // type de déplacements (same domain etc) - int seeker; // up & down - int depth; // nombre de niveaux de récursion - int extdepth; // nombre de niveaux de récursion à l'éxtérieur - int urlmode; // liens relatifs etc - int debug; // mode débug log - int getmode; // sauver html, images.. - FILE* log; // fichier log - FILE* errlog; // et erreur - LLint maxsite; // taille max site - LLint maxfile_nonhtml; // taille max non html - LLint maxfile_html; // taille max html - int maxsoc; // nbre sockets - LLint fragment; // fragmentation d'un site - int nearlink; // prendre les images/data proche d'une page mais à l'extérieur - int makeindex; // faire un index - int kindex; // et un index 'keyword' - int delete_old; // effacer anciens fichiers - int timeout; // nombre de secondes de timeout - int rateout; // nombre d'octets minium pour le transfert - int maxtime; // temps max en secondes - int maxrate; // taux de transfert max - int mms_maxtime; // max duration of a mms file - float maxconn; // nombre max de connexions/s - int waittime; // démarrage programmé - int cache; // génération d'un cache + int wizard; // wizard aucun/grand/petit + int flush; // fflush sur les fichiers log + int travel; // type de déplacements (same domain etc) + int seeker; // up & down + int depth; // nombre de niveaux de récursion + int extdepth; // nombre de niveaux de récursion à l'éxtérieur + int urlmode; // liens relatifs etc + int debug; // mode débug log + int getmode; // sauver html, images.. + FILE *log; // fichier log + FILE *errlog; // et erreur + LLint maxsite; // taille max site + LLint maxfile_nonhtml; // taille max non html + LLint maxfile_html; // taille max html + int maxsoc; // nbre sockets + LLint fragment; // fragmentation d'un site + int nearlink; // prendre les images/data proche d'une page mais à l'extérieur + int makeindex; // faire un index + int kindex; // et un index 'keyword' + int delete_old; // effacer anciens fichiers + int timeout; // nombre de secondes de timeout + int rateout; // nombre d'octets minium pour le transfert + int maxtime; // temps max en secondes + int maxrate; // taux de transfert max + int mms_maxtime; // max duration of a mms file + float maxconn; // nombre max de connexions/s + int waittime; // démarrage programmé + int cache; // génération d'un cache //int aff_progress; // barre de progression - int shell; // gestion d'un shell par pipe stdin/stdout - t_proxy proxy; // configuration du proxy - int savename_83; // conversion 8-3 pour les noms de fichiers - int savename_type; // type de noms: structure originale/html-images en un seul niveau - String savename_userdef; // structure userdef (ex: %h%p/%n%q.%t) - int savename_delayed; // delayed type check - int delayed_cached; // delayed type check can be cached to speedup updates - int mimehtml; // MIME-html - int user_agent_send; // user agent (ex: httrack/1.0 [sun]) - String user_agent; // - String referer; // referer - String from; // from - String path_log; // chemin pour cache et log - String path_html; // chemin pour miroir - String path_html_utf8; // chemin pour miroir, UTF-8 - String path_bin; // chemin pour templates - int retry; // nombre d'essais supplémentaires en cas d'échec - int makestat; // mettre à jour un fichier log de statistiques de transfert - int maketrack; // mettre à jour un fichier log de statistiques d'opérations - int parsejava; // parsing des classes java pour récupérer les class, gif & cie ; see htsparsejava_flags - int hostcontrol; // abandon d'un host trop lent etc. - int errpage; // générer une page d'erreur en cas de 404 etc. - int check_type; // si type inconnu (cgi,asp,/) alors tester lien (et gérer moved éventuellement) - int all_in_cache; // tout mettre en cache! - int robots; // traitement des robots - int external; // pages externes->pages d'erreur - int passprivacy; // pas de mot de pass dans les liens externes? - int includequery; // include la query-string - int mirror_first_page; // miroir des liens - String sys_com; // commande système - int sys_com_exec; // executer commande - int accept_cookie; // gestion des cookies - t_cookie* cookie; - int http10; // forcer http 1.0 - int nokeepalive; // pas de keep-alive - int nocompression; // pas de compression - int sizehack; // forcer réponse "mis à jour" si taille identique - int urlhack; // force "url normalization" to avoid loops - int tolerant; // accepter content-length incorrect - int parseall; // essayer de tout parser (tags inconnus contenant des liens, par exemple) - int parsedebug; // débugger parser (debug!) - int norecatch; // ne pas reprendre les fichiers effacés localement par l'utilisateur - int verbosedisplay; // animation textuelle - String footer; // ligne d'infos - int maxcache; // maximum en mémoire au niveau du cache (backing) + int shell; // gestion d'un shell par pipe stdin/stdout + t_proxy proxy; // configuration du proxy + int savename_83; // conversion 8-3 pour les noms de fichiers + int savename_type; // type de noms: structure originale/html-images en un seul niveau + String savename_userdef; // structure userdef (ex: %h%p/%n%q.%t) + int savename_delayed; // delayed type check + int delayed_cached; // delayed type check can be cached to speedup updates + int mimehtml; // MIME-html + int user_agent_send; // user agent (ex: httrack/1.0 [sun]) + String user_agent; // + String referer; // referer + String from; // from + String path_log; // chemin pour cache et log + String path_html; // chemin pour miroir + String path_html_utf8; // chemin pour miroir, UTF-8 + String path_bin; // chemin pour templates + int retry; // nombre d'essais supplémentaires en cas d'échec + int makestat; // mettre à jour un fichier log de statistiques de transfert + int maketrack; // mettre à jour un fichier log de statistiques d'opérations + int parsejava; // parsing des classes java pour récupérer les class, gif & cie ; see htsparsejava_flags + int hostcontrol; // abandon d'un host trop lent etc. + int errpage; // générer une page d'erreur en cas de 404 etc. + int check_type; // si type inconnu (cgi,asp,/) alors tester lien (et gérer moved éventuellement) + int all_in_cache; // tout mettre en cache! + int robots; // traitement des robots + int external; // pages externes->pages d'erreur + int passprivacy; // pas de mot de pass dans les liens externes? + int includequery; // include la query-string + int mirror_first_page; // miroir des liens + String sys_com; // commande système + int sys_com_exec; // executer commande + int accept_cookie; // gestion des cookies + t_cookie *cookie; + int http10; // forcer http 1.0 + int nokeepalive; // pas de keep-alive + int nocompression; // pas de compression + int sizehack; // forcer réponse "mis à jour" si taille identique + int urlhack; // force "url normalization" to avoid loops + int tolerant; // accepter content-length incorrect + int parseall; // essayer de tout parser (tags inconnus contenant des liens, par exemple) + int parsedebug; // débugger parser (debug!) + int norecatch; // ne pas reprendre les fichiers effacés localement par l'utilisateur + int verbosedisplay; // animation textuelle + String footer; // ligne d'infos + int maxcache; // maximum en mémoire au niveau du cache (backing) //int maxcache_anticipate; // maximum de liens à anticiper (majorant) - int ftp_proxy; // proxy http pour ftp - String filelist; // fichier liste URL à inclure - String urllist; // fichier liste de filtres à inclure - htsfilters filters; // contient les pointeurs pour les filtres - hash_struct* hash; // hash structure - robots_wizard* robotsptr; // robots ptr - String lang_iso; // en, fr .. - String mimedefs; // ext1=mimetype1\next2=mimetype2.. - String mod_blacklist; // (3.41) - int convert_utf8; // filenames UTF-8 conversion (3.46) + int ftp_proxy; // proxy http pour ftp + String filelist; // fichier liste URL à inclure + String urllist; // fichier liste de filtres à inclure + htsfilters filters; // contient les pointeurs pour les filtres + hash_struct *hash; // hash structure + robots_wizard *robotsptr; // robots ptr + String lang_iso; // en, fr .. + String mimedefs; // ext1=mimetype1\next2=mimetype2.. + String mod_blacklist; // (3.41) + int convert_utf8; // filenames UTF-8 conversion (3.46) // - int maxlink; // nombre max de liens - int maxfilter; // nombre max de filtres + int maxlink; // nombre max de liens + int maxfilter; // nombre max de filtres // - char* exec; // adresse du nom de l'éxecutable + char *exec; // adresse du nom de l'éxecutable // - int quiet; // poser des questions autres que wizard? - int keyboard; // vérifier stdin - int bypass_limits; // bypass built-in limits - int background_on_suspend; // background process on suspend signal + int quiet; // poser des questions autres que wizard? + int keyboard; // vérifier stdin + int bypass_limits; // bypass built-in limits + int background_on_suspend; // background process on suspend signal // - int is_update; // c'est une update (afficher "File updated...") - int dir_topindex; // reconstruire top index par la suite + int is_update; // c'est une update (afficher "File updated...") + int dir_topindex; // reconstruire top index par la suite // // callbacks - t_hts_htmlcheck_callbacks *callbacks_fun; + t_hts_htmlcheck_callbacks *callbacks_fun; // store library handles htslibhandles libHandles; // - htsoptstate state; // state + htsoptstate state; // state }; // stats for httrack @@ -392,39 +388,38 @@ struct httrackp { typedef struct hts_stat_struct hts_stat_struct; #endif struct hts_stat_struct { - LLint HTS_TOTAL_RECV; // flux entrant reçu - LLint stat_bytes; // octets écrits sur disque + LLint HTS_TOTAL_RECV; // flux entrant reçu + LLint stat_bytes; // octets écrits sur disque // int HTS_TOTAL_RECV_STATE; // status: 0 tout va bien 1: ralentir un peu 2: ralentir 3: beaucoup - TStamp stat_timestart; // départ + TStamp stat_timestart; // départ // - LLint total_packed; // flux entrant compressé reçu - LLint total_unpacked; // flux entrant compressé reçu - int total_packedfiles; // fichiers compressés + LLint total_packed; // flux entrant compressé reçu + LLint total_unpacked; // flux entrant compressé reçu + int total_packedfiles; // fichiers compressés // - TStamp istat_timestart[2]; // départ pour calcul instantanné - LLint istat_bytes[2]; // calcul pour instantanné - TStamp istat_reference01; // top départ donné par #0 à #1 - int istat_idlasttimer; // id du timer qui a récemment donné une stat + TStamp istat_timestart[2]; // départ pour calcul instantanné + LLint istat_bytes[2]; // calcul pour instantanné + TStamp istat_reference01; // top départ donné par #0 à #1 + int istat_idlasttimer; // id du timer qui a récemment donné une stat // - int stat_files; // nombre de fichiers écrits - int stat_updated_files; // nombre de fichiers mis à jour - int stat_background; // nombre de fichiers écrits en arrière plan + int stat_files; // nombre de fichiers écrits + int stat_updated_files; // nombre de fichiers mis à jour + int stat_background; // nombre de fichiers écrits en arrière plan // - int stat_nrequests; // nombre de requêtes sur socket - int stat_sockid; // nombre de sockets allouées au total - int stat_nsocket; // nombre de sockets - int stat_errors; // nombre d'erreurs - int stat_errors_front; // idem, mais au tout premier niveau - int stat_warnings; // '' warnings - int stat_infos; // '' infos - int nbk; // fichiers anticipés en arrière plan et terminés - LLint nb; // données transférées actuellement (estimation) + int stat_nrequests; // nombre de requêtes sur socket + int stat_sockid; // nombre de sockets allouées au total + int stat_nsocket; // nombre de sockets + int stat_errors; // nombre d'erreurs + int stat_errors_front; // idem, mais au tout premier niveau + int stat_warnings; // '' warnings + int stat_infos; // '' infos + int nbk; // fichiers anticipés en arrière plan et terminés + LLint nb; // données transférées actuellement (estimation) // LLint rate; // - TStamp last_connect; // last connect() call - TStamp last_request; // last request issued + TStamp last_connect; // last connect() call + TStamp last_request; // last request issued }; #endif - diff --git a/src/htsparse.h b/src/htsparse.h index ee797da..9c0f401 100644 --- a/src/htsparse.h +++ b/src/htsparse.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsparse.h parser */ /* html/javascript/css parser */ @@ -58,29 +55,29 @@ typedef struct htsmoduleStructExtended htsmoduleStructExtended; #endif struct htsmoduleStructExtended { /* Main object */ - htsblk* r_; + htsblk *r_; /* Error handling */ - int* error_; - int* exit_xh_; - int* store_errpage_; + int *error_; + int *exit_xh_; + int *store_errpage_; /* Structural */ - int* filptr_; - char*** filters_; - robots_wizard* robots_; - hash_struct* hash_; - int* lien_max_; + int *filptr_; + char ***filters_; + robots_wizard *robots_; + hash_struct *hash_; + int *lien_max_; /* Base & codebase */ - char* base; - char* codebase; + char *base; + char *codebase; /* Index */ - int* makeindex_done_; - FILE** makeindex_fp_; - int* makeindex_links_; - char* makeindex_firstlink_; + int *makeindex_done_; + FILE **makeindex_fp_; + int *makeindex_links_; + char *makeindex_firstlink_; /* Html templates */ char *template_header_; @@ -88,21 +85,20 @@ struct htsmoduleStructExtended { char *template_footer_; /* Specific to downloads */ - LLint* stat_fragment_; + LLint *stat_fragment_; TStamp makestat_time; - FILE* makestat_fp; - LLint* makestat_total_; - int* makestat_lnk_; - FILE* maketrack_fp; + FILE *makestat_fp; + LLint *makestat_total_; + int *makestat_lnk_; + FILE *maketrack_fp; /* Function-dependant */ - char* loc_; - TStamp* last_info_shell_; - int* info_shell_; + char *loc_; + TStamp *last_info_shell_; + int *info_shell_; }; - /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE @@ -110,7 +106,7 @@ struct htsmoduleStructExtended { Main parser, attempt to scan links inside the html/css/js file Parameters: The public module structure, and the private module variables */ -int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre); +int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre); /* Check for 301,302.. errors ("moved") and handle them; re-isuue requests, make @@ -118,30 +114,31 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre); Parameters: The public module structure, and the private module variables Returns 0 upon success */ -int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre); +int hts_mirror_check_moved(htsmoduleStruct * str, + htsmoduleStructExtended * stre); /* Process user intercations: pause, add link, delete link.. */ -void hts_mirror_process_user_interaction(htsmoduleStruct* str, htsmoduleStructExtended* stre); +void hts_mirror_process_user_interaction(htsmoduleStruct * str, + htsmoduleStructExtended * stre); /* Get the next file on the queue, waiting for it, handling other files in background.. Parameters: The public module structure, and the private module variables Returns 0 upon success */ -int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* stre); +int hts_mirror_wait_for_next_file(htsmoduleStruct * str, + htsmoduleStructExtended * stre); /* Wait for (adr, fil, save) to be started, that is, to be ready for naming, having its header MIME type If the final URL is to be forbidden, sets 'forbidden_url' to the corresponding value */ -int hts_wait_delayed(htsmoduleStruct* str, - char* adr, char* fil, char* save, - char* parent_adr, char* parent_fil, - char* former_adr, char* former_fil, - int* forbidden_url); +int hts_wait_delayed(htsmoduleStruct * str, char *adr, char *fil, char *save, + char *parent_adr, char *parent_fil, char *former_adr, + char *former_fil, int *forbidden_url); /* Context state */ diff --git a/src/htsrobots.c b/src/htsrobots.c index 589f70c..54367e6 100644 --- a/src/htsrobots.c +++ b/src/htsrobots.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* robots.txt (website robot file) */ @@ -49,53 +46,55 @@ Please visit our Website: http://www.httrack.com // -- robots -- // fil="" : vérifier si règle déja enregistrée -int checkrobots(robots_wizard* robots,char* adr,char* fil) { +int checkrobots(robots_wizard * robots, char *adr, char *fil) { while(robots) { - if (strfield2(robots->adr,adr)) { + if (strfield2(robots->adr, adr)) { if (fil[0]) { - int ptr=0; + int ptr = 0; char line[250]; + if (strnotempty(robots->token)) { do { - ptr+=binput(robots->token+ptr,line,200); - if (line[0]=='/') { // absolu - if (strfield(fil,line)) { // commence avec ligne - return -1; // interdit + ptr += binput(robots->token + ptr, line, 200); + if (line[0] == '/') { // absolu + if (strfield(fil, line)) { // commence avec ligne + return -1; // interdit } - } else { // relatif - if (strstrcase(fil,line)) { + } else { // relatif + if (strstrcase(fil, line)) { return -1; } } - } while( (strnotempty(line)) && (ptr<(int) strlen(robots->token)) ); + } while((strnotempty(line)) && (ptr < (int) strlen(robots->token))); } } else { return -1; } } - robots=robots->next; + robots = robots->next; } return 0; } -int checkrobots_set(robots_wizard* robots,char* adr,char* data) { - if (((int) strlen(adr)) >= sizeof(robots->adr) - 2) return 0; - if (((int) strlen(data)) >= sizeof(robots->token) - 2) return 0; +int checkrobots_set(robots_wizard * robots, char *adr, char *data) { + if (((int) strlen(adr)) >= sizeof(robots->adr) - 2) + return 0; + if (((int) strlen(data)) >= sizeof(robots->token) - 2) + return 0; while(robots) { - if (strfield2(robots->adr,adr)) { // entrée existe - strcpybuff(robots->token,data); + if (strfield2(robots->adr, adr)) { // entrée existe + strcpybuff(robots->token, data); #if DEBUG_ROBOTS - printf("robots.txt: set %s to %s\n",adr,data); + printf("robots.txt: set %s to %s\n", adr, data); #endif return -1; - } - else if (!robots->next) { - robots->next=(robots_wizard*) calloct(1,sizeof(robots_wizard)); + } else if (!robots->next) { + robots->next = (robots_wizard *) calloct(1, sizeof(robots_wizard)); if (robots->next) { - robots->next->next=NULL; - strcpybuff(robots->next->adr,adr); - strcpybuff(robots->next->token,data); + robots->next->next = NULL; + strcpybuff(robots->next->adr, adr); + strcpybuff(robots->next->token, data); #if DEBUG_ROBOTS - printf("robots.txt: new set %s to %s\n",adr,data); + printf("robots.txt: new set %s to %s\n", adr, data); #endif } #if DEBUG_ROBOTS @@ -103,15 +102,15 @@ int checkrobots_set(robots_wizard* robots,char* adr,char* data) { printf("malloc error!!\n"); #endif } - robots=robots->next; + robots = robots->next; } return 0; } -void checkrobots_free(robots_wizard* robots) { +void checkrobots_free(robots_wizard * robots) { if (robots->next) { checkrobots_free(robots->next); freet(robots->next); - robots->next=NULL; + robots->next = NULL; } } diff --git a/src/htsrobots.h b/src/htsrobots.h index b7d051d..67cc34f 100644 --- a/src/htsrobots.h +++ b/src/htsrobots.h @@ -17,14 +17,12 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ @@ -34,10 +32,8 @@ Please visit our Website: http://www.httrack.com /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - - #ifndef HTSROBOTS_DEFH -#define HTSROBOTS_DEFH +#define HTSROBOTS_DEFH // robots wizard #ifndef HTS_DEF_FWSTRUCT_robots_wizard @@ -47,15 +43,14 @@ typedef struct robots_wizard robots_wizard; struct robots_wizard { char adr[128]; char token[4096]; - struct robots_wizard* next; + struct robots_wizard *next; }; - /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -int checkrobots(robots_wizard* robots,char* adr,char* fil); -void checkrobots_free(robots_wizard* robots); -int checkrobots_set(robots_wizard* robots,char* adr,char* data); +int checkrobots(robots_wizard * robots, char *adr, char *fil); +void checkrobots_free(robots_wizard * robots); +int checkrobots_set(robots_wizard * robots, char *adr, char *data); #endif #endif diff --git a/src/htsserver.c b/src/htsserver.c index 39522dd..01562d7 100644 --- a/src/htsserver.c +++ b/src/htsserver.c @@ -17,30 +17,26 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Mini-server */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - /* specific definitions */ /* specific definitions */ /* Bypass internal definition protection */ #define HTS_INTERNAL_BYTECODE - #include "htsbase.h" +#include "htsbase.h" #undef HTS_INTERNAL_BYTECODE #include "htsnet.h" @@ -72,96 +68,95 @@ Please visit our Website: http://www.httrack.com /* Bypass internal definition protection */ #define HTS_INTERNAL_BYTECODE - #include "htsinthash.h" +#include "htsinthash.h" #undef HTS_INTERNAL_BYTECODE -int NewLangStrSz=1024; -inthash NewLangStr=NULL; -int NewLangStrKeysSz=1024; -inthash NewLangStrKeys=NULL; -int NewLangListSz=1024; -inthash NewLangList=NULL; +int NewLangStrSz = 1024; +inthash NewLangStr = NULL; +int NewLangStrKeysSz = 1024; +inthash NewLangStrKeys = NULL; +int NewLangListSz = 1024; +inthash NewLangList = NULL; + /* Language files */ #include "htsserver.h" -char* gethomedir(void); +char *gethomedir(void); int commandRunning = 0; int commandEndRequested = 0; int commandEnd = 0; int commandReturn = 0; -char* commandReturnMsg = NULL; -char* commandReturnCmdl = NULL; +char *commandReturnMsg = NULL; +char *commandReturnCmdl = NULL; int commandReturnSet = 0; httrackp *global_opt = NULL; /* Extern */ -extern void webhttrack_main(char* cmd); +extern void webhttrack_main(char *cmd); extern void webhttrack_lock(void); extern void webhttrack_release(void); -static int is_image(char* file) { - return ( (strstr(file, ".gif") != NULL) ); +static int is_image(char *file) { + return ((strstr(file, ".gif") != NULL)); } -static int is_text(char* file) { - return ( (strstr(file, ".txt") != NULL) ); +static int is_text(char *file) { + return ((strstr(file, ".txt") != NULL)); } -static int is_html(char* file) { - return ( (strstr(file, ".htm") != NULL) ); +static int is_html(char *file) { + return ((strstr(file, ".htm") != NULL)); } -static void sig_brpipe( int code ) { +static void sig_brpipe(int code) { /* ignore */ } static int check_readinput_t(T_SOC soc, int timeout); -static int recv_bl(T_SOC soc, void* buffer, size_t len, int timeout); -static int linputsoc(T_SOC soc, char* s, int max); -static int check_readinput(htsblk* r); -static int linputsoc_t(T_SOC soc, char* s, int max, int timeout); - - -static int linput(FILE* fp,char* s,int max); +static int recv_bl(T_SOC soc, void *buffer, size_t len, int timeout); +static int linputsoc(T_SOC soc, char *s, int max); +static int check_readinput(htsblk * r); +static int linputsoc_t(T_SOC soc, char *s, int max, int timeout); +static int linput(FILE * fp, char *s, int max); // URL Link catcher // 0- Init the URL catcher with standard port // smallserver_init(&port,&return_host); -T_SOC smallserver_init_std(int* port_prox, char* adr_prox, int defaultPort) { - T_SOC soc; - if (defaultPort <= 0) { - int try_to_listen_to[]={8080,8081,8082,8083,8084,8085,8086,8087,8088,8089, - 32000,32001,32002,32003,32004,32006,32006,32007,32008,32009, - 42000,42001,42002,42003,42004,42006,42006,42007,42008,42009, - 0,-1}; - int i=0; - do { - soc=smallserver_init(&try_to_listen_to[i],adr_prox); - *port_prox=try_to_listen_to[i]; - i++; - } while( (soc == INVALID_SOCKET) && (try_to_listen_to[i]>=0)); - } else { - soc=smallserver_init(&defaultPort, adr_prox); - *port_prox = defaultPort; - } - return soc; -} +T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort) { + T_SOC soc; + + if (defaultPort <= 0) { + int try_to_listen_to[] = + { 8080, 8081, 8082, 8083, 8084, 8085, 8086, 8087, 8088, 8089, + 32000, 32001, 32002, 32003, 32004, 32006, 32006, 32007, 32008, 32009, + 42000, 42001, 42002, 42003, 42004, 42006, 42006, 42007, 42008, 42009, + 0, -1 + }; + int i = 0; + do { + soc = smallserver_init(&try_to_listen_to[i], adr_prox); + *port_prox = try_to_listen_to[i]; + i++; + } while((soc == INVALID_SOCKET) && (try_to_listen_to[i] >= 0)); + } else { + soc = smallserver_init(&defaultPort, adr_prox); + *port_prox = defaultPort; + } + return soc; +} // 1- Init the URL catcher // smallserver_init(&port,&return_host); -T_SOC smallserver_init(int* port,char* adr) { +T_SOC smallserver_init(int *port, char *adr) { T_SOC soc = INVALID_SOCKET; - char h_loc[256+2]; - - commandRunning = - commandEnd = - commandReturn = - commandReturnSet = + char h_loc[256 + 2]; + + commandRunning = commandEnd = commandReturn = commandReturnSet = commandEndRequested = 0; if (commandReturnMsg) free(commandReturnMsg); @@ -170,62 +165,64 @@ T_SOC smallserver_init(int* port,char* adr) { free(commandReturnCmdl); commandReturnCmdl = NULL; - if (gethostname(h_loc,256)==0) { // host name + if (gethostname(h_loc, 256) == 0) { // host name SOCaddr server; - int server_size=sizeof(server); + int server_size = sizeof(server); + /*t_hostent* hp_loc; - t_fullhostent buffer;*/ + t_fullhostent buffer; */ // effacer structure memset(&server, 0, sizeof(server)); - - /*if ( (hp_loc=vxgethostbyname(h_loc, &buffer)) )*/ { // notre host + + /*if ( (hp_loc=vxgethostbyname(h_loc, &buffer)) ) */ { + // notre host // copie adresse // NO (bind all) // SOCaddr_copyaddr(server, server_size, hp_loc->h_addr_list[0], hp_loc->h_length); - SOCaddr_initany(server, server_size); - if ( (soc = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0)) != INVALID_SOCKET) { + SOCaddr_initany(server, server_size); + if ((soc = + (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, + 0)) != INVALID_SOCKET) { SOCaddr_initport(server, *port); - if ( bind(soc,(struct sockaddr*) &server, server_size) == 0 ) { + if (bind(soc, (struct sockaddr *) &server, server_size) == 0) { /*SOClen len; - SOCaddr server2; - len=sizeof(server2);*/ + SOCaddr server2; + len=sizeof(server2); */ // effacer structure /*memset(&server2, 0, sizeof(server2)); - if (getsockname(soc,(struct sockaddr*) &server2,&len) == 0) { - *port=ntohs(SOCaddr_sinport(server)); // récupérer port*/ - if (listen(soc,10)>=0) { // au pif le 10 - // SOCaddr_inetntoa(adr, 128, server2, len); - strcpy(adr, h_loc); - } else { -#ifdef _WIN32 - closesocket(soc); -#else - close(soc); -#endif - soc=INVALID_SOCKET; - } - - - /*} else { + if (getsockname(soc,(struct sockaddr*) &server2,&len) == 0) { + *port=ntohs(SOCaddr_sinport(server)); // récupérer port */ + if (listen(soc, 10) >= 0) { // au pif le 10 + // SOCaddr_inetntoa(adr, 128, server2, len); + strcpy(adr, h_loc); + } else { #ifdef _WIN32 closesocket(soc); #else close(soc); #endif - soc=INVALID_SOCKET; - }*/ - - + soc = INVALID_SOCKET; + } + + /*} else { + #ifdef _WIN32 + closesocket(soc); + #else + close(soc); + #endif + soc=INVALID_SOCKET; + } */ + } else { #ifdef _WIN32 closesocket(soc); #else close(soc); #endif - soc=INVALID_SOCKET; + soc = INVALID_SOCKET; } } } @@ -235,7 +232,6 @@ T_SOC smallserver_init(int* port,char* adr) { // 2 - Wait for URL - // check if data is available // smallserver @@ -244,12 +240,12 @@ T_SOC smallserver_init(int* port,char* adr) { // data: 32Kb typedef struct { - char* name; + char *name; int value; } initIntElt; typedef struct { - char* name; - char* value; + char *name; + char *value; } initStrElt; #define SET_ERROR(err) do { \ @@ -257,23 +253,24 @@ typedef struct { error_redirect = "/server/error.html"; \ } while(0) -int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { - int timeout=30; - int retour=0; - int willexit=0; +int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) { + int timeout = 30; + int retour = 0; + int willexit = 0; int buffer_size = 32768; - char* buffer = (char*)malloc(buffer_size); + char *buffer = (char *) malloc(buffer_size); String headers = STRING_EMPTY; String output = STRING_EMPTY; String tmpbuff = STRING_EMPTY; String tmpbuff2 = STRING_EMPTY; String fspath = STRING_EMPTY; - char catbuff[CATBUFF_SIZE]; + char catbuff[CATBUFF_SIZE]; /* Load strings */ htslang_init(); if (!htslang_load(NULL, path)) { - fprintf(stderr, "unable to find lang.def and/or lang/ strings in %s\n", path); + fprintf(stderr, "unable to find lang.def and/or lang/ strings in %s\n", + path); return 0; } LANG_T(path, 0); @@ -281,46 +278,52 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { /* Init various values */ { char pth[1024]; - char* initOn[] = { "parseall", "Cache", "ka", - "cookies", "parsejava", "testall", "updhack", "urlhack", "index", NULL }; + + char *initOn[] = { "parseall", "Cache", "ka", + "cookies", "parsejava", "testall", "updhack", "urlhack", "index", NULL + }; initIntElt initInt[] = { - { "filter", 4 }, - { "travel", 2 }, - { "travel2", 1 }, - { "travel3", 1 }, + {"filter", 4}, + {"travel", 2}, + {"travel2", 1}, + {"travel3", 1}, /* */ - { "connexion", 4 }, + {"connexion", 4}, /* */ - { "maxrate", 25000 }, + {"maxrate", 25000}, /* */ - { "build", 1 }, + {"build", 1}, /* */ - { "checktype", 2}, - { "robots", 3 }, + {"checktype", 2}, + {"robots", 3}, - { NULL, 0 } + {NULL, 0} }; initStrElt initStr[] = { - { "user", "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" }, - { "footer", "<!-- Mirrored from %s%s by HTTrack Website Copier/3.x [XR&CO'2013], %s -->" }, - { "url2", "+*.png +*.gif +*.jpg +*.css +*.js -ad.doubleclick.net/*" }, - { NULL, NULL } + {"user", "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"}, + {"footer", + "<!-- Mirrored from %s%s by HTTrack Website Copier/3.x [XR&CO'2013], %s -->"}, + {"url2", "+*.png +*.gif +*.jpg +*.css +*.js -ad.doubleclick.net/*"}, + {NULL, NULL} }; int i = 0; - for(i = 0 ; initInt[i].name ; i++) { + + for(i = 0; initInt[i].name; i++) { char tmp[32]; + sprintf(tmp, "%d", initInt[i].value); - inthash_write(NewLangList, initInt[i].name, (intptr_t)strdup(tmp)); + inthash_write(NewLangList, initInt[i].name, (intptr_t) strdup(tmp)); } - for(i = 0 ; initOn[i] ; i++) { - inthash_write(NewLangList, initOn[i], (intptr_t)strdup("1")); /* "on" */ + for(i = 0; initOn[i]; i++) { + inthash_write(NewLangList, initOn[i], (intptr_t) strdup("1")); /* "on" */ } - for(i = 0 ; initStr[i].name ; i++) { - inthash_write(NewLangList, initStr[i].name, (intptr_t)strdup(initStr[i].value)); + for(i = 0; initStr[i].name; i++) { + inthash_write(NewLangList, initStr[i].name, + (intptr_t) strdup(initStr[i].value)); } strcpybuff(pth, gethomedir()); strcatbuff(pth, "/websites"); - inthash_write(NewLangList, "path", (intptr_t)strdup(pth)); + inthash_write(NewLangList, "path", (intptr_t) strdup(pth)); } /* Lock */ @@ -333,7 +336,7 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { char line2[1024]; T_SOC soc_c; LLint length = 0; - char* error_redirect = NULL; + char *error_redirect = NULL; line[0] = '\0'; buffer[0] = '\0'; @@ -353,22 +356,23 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { /* sigpipe */ #ifndef _WIN32 - signal( SIGPIPE , sig_brpipe ); + signal(SIGPIPE, sig_brpipe); #endif /* Accept */ - while ( (soc_c = (T_SOC) accept(soc, NULL, NULL)) == INVALID_SOCKET); + while((soc_c = (T_SOC) accept(soc, NULL, NULL)) == INVALID_SOCKET) ; /* Lock */ webhttrack_lock(); - if(linputsoc_t(soc_c, line1, sizeof(line1) - 2, timeout) > 0) { + if (linputsoc_t(soc_c, line1, sizeof(line1) - 2, timeout) > 0) { int meth = 0; + if (strfield(line1, "get ")) { meth = 1; } else if (strfield(line1, "post ")) { meth = 2; - } else if (strfield(line1, "head ")) { /* yes, we can do that */ + } else if (strfield(line1, "head ")) { /* yes, we can do that */ meth = 10; } else { #ifdef _DEBUG @@ -380,25 +384,29 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { length = buffer_size - 2; while(linputsoc_t(soc_c, line, sizeof(line) - 2, timeout) > 0) { int p; - if ((p=strfield(line,"Content-length:"))!=0) { - sscanf(line+p, LLintP, &(length)); - } - else if ((p=strfield(line,"Accept-language:"))!=0) { + + if ((p = strfield(line, "Content-length:")) != 0) { + sscanf(line + p, LLintP, &(length)); + } else if ((p = strfield(line, "Accept-language:")) != 0) { char tmp[32]; - char* s = line + p; - /*int l;*/ - while(*s == ' ') s++; + char *s = line + p; + + /*int l; */ + while(*s == ' ') + s++; tmp[0] = '\0'; strncatbuff(tmp, s, 2); - /*l = LANG_SEARCH(path, tmp);*/ + /*l = LANG_SEARCH(path, tmp); */ } } if (meth == 2) { int sz = 0; + if (length > buffer_size - 2) { length = buffer_size - 2; } - if (length > 0 && (sz=recv_bl(soc_c, buffer, (int)length, timeout)) < 0) { + if (length > 0 + && (sz = recv_bl(soc_c, buffer, (int) length, timeout)) < 0) { meth = 0; } else { buffer[sz] = '\0'; @@ -411,52 +419,59 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { commandReturnSet = 1; if (commandReturn) { char tmp[32]; + sprintf(tmp, "%d", commandReturn); - inthash_write(NewLangList, "commandReturn", (intptr_t)strdup(tmp)); - inthash_write(NewLangList, "commandReturnMsg", (intptr_t)commandReturnMsg); - inthash_write(NewLangList, "commandReturnCmdl", (intptr_t)commandReturnCmdl); + inthash_write(NewLangList, "commandReturn", (intptr_t) strdup(tmp)); + inthash_write(NewLangList, "commandReturnMsg", + (intptr_t) commandReturnMsg); + inthash_write(NewLangList, "commandReturnCmdl", + (intptr_t) commandReturnCmdl); } else { - inthash_write(NewLangList, "commandReturn", (intptr_t)NULL); - inthash_write(NewLangList, "commandReturnMsg", (intptr_t)NULL); - inthash_write(NewLangList, "commandReturnCmdl", (intptr_t)NULL); + inthash_write(NewLangList, "commandReturn", (intptr_t) NULL); + inthash_write(NewLangList, "commandReturnMsg", (intptr_t) NULL); + inthash_write(NewLangList, "commandReturnCmdl", (intptr_t) NULL); } } /* SID check */ { intptr_t adr = 0; + if (inthash_readptr(NewLangList, "_sid", &adr)) { - if (inthash_write(NewLangList, "sid", (intptr_t)strdup((char*)adr))) { + if (inthash_write + (NewLangList, "sid", (intptr_t) strdup((char *) adr))) { } } } /* check variables */ if (meth && buffer[0]) { - char* s = buffer; + char *s = buffer; char *e, *f; + strcatbuff(buffer, "&"); - while( s && (e = strchr(s, '=')) && (f = strchr(s, '&')) ) { - char* ua; + while(s && (e = strchr(s, '=')) && (f = strchr(s, '&'))) { + char *ua; String sua = STRING_EMPTY; + *e = *f = '\0'; ua = e + 1; - if (strfield2(ua, "on")) /* hack : "on" == 1 */ + if (strfield2(ua, "on")) /* hack : "on" == 1 */ ua = "1"; unescapehttp(ua, &sua); - inthash_write(NewLangList, s, (intptr_t)StringAcquire(&sua)); + inthash_write(NewLangList, s, (intptr_t) StringAcquire(&sua)); s = f + 1; } } - /* Error check */ { intptr_t adr = 0; intptr_t adr2 = 0; + if (inthash_readptr(NewLangList, "sid", &adr)) { if (inthash_readptr(NewLangList, "_sid", &adr2)) { - if (strcmp((char*)adr, (char*)adr2) != 0) { + if (strcmp((char *) adr, (char *) adr2) != 0) { meth = 0; } } @@ -465,57 +480,65 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { /* Check variables (internal) */ if (meth) { - int doLoad=0; + int doLoad = 0; intptr_t adr = 0; + if (inthash_readptr(NewLangList, "lang", &adr)) { int n = 0; - if (sscanf((char*)adr, "%d", &n) == 1 && n > 0 && n - 1 != LANG_T(path, -1)) { + + if (sscanf((char *) adr, "%d", &n) == 1 && n > 0 + && n - 1 != LANG_T(path, -1)) { LANG_T(path, n - 1); /* make a backup, because the GUI will override it */ - inthash_write(NewLangList, "lang_", (intptr_t)strdup((char*)adr)); + inthash_write(NewLangList, "lang_", + (intptr_t) strdup((char *) adr)); } } /* Load existing project settings */ if (inthash_readptr(NewLangList, "loadprojname", &adr)) { - char* pname = (char*) adr; + char *pname = (char *) adr; + if (*pname) { - inthash_write(NewLangList, "projname", (intptr_t)strdup(pname)); + inthash_write(NewLangList, "projname", (intptr_t) strdup(pname)); } - inthash_write(NewLangList, "loadprojname", (intptr_t)NULL); - doLoad=1; - } - else if (inthash_readptr(NewLangList, "loadprojcateg", &adr)) { - char* pname = (char*) adr; + inthash_write(NewLangList, "loadprojname", (intptr_t) NULL); + doLoad = 1; + } else if (inthash_readptr(NewLangList, "loadprojcateg", &adr)) { + char *pname = (char *) adr; + if (*pname) { - inthash_write(NewLangList, "projcateg", (intptr_t)strdup(pname)); + inthash_write(NewLangList, "projcateg", (intptr_t) strdup(pname)); } - inthash_write(NewLangList, "loadprojcateg", (intptr_t)NULL); + inthash_write(NewLangList, "loadprojcateg", (intptr_t) NULL); } - + /* intial configuration */ { if (!inthash_read(NewLangList, "conf_file_loaded", NULL)) { - inthash_write(NewLangList, "conf_file_loaded", (intptr_t)strdup("true")); + inthash_write(NewLangList, "conf_file_loaded", + (intptr_t) strdup("true")); doLoad = 2; } } - + /* path : <path>/<project> */ if (!commandRunning) { intptr_t adrpath = 0, adrprojname = 0; + if (inthash_readptr(NewLangList, "path", &adrpath) - && inthash_readptr(NewLangList, "projname", &adrprojname)) { + && inthash_readptr(NewLangList, "projname", &adrprojname)) { StringClear(fspath); - StringCat(fspath, (char*)adrpath); + StringCat(fspath, (char *) adrpath); StringCat(fspath, "/"); - StringCat(fspath, (char*)adrprojname); + StringCat(fspath, (char *) adrprojname); } } - + /* Load existing project settings */ if (doLoad) { - FILE* fp; + FILE *fp; + if (doLoad == 1) { StringCat(fspath, "/hts-cache/winprofile.ini"); } else if (doLoad == 2) { @@ -530,85 +553,96 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { if (fp) { /* Read file */ while(!feof(fp)) { - char* str = line; - char* pos; + char *str = line; + char *pos; + if (!linput(fp, line, sizeof(line) - 2)) { *str = '\0'; } - pos=strchr(line, '='); + pos = strchr(line, '='); if (pos) { String escline = STRING_EMPTY; - *pos++='\0'; + + *pos++ = '\0'; if (pos[0] == '0' && pos[1] == '\0') - *pos = '\0'; /* 0 => empty */ + *pos = '\0'; /* 0 => empty */ unescapeini(pos, &escline); - inthash_write(NewLangList, line, (intptr_t)StringAcquire(&escline)); + inthash_write(NewLangList, line, + (intptr_t) StringAcquire(&escline)); } } - + fclose(fp); } } - + } /* Execute command */ { intptr_t adr = 0; int p = 0; + if (inthash_readptr(NewLangList, "command", &adr)) { - if (strcmp((char*)adr, "cancel") == 0) { + if (strcmp((char *) adr, "cancel") == 0) { if (commandRunning) { if (!commandEndRequested) { - commandEndRequested=1; + commandEndRequested = 1; hts_request_stop(global_opt, 0); } else { - hts_request_stop(global_opt, 1); /* note: the force flag does not have anyeffect yet */ - commandEndRequested=2; /* will break the loop() callback */ + hts_request_stop(global_opt, 1); /* note: the force flag does not have anyeffect yet */ + commandEndRequested = 2; /* will break the loop() callback */ } } - } else if ((p=strfield((char*)adr, "cancel-file="))) { + } else if ((p = strfield((char *) adr, "cancel-file="))) { if (commandRunning) { - hts_cancel_file_push(global_opt, (char*)adr + p); + hts_cancel_file_push(global_opt, (char *) adr + p); } - } else if (strcmp((char*)adr, "cancel-parsing") == 0) { + } else if (strcmp((char *) adr, "cancel-parsing") == 0) { if (commandRunning) { hts_cancel_parsing(global_opt); } - } else if ((p=strfield((char*)adr, "pause="))) { + } else if ((p = strfield((char *) adr, "pause="))) { if (commandRunning) { hts_setpause(global_opt, 1); } - } else if ((p=strfield((char*)adr, "unpause"))) { + } else if ((p = strfield((char *) adr, "unpause"))) { if (commandRunning) { hts_setpause(global_opt, 0); } - } else if (strcmp((char*)adr, "abort") == 0) { + } else if (strcmp((char *) adr, "abort") == 0) { if (commandRunning) { hts_request_stop(global_opt, 1); - commandEndRequested=2; /* will break the loop() callback */ + commandEndRequested = 2; /* will break the loop() callback */ } - } else if ((p=strfield((char*)adr, "add-url="))) { + } else if ((p = strfield((char *) adr, "add-url="))) { if (commandRunning) { - char* ptraddr[2]; - ptraddr[0] = (char*)adr + p; + char *ptraddr[2]; + + ptraddr[0] = (char *) adr + p; ptraddr[1] = NULL; hts_addurl(global_opt, ptraddr); } - } else if ((p=strfield((char*)adr, "httrack"))) { + } else if ((p = strfield((char *) adr, "httrack"))) { if (!commandRunning) { intptr_t adrcd = 0; + if (inthash_readptr(NewLangList, "command_do", &adrcd)) { intptr_t adrw = 0, adrpath = 0; + if (inthash_readptr(NewLangList, "winprofile", &adrw)) { /* User general profile */ intptr_t adruserprofile = 0; - if (inthash_readptr(NewLangList, "userprofile", &adruserprofile) - && adruserprofile != 0) { - int count = (int) strlen((char*)adruserprofile); + + if (inthash_readptr + (NewLangList, "userprofile", &adruserprofile) + && adruserprofile != 0) { + int count = (int) strlen((char *) adruserprofile); + if (count > 0) { - FILE* fp; + FILE *fp; + StringClear(tmpbuff); StringCopy(tmpbuff, gethomedir()); #ifdef _WIN32 @@ -618,87 +652,100 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { #endif fp = fopen(StringBuff(tmpbuff), "wb"); if (fp != NULL) { - (void)((int)fwrite((void*)adruserprofile, 1, count, fp)); + (void) ((int) + fwrite((void *) adruserprofile, 1, count, fp)); fclose(fp); } } } - + /* Profile */ StringClear(tmpbuff); StringCat(tmpbuff, StringBuff(fspath)); StringCat(tmpbuff, "/hts-cache/"); - + /* Create minimal directory structure */ if (!structcheck(StringBuff(tmpbuff))) { - FILE* fp; + FILE *fp; + StringCat(tmpbuff, "winprofile.ini"); fp = fopen(StringBuff(tmpbuff), "wb"); if (fp != NULL) { - int count = (int) strlen((char*)adrw); - if ((int)fwrite((void*)adrw, 1, count, fp) == count) { - - /* Wipe the doit.log file, useless here (all options are replicated) and - even a bit annoying (duplicate/ghost options) - The behaviour is exactly the same as in WinHTTrack - */ + int count = (int) strlen((char *) adrw); + + if ((int) fwrite((void *) adrw, 1, count, fp) == count) { + + /* Wipe the doit.log file, useless here (all options are replicated) and + even a bit annoying (duplicate/ghost options) + The behaviour is exactly the same as in WinHTTrack + */ StringClear(tmpbuff); StringCat(tmpbuff, StringBuff(fspath)); StringCat(tmpbuff, "/hts-cache/doit.log"); remove(StringBuff(tmpbuff)); - + /* - RUN THE SERVER - */ - if (strcmp((char*)adrcd, "start") == 0) { - webhttrack_main((char*)adr + p); + RUN THE SERVER + */ + if (strcmp((char *) adrcd, "start") == 0) { + webhttrack_main((char *) adr + p); } else { commandRunning = 0; commandEnd = 1; } } else { char tmp[1024]; - sprintf(tmp, "Unable to write %d bytes in the the init file %s", count, StringBuff(fspath)); + + sprintf(tmp, + "Unable to write %d bytes in the the init file %s", + count, StringBuff(fspath)); SET_ERROR(tmp); } fclose(fp); } else { char tmp[1024]; - sprintf(tmp, "Unable to create the init file %s", StringBuff(fspath)); + + sprintf(tmp, "Unable to create the init file %s", + StringBuff(fspath)); SET_ERROR(tmp); } } else { char tmp[1024]; - sprintf(tmp, "Unable to create the directory structure in %s", StringBuff(fspath)); + + sprintf(tmp, + "Unable to create the directory structure in %s", + StringBuff(fspath)); SET_ERROR(tmp); } - + } else { - SET_ERROR("Internal server error: unable to fetch project name or path"); + SET_ERROR + ("Internal server error: unable to fetch project name or path"); } } } - } else if (strcmp((char*)adr, "quit") == 0) { - willexit=1; + } else if (strcmp((char *) adr, "quit") == 0) { + willexit = 1; } - inthash_write(NewLangList, "command", (intptr_t)NULL); + inthash_write(NewLangList, "command", (intptr_t) NULL); } } /* Response */ if (meth) { int virtualpath = 0; - char* pos; - char* url = strchr(line1, ' '); - if (url && *++url == '/' && (pos = strchr(url, ' ')) && !(*pos = '\0') ) { + char *pos; + char *url = strchr(line1, ' '); + + if (url && *++url == '/' && (pos = strchr(url, ' ')) && !(*pos = '\0')) { char fsfile[1024]; - char* file; - FILE* fp; - char* qpos; + char *file; + FILE *fp; + char *qpos; /* get the URL */ if (error_redirect == NULL) { - if ( (qpos = strchr(url, '?')) ) { + if ((qpos = strchr(url, '?'))) { *qpos = '\0'; } fsfile[0] = '\0'; @@ -712,7 +759,7 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { file = error_redirect; meth = 2; } - + if (strncmp(file, "/website/", 9) == 0) { virtualpath = 1; } @@ -726,39 +773,35 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { file = "/server/finished.html"; } } - + if (strlen(path) + strlen(file) + 32 < sizeof(fsfile)) { if (strncmp(file, "/website/", 9) != 0) { sprintf(fsfile, "%shtml%s", path, file); } else { intptr_t adr = 0; + if (inthash_readptr(NewLangList, "projpath", &adr)) { - sprintf(fsfile, "%s%s", (char*)adr, file + 9); + sprintf(fsfile, "%s%s", (char *) adr, file + 9); } } } - - if (fsfile[0] && strstr(file, "..") == NULL && (fp = fopen(fsfile, "rb"))) { - char ok[] = "HTTP/1.0 200 OK\r\n" - "Connection: close\r\n" - "Server: httrack-small-server\r\n" - "Content-type: text/html\r\n" + + if (fsfile[0] && strstr(file, "..") == NULL + && (fp = fopen(fsfile, "rb"))) { + char ok[] = + "HTTP/1.0 200 OK\r\n" "Connection: close\r\n" + "Server: httrack-small-server\r\n" "Content-type: text/html\r\n" "Cache-Control: no-cache, must-revalidate, private\r\n" - "Pragma: no-cache\r\n" - ; - char ok_img[] = "HTTP/1.0 200 OK\r\n" - "Connection: close\r\n" - "Server: httrack small server\r\n" - "Content-type: image/gif\r\n" - ; - char ok_text[] = "HTTP/1.0 200 OK\r\n" - "Connection: close\r\n" - "Server: httrack small server\r\n" - "Content-type: text/plain\r\n" - ; + "Pragma: no-cache\r\n"; + char ok_img[] = + "HTTP/1.0 200 OK\r\n" "Connection: close\r\n" + "Server: httrack small server\r\n" "Content-type: image/gif\r\n"; + char ok_text[] = + "HTTP/1.0 200 OK\r\n" "Connection: close\r\n" + "Server: httrack small server\r\n" "Content-type: text/plain\r\n"; /* register current page */ - inthash_write(NewLangList, "thisfile", (intptr_t)strdup(file)); + inthash_write(NewLangList, "thisfile", (intptr_t) strdup(file)); /* Force GET for the last request */ if (meth == 2 && willexit) { @@ -767,13 +810,15 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { /* posted data are redirected to get protocol */ if (meth == 2) { - char redir[] = "HTTP/1.0 302 Redirect\r\n" - "Connection: close\r\n" + char redir[] = + "HTTP/1.0 302 Redirect\r\n" "Connection: close\r\n" "Server: httrack-small-server\r\n"; intptr_t adr = 0; - char* newfile = file; + char *newfile = file; + if (inthash_readptr(NewLangList, "redirect", &adr) && adr != 0) { - char* newadr = (char*)adr; + char *newadr = (char *) adr; + if (*newadr) { newfile = newadr; } @@ -781,20 +826,22 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { StringMemcat(headers, redir, strlen(redir)); { char tmp[256]; + if (strlen(file) < sizeof(tmp) - 32) { sprintf(tmp, "Location: %s\r\n", newfile); StringMemcat(headers, tmp, strlen(tmp)); } } - inthash_write(NewLangList, "redirect", (intptr_t)NULL); - } - else if (is_html(file)) { + inthash_write(NewLangList, "redirect", (intptr_t) NULL); + } else if (is_html(file)) { int outputmode = 0; + StringMemcat(headers, ok, sizeof(ok) - 1); while(!feof(fp)) { - char* str = line; + char *str = line; int prevlen = (int) StringLength(output); int nocr = 0; + if (!linput(fp, line, sizeof(line) - 2)) { *str = '\0'; } @@ -803,34 +850,34 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { str[strlen(str) - 1] = '\0'; } while(*str) { - char* pos; + char *pos; size_t n; - if (*str == '$' && *++str == '{' && (pos = strchr(++str, '}')) && (n = (pos - str) ) && n < 1024 ) { + + if (*str == '$' && *++str == '{' && (pos = strchr(++str, '}')) + && (n = (pos - str)) && n < 1024) { char name_[1024 + 2]; - char* name = name_; - char* langstr = NULL; + char *name = name_; + char *langstr = NULL; int p; int format = 0; int listDefault = 0; + name[0] = '\0'; strncatbuff(name, str, n); if (strncmp(name, "/*", 2) == 0) { /* comments */ - } - else if (( p = strfield(name, "html:"))) { + } else if ((p = strfield(name, "html:"))) { name += p; format = 1; - } - else if (( p = strfield(name, "list:"))) { + } else if ((p = strfield(name, "list:"))) { name += p; format = 2; - } - else if (( p = strfield(name, "liststr:"))) { + } else if ((p = strfield(name, "liststr:"))) { name += p; format = -2; - } - else if (( p = strfield(name, "file-exists:"))) { - char* pos2; + } else if ((p = strfield(name, "file-exists:"))) { + char *pos2; + name += p; format = 0; pos2 = strchr(name, ':'); @@ -843,9 +890,9 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { } } } - } - else if (( p = strfield(name, "do:"))) { - char* pos2; + } else if ((p = strfield(name, "do:"))) { + char *pos2; + name += p; format = 1; pos2 = strchr(name, ':'); @@ -854,15 +901,15 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { *pos2 = '\0'; pos2++; } else { - pos2=""; + pos2 = ""; } if (strcmp(name, "output-mode") == 0) { if (strcmp(pos2, "html") == 0) { outputmode = 1; } else if (strcmp(pos2, "inifile") == 0) { - outputmode = 2; + outputmode = 2; } else if (strcmp(pos2, "html-urlescaped") == 0) { - outputmode = 3; + outputmode = 3; } else { outputmode = 0; } @@ -874,7 +921,8 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { } } else if (strcmp(name, "if-project-file-exists") == 0) { if (strstr(pos2, "..") == NULL) { - if (!fexist(fconcat(catbuff, StringBuff(fspath), pos2))) { + if (!fexist + (fconcat(catbuff, StringBuff(fspath), pos2))) { outputmode = -1; } } @@ -886,106 +934,125 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { } } else if (strcmp(name, "if-not-empty") == 0) { intptr_t adr = 0; - if (!inthash_readptr(NewLangList, pos2, &adr) || *((char*)adr) == 0 ) { + + if (!inthash_readptr(NewLangList, pos2, &adr) + || *((char *) adr) == 0) { outputmode = -1; } } else if (strcmp(name, "if-empty") == 0) { intptr_t adr = 0; - if (inthash_readptr(NewLangList, pos2, &adr) && *((char*)adr) != 0 ) { + + if (inthash_readptr(NewLangList, pos2, &adr) + && *((char *) adr) != 0) { outputmode = -1; } } else if (strcmp(name, "end-if") == 0) { outputmode = 0; } else if (strcmp(name, "loadhash") == 0) { intptr_t adr = 0; + if (inthash_readptr(NewLangList, "path", &adr)) { - char* rpath = (char*) adr; + char *rpath = (char *) adr; + //find_handle h; if (rpath[0]) { - if (rpath[strlen(rpath)-1]=='/') { - rpath[strlen(rpath)-1]='\0'; /* note: patching stored (inhash) value */ + if (rpath[strlen(rpath) - 1] == '/') { + rpath[strlen(rpath) - 1] = '\0'; /* note: patching stored (inhash) value */ } } { - const char* profiles = hts_getcategories(rpath, 0); - const char* categ = hts_getcategories(rpath,1 ); - inthash_write(NewLangList, "winprofile", (intptr_t)profiles); - inthash_write(NewLangList, "wincateg", (intptr_t)categ); + const char *profiles = hts_getcategories(rpath, 0); + const char *categ = hts_getcategories(rpath, 1); + + inthash_write(NewLangList, "winprofile", + (intptr_t) profiles); + inthash_write(NewLangList, "wincateg", + (intptr_t) categ); } - } + } } else if (strcmp(name, "copy") == 0) { if (*pos2) { - char* pos3 = strchr(pos2, ':'); - if ( pos3 && *(pos3 + 1) ) { + char *pos3 = strchr(pos2, ':'); + + if (pos3 && *(pos3 + 1)) { intptr_t adr = 0; + *pos3++ = '\0'; if (inthash_readptr(NewLangList, pos2, &adr)) { - inthash_write(NewLangList, pos3, (intptr_t)strdup((char*)adr)); - inthash_write(NewLangList, pos2, (intptr_t)NULL); + inthash_write(NewLangList, pos3, + (intptr_t) strdup((char *) adr)); + inthash_write(NewLangList, pos2, (intptr_t) NULL); } } } } else if (strcmp(name, "set") == 0) { if (*pos2) { - char* pos3 = strchr(pos2, ':'); - if ( pos3 ) { + char *pos3 = strchr(pos2, ':'); + + if (pos3) { *pos3++ = '\0'; - inthash_write(NewLangList, pos2, (intptr_t)strdup(pos3)); + inthash_write(NewLangList, pos2, + (intptr_t) strdup(pos3)); } else { - inthash_write(NewLangList, pos2, (intptr_t)NULL); + inthash_write(NewLangList, pos2, (intptr_t) NULL); } } } } /* - test:<if exist> - test:<if ==0>:<if ==1>:<if == 2>.. - ztest:<if == 0 || !exist>:<if == 1>:<if == 2>.. - */ - else if ( ( p = strfield(name, "test:")) || ( p = strfield(name, "ztest:")) ) { + test:<if exist> + test:<if ==0>:<if ==1>:<if == 2>.. + ztest:<if == 0 || !exist>:<if == 1>:<if == 2>.. + */ + else if ((p = strfield(name, "test:")) + || (p = strfield(name, "ztest:"))) { intptr_t adr = 0; - char* pos2; + char *pos2; int ztest = (name[0] == 'z'); + langstr = ""; name += p; pos2 = strchr(name, ':'); if (pos2 != NULL) { *pos2 = '\0'; if (inthash_readptr(NewLangList, name, &adr) || ztest) { - char* newadr = (char*)adr; + char *newadr = (char *) adr; + if (!newadr) newadr = ""; if (*newadr || ztest) { int npos = 0; + name = pos2 + 1; format = 4; if (strchr(name, ':') == NULL) { - npos = 0; /* first is good if only one : */ + npos = 0; /* first is good if only one : */ format = 0; } else { if (sscanf(newadr, "%d", &npos) != 1) { if (strfield(newadr, "on")) { npos = 1; } else { - npos = 0; /* first one will be ok */ + npos = 0; /* first one will be ok */ format = 0; } } } - while( *name && *name != '}' && npos >= 0) { - int end=0; - char* fpos = strchr(name, ':'); + while(*name && *name != '}' && npos >= 0) { + int end = 0; + char *fpos = strchr(name, ':'); int n2; + if (fpos == NULL) { fpos = name + strlen(name); - end=1; + end = 1; } n2 = (int) (fpos - name); if (npos == 0) { langstr = name; - *fpos='\0'; + *fpos = '\0'; } else if (end) { - npos=0; + npos = 0; } name += n2 + 1; npos--; @@ -993,30 +1060,32 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { } } } - } - else if (( p = strfield(name, "listid:"))) { - char* pos2; + } else if ((p = strfield(name, "listid:"))) { + char *pos2; + name += p; format = 2; pos2 = strchr(name, ':'); if (pos2) { char dname[32]; int n2 = (int) (pos2 - name); + if (n2 > 0 && n2 < sizeof(dname) - 2) { intptr_t adr = 0; + dname[0] = '\0'; strncatbuff(dname, name, n2); if (inthash_readptr(NewLangList, dname, &adr)) { int n = 0; - if (sscanf((char*)adr, "%d", &n) == 1) { + + if (sscanf((char *) adr, "%d", &n) == 1) { listDefault = n; } } name += n2 + 1; } } - } - else if (( p = strfield(name, "checked:"))) { + } else if ((p = strfield(name, "checked:"))) { name += p; format = 3; } @@ -1030,24 +1099,29 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { langstr = LANGSEL(name); if (langstr == NULL || *langstr == '\0') { intptr_t adr = 0; + if (inthash_readptr(NewLangList, name, &adr)) { - char* newadr = (char*)adr; + char *newadr = (char *) adr; + langstr = newadr; } } } } if (langstr && outputmode != -1) { - switch(format) { + switch (format) { case 0: { - char* a = langstr; + char *a = langstr; + while(*a) { - if (a[0] == '\\' && isxdigit(a[1]) && isxdigit(a[2])) { + if (a[0] == '\\' && isxdigit(a[1]) + && isxdigit(a[2])) { int n; char c; - if (sscanf(a+1, "%x", &n) == 1) { - c = (char)n; + + if (sscanf(a + 1, "%x", &n) == 1) { + c = (char) n; StringMemcat(output, &c, 1); } a += 2; @@ -1059,9 +1133,11 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { StringCat(output, "&"); } else if (outputmode == 3 && a[0] == ' ') { StringCat(output, "%20"); - } else if (outputmode >= 2 && ((unsigned char)a[0]) < 32) { + } else if (outputmode >= 2 + && ((unsigned char) a[0]) < 32) { char tmp[32]; - sprintf(tmp, "%%%02x", (unsigned char)a[0]); + + sprintf(tmp, "%%%02x", (unsigned char) a[0]); StringCat(output, tmp); } else if (outputmode == 2 && a[0] == '%') { StringCat(output, "%%"); @@ -1079,10 +1155,11 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { StringCat(output, "checked"); } break; - default: + default: if (*langstr) { - int id=1; - char* fstr = langstr; + int id = 1; + char *fstr = langstr; + StringClear(tmpbuff); if (format == 2) { StringCat(output, "<option value=1>"); @@ -1090,8 +1167,9 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { StringCat(output, "<option value=\""); } while(*fstr) { - switch(*fstr) { - case 13: break; + switch (*fstr) { + case 13: + break; case 10: if (format == 1) { StringCat(output, StringBuff(tmpbuff)); @@ -1104,6 +1182,7 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { StringCat(output, "<option value=\""); } else { char tmp[32]; + sprintf(tmp, "%d", ++id); StringCat(output, StringBuff(tmpbuff)); StringCat(output, "</option>\r\n"); @@ -1160,14 +1239,16 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { } #ifdef _DEBUG { - int len = (int)strlen((char*)StringBuff(output)); - assert(len == (int)StringLength(output)); + int len = (int) strlen((char *) StringBuff(output)); + + assert(len == (int) StringLength(output)); } #endif } else if (is_text(file)) { StringMemcat(headers, ok_text, sizeof(ok_text) - 1); while(!feof(fp)) { int n = (int) fread(line, 1, sizeof(line) - 2, fp); + if (n > 0) { StringMemcat(output, line, n); } @@ -1176,6 +1257,7 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { StringMemcat(headers, ok_img, sizeof(ok_img) - 1); while(!feof(fp)) { int n = (int) fread(line, 1, sizeof(line) - 2, fp); + if (n > 0) { StringMemcat(output, line, n); } @@ -1183,11 +1265,11 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { } fclose(fp); } else { - char error_hdr[] = "HTTP/1.0 404 Not Found\r\n" - "Server: httrack small server\r\n" + char error_hdr[] = + "HTTP/1.0 404 Not Found\r\n" "Server: httrack small server\r\n" "Content-type: text/html\r\n"; - char error[] = - "Page not found.\r\n"; + char error[] = "Page not found.\r\n"; + StringCat(headers, error_hdr); StringCat(output, error); //assert(file == NULL); @@ -1195,25 +1277,27 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { } } else { #ifdef _DEBUG - char error_hdr[] = "HTTP/1.0 500 Server Error\r\n" - "Server: httrack small server\r\n" + char error_hdr[] = + "HTTP/1.0 500 Server Error\r\n" "Server: httrack small server\r\n" "Content-type: text/html\r\n"; - char error[] = - "Server error.\r\n"; + char error[] = "Server error.\r\n"; + StringCat(headers, error_hdr); StringCat(output, error); #endif } { char tmp[256]; + sprintf(tmp, "Content-length: %d\r\n", (int) StringLength(output)); StringCat(headers, tmp); } StringCat(headers, "\r\n"); - if ( - (send(soc_c, StringBuff(headers), (int) StringLength(headers), 0) != StringLength(headers)) - || - ( (meth == 1) && (send(soc_c, StringBuff(output), (int) StringLength(output), 0) != StringLength(output)) ) + if ((send(soc_c, StringBuff(headers), (int) StringLength(headers), 0) != + StringLength(headers)) + || ((meth == 1) + && (send(soc_c, StringBuff(output), (int) StringLength(output), 0) + != StringLength(output))) ) { #ifdef _DEBUG //assert(FALSE); @@ -1224,17 +1308,18 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { // assert(FALSE); #endif } - + /* Shutdown (FIN) and wait until confirmed */ { char c; + #ifdef _WIN32 shutdown(soc_c, SD_SEND); #else shutdown(soc_c, 1); #endif /* This is necessary as IE sometimes (!) sends an additional CRLF after POST data */ - while(recv(soc_c, ((char*)&c), 1, 0) > 0); + while(recv(soc_c, ((char *) &c), 1, 0) > 0) ; } #ifdef _WIN32 @@ -1274,110 +1359,116 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { return retour; } - - /* Language files */ - int htslang_init(void) { - if (NewLangList==NULL) { - NewLangList=inthash_new(NewLangListSz); - if (NewLangList==NULL) { + if (NewLangList == NULL) { + NewLangList = inthash_new(NewLangListSz); + if (NewLangList == NULL) { abortLog("Error in lang.h: not enough memory"); } else { - inthash_value_is_malloc(NewLangList,1); + inthash_value_is_malloc(NewLangList, 1); } } return 1; } int htslang_uninit(void) { - if (NewLangList!=NULL) { + if (NewLangList != NULL) { inthash_delete(&NewLangList); } return 1; } -int smallserver_setkey(char* key, char* value) { - return inthash_write(NewLangList, key, (intptr_t)strdup(value)); +int smallserver_setkey(char *key, char *value) { + return inthash_write(NewLangList, key, (intptr_t) strdup(value)); } -int smallserver_setkeyint(char* key, LLint value) { +int smallserver_setkeyint(char *key, LLint value) { char tmp[256]; + sprintf(tmp, LLintP, value); - return inthash_write(NewLangList, key, (intptr_t)strdup(tmp)); + return inthash_write(NewLangList, key, (intptr_t) strdup(tmp)); } -int smallserver_setkeyarr(char* key, int id, char* key2, char* value) { +int smallserver_setkeyarr(char *key, int id, char *key2, char *value) { char tmp[256]; + sprintf(tmp, "%s%d%s", key, id, key2); - return inthash_write(NewLangList, tmp, (intptr_t)strdup(value)); + return inthash_write(NewLangList, tmp, (intptr_t) strdup(value)); } -static int htslang_load(char* limit_to, char* path) { - char* hashname; - char catbuff[CATBUFF_SIZE]; +static int htslang_load(char *limit_to, char *path) { + char *hashname; + char catbuff[CATBUFF_SIZE]; + // - int selected_lang=LANG_T(path, -1); + int selected_lang = LANG_T(path, -1); + // if (!limit_to) { LANG_DELETE(); - NewLangStr=inthash_new(NewLangStrSz); - NewLangStrKeys=inthash_new(NewLangStrKeysSz); - if ((NewLangStr==NULL) || (NewLangStrKeys==NULL)) { + NewLangStr = inthash_new(NewLangStrSz); + NewLangStrKeys = inthash_new(NewLangStrKeysSz); + if ((NewLangStr == NULL) || (NewLangStrKeys == NULL)) { abortLog("Error in lang.h: not enough memory"); } else { - inthash_value_is_malloc(NewLangStr,1); - inthash_value_is_malloc(NewLangStrKeys,1); + inthash_value_is_malloc(NewLangStr, 1); + inthash_value_is_malloc(NewLangStrKeys, 1); } } /* Load master file (list of keys and internal keys) */ if (!limit_to) { - char* mname = "lang.def"; - FILE* fp=fopen(fconcat(catbuff, path, mname),"rb"); + char *mname = "lang.def"; + FILE *fp = fopen(fconcat(catbuff, path, mname), "rb"); + if (fp) { char intkey[8192]; char key[8192]; + while(!feof(fp)) { - linput_cpp(fp,intkey,8000); - linput_cpp(fp,key,8000); + linput_cpp(fp, intkey, 8000); + linput_cpp(fp, key, 8000); if (strnotempty(intkey) && strnotempty(key)) { - char* test=LANGINTKEY(key); + char *test = LANGINTKEY(key); /* Increment for multiple definitions */ if (strnotempty(test)) { - int increment=0; + int increment = 0; size_t pos = strlen(key); + do { increment++; - sprintf(key+pos,"%d",increment); - test=LANGINTKEY(key); - } while (strnotempty(test)); + sprintf(key + pos, "%d", increment); + test = LANGINTKEY(key); + } while(strnotempty(test)); } - if (!strnotempty(test)) { // éviter doublons + if (!strnotempty(test)) { // éviter doublons // conv_printf(key,key); int len; - char* buff; + char *buff; + len = (int) strlen(intkey); - buff=(char*)malloc(len+2); + buff = (char *) malloc(len + 2); if (buff) { - strcpybuff(buff,intkey); - inthash_add(NewLangStrKeys,key,(intptr_t)buff); + strcpybuff(buff, intkey); + inthash_add(NewLangStrKeys, key, (intptr_t) buff); } } - } // if - } // while + } // if + } // while fclose(fp); } else { return 0; } } - + /* Language Name? */ { char name[256]; - sprintf(name,"LANGUAGE_%d",selected_lang+1); - hashname=LANGINTKEY(name); + + sprintf(name, "LANGUAGE_%d", selected_lang + 1); + hashname = LANGINTKEY(name); } /* Get only language name */ @@ -1396,50 +1487,56 @@ static int htslang_load(char* limit_to, char* path) { /* Load specific language file */ { int loops; + // 2nd loop: load undefined strings - for(loops=0;loops<2;loops++) { - FILE* fp; + for(loops = 0; loops < 2; loops++) { + FILE *fp; char lbasename[1024]; + { char name[256]; - sprintf(name,"LANGUAGE_%d",(loops==0)?(selected_lang+1):1); - hashname=LANGINTKEY(name); + + sprintf(name, "LANGUAGE_%d", (loops == 0) ? (selected_lang + 1) : 1); + hashname = LANGINTKEY(name); } - sprintf(lbasename, "lang/%s.txt",hashname); - fp=fopen(fconcat(catbuff, path, lbasename), "rb"); + sprintf(lbasename, "lang/%s.txt", hashname); + fp = fopen(fconcat(catbuff, path, lbasename), "rb"); if (fp) { char extkey[8192]; char value[8192]; + while(!feof(fp)) { - linput_cpp(fp,extkey,8000); - linput_cpp(fp,value,8000); + linput_cpp(fp, extkey, 8000); + linput_cpp(fp, value, 8000); if (strnotempty(extkey) && strnotempty(value)) { int len; - char* buff; - char* intkey; - - intkey=LANGINTKEY(extkey); - + char *buff; + char *intkey; + + intkey = LANGINTKEY(extkey); + if (strnotempty(intkey)) { - + /* Increment for multiple definitions */ { - char* test=LANGSEL(intkey); + char *test = LANGSEL(intkey); + if (strnotempty(test)) { if (loops == 0) { - int increment=0; - size_t pos=strlen(extkey); + int increment = 0; + size_t pos = strlen(extkey); + do { increment++; - sprintf(extkey+pos,"%d",increment); - intkey=LANGINTKEY(extkey); + sprintf(extkey + pos, "%d", increment); + intkey = LANGINTKEY(extkey); if (strnotempty(intkey)) - test=LANGSEL(intkey); + test = LANGSEL(intkey); else - test=""; - } while (strnotempty(test)); + test = ""; + } while(strnotempty(test)); } else - intkey=""; + intkey = ""; } else { if (loops > 0) { //err_msg += intkey; @@ -1447,20 +1544,20 @@ static int htslang_load(char* limit_to, char* path) { } } } - + /* Add key */ if (strnotempty(intkey)) { len = (int) strlen(value); - buff=(char*)malloc(len+2); + buff = (char *) malloc(len + 2); if (buff) { - conv_printf(value,buff); - inthash_add(NewLangStr,intkey,(intptr_t)buff); + conv_printf(value, buff); + inthash_add(NewLangStr, intkey, (intptr_t) buff); } } - + } - } // if - } // while + } // if + } // while fclose(fp); } else { return 0; @@ -1470,45 +1567,71 @@ static int htslang_load(char* limit_to, char* path) { // Control limit_to if (limit_to) - limit_to[0]='\0'; + limit_to[0] = '\0'; return 1; } /* NOTE : also contains the "webhttrack" hack */ -static void conv_printf(char* from,char* to) { - int i=0,j=0,len; +static void conv_printf(char *from, char *to) { + int i = 0, j = 0, len; + len = (int) strlen(from); - while(i<len) { - switch(from[i]) { - case '\\': + while(i < len) { + switch (from[i]) { + case '\\': i++; - switch(from[i]) { - case 'a': to[j]='\a'; break; - case 'b': to[j]='\b'; break; - case 'f': to[j]='\f'; break; - case 'n': to[j]='\n'; break; - case 'r': to[j]='\r'; break; - case 't': to[j]='\t'; break; - case 'v': to[j]='\v'; break; - case '\'': to[j]='\''; break; - case '\"': to[j]='\"'; break; - case '\\': to[j]='\\'; break; - case '?': to[j]='\?'; break; - default: to[j]=from[i]; break; + switch (from[i]) { + case 'a': + to[j] = '\a'; + break; + case 'b': + to[j] = '\b'; + break; + case 'f': + to[j] = '\f'; + break; + case 'n': + to[j] = '\n'; + break; + case 'r': + to[j] = '\r'; + break; + case 't': + to[j] = '\t'; + break; + case 'v': + to[j] = '\v'; + break; + case '\'': + to[j] = '\''; + break; + case '\"': + to[j] = '\"'; + break; + case '\\': + to[j] = '\\'; + break; + case '?': + to[j] = '\?'; + break; + default: + to[j] = from[i]; + break; } break; - default: - to[j]=from[i]; - break; + default: + to[j] = from[i]; + break; } i++; j++; } - to[j++]='\0'; + to[j++] = '\0'; /* Dirty hack */ { - char * a = to; + char *a = to; + while((a = strstr(a, "WinHTTrack"))) { a[0] = 'W'; a[1] = 'e'; @@ -1524,30 +1647,31 @@ static void LANG_DELETE(void) { } // sélection de la langue -static void LANG_INIT(char* path) { +static void LANG_INIT(char *path) { //CWinApp* pApp = AfxGetApp(); //if (pApp) { - /* pApp->GetProfileInt("Language","IntId",0); */ - LANG_T(path, 0 /*pApp->GetProfileInt("Language","IntId",0)*/ ); + /* pApp->GetProfileInt("Language","IntId",0); */ + LANG_T(path, 0 /*pApp->GetProfileInt("Language","IntId",0) */ ); //} } -static int LANG_T(char* path, int l) { - if (l>=0) { +static int LANG_T(char *path, int l) { + if (l >= 0) { QLANG_T(l); htslang_load(NULL, path); } - return QLANG_T(-1); // 0=default (english) + return QLANG_T(-1); // 0=default (english) } -static int LANG_SEARCH(char* path, char* iso) { +static int LANG_SEARCH(char *path, char *iso) { char lang_str[1024]; int i = 0; - int curr_lng=LANG_T(path, -1); + int curr_lng = LANG_T(path, -1); int found = 0; + do { QLANG_T(i); - strcpybuff(lang_str,"LANGUAGE_ISO"); + strcpybuff(lang_str, "LANGUAGE_ISO"); htslang_load(lang_str, path); if (strfield(iso, lang_str)) { found = i; @@ -1558,10 +1682,11 @@ static int LANG_SEARCH(char* path, char* iso) { return found; } -static int LANG_LIST(char* path, char* buffer) { +static int LANG_LIST(char *path, char *buffer) { char lang_str[1024]; int i = 0; - int curr_lng=LANG_T(path, -1); + int curr_lng = LANG_T(path, -1); + buffer[0] = '\0'; do { QLANG_T(i); @@ -1579,73 +1704,70 @@ static int LANG_LIST(char* path, char* buffer) { } static int QLANG_T(int l) { - static int lng=0; - if (l>=0) { - lng=l; + static int lng = 0; + + if (l >= 0) { + lng = l; } - return lng; // 0=default (english) + return lng; // 0=default (english) } -static char* LANGSEL(char* name) { +static char *LANGSEL(char *name) { intptr_t adr = 0; + if (NewLangStr) - if (!inthash_read(NewLangStr,name,&adr)) - adr=0; + if (!inthash_read(NewLangStr, name, &adr)) + adr = 0; if (adr) { - return (char*)adr; + return (char *) adr; } return ""; } -static char* LANGINTKEY(char* name) { - intptr_t adr=0; +static char *LANGINTKEY(char *name) { + intptr_t adr = 0; + if (NewLangStrKeys) - if (!inthash_read(NewLangStrKeys,name,&adr)) - adr=0; + if (!inthash_read(NewLangStrKeys, name, &adr)) + adr = 0; if (adr) { - return (char*)adr; + return (char *) adr; } return ""; } - - - /* *** Various functions *** */ - - - - -static int recv_bl(T_SOC soc, void* buffer, size_t len, int timeout) { +static int recv_bl(T_SOC soc, void *buffer, size_t len, int timeout) { if (check_readinput_t(soc, timeout)) { int n = 1; size_t size = len; size_t offs = 0; + while(n > 0 && size > 0) { - n = recv(soc, ((char*)buffer) + offs, (int) size, 0); + n = recv(soc, ((char *) buffer) + offs, (int) size, 0); if (n > 0) { offs += n; size -= n; } } - return (int)offs; + return (int) offs; } return -1; } - // check if data is available -static int check_readinput(htsblk* r) { +static int check_readinput(htsblk * r) { if (r->soc != INVALID_SOCKET) { - fd_set fds; // poll structures + fd_set fds; // poll structures struct timeval tv; // structure for select + FD_ZERO(&fds); - FD_SET(r->soc,&fds); - tv.tv_sec=0; - tv.tv_usec=0; - select(r->soc + 1,&fds,NULL,NULL,&tv); - if (FD_ISSET(r->soc,&fds)) + FD_SET(r->soc, &fds); + tv.tv_sec = 0; + tv.tv_usec = 0; + select(r->soc + 1, &fds, NULL, NULL, &tv); + if (FD_ISSET(r->soc, &fds)) return 1; else return 0; @@ -1653,7 +1775,6 @@ static int check_readinput(htsblk* r) { return 0; } - /*int strfield(const char* f,const char* s) { int r=0; while (streql(*f,*s) && ((*f)!=0) && ((*s)!=0)) { f++; s++; r++; } @@ -1664,4 +1785,3 @@ static int check_readinput(htsblk* r) { }*/ /* same, except + */ - diff --git a/src/htsserver.h b/src/htsserver.h index da176f5..29a8bb9 100644 --- a/src/htsserver.h +++ b/src/htsserver.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Mini-server */ /* Author: Xavier Roche */ @@ -37,19 +34,18 @@ Please visit our Website: http://www.httrack.com // Fichier intercepteur d'URL .h #ifndef HTS_SERVER_DEFH -#define HTS_SERVER_DEFH +#define HTS_SERVER_DEFH #include "htsnet.h" /* String */ #include "htsstrings.h" - // Fonctions -void socinput(T_SOC soc,char* s,int max); -T_SOC smallserver_init_std(int* port_prox,char* adr_prox,int defaultPort); -T_SOC smallserver_init(int* port,char* adr); -int smallserver(T_SOC soc,char* url,char* method,char* data, char* path); +void socinput(T_SOC soc, char *s, int max); +T_SOC smallserver_init_std(int *port_prox, char *adr_prox, int defaultPort); +T_SOC smallserver_init(int *port, char *adr); +int smallserver(T_SOC soc, char *url, char *method, char *data, char *path); #define CATCH_RESPONSE \ "HTTP/1.0 200 OK\r\n"\ @@ -90,67 +86,76 @@ extern httrackp *global_opt; #define is_quote(c) ( ((c)=='\"') || ((c)=='\'') ) #define is_retorsep(c) ( ((c)==10) || ((c)==13) || ((c)==9) ) -extern int smallserver_setkey(char* key, char* value); -extern int smallserver_setkeyint(char* key, LLint value); -extern int smallserver_setkeyarr(char* key, int id, char* key2, char* value); - +extern int smallserver_setkey(char *key, char *value); +extern int smallserver_setkeyint(char *key, LLint value); +extern int smallserver_setkeyarr(char *key, int id, char *key2, char *value); /* Language files */ -static int htslang_load(char* limit_to, char* apppath); -static void conv_printf(char* from,char* to); +static int htslang_load(char *limit_to, char *apppath); +static void conv_printf(char *from, char *to); static void LANG_DELETE(void); -static void LANG_INIT(char* path); -static int LANG_T(char* path, int l); +static void LANG_INIT(char *path); +static int LANG_T(char *path, int l); static int QLANG_T(int l); -static char* LANGSEL(char* name); -static char* LANGINTKEY(char* name); -static int LANG_SEARCH(char* path, char* iso); -static int LANG_LIST(char* path, char* buffer); +static char *LANGSEL(char *name); +static char *LANGINTKEY(char *name); +static int LANG_SEARCH(char *path, char *iso); +static int LANG_LIST(char *path, char *buffer); int htslang_init(void); int htslang_uninit(void); /* Static definitions */ -static char* gethomedir(void); -static int linput_cpp(FILE* fp,char* s,int max); -static int linput_trim(FILE* fp,char* s,int max); -static int fexist(const char* s); -static int linput(FILE* fp,char* s,int max); +static char *gethomedir(void); +static int linput_cpp(FILE * fp, char *s, int max); +static int linput_trim(FILE * fp, char *s, int max); +static int fexist(const char *s); +static int linput(FILE * fp, char *s, int max); -static int linputsoc(T_SOC soc, char* s, int max) { +static int linputsoc(T_SOC soc, char *s, int max) { int c; - int j=0; + int j = 0; + do { unsigned char ch; + if (recv(soc, &ch, 1, 0) == 1) { c = ch; } else { c = EOF; } - if (c!=EOF) { - switch(c) { - case 13: break; // sauter CR - case 10: c=-1; break; - case 9: case 12: break; // sauter ces caractères - default: s[j++]=(char) c; break; + if (c != EOF) { + switch (c) { + case 13: + break; // sauter CR + case 10: + c = -1; + break; + case 9: + case 12: + break; // sauter ces caractères + default: + s[j++] = (char) c; + break; } } - } while((c!=-1) && (c!=EOF) && (j<(max-1))); - s[j]='\0'; + } while((c != -1) && (c != EOF) && (j < (max - 1))); + s[j] = '\0'; return j; } static int check_readinput_t(T_SOC soc, int timeout) { if (soc != INVALID_SOCKET) { - fd_set fds; // poll structures + fd_set fds; // poll structures struct timeval tv; // structure for select + FD_ZERO(&fds); - FD_SET(soc,&fds); - tv.tv_sec=timeout; - tv.tv_usec=0; - select(soc + 1,&fds,NULL,NULL,&tv); - if (FD_ISSET(soc,&fds)) + FD_SET(soc, &fds); + tv.tv_sec = timeout; + tv.tv_usec = 0; + select(soc + 1, &fds, NULL, NULL, &tv); + if (FD_ISSET(soc, &fds)) return 1; else return 0; @@ -158,38 +163,42 @@ static int check_readinput_t(T_SOC soc, int timeout) { return 0; } -static int linputsoc_t(T_SOC soc, char* s, int max, int timeout) { +static int linputsoc_t(T_SOC soc, char *s, int max, int timeout) { if (check_readinput_t(soc, timeout)) { return linputsoc(soc, s, max); } return -1; } -static char* gethomedir(void) { - char* home = getenv( "HOME" ); +static char *gethomedir(void) { + char *home = getenv("HOME"); + if (home) return home; else return "."; } -static int linput_cpp(FILE* fp,char* s,int max) { - int rlen=0; - s[0]='\0'; +static int linput_cpp(FILE * fp, char *s, int max) { + int rlen = 0; + + s[0] = '\0'; do { int ret; - if (rlen>0) - if (s[rlen-1]=='\\') - s[--rlen]='\0'; // couper \ final + + if (rlen > 0) + if (s[rlen - 1] == '\\') + s[--rlen] = '\0'; // couper \ final // lire ligne - ret=linput_trim(fp,s+rlen,max-rlen); - if (ret>0) - rlen+=ret; - } while((s[max(rlen-1,0)]=='\\') && (rlen<max)); + ret = linput_trim(fp, s + rlen, max - rlen); + if (ret > 0) + rlen += ret; + } while((s[max(rlen - 1, 0)] == '\\') && (rlen < max)); return rlen; } -static int fexist(const char* s) { +static int fexist(const char *s) { struct stat st; + memset(&st, 0, sizeof(st)); if (stat(s, &st) == 0) { if (S_ISREG(st.st_mode)) { @@ -197,45 +206,56 @@ static int fexist(const char* s) { } } return 0; -} -static int linput(FILE* fp,char* s,int max) { +} +static int linput(FILE * fp, char *s, int max) { int c; - int j=0; + int j = 0; + do { - c=fgetc(fp); - if (c!=EOF) { - switch(c) { - case 13: break; // sauter CR - case 10: c=-1; break; - case 0: case 9: case 12: break; // sauter ces caractères - default: s[j++]=(char) c; break; + c = fgetc(fp); + if (c != EOF) { + switch (c) { + case 13: + break; // sauter CR + case 10: + c = -1; + break; + case 0: + case 9: + case 12: + break; // sauter ces caractères + default: + s[j++] = (char) c; + break; } } - } while((c!=-1) && (c!=EOF) && (j<(max-1))); - s[j]='\0'; + } while((c != -1) && (c != EOF) && (j < (max - 1))); + s[j] = '\0'; return j; } -static int linput_trim(FILE* fp,char* s,int max) { - int rlen=0; - char* ls=(char*) malloct(max+2); - s[0]='\0'; +static int linput_trim(FILE * fp, char *s, int max) { + int rlen = 0; + char *ls = (char *) malloct(max + 2); + + s[0] = '\0'; if (ls) { - char* a; + char *a; + // lire ligne - rlen=linput(fp,ls,max); + rlen = linput(fp, ls, max); if (rlen) { // sauter espaces et tabs en fin - while( (rlen>0) && is_realspace(ls[max(rlen-1,0)]) ) - ls[--rlen]='\0'; + while((rlen > 0) && is_realspace(ls[max(rlen - 1, 0)])) + ls[--rlen] = '\0'; // sauter espaces en début - a=ls; - while((rlen>0) && ((*a==' ') || (*a=='\t'))) { + a = ls; + while((rlen > 0) && ((*a == ' ') || (*a == '\t'))) { a++; rlen--; } - if (rlen>0) { - memcpy(s,a,rlen); // can copy \0 chars - s[rlen]='\0'; + if (rlen > 0) { + memcpy(s, a, rlen); // can copy \0 chars + s[rlen] = '\0'; } } // @@ -245,57 +265,60 @@ static int linput_trim(FILE* fp,char* s,int max) { } static int ehexh(char c) { - if ((c>='0') && (c<='9')) return c-'0'; - if ((c>='a') && (c<='f')) c-=('a'-'A'); - if ((c>='A') && (c<='F')) return (c-'A'+10); + if ((c >= '0') && (c <= '9')) + return c - '0'; + if ((c >= 'a') && (c <= 'f')) + c -= ('a' - 'A'); + if ((c >= 'A') && (c <= 'F')) + return (c - 'A' + 10); return 0; } -static int ehex(char* s) { - return 16*ehexh(*s)+ehexh(*(s+1)); +static int ehex(char *s) { + return 16 * ehexh(*s) + ehexh(*(s + 1)); } -static void unescapehttp(char* s, String* tempo) { +static void unescapehttp(char *s, String * tempo) { int i; - for (i=0;i<(int) strlen(s);i++) { - if (s[i]=='%' && s[i+1]=='%') { + + for(i = 0; i < (int) strlen(s); i++) { + if (s[i] == '%' && s[i + 1] == '%') { i++; StringAddchar(*tempo, '%'); - } else if (s[i]=='%') { + } else if (s[i] == '%') { char hc; + i++; - hc = (char) ehex(s+i); + hc = (char) ehex(s + i); StringAddchar(*tempo, (char) hc); - i++; // sauter 2 caractères finalement - } - else if (s[i]=='+') { + i++; // sauter 2 caractères finalement + } else if (s[i] == '+') { StringAddchar(*tempo, ' '); - } - else + } else StringAddchar(*tempo, s[i]); } } -static void unescapeini(char* s, String* tempo) { +static void unescapeini(char *s, String * tempo) { int i; - char lastc=0; - for (i=0;i<(int) strlen(s);i++) { - if (s[i]=='%' && s[i+1]=='%') { + char lastc = 0; + + for(i = 0; i < (int) strlen(s); i++) { + if (s[i] == '%' && s[i + 1] == '%') { i++; StringAddchar(*tempo, lastc = '%'); - } else if (s[i]=='%') { + } else if (s[i] == '%') { char hc; + i++; - hc = (char) ehex(s+i); + hc = (char) ehex(s + i); if (!is_retorsep(hc) || !is_retorsep(lastc)) { StringAddchar(*tempo, lastc = (char) hc); } - i++; // sauter 2 caractères finalement - } - else + i++; // sauter 2 caractères finalement + } else StringAddchar(*tempo, lastc = s[i]); } } #endif - diff --git a/src/htsstrings.h b/src/htsstrings.h index 1102219..d4c3641 100755 --- a/src/htsstrings.h +++ b/src/htsstrings.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Strings */ /* Author: Xavier Roche */ @@ -37,7 +34,7 @@ Please visit our Website: http://www.httrack.com /* Safer Strings ; standalone .h library */ #ifndef HTS_STRINGS_DEFSTATIC -#define HTS_STRINGS_DEFSTATIC +#define HTS_STRINGS_DEFSTATIC /* System definitions. */ #include <string.h> @@ -63,7 +60,7 @@ typedef struct String String; #ifndef HTS_DEF_STRUCT_String #define HTS_DEF_STRUCT_String struct String { - char* buffer_; + char *buffer_; size_t length_; size_t capacity_; }; @@ -135,8 +132,8 @@ struct String { #define StringRoom(BLK, SIZE) StringRoomTotal(BLK, StringLength(BLK) + (SIZE) + 1) /** Return the RW buffer for a strcat() operation of at most SIZE characters. **/ -#define StringBuffN(BLK, SIZE) StringBuffN_(&(BLK), SIZE) -HTS_STATIC char* StringBuffN_(String* blk, int size) { +#define StringBuffN(BLK, SIZE) StringBuffN_(&(BLK), SIZE) +HTS_STATIC char *StringBuffN_(String * blk, int size) { StringRoom(*blk, size); return StringBuffRW(*blk); } @@ -212,8 +209,9 @@ The pointer _MUST_ be compatible with STRING_REALLOC() and STRING_FREE() **/ } while(0) /** Acquire a string ; it's the client's responsability to free() it **/ -HTS_STATIC char* StringAcquire(String* blk) { - char* buff = StringBuffRW(*blk); +HTS_STATIC char *StringAcquire(String * blk) { + char *buff = StringBuffRW(*blk); + StringBuffRW(*blk) = NULL; StringCapacity(*blk) = 0; StringLength(*blk) = 0; @@ -221,14 +219,15 @@ HTS_STATIC char* StringAcquire(String* blk) { } /** Clone a string. **/ -HTS_STATIC String StringDup(const String* src) { +HTS_STATIC String StringDup(const String * src) { String s = STRING_EMPTY; + StringMemcat(s, StringBuff(*src), StringLength(*src)); return s; } /** Attach a string using a pointer. **/ -HTS_STATIC void StringAttach(String* blk, char** str) { +HTS_STATIC void StringAttach(String * blk, char **str) { StringFree(*blk); if (str != NULL && *str != NULL) { StringBuffRW(*blk) = *str; diff --git a/src/htsthread.c b/src/htsthread.c index 58f2917..c0feb8d 100644 --- a/src/htsthread.c +++ b/src/htsthread.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Threads */ /* Author: Xavier Roche */ @@ -50,16 +47,17 @@ Please visit our Website: http://www.httrack.com static int process_chain = 0; static htsmutex process_chain_mutex = HTSMUTEX_INIT; -HTSEXT_API void htsthread_wait(void ) { +HTSEXT_API void htsthread_wait(void) { htsthread_wait_n(0); } HTSEXT_API void htsthread_wait_n(int n_wait) { #if USE_BEGINTHREAD int wait = 0; + do { hts_mutexlock(&process_chain_mutex); - wait = (process_chain > n_wait ); + wait = (process_chain > n_wait); hts_mutexrelease(&process_chain_mutex); if (wait) Sleep(100); @@ -68,7 +66,7 @@ HTSEXT_API void htsthread_wait_n(int n_wait) { } /* ensure initialized */ -HTSEXT_API void htsthread_init(void ) { +HTSEXT_API void htsthread_init(void) { #if USE_BEGINTHREAD #if (defined(_DEBUG) || defined(DEBUG)) assertf(process_chain == 0); @@ -79,7 +77,7 @@ HTSEXT_API void htsthread_init(void ) { #endif } -HTSEXT_API void htsthread_uninit(void ) { +HTSEXT_API void htsthread_uninit(void) { htsthread_wait(); #if USE_BEGINTHREAD hts_mutexfree(&process_chain_mutex); @@ -88,18 +86,19 @@ HTSEXT_API void htsthread_uninit(void ) { typedef struct hts_thread_s { void *arg; - void (*fun)(void *arg); + void (*fun) (void *arg); } hts_thread_s; #ifdef _WIN32 static unsigned int __stdcall hts_entry_point(void *tharg) #else -static void* hts_entry_point(void *tharg) +static void *hts_entry_point(void *tharg) #endif { - hts_thread_s *s_args = (hts_thread_s*) tharg; - void * const arg = s_args->arg; - void (*fun)(void *arg) = s_args->fun; + hts_thread_s *s_args = (hts_thread_s *) tharg; + void *const arg = s_args->arg; + void (*fun) (void *arg) = s_args->fun; + free(tharg); hts_mutexlock(&process_chain_mutex); @@ -122,16 +121,17 @@ static void* hts_entry_point(void *tharg) } /* create a thread */ -HTSEXT_API int hts_newthread( void (*fun)(void *arg), void *arg) -{ +HTSEXT_API int hts_newthread(void (*fun) (void *arg), void *arg) { hts_thread_s *s_args = malloc(sizeof(hts_thread_s)); + assertf(s_args != NULL); s_args->arg = arg; s_args->fun = fun; #ifdef _WIN32 { unsigned int idt; - HANDLE handle = (HANDLE) _beginthreadex(NULL, 0, hts_entry_point, s_args, 0, &idt); + HANDLE handle = + (HANDLE) _beginthreadex(NULL, 0, hts_entry_point, s_args, 0, &idt); if (handle == 0) { free(s_args); return -1; @@ -142,13 +142,15 @@ HTSEXT_API int hts_newthread( void (*fun)(void *arg), void *arg) } #else { - const size_t stackSize = 1024*1024*8; + const size_t stackSize = 1024 * 1024 * 8; pthread_attr_t attr; pthread_t handle = 0; int retcode; + if (pthread_attr_init(&attr) != 0 - || pthread_attr_setstacksize(&attr, stackSize) != 0 - || (retcode = pthread_create(&handle, &attr, hts_entry_point, s_args)) != 0) { + || pthread_attr_setstacksize(&attr, stackSize) != 0 + || (retcode = + pthread_create(&handle, &attr, hts_entry_point, s_args)) != 0) { free(s_args); return -1; } else { @@ -165,8 +167,9 @@ HTSEXT_API int hts_newthread( void (*fun)(void *arg), void *arg) /* Note: new 3.41 cleaned up functions. */ -HTSEXT_API void hts_mutexinit(htsmutex* mutex) { - htsmutex_s* smutex = malloct(sizeof(htsmutex_s)); +HTSEXT_API void hts_mutexinit(htsmutex * mutex) { + htsmutex_s *smutex = malloct(sizeof(htsmutex_s)); + #ifdef _WIN32 smutex->handle = CreateMutex(NULL, FALSE, NULL); #else @@ -175,21 +178,21 @@ HTSEXT_API void hts_mutexinit(htsmutex* mutex) { *mutex = smutex; } -HTSEXT_API void hts_mutexfree(htsmutex* mutex) { +HTSEXT_API void hts_mutexfree(htsmutex * mutex) { if (mutex != NULL && *mutex != NULL) { #ifdef _WIN32 CloseHandle((*mutex)->handle); #else - pthread_mutex_destroy(& ( (*mutex)->handle ) ); + pthread_mutex_destroy(&((*mutex)->handle)); #endif freet(*mutex); *mutex = NULL; } } -HTSEXT_API void hts_mutexlock(htsmutex* mutex) { +HTSEXT_API void hts_mutexlock(htsmutex * mutex) { assertf(mutex != NULL); - if (*mutex == HTSMUTEX_INIT) { /* must be initialized */ + if (*mutex == HTSMUTEX_INIT) { /* must be initialized */ hts_mutexinit(mutex); } assertf(*mutex != NULL); @@ -201,7 +204,7 @@ HTSEXT_API void hts_mutexlock(htsmutex* mutex) { #endif } -HTSEXT_API void hts_mutexrelease(htsmutex* mutex) { +HTSEXT_API void hts_mutexrelease(htsmutex * mutex) { assertf(mutex != NULL && *mutex != NULL); #ifdef _WIN32 assert((*mutex)->handle != NULL); diff --git a/src/htsthread.h b/src/htsthread.h index a8c792e..e609ced 100644 --- a/src/htsthread.h +++ b/src/htsthread.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Threads */ /* Author: Xavier Roche */ @@ -64,24 +61,25 @@ typedef struct htsmutex_s htsmutex_s, *htsmutex; struct htsmutex_s { HANDLE handle; }; -#else /* #ifdef _WIN32 */ +#else /* #ifdef _WIN32 */ struct htsmutex_s { pthread_mutex_t handle; }; -#endif /* #ifdef _WIN32 */ +#endif /* #ifdef _WIN32 */ /* Library internal definictions */ -HTSEXT_API int hts_newthread( void (*fun)(void *arg), void *arg); +HTSEXT_API int hts_newthread(void (*fun) (void *arg), void *arg); + #ifndef HTTRACK_DEFLIB -HTSEXT_API void htsthread_wait(void ); +HTSEXT_API void htsthread_wait(void); #endif HTSEXT_API void htsthread_wait_n(int n_wait); /* Locking functions */ -HTSEXT_API void hts_mutexinit(htsmutex* mutex); -HTSEXT_API void hts_mutexfree(htsmutex* mutex); -HTSEXT_API void hts_mutexlock(htsmutex* mutex); -HTSEXT_API void hts_mutexrelease(htsmutex* mutex); +HTSEXT_API void hts_mutexinit(htsmutex * mutex); +HTSEXT_API void hts_mutexfree(htsmutex * mutex); +HTSEXT_API void hts_mutexlock(htsmutex * mutex); +HTSEXT_API void hts_mutexrelease(htsmutex * mutex); #ifdef HTS_INTERNAL_BYTECODE /* Thread initialization */ diff --git a/src/htstools.c b/src/htstools.c index f6da19e..885bdc8 100644 --- a/src/htstools.c +++ b/src/htstools.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* various tools (filename analyzing ..) */ @@ -65,8 +62,8 @@ struct find_handle_struct { }; #else struct find_handle_struct { - DIR * hdir; - struct dirent* dirp; + DIR *hdir; + struct dirent *dirp; STRUCT_STAT filestat; char path[2048]; }; @@ -76,43 +73,45 @@ struct find_handle_struct { typedef struct topindex_chain topindex_chain; #endif struct topindex_chain { - int level; /* sort level */ - char* category; /* category */ - char name[2048]; /* path */ - struct topindex_chain* next; /* next element */ + int level; /* sort level */ + char *category; /* category */ + char name[2048]; /* path */ + struct topindex_chain *next; /* next element */ }; - /* Tools */ static int ehexh(char c) { - if ((c>='0') && (c<='9')) return c-'0'; - if ((c>='a') && (c<='f')) c-=('a'-'A'); - if ((c>='A') && (c<='F')) return (c-'A'+10); + if ((c >= '0') && (c <= '9')) + return c - '0'; + if ((c >= 'a') && (c <= 'f')) + c -= ('a' - 'A'); + if ((c >= 'A') && (c <= 'F')) + return (c - 'A' + 10); return 0; } -static int ehex(char* s) { - return 16*ehexh(*s)+ehexh(*(s+1)); +static int ehex(char *s) { + return 16 * ehexh(*s) + ehexh(*(s + 1)); } -static void unescapehttp(char* s, String* tempo) { +static void unescapehttp(char *s, String * tempo) { int i; - for (i=0;i<(int) strlen(s);i++) { - if (s[i]=='%' && s[i+1]=='%') { + + for(i = 0; i < (int) strlen(s); i++) { + if (s[i] == '%' && s[i + 1] == '%') { i++; StringAddchar(*tempo, '%'); - } else if (s[i]=='%') { + } else if (s[i] == '%') { char hc; + i++; - hc = (char) ehex(s+i); + hc = (char) ehex(s + i); StringAddchar(*tempo, (char) hc); - i++; // sauter 2 caractères finalement - } - else if (s[i]=='+') { + i++; // sauter 2 caractères finalement + } else if (s[i] == '+') { StringAddchar(*tempo, ' '); - } - else + } else StringAddchar(*tempo, s[i]); } } @@ -122,223 +121,247 @@ static void unescapehttp(char* s, String* tempo) { // 0 : ok // -1 : erreur // -2 : protocole non supporté (ftp) -int ident_url_relatif(const char *lien,const char* origin_adr,const char* origin_fil,char* adr,char* fil) { - int ok=0; - int scheme=0; +int ident_url_relatif(const char *lien, const char *origin_adr, + const char *origin_fil, char *adr, char *fil) { + int ok = 0; + int scheme = 0; - adr[0]='\0'; fil[0]='\0'; //effacer buffers + adr[0] = '\0'; + fil[0] = '\0'; //effacer buffers // lien non vide! - if (strnotempty(lien)==0) return -1; // erreur! + if (strnotempty(lien) == 0) + return -1; // erreur! // Scheme? { - const char* a=lien; - while (isalpha((unsigned char)*a)) + const char *a = lien; + + while(isalpha((unsigned char) *a)) a++; if (*a == ':') - scheme=1; + scheme = 1; } // filtrer les parazites (mailto & cie) // scheme+authority (//) - if ( - (strfield(lien,"http://")) // scheme+// - || (strfield(lien,"file://")) // scheme+// - || (strncmp(lien,"//",2)==0) // // sans scheme (-> default) - ) { - if (ident_url_absolute(lien,adr,fil)==-1) { - ok=-1; // erreur URL + if ((strfield(lien, "http://")) // scheme+// + || (strfield(lien, "file://")) // scheme+// + || (strncmp(lien, "//", 2) == 0) // // sans scheme (-> default) + ) { + if (ident_url_absolute(lien, adr, fil) == -1) { + ok = -1; // erreur URL } - } - else if (strfield(lien,"ftp://")) { + } else if (strfield(lien, "ftp://")) { // Note: ftp:foobar.gif is not valid - if (ftp_available()) { // ftp supporté - if (ident_url_absolute(lien,adr,fil)==-1) { - ok=-1; // erreur URL + if (ftp_available()) { // ftp supporté + if (ident_url_absolute(lien, adr, fil) == -1) { + ok = -1; // erreur URL } } else { - ok=-2; // non supporté + ok = -2; // non supporté } #if HTS_USEMMS - } else if (strfield(lien,"mms://")) { - if (ident_url_absolute(lien,adr,fil)==-1) { - ok=-1; // erreur URL - } + } else if (strfield(lien, "mms://")) { + if (ident_url_absolute(lien, adr, fil) == -1) { + ok = -1; // erreur URL + } #endif #if HTS_USEOPENSSL - } else if (strfield(lien,"https://")) { + } else if (strfield(lien, "https://")) { if (SSL_is_available) { // Note: ftp:foobar.gif is not valid - if (ident_url_absolute(lien,adr,fil)==-1) { - ok=-1; // erreur URL + if (ident_url_absolute(lien, adr, fil) == -1) { + ok = -1; // erreur URL } } else { - ok=-1; + ok = -1; } #endif - } else if ((scheme) && ( - (!strfield(lien,"http:")) - && (!strfield(lien,"https:")) - && (!strfield(lien,"ftp:")) + } else if ((scheme) && ((!strfield(lien, "http:")) + && (!strfield(lien, "https:")) + && (!strfield(lien, "ftp:")) #if HTS_USEMMS - && (!strfield(lien,"mms:")) + && (!strfield(lien, "mms:")) #endif - )) { - ok=-1; // unknown scheme - } else { // c'est un lien relatif + )) { + ok = -1; // unknown scheme + } else { // c'est un lien relatif // On forme l'URL complète à partie de l'url actuelle // et du chemin actuel si besoin est. - + // copier adresse - if (((int) strlen(origin_adr)<HTS_URLMAXSIZE) && ((int) strlen(origin_fil)<HTS_URLMAXSIZE) && ((int) strlen(lien)<HTS_URLMAXSIZE)) { + if (((int) strlen(origin_adr) < HTS_URLMAXSIZE) + && ((int) strlen(origin_fil) < HTS_URLMAXSIZE) + && ((int) strlen(lien) < HTS_URLMAXSIZE)) { /* patch scheme if necessary */ - if (strfield(lien,"http:")) { - lien+=5; - strcpybuff(adr, jump_protocol(origin_adr)); // même adresse ; protocole vide (http) - } else if (strfield(lien,"https:")) { - lien+=6; - strcpybuff(adr, "https://"); // même adresse forcée en https + if (strfield(lien, "http:")) { + lien += 5; + strcpybuff(adr, jump_protocol(origin_adr)); // même adresse ; protocole vide (http) + } else if (strfield(lien, "https:")) { + lien += 6; + strcpybuff(adr, "https://"); // même adresse forcée en https strcatbuff(adr, jump_protocol(origin_adr)); - } else if (strfield(lien,"ftp:")) { - lien+=4; - strcpybuff(adr, "ftp://"); // même adresse forcée en ftp + } else if (strfield(lien, "ftp:")) { + lien += 4; + strcpybuff(adr, "ftp://"); // même adresse forcée en ftp strcatbuff(adr, jump_protocol(origin_adr)); #if HTS_USEMMS - } else if (strfield(lien,"mms:")) { - lien+=4; - strcpybuff(adr, "mms://"); // même adresse forcée en ftp + } else if (strfield(lien, "mms:")) { + lien += 4; + strcpybuff(adr, "mms://"); // même adresse forcée en ftp strcatbuff(adr, jump_protocol(origin_adr)); #endif } else { - strcpybuff(adr,origin_adr); // même adresse ; et même éventuel protocole + strcpybuff(adr, origin_adr); // même adresse ; et même éventuel protocole } - - if (*lien!='/') { // sinon c'est un lien absolu + + if (*lien != '/') { // sinon c'est un lien absolu if (*lien == '\0') { - strcpybuff(fil,origin_fil); - } else if (*lien == '?') { // example: a href="?page=2" - char* a; - strcpybuff(fil,origin_fil); - a=strchr(fil,'?'); - if (a) *a='\0'; - strcatbuff(fil,lien); + strcpybuff(fil, origin_fil); + } else if (*lien == '?') { // example: a href="?page=2" + char *a; + + strcpybuff(fil, origin_fil); + a = strchr(fil, '?'); + if (a) + *a = '\0'; + strcatbuff(fil, lien); } else { - const char *a=strchr(origin_fil,'?'); - if (a == NULL) a=origin_fil+strlen(origin_fil); - while((*a!='/') && ( a > origin_fil) ) a--; - if (*a=='/') { // ok on a un '/' - if ( (((int) (a - origin_fil))+1+strlen(lien)) < HTS_URLMAXSIZE) { + const char *a = strchr(origin_fil, '?'); + + if (a == NULL) + a = origin_fil + strlen(origin_fil); + while((*a != '/') && (a > origin_fil)) + a--; + if (*a == '/') { // ok on a un '/' + if ((((int) (a - origin_fil)) + 1 + strlen(lien)) < HTS_URLMAXSIZE) { // copier chemin - strncpy(fil,origin_fil,((int) (a - origin_fil))+1); - *(fil + ((int) (a - origin_fil))+1)='\0'; - + strncpy(fil, origin_fil, ((int) (a - origin_fil)) + 1); + *(fil + ((int) (a - origin_fil)) + 1) = '\0'; + // copier chemin relatif - if (((int) strlen(fil)+(int) strlen(lien)) < HTS_URLMAXSIZE) { - strcatbuff(fil,lien + ((*lien=='/')?1:0) ); + if (((int) strlen(fil) + (int) strlen(lien)) < HTS_URLMAXSIZE) { + strcatbuff(fil, lien + ((*lien == '/') ? 1 : 0)); // simplifier url pour les ../ fil_simplifie(fil); } else - ok=-1; // erreur - } else { // erreur - ok=-1; // erreur URL + ok = -1; // erreur + } else { // erreur + ok = -1; // erreur URL } - } else { // erreur - ok=-1; // erreur URL + } else { // erreur + ok = -1; // erreur URL } } - } else { // chemin absolu + } else { // chemin absolu // copier chemin directement - strcatbuff(fil,lien); + strcatbuff(fil, lien); fil_simplifie(fil); - } // *lien!='/' + } // *lien!='/' } else - ok=-1; - - } // test news: etc. + ok = -1; + + } // test news: etc. // case insensitive pour adresse { - char *a=jump_identification(adr); + char *a = jump_identification(adr); + while(*a) { - if ((*a>='A') && (*a<='Z')) - *a+='a'-'A'; + if ((*a >= 'A') && (*a <= 'Z')) + *a += 'a' - 'A'; a++; } } - + return ok; } - - - - // créer dans s, à partir du chemin courant curr_fil, le lien vers link (absolu) // un ident_url_relatif a déja été fait avant, pour que link ne soit pas un chemin relatif -int lienrelatif(char* s,const char* link,const char* curr_fil) { - char BIGSTK _curr[HTS_URLMAXSIZE*2]; - char BIGSTK newcurr_fil[HTS_URLMAXSIZE*2],newlink[HTS_URLMAXSIZE*2]; - char* curr; +int lienrelatif(char *s, const char *link, const char *curr_fil) { + char BIGSTK _curr[HTS_URLMAXSIZE * 2]; + char BIGSTK newcurr_fil[HTS_URLMAXSIZE * 2], newlink[HTS_URLMAXSIZE * 2]; + char *curr; + //int n=0; - char* a; - int slash=0; + char *a; + int slash = 0; + // - newcurr_fil[0]='\0'; newlink[0]='\0'; + newcurr_fil[0] = '\0'; + newlink[0] = '\0'; // // patch: éliminer les ? (paramètres) sinon bug { - const char* a; - if ( (a=strchr(curr_fil,'?')) ) { - strncatbuff(newcurr_fil,curr_fil,(int) (a - curr_fil)); + const char *a; + + if ((a = strchr(curr_fil, '?'))) { + strncatbuff(newcurr_fil, curr_fil, (int) (a - curr_fil)); curr_fil = newcurr_fil; } - if ( (a=strchr(link,'?')) ) { - strncatbuff(newlink,link,(int) (a - link)); + if ((a = strchr(link, '?'))) { + strncatbuff(newlink, link, (int) (a - link)); link = newlink; } } // recopier uniquement le chemin courant - curr=_curr; - strcpybuff(curr,curr_fil); - if ((a=strchr(curr,'?'))==NULL) // couper au ? (params) - a=curr+strlen(curr)-1; // pas de params: aller à la fin - while((*a!='/') && ( a> curr)) a--; // chercher dernier / du chemin courant - if (*a=='/') *(a+1)='\0'; // couper dernier / - + curr = _curr; + strcpybuff(curr, curr_fil); + if ((a = strchr(curr, '?')) == NULL) // couper au ? (params) + a = curr + strlen(curr) - 1; // pas de params: aller à la fin + while((*a != '/') && (a > curr)) + a--; // chercher dernier / du chemin courant + if (*a == '/') + *(a + 1) = '\0'; // couper dernier / + // "effacer" s - s[0]='\0'; - + s[0] = '\0'; + // sauter ce qui est commun aux 2 chemins { const char *l; - if (*link=='/') link++; // sauter slash - if (*curr=='/') curr++; - l=link; + + if (*link == '/') + link++; // sauter slash + if (*curr == '/') + curr++; + l = link; //c=curr; // couper ce qui est commun - while ((streql(*link,*curr)) && (*link!=0)) {link++; curr++; } + while((streql(*link, *curr)) && (*link != 0)) { + link++; + curr++; + } // mais on veut un répertoirer entier! // si on a /toto/.. et /toto2/.. on ne veut pas sauter /toto ! - while(((*link!='/') || (*curr!='/')) && ( link > l)) { link--; curr--; } + while(((*link != '/') || (*curr != '/')) && (link > l)) { + link--; + curr--; + } //if (*link=='/') link++; //if (*curr=='/') curr++; } - + // calculer la profondeur du répertoire courant et remonter // LES ../ ONT ETE SIMPLIFIES - a=curr; - if (*a=='/') a++; - while(*a) if (*(a++)=='/') strcatbuff(s,"../"); + a = curr; + if (*a == '/') + a++; + while(*a) + if (*(a++) == '/') + strcatbuff(s, "../"); //if (strlen(s)==0) strcatbuff(s,"/"); - if (slash) strcatbuff(s,"/"); // garder absolu!! - + if (slash) + strcatbuff(s, "/"); // garder absolu!! + // on est dans le répertoire de départ, copier - strcatbuff(s,link + ((*link=='/')?1:0) ); + strcatbuff(s, link + ((*link == '/') ? 1 : 0)); /* Security check */ if (strlen(s) >= HTS_URLMAXSIZE) @@ -349,26 +372,28 @@ int lienrelatif(char* s,const char* link,const char* curr_fil) { } /* Is the link absolute (http://www..) or relative (/bar/foo.html) ? */ -int link_has_authority(const char* lien) { - const char* a=lien; - if (isalpha((unsigned char)*a)) { +int link_has_authority(const char *lien) { + const char *a = lien; + + if (isalpha((unsigned char) *a)) { // Skip scheme? - while (isalpha((unsigned char)*a)) + while(isalpha((unsigned char) *a)) a++; if (*a == ':') a++; else return 0; } - if (strncmp(a,"//",2) == 0) + if (strncmp(a, "//", 2) == 0) return 1; return 0; } -int link_has_authorization(const char* lien) { - const char* adr = jump_protocol(lien); - const char* firstslash = strchr(adr, '/'); - const char* detect = strchr(adr, '@'); +int link_has_authorization(const char *lien) { + const char *adr = jump_protocol(lien); + const char *firstslash = strchr(adr, '/'); + const char *detect = strchr(adr, '@'); + if (firstslash) { if (detect) { return (detect < firstslash); @@ -379,154 +404,182 @@ int link_has_authorization(const char* lien) { return 0; } - // conversion chemin de fichier/dossier vers 8-3 ou ISO9660 -void long_to_83(int mode,char* n83,char* save) { - n83[0]='\0'; +void long_to_83(int mode, char *n83, char *save) { + n83[0] = '\0'; while(*save) { - char fn83[256],fnl[256]; - int i=0; - fn83[0]=fnl[0]='\0'; - while((save[i]) && (save[i]!='/')) { fnl[i]=save[i]; i++; } - fnl[i]='\0'; + char fn83[256], fnl[256]; + int i = 0; + + fn83[0] = fnl[0] = '\0'; + while((save[i]) && (save[i] != '/')) { + fnl[i] = save[i]; + i++; + } + fnl[i] = '\0'; // conversion - longfile_to_83(mode,fn83,fnl); - strcatbuff(n83,fn83); + longfile_to_83(mode, fn83, fnl); + strcatbuff(n83, fn83); - save+=i; - if (*save=='/') { strcatbuff(n83,"/"); save++; } + save += i; + if (*save == '/') { + strcatbuff(n83, "/"); + save++; + } } } - // conversion nom de fichier/dossier isolé vers 8-3 ou ISO9660 -void longfile_to_83(int mode,char* n83,char* save) { - int j=0,max=0; +void longfile_to_83(int mode, char *n83, char *save) { + int j = 0, max = 0; int i = 0; char nom[256]; char ext[256]; - nom[0]=ext[0]='\0'; - - switch(mode) { + + nom[0] = ext[0] = '\0'; + + switch (mode) { case 1: - max=8; + max = 8; break; case 2: - max=31; + max = 31; break; default: - max=8; + max = 8; break; } /* No starting . */ if (save[0] == '.') { - save[0]='_'; + save[0] = '_'; } /* No multiple dots */ { - char* last_dot=strrchr(save, '.'); - char* dot; - while((dot=strchr(save, '.'))) { + char *last_dot = strrchr(save, '.'); + char *dot; + + while((dot = strchr(save, '.'))) { *dot = '_'; } if (last_dot) { - *last_dot='.'; + *last_dot = '.'; } } /* - Avoid: (ISO9660, but also suitable for 8-3) - (Thanks to jonat@cellcast.com for te hint) - /:;?\#*~ - 0x00-0x1f and 0x80-0xff - */ - for(i = 0 ; save[i] != 0 ; i++) { + Avoid: (ISO9660, but also suitable for 8-3) + (Thanks to jonat@cellcast.com for te hint) + /:;?\#*~ + 0x00-0x1f and 0x80-0xff + */ + for(i = 0; save[i] != 0; i++) { char a = save[i]; + if (a >= 'a' && a <= 'z') { a -= 'a' - 'A'; - } - else if ( ! ( (a >= 'A' && a <= 'Z') || (a >= '0' && a <= '9') || a == '_' || a == '.') ) { + } else + if (! + ((a >= 'A' && a <= 'Z') || (a >= '0' && a <= '9') || a == '_' + || a == '.')) { a = '_'; } save[i] = a; } - i=j=0; - while((i<max) && (save[j]) && (save[j]!='.')) { - if (save[j]!=' ') { - nom[i]=save[j]; - i++; - } - j++; - } // recopier nom - nom[i]='\0'; - if (save[j]) { // il reste au moins un point - i = (int) strlen(save)-1; - while((i>0) && (save[i]!='.') && (save[i]!='/')) i--; // rechercher dernier . - if (save[i]=='.') { // point! - int j=0; + i = j = 0; + while((i < max) && (save[j]) && (save[j] != '.')) { + if (save[j] != ' ') { + nom[i] = save[j]; + i++; + } + j++; + } // recopier nom + nom[i] = '\0'; + if (save[j]) { // il reste au moins un point + i = (int) strlen(save) - 1; + while((i > 0) && (save[i] != '.') && (save[i] != '/')) + i--; // rechercher dernier . + if (save[i] == '.') { // point! + int j = 0; + i++; - while((j<3) && (save[i]) ) { if (save[i]!=' ') { ext[j]=save[i]; j++; } i++; } - ext[j]='\0'; + while((j < 3) && (save[i])) { + if (save[i] != ' ') { + ext[j] = save[i]; + j++; + } + i++; + } + ext[j] = '\0'; } } // corriger vers 8-3 - n83[0]='\0'; - strncatbuff(n83,nom,max); + n83[0] = '\0'; + strncatbuff(n83, nom, max); if (strnotempty(ext)) { - strcatbuff(n83,"."); - strncatbuff(n83,ext,3); + strcatbuff(n83, "."); + strncatbuff(n83, ext, 3); } } // écrire backblue.gif /* Note: utf-8 */ -int verif_backblue(httrackp* opt, const char* base) { - int* done = &opt->state.verif_backblue_done; - int ret=0; +int verif_backblue(httrackp * opt, const char *base) { + int *done = &opt->state.verif_backblue_done; + int ret = 0; + // - if (!base) { // init - *done=0; + if (!base) { // init + *done = 0; return 0; } - if ( (!*done) - || (fsize_utf8(fconcat(OPT_GET_BUFF(opt), base,"backblue.gif")) != HTS_DATA_BACK_GIF_LEN)) { - FILE* fp = filecreate(&opt->state.strc, fconcat(OPT_GET_BUFF(opt), base,"backblue.gif")); - *done=1; + if ((!*done) + || (fsize_utf8(fconcat(OPT_GET_BUFF(opt), base, "backblue.gif")) != + HTS_DATA_BACK_GIF_LEN)) { + FILE *fp = + filecreate(&opt->state.strc, + fconcat(OPT_GET_BUFF(opt), base, "backblue.gif")); + *done = 1; if (fp) { - if (fwrite(HTS_DATA_BACK_GIF,HTS_DATA_BACK_GIF_LEN,1,fp) != HTS_DATA_BACK_GIF_LEN) - ret=1; + if (fwrite(HTS_DATA_BACK_GIF, HTS_DATA_BACK_GIF_LEN, 1, fp) != + HTS_DATA_BACK_GIF_LEN) + ret = 1; fclose(fp); - usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt), base,"backblue.gif"),"",""); + usercommand(opt, 0, NULL, + fconcat(OPT_GET_BUFF(opt), base, "backblue.gif"), "", ""); } else - ret=1; + ret = 1; // - fp = filecreate(&opt->state.strc, fconcat(OPT_GET_BUFF(opt), base,"fade.gif")); + fp = + filecreate(&opt->state.strc, + fconcat(OPT_GET_BUFF(opt), base, "fade.gif")); if (fp) { - if (fwrite(HTS_DATA_FADE_GIF,HTS_DATA_FADE_GIF_LEN,1,fp) != HTS_DATA_FADE_GIF_LEN) - ret=1; + if (fwrite(HTS_DATA_FADE_GIF, HTS_DATA_FADE_GIF_LEN, 1, fp) != + HTS_DATA_FADE_GIF_LEN) + ret = 1; fclose(fp); - usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt), base,"fade.gif"),"",""); + usercommand(opt, 0, NULL, fconcat(OPT_GET_BUFF(opt), base, "fade.gif"), + "", ""); } else - ret=1; - } + ret = 1; + } return ret; } // flag -int verif_external(httrackp* opt,int nb,int test) { - int* status = &opt->state.verif_external_status; +int verif_external(httrackp * opt, int nb, int test) { + int *status = &opt->state.verif_external_status; + if (!test) - status[nb]=0; // reset + status[nb] = 0; // reset else if (!status[nb]) { - status[nb]=1; + status[nb] = 1; return 1; } return 0; } - // recherche chaîne de type truc<espaces>= // renvoi décalage à effectuer ou 0 si non trouvé /* SECTION OPTIMISEE: @@ -550,86 +603,96 @@ HTS_INLINE int rech_tageq(const char* adr,const char* s) { } */ // Deuxième partie -HTS_INLINE int __rech_tageq(const char* adr,const char* s) { +HTS_INLINE int __rech_tageq(const char *adr, const char *s) { int p; - p=strfield(adr,s); + + p = strfield(adr, s); if (p) { - while(is_space(adr[p])) p++; - if (adr[p]=='=') { - return p+1; + while(is_space(adr[p])) + p++; + if (adr[p] == '=') { + return p + 1; } } return 0; } -HTS_INLINE int rech_tageq_all(const char* adr, const char* s) { - int p; - char quot = 0; - const char *token = NULL; - int s_len = (int) strlen(s); - if (adr == NULL) { - return 0; - } - for(p = 0 ; adr[p] != 0 ; p++) { - if (quot == 0) { - if (adr[p] == '"' || adr[p] == '\'' ) { - quot = adr[p]; - } else if (adr[p] == '=' || is_realspace(adr[p]) ) { - token = NULL; - } else if (adr[p] == '>') { - break; - } else { /* note: bogus for bogus foo = bar */ - if (token == NULL) { - if (strncasecmp(&adr[p], s, s_len) == 0 - && (is_realspace(adr[p + s_len]) || adr[p + s_len] == '=') - ) { - for( p += s_len ; is_realspace(adr[p]) || adr[p] == '=' ; p++ ); - return p; - } - token = &adr[p]; - } - } - } else if (adr[p] == quot) { - quot = 0; - } - } +HTS_INLINE int rech_tageq_all(const char *adr, const char *s) { + int p; + char quot = 0; + const char *token = NULL; + int s_len = (int) strlen(s); + + if (adr == NULL) { + return 0; + } + for(p = 0; adr[p] != 0; p++) { + if (quot == 0) { + if (adr[p] == '"' || adr[p] == '\'') { + quot = adr[p]; + } else if (adr[p] == '=' || is_realspace(adr[p])) { + token = NULL; + } else if (adr[p] == '>') { + break; + } else { /* note: bogus for bogus foo = bar */ + if (token == NULL) { + if (strncasecmp(&adr[p], s, s_len) == 0 + && (is_realspace(adr[p + s_len]) || adr[p + s_len] == '=') + ) { + for(p += s_len; is_realspace(adr[p]) || adr[p] == '='; p++) ; + return p; + } + token = &adr[p]; + } + } + } else if (adr[p] == quot) { + quot = 0; + } + } return 0; } -HTS_INLINE int rech_endtoken(const char* adr, const char** start) { +HTS_INLINE int rech_endtoken(const char *adr, const char **start) { char quote = '\0'; int length = 0; - while(is_space(*adr)) adr++; - if (*adr == '"' || *adr == '\'') + + while(is_space(*adr)) + adr++; + if (*adr == '"' || *adr == '\'') quote = *adr++; *start = adr; - while(*adr != 0 && *adr != quote && (quote != '\0' || !is_space(*adr)) ) { + while(*adr != 0 && *adr != quote && (quote != '\0' || !is_space(*adr))) { length++; - adr++; + adr++; } return length; } + // same, but check beginning of adr with s (for <object src="bar.mov" .. hotspot123="foo.html">) -HTS_INLINE int __rech_tageqbegdigits(const char* adr,const char* s) { +HTS_INLINE int __rech_tageqbegdigits(const char *adr, const char *s) { int p; - p=strfield(adr,s); + + p = strfield(adr, s); if (p) { - while(isdigit((unsigned char)adr[p])) p++; // jump digits - while(is_space(adr[p])) p++; - if (adr[p]=='=') { - return p+1; + while(isdigit((unsigned char) adr[p])) + p++; // jump digits + while(is_space(adr[p])) + p++; + if (adr[p] == '=') { + return p + 1; } } return 0; } // tag sans = -HTS_INLINE int rech_sampletag(const char* adr,const char* s) { +HTS_INLINE int rech_sampletag(const char *adr, const char *s) { int p; - if ( (*(adr-1)=='<') || (is_space(*(adr-1))) ) { // <tag < tag etc - p=strfield(adr,s); + + if ((*(adr - 1) == '<') || (is_space(*(adr - 1)))) { // <tag < tag etc + p = strfield(adr, s); if (p) { - if (!isalnum((unsigned char)adr[p])) { // <srcbis n'est pas <src + if (!isalnum((unsigned char) adr[p])) { // <srcbis n'est pas <src return 1; } return 0; @@ -639,40 +702,47 @@ HTS_INLINE int rech_sampletag(const char* adr,const char* s) { } // teste si le tag contenu dans from est égal à "tag" -HTS_INLINE int check_tag(char* from,const char* tag) { - char* a=from+1; - int i=0; +HTS_INLINE int check_tag(char *from, const char *tag) { + char *a = from + 1; + int i = 0; char s[256]; - while(is_space(*a)) a++; - while((isalnum((unsigned char)*a) || (*a=='/')) && (i<250)) { s[i++]=*a; a++; } - s[i++]='\0'; - return (strfield2(s,tag)); // comparer + + while(is_space(*a)) + a++; + while((isalnum((unsigned char) *a) || (*a == '/')) && (i < 250)) { + s[i++] = *a; + a++; + } + s[i++] = '\0'; + return (strfield2(s, tag)); // comparer } // teste si un fichier dépasse le quota -int istoobig(httrackp *opt,LLint size,LLint maxhtml,LLint maxnhtml,char* type) { - int ok=1; - if (size>0) { - if (is_hypertext_mime(opt,type, "")) { - if (maxhtml>0) { - if (size>maxhtml) - ok=0; +int istoobig(httrackp * opt, LLint size, LLint maxhtml, LLint maxnhtml, + char *type) { + int ok = 1; + + if (size > 0) { + if (is_hypertext_mime(opt, type, "")) { + if (maxhtml > 0) { + if (size > maxhtml) + ok = 0; } } else { - if (maxnhtml>0) { - if (size>maxnhtml) - ok=0; + if (maxnhtml > 0) { + if (size > maxnhtml) + ok = 0; } } } return (!ok); } - -static int sortTopIndexFnc(const void * a_, const void * b_) { +static int sortTopIndexFnc(const void *a_, const void *b_) { int cmp; - topindex_chain** a = (topindex_chain**) a_; - topindex_chain** b = (topindex_chain**) b_; + topindex_chain **a = (topindex_chain **) a_; + topindex_chain **b = (topindex_chain **) b_; + /* Category first, then name */ if ((cmp = (*a)->level - (*b)->level) == 0) { if ((cmp = strcmpnocase((*a)->category, (*b)->category)) == 0) { @@ -682,62 +752,75 @@ static int sortTopIndexFnc(const void * a_, const void * b_) { return cmp; } -HTSEXT_API char* hts_getcategory(const char* filename); +HTSEXT_API char *hts_getcategory(const char *filename); /* Note: NOT utf-8 */ -HTSEXT_API int hts_buildtopindex(httrackp* opt,const char* path,const char* binpath) { - FILE* fpo; - int retval=0; - char BIGSTK rpath[1024*2]; - char *toptemplate_header=NULL,*toptemplate_body=NULL,*toptemplate_footer=NULL,*toptemplate_bodycat=NULL; - char catbuff[CATBUFF_SIZE]; +HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path, + const char *binpath) { + FILE *fpo; + int retval = 0; + char BIGSTK rpath[1024 * 2]; + char *toptemplate_header = NULL, *toptemplate_body = + NULL, *toptemplate_footer = NULL, *toptemplate_bodycat = NULL; + char catbuff[CATBUFF_SIZE]; // et templates html - toptemplate_header=readfile_or(fconcat(catbuff, binpath,"templates/topindex-header.html"),HTS_INDEX_HEADER); - toptemplate_body=readfile_or(fconcat(catbuff, binpath,"templates/topindex-body.html"),HTS_INDEX_BODY); - toptemplate_bodycat=readfile_or(fconcat(catbuff, binpath,"templates/topindex-bodycat.html"),HTS_INDEX_BODYCAT); - toptemplate_footer=readfile_or(fconcat(catbuff, binpath,"templates/topindex-footer.html"),HTS_INDEX_FOOTER); - - if (toptemplate_header && toptemplate_body && toptemplate_footer && toptemplate_bodycat) { - - strcpybuff(rpath,path); + toptemplate_header = + readfile_or(fconcat(catbuff, binpath, "templates/topindex-header.html"), + HTS_INDEX_HEADER); + toptemplate_body = + readfile_or(fconcat(catbuff, binpath, "templates/topindex-body.html"), + HTS_INDEX_BODY); + toptemplate_bodycat = + readfile_or(fconcat(catbuff, binpath, "templates/topindex-bodycat.html"), + HTS_INDEX_BODYCAT); + toptemplate_footer = + readfile_or(fconcat(catbuff, binpath, "templates/topindex-footer.html"), + HTS_INDEX_FOOTER); + + if (toptemplate_header && toptemplate_body && toptemplate_footer + && toptemplate_bodycat) { + + strcpybuff(rpath, path); if (rpath[0]) { - if (rpath[strlen(rpath)-1]=='/') - rpath[strlen(rpath)-1]='\0'; + if (rpath[strlen(rpath) - 1] == '/') + rpath[strlen(rpath) - 1] = '\0'; } - - fpo=fopen(fconcat(catbuff, rpath,"/index.html"),"wb"); + + fpo = fopen(fconcat(catbuff, rpath, "/index.html"), "wb"); if (fpo) { find_handle h; - verif_backblue(opt,concat(catbuff, rpath,"/")); // générer gif + + verif_backblue(opt, concat(catbuff, rpath, "/")); // générer gif // Header - fprintf(fpo,toptemplate_header, - "<!-- Mirror and index made by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->" - ); - + fprintf(fpo, toptemplate_header, + "<!-- Mirror and index made by HTTrack Website Copier/" + HTTRACK_VERSION " " HTTRACK_AFF_AUTHORS " -->"); + /* Find valid project names */ h = hts_findfirst(rpath); if (h) { - struct topindex_chain * chain=NULL; - struct topindex_chain * startchain=NULL; + struct topindex_chain *chain = NULL; + struct topindex_chain *startchain = NULL; String iname = STRING_EMPTY; int chainSize = 0; + do { if (hts_findisdir(h)) { - StringCopy(iname,rpath); - StringCat(iname,"/"); - StringCat(iname,hts_findgetname(h)); - StringCat(iname,"/index.html"); + StringCopy(iname, rpath); + StringCat(iname, "/"); + StringCat(iname, hts_findgetname(h)); + StringCat(iname, "/index.html"); if (fexist(StringBuff(iname))) { int level = 0; - char* category = NULL; - struct topindex_chain * oldchain=chain; - + char *category = NULL; + struct topindex_chain *oldchain = chain; + /* Check for an existing category */ - StringCopy(iname,rpath); - StringCat(iname,"/"); - StringCat(iname,hts_findgetname(h)); - StringCat(iname,"/hts-cache/winprofile.ini"); + StringCopy(iname, rpath); + StringCat(iname, "/"); + StringCat(iname, hts_findgetname(h)); + StringCat(iname, "/hts-cache/winprofile.ini"); if (fexist(StringBuff(iname))) { category = hts_getcategory(StringBuff(iname)); if (category != NULL) { @@ -752,85 +835,86 @@ HTSEXT_API int hts_buildtopindex(httrackp* opt,const char* path,const char* binp level = 1; } - chain=calloc(sizeof(struct topindex_chain), 1); + chain = calloc(sizeof(struct topindex_chain), 1); chainSize++; if (!startchain) { - startchain=chain; + startchain = chain; } if (chain) { if (oldchain) { - oldchain->next=chain; + oldchain->next = chain; } - chain->next=NULL; + chain->next = NULL; strcpybuff(chain->name, hts_findgetname(h)); chain->category = category; chain->level = level; } } - + } } while(hts_findnext(h)); hts_findclose(h); StringFree(iname); - + /* Sort chain */ { - struct topindex_chain** sortedElts = (struct topindex_chain**) calloct(sizeof(topindex_chain*), chainSize); + struct topindex_chain **sortedElts = + (struct topindex_chain **) calloct(sizeof(topindex_chain *), + chainSize); assertf(sortedElts != NULL); if (sortedElts != NULL) { int i; - char* category = ""; - + char *category = ""; + /* Build array */ - struct topindex_chain * chain = startchain; - for(i = 0 ; i < chainSize ; i++) { + struct topindex_chain *chain = startchain; + + for(i = 0; i < chainSize; i++) { assertf(chain != NULL); sortedElts[i] = chain; chain = chain->next; } - qsort(sortedElts, chainSize, sizeof(topindex_chain*), sortTopIndexFnc); - + qsort(sortedElts, chainSize, sizeof(topindex_chain *), + sortTopIndexFnc); + /* Build sorted index */ - for(i = 0 ; i < chainSize ; i++) { - char BIGSTK hname[HTS_URLMAXSIZE*2]; - strcpybuff(hname,sortedElts[i]->name); + for(i = 0; i < chainSize; i++) { + char BIGSTK hname[HTS_URLMAXSIZE * 2]; + + strcpybuff(hname, sortedElts[i]->name); escape_check_url(hname); /* Changed category */ if (strcmp(category, sortedElts[i]->category) != 0) { category = sortedElts[i]->category; - fprintf(fpo,toptemplate_bodycat, category); + fprintf(fpo, toptemplate_bodycat, category); } - fprintf(fpo,toptemplate_body, - hname, - sortedElts[i]->name - ); + fprintf(fpo, toptemplate_body, hname, sortedElts[i]->name); } - + /* Wipe elements */ - for(i = 0 ; i < chainSize ; i++) { + for(i = 0; i < chainSize; i++) { freet(sortedElts[i]->category); freet(sortedElts[i]); sortedElts[i] = NULL; } freet(sortedElts); - + /* Return value */ - retval=1; + retval = 1; } } - + } - // Footer - fprintf(fpo,toptemplate_footer, - "<!-- Mirror and index made by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->" - ); - + fprintf(fpo, toptemplate_footer, + "<!-- Mirror and index made by HTTrack Website Copier/" + HTTRACK_VERSION " " HTTRACK_AFF_AUTHORS " -->"); + fclose(fpo); - + } - + } if (toptemplate_header) @@ -841,24 +925,28 @@ HTSEXT_API int hts_buildtopindex(httrackp* opt,const char* path,const char* binp freet(toptemplate_footer); if (toptemplate_body) freet(toptemplate_body); - + return retval; } /* Note: NOT utf-8 */ -HTSEXT_API char* hts_getcategory(const char* filename) { +HTSEXT_API char *hts_getcategory(const char *filename) { String categ = STRING_EMPTY; + if (fexist(filename)) { - FILE* fp = fopen(filename, "rb"); + FILE *fp = fopen(filename, "rb"); + if (fp != NULL) { - int done=0; + int done = 0; + while(!feof(fp) && !done) { char BIGSTK line[1024]; int n = linput(fp, line, sizeof(line) - 2); + if (n > 0) { if (strfield(line, "category=")) { - unescapehttp(line+9, &categ); - done=1; + unescapehttp(line + 9, &categ); + done = 1; } } } @@ -869,20 +957,22 @@ HTSEXT_API char* hts_getcategory(const char* filename) { } /* Note: NOT utf-8 */ -HTSEXT_API char* hts_getcategories(char* path, int type) { +HTSEXT_API char *hts_getcategories(char *path, int type) { String categ = STRING_EMPTY; String profiles = STRING_EMPTY; - char* rpath = path; + char *rpath = path; find_handle h; inthash hashCateg = NULL; + if (rpath[0]) { - if (rpath[strlen(rpath)-1]=='/') { - rpath[strlen(rpath)-1]='\0'; /* note: patching stored (inhash) value */ + if (rpath[strlen(rpath) - 1] == '/') { + rpath[strlen(rpath) - 1] = '\0'; /* note: patching stored (inhash) value */ } } h = hts_findfirst(rpath); if (h) { String iname = STRING_EMPTY; + if (type == 1) { hashCateg = inthash_new(127); StringCat(categ, "Test category 1"); @@ -891,29 +981,33 @@ HTSEXT_API char* hts_getcategories(char* path, int type) { do { if (hts_findisdir(h)) { char BIGSTK line2[1024]; - StringCopy(iname,rpath); - StringCat(iname,"/"); - StringCat(iname,hts_findgetname(h)); - StringCat(iname,"/hts-cache/winprofile.ini"); + + StringCopy(iname, rpath); + StringCat(iname, "/"); + StringCat(iname, hts_findgetname(h)); + StringCat(iname, "/hts-cache/winprofile.ini"); if (fexist(StringBuff(iname))) { if (type == 1) { - FILE* fp = fopen(StringBuff(iname), "rb"); + FILE *fp = fopen(StringBuff(iname), "rb"); + if (fp != NULL) { - int done=0; + int done = 0; + while(!feof(fp) && !done) { int n = linput(fp, line2, sizeof(line2) - 2); + if (n > 0) { if (strfield(line2, "category=")) { - if (*(line2+9)) { - if (!inthash_read(hashCateg, line2+9, NULL)) { - inthash_write(hashCateg, line2+9, 0); + if (*(line2 + 9)) { + if (!inthash_read(hashCateg, line2 + 9, NULL)) { + inthash_write(hashCateg, line2 + 9, 0); if (StringLength(categ) > 0) { StringCat(categ, "\r\n"); } - unescapehttp(line2+9, &categ); + unescapehttp(line2 + 9, &categ); } } - done=1; + done = 1; } } } @@ -927,7 +1021,7 @@ HTSEXT_API char* hts_getcategories(char* path, int type) { StringCat(profiles, hts_findgetname(h)); } } - + } } while(hts_findnext(h)); hts_findclose(h); @@ -943,9 +1037,6 @@ HTSEXT_API char* hts_getcategories(char* path, int type) { return StringBuffRW(profiles); } - - - // Portable directory find functions /* // Example: @@ -960,57 +1051,61 @@ if (h) { hts_findclose(h); } */ -HTSEXT_API find_handle hts_findfirst(char* path) { +HTSEXT_API find_handle hts_findfirst(char *path) { if (path) { if (strnotempty(path)) { - find_handle_struct* find = (find_handle_struct*) calloc(1,sizeof(find_handle_struct)); + find_handle_struct *find = + (find_handle_struct *) calloc(1, sizeof(find_handle_struct)); if (find) { memset(find, 0, sizeof(find_handle_struct)); #ifdef _WIN32 { - char BIGSTK rpath[1024*2]; - strcpybuff(rpath,path); + char BIGSTK rpath[1024 * 2]; + + strcpybuff(rpath, path); if (rpath[0]) { - if (rpath[strlen(rpath)-1]!='\\') - strcatbuff(rpath,"\\"); + if (rpath[strlen(rpath) - 1] != '\\') + strcatbuff(rpath, "\\"); } - strcatbuff(rpath,"*.*"); - find->handle = FindFirstFileA(rpath,&find->hdata); + strcatbuff(rpath, "*.*"); + find->handle = FindFirstFileA(rpath, &find->hdata); if (find->handle != INVALID_HANDLE_VALUE) return find; } #else - strcpybuff(find->path,path); + strcpybuff(find->path, path); { if (find->path[0]) { - if (find->path[strlen(find->path)-1]!='/') - strcatbuff(find->path,"/"); + if (find->path[strlen(find->path) - 1] != '/') + strcatbuff(find->path, "/"); } } - find->hdir=opendir(path); + find->hdir = opendir(path); if (find->hdir != NULL) { if (hts_findnext(find) == 1) return find; } #endif - free((void*)find); + free((void *) find); } } } - return NULL; + return NULL; } HTSEXT_API int hts_findnext(find_handle find) { if (find) { #ifdef _WIN32 - if ( (FindNextFileA(find->handle,&find->hdata))) + if ((FindNextFileA(find->handle, &find->hdata))) return 1; #else - char catbuff[CATBUFF_SIZE]; + char catbuff[CATBUFF_SIZE]; + memset(&(find->filestat), 0, sizeof(find->filestat)); - if ((find->dirp=readdir(find->hdir))) + if ((find->dirp = readdir(find->hdir))) if (find->dirp->d_name) - if (!STAT(concat(catbuff, find->path,find->dirp->d_name),&find->filestat)) + if (!STAT + (concat(catbuff, find->path, find->dirp->d_name), &find->filestat)) return 1; #endif } @@ -1022,20 +1117,20 @@ HTSEXT_API int hts_findclose(find_handle find) { #ifdef _WIN32 if (find->handle) { FindClose(find->handle); - find->handle=NULL; + find->handle = NULL; } #else if (find->hdir) { - closedir (find->hdir); - find->hdir=NULL; + closedir(find->hdir); + find->hdir = NULL; } #endif - free((void*)find); + free((void *) find); } return 0; } -HTSEXT_API char* hts_findgetname(find_handle find) { +HTSEXT_API char *hts_findgetname(find_handle find) { if (find) { #ifdef _WIN32 return find->hdata.cFileName; @@ -1062,7 +1157,7 @@ HTSEXT_API int hts_findisdir(find_handle find) { if (find) { if (!hts_findissystem(find)) { #ifdef _WIN32 - if (find->hdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + if (find->hdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) return 1; #else if (S_ISDIR(find->filestat.st_mode)) @@ -1076,7 +1171,7 @@ HTSEXT_API int hts_findisfile(find_handle find) { if (find) { if (!hts_findissystem(find)) { #ifdef _WIN32 - if (!(find->hdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + if (!(find->hdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) return 1; #else if (S_ISREG(find->filestat.st_mode)) @@ -1089,22 +1184,22 @@ HTSEXT_API int hts_findisfile(find_handle find) { HTSEXT_API int hts_findissystem(find_handle find) { if (find) { #ifdef _WIN32 - if (find->hdata.dwFileAttributes & (FILE_ATTRIBUTE_SYSTEM|FILE_ATTRIBUTE_HIDDEN|FILE_ATTRIBUTE_TEMPORARY)) + if (find->hdata. + dwFileAttributes & (FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN | + FILE_ATTRIBUTE_TEMPORARY)) return 1; - else if ( (!strcmp(find->hdata.cFileName,"..")) || (!strcmp(find->hdata.cFileName,".")) ) + else if ((!strcmp(find->hdata.cFileName, "..")) + || (!strcmp(find->hdata.cFileName, "."))) return 1; #else - if ( - (S_ISCHR(find->filestat.st_mode)) - || - (S_ISBLK(find->filestat.st_mode)) - || - (S_ISFIFO(find->filestat.st_mode)) - || - (S_ISSOCK(find->filestat.st_mode)) + if ((S_ISCHR(find->filestat.st_mode)) + || (S_ISBLK(find->filestat.st_mode)) + || (S_ISFIFO(find->filestat.st_mode)) + || (S_ISSOCK(find->filestat.st_mode)) ) return 1; - else if ( (!strcmp(find->dirp->d_name,"..")) || (!strcmp(find->dirp->d_name,".")) ) + else if ((!strcmp(find->dirp->d_name, "..")) + || (!strcmp(find->dirp->d_name, "."))) return 1; #endif } diff --git a/src/htstools.h b/src/htstools.h index ec0d3f0..02e73bc 100644 --- a/src/htstools.h +++ b/src/htstools.h @@ -17,27 +17,23 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* various tools (filename analyzing ..) */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - #ifndef HTSTOOLS_DEFH -#define HTSTOOLS_DEFH +#define HTSTOOLS_DEFH /* specific definitions */ #include "htsglobal.h" @@ -50,20 +46,22 @@ typedef struct httrackp httrackp; #ifndef HTS_DEF_FWSTRUCT_find_handle_struct #define HTS_DEF_FWSTRUCT_find_handle_struct typedef struct find_handle_struct find_handle_struct; -typedef find_handle_struct* find_handle; +typedef find_handle_struct *find_handle; #endif /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -int ident_url_relatif(const char *lien, const char* urladr, const char* urlfil, char* adr, char* fil); -int lienrelatif(char* s,const char* link,const char* curr); -int link_has_authority(const char* lien); -int link_has_authorization(const char* lien); -void long_to_83(int mode,char* n83,char* save); -void longfile_to_83(int mode,char* n83,char* save); -HTS_INLINE int __rech_tageq(const char* adr,const char* s); -HTS_INLINE int __rech_tageqbegdigits(const char* adr,const char* s); -HTS_INLINE int rech_tageq_all(const char* adr, const char* s); +int ident_url_relatif(const char *lien, const char *urladr, const char *urlfil, + char *adr, char *fil); +int lienrelatif(char *s, const char *link, const char *curr); +int link_has_authority(const char *lien); +int link_has_authorization(const char *lien); +void long_to_83(int mode, char *n83, char *save); +void longfile_to_83(int mode, char *n83, char *save); +HTS_INLINE int __rech_tageq(const char *adr, const char *s); +HTS_INLINE int __rech_tageqbegdigits(const char *adr, const char *s); +HTS_INLINE int rech_tageq_all(const char *adr, const char *s); + #define rech_tageq(adr,s) \ ( \ ( (*((adr)-1)=='<') || (is_space(*((adr)-1))) ) ? \ @@ -85,30 +83,33 @@ HTS_INLINE int rech_tageq_all(const char* adr, const char* s); : 0\ ) //HTS_INLINE int rech_tageq(const char* adr,const char* s); -HTS_INLINE int rech_sampletag(const char* adr,const char* s); -HTS_INLINE int rech_endtoken(const char* adr, const char** start); -HTS_INLINE int check_tag(char* from,const char* tag); -int verif_backblue(httrackp* opt, const char* base); -int verif_external(httrackp *opt,int nb,int test); +HTS_INLINE int rech_sampletag(const char *adr, const char *s); +HTS_INLINE int rech_endtoken(const char *adr, const char **start); +HTS_INLINE int check_tag(char *from, const char *tag); +int verif_backblue(httrackp * opt, const char *base); +int verif_external(httrackp * opt, int nb, int test); -int istoobig(httrackp *opt,LLint size,LLint maxhtml,LLint maxnhtml,char* type); -HTSEXT_API int hts_buildtopindex(httrackp* opt,const char* path,const char* binpath); +int istoobig(httrackp * opt, LLint size, LLint maxhtml, LLint maxnhtml, + char *type); +HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path, + const char *binpath); // Portable directory find functions // Directory find functions -HTSEXT_API find_handle hts_findfirst(char* path); +HTSEXT_API find_handle hts_findfirst(char *path); HTSEXT_API int hts_findnext(find_handle find); HTSEXT_API int hts_findclose(find_handle find); + // -HTSEXT_API char* hts_findgetname(find_handle find); +HTSEXT_API char *hts_findgetname(find_handle find); HTSEXT_API int hts_findgetsize(find_handle find); HTSEXT_API int hts_findisdir(find_handle find); HTSEXT_API int hts_findisfile(find_handle find); HTSEXT_API int hts_findissystem(find_handle find); #ifndef HTTRACK_DEFLIB -HTSEXT_API char* hts_getcategory(const char* filename); -HTSEXT_API char* hts_getcategories(char* path, int type); +HTSEXT_API char *hts_getcategory(const char *filename); +HTSEXT_API char *hts_getcategories(char *path, int type); #endif #endif diff --git a/src/htsweb.c b/src/htsweb.c index 577e77d..254d204 100644 --- a/src/htsweb.c +++ b/src/htsweb.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: webhttrack.c routines */ /* Author: Xavier Roche */ @@ -76,43 +73,44 @@ Please visit our Website: http://www.httrack.com #ifdef _WIN32 #ifndef __cplusplus // DOS -#include <process.h> /* _beginthread, _endthread */ +#include <process.h> /* _beginthread, _endthread */ #endif #else #endif static htsmutex refreshMutex = HTSMUTEX_INIT; -static int help_server(char* dest_path, int defaultPort); +static int help_server(char *dest_path, int defaultPort); extern int commandRunning; extern int commandEnd; extern int commandReturn; extern int commandEndRequested; -extern char* commandReturnMsg; -extern char* commandReturnCmdl; +extern char *commandReturnMsg; +extern char *commandReturnCmdl; -static void htsweb_sig_brpipe( int code ) { +static void htsweb_sig_brpipe(int code) { /* ignore */ } -int main(int argc, char* argv[]) -{ +int main(int argc, char *argv[]) { int i; int ret = 0; int defaultPort = 0; + printf("Initialzing the server..\n"); #ifdef _WIN32 { - WORD wVersionRequested; // requested version WinSock API + WORD wVersionRequested; // requested version WinSock API WSADATA wsadata; // Windows Sockets API data int stat; + wVersionRequested = 0x0101; - stat = WSAStartup( wVersionRequested, &wsadata ); + stat = WSAStartup(wVersionRequested, &wsadata); if (stat != 0) { fprintf(stderr, "Winsock not found!\n"); return -1; - } else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) { + } else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) { fprintf(stderr, "WINSOCK.DLL does not support version 1.1\n"); WSACleanup(); return -1; @@ -122,7 +120,9 @@ int main(int argc, char* argv[]) if (argc < 2 || (argc % 2) != 0) { fprintf(stderr, "** Warning: use the webhttrack frontend if available\n"); - fprintf(stderr, "usage: %s [--port <port>] <path-to-html-root-dir> [key value [key value]..]\n", argv[0]); + fprintf(stderr, + "usage: %s [--port <port>] <path-to-html-root-dir> [key value [key value]..]\n", + argv[0]); fprintf(stderr, "example: %s /usr/share/httrack/\n", argv[0]); return 1; } @@ -173,37 +173,40 @@ int main(int argc, char* argv[]) smallserver_setkey("HTTRACK_AFF_VERSION", HTTRACK_AFF_VERSION); { char tmp[32]; + sprintf(tmp, "%d", -1); smallserver_setkey("HTS_PLATFORM", tmp); } - smallserver_setkey("HTTRACK_WEB", HTTRACK_WEB); + smallserver_setkey("HTTRACK_WEB", HTTRACK_WEB); /* protected session-id */ { char buff[1024]; char digest[32 + 2]; - srand((unsigned int)time(NULL)); - sprintf(buff, "%d-%d", (int)time(NULL), (int)rand()); - domd5mem(buff,strlen(buff),digest,1); + + srand((unsigned int) time(NULL)); + sprintf(buff, "%d-%d", (int) time(NULL), (int) rand()); + domd5mem(buff, strlen(buff), digest, 1); smallserver_setkey("sid", digest); smallserver_setkey("_sid", digest); } /* set commandline keys */ - for(i = 2 ; i < argc ; i += 2) { - if (strcmp(argv[i], "--port") == 0) { - if (sscanf(argv[i + 1], "%d", &defaultPort) != 1 || defaultPort < 0 || defaultPort >= 65535 ) { - fprintf(stderr, "couldn't set the port number to %s\n", argv[i + 1]); - return -1; - } - } else { - smallserver_setkey(argv[i], argv[i + 1]); - } + for(i = 2; i < argc; i += 2) { + if (strcmp(argv[i], "--port") == 0) { + if (sscanf(argv[i + 1], "%d", &defaultPort) != 1 || defaultPort < 0 + || defaultPort >= 65535) { + fprintf(stderr, "couldn't set the port number to %s\n", argv[i + 1]); + return -1; + } + } else { + smallserver_setkey(argv[i], argv[i + 1]); + } } /* sigpipe */ #ifndef _WIN32 - signal( SIGPIPE , htsweb_sig_brpipe ); // broken pipe (write into non-opened socket) + signal(SIGPIPE, htsweb_sig_brpipe); // broken pipe (write into non-opened socket) #endif /* launch */ @@ -219,14 +222,15 @@ int main(int argc, char* argv[]) return ret; } -static int webhttrack_runmain(httrackp *opt, int argc, char** argv); -static void back_launch_cmd( void* pP ) { - char* cmd = (char*) pP; - char** argv = (char**) malloct(1024 * sizeof(char*)); +static int webhttrack_runmain(httrackp * opt, int argc, char **argv); +static void back_launch_cmd(void *pP) { + char *cmd = (char *) pP; + char **argv = (char **) malloct(1024 * sizeof(char *)); int argc = 0; int i = 0; int g = 0; - // + + // httrackp *opt; /* copy commandline */ @@ -235,9 +239,9 @@ static void back_launch_cmd( void* pP ) { commandReturnCmdl = strdup(cmd); /* split */ - argv[0]="webhttrack"; - argv[1]=cmd; - argc++; + argv[0] = "webhttrack"; + argv[1] = cmd; + argc++; i = 0; while(cmd[i]) { if (cmd[i] == '\t' || cmd[i] == '\r' || cmd[i] == '\n') { @@ -246,19 +250,20 @@ static void back_launch_cmd( void* pP ) { i++; } i = 0; - while(cmd[i]) { - if(cmd[i]=='\"') g=!g; - if(cmd[i]==' ') { - if(!g){ - cmd[i]='\0'; - argv[argc++]=cmd+i+1; + while(cmd[i]) { + if (cmd[i] == '\"') + g = !g; + if (cmd[i] == ' ') { + if (!g) { + cmd[i] = '\0'; + argv[argc++] = cmd + i + 1; } - } + } i++; } - /* init */ - hts_init(); + /* init */ + hts_init(); global_opt = opt = hts_create_opt(); /* run */ @@ -269,9 +274,9 @@ static void back_launch_cmd( void* pP ) { commandReturnMsg = strdup(hts_errmsg(opt)); } - /* free */ - global_opt = NULL; - hts_free_opt(opt); + /* free */ + global_opt = NULL; + hts_free_opt(opt); hts_uninit(); /* okay */ @@ -283,12 +288,12 @@ static void back_launch_cmd( void* pP ) { /* free */ free(cmd); freet(argv); - return ; + return; } -void webhttrack_main(char* cmd) { +void webhttrack_main(char *cmd) { commandRunning = 1; - hts_newthread(back_launch_cmd, (void*) strdup(cmd)); + hts_newthread(back_launch_cmd, (void *) strdup(cmd)); } void webhttrack_lock(void) { @@ -299,8 +304,8 @@ void webhttrack_release(void) { hts_mutexrelease(&refreshMutex); } -static int webhttrack_runmain(httrackp *opt, int argc, char** argv) { - int ret; +static int webhttrack_runmain(httrackp * opt, int argc, char **argv) { + int ret; CHAIN_FUNCTION(opt, init, htsshow_init, NULL); CHAIN_FUNCTION(opt, uninit, htsshow_uninit, NULL); @@ -326,37 +331,42 @@ static int webhttrack_runmain(httrackp *opt, int argc, char** argv) { CHAIN_FUNCTION(opt, sendhead, htsshow_sendheader, NULL); CHAIN_FUNCTION(opt, receivehead, htsshow_receiveheader, NULL); - ret = hts_main2(argc, argv, opt); + ret = hts_main2(argc, argv, opt); htsthread_wait_n(1); - - return ret; + + return ret; } -static int help_server(char* dest_path, int defaultPort) { +static int help_server(char *dest_path, int defaultPort) { int returncode = 0; - char adr_prox[HTS_URLMAXSIZE*2]; + char adr_prox[HTS_URLMAXSIZE * 2]; int port_prox; - T_SOC soc=smallserver_init_std(&port_prox, adr_prox, defaultPort); - if (soc!=INVALID_SOCKET) { - char url[HTS_URLMAXSIZE*2]; + T_SOC soc = smallserver_init_std(&port_prox, adr_prox, defaultPort); + + if (soc != INVALID_SOCKET) { + char url[HTS_URLMAXSIZE * 2]; char method[32]; char data[32768]; - url[0]=method[0]=data[0]='\0'; + + url[0] = method[0] = data[0] = '\0'; // printf("Okay, temporary server installed.\nThe URL is:\n"); printf("URL=http://%s:%d/\n", adr_prox, port_prox); #ifndef _WIN32 { pid_t pid = getpid(); - printf("PID=%d\n", (int)pid); + + printf("PID=%d\n", (int) pid); } #endif fflush(stdout); fflush(stderr); // - if (!smallserver(soc,url,method,data,dest_path)) { + if (!smallserver(soc, url, method, data, dest_path)) { int last_errno = errno; - fprintf(stderr, "Unable to create the server: %s\n", strerror(last_errno)); + + fprintf(stderr, "Unable to create the server: %s\n", + strerror(last_errno)); #ifdef _WIN32 closesocket(soc); #else @@ -368,7 +378,8 @@ static int help_server(char* dest_path, int defaultPort) { returncode = 0; } } else { - fprintf(stderr, "Unable to initialize a temporary server (no remaining port)\n"); + fprintf(stderr, + "Unable to initialize a temporary server (no remaining port)\n"); returncode = 1; } printf("EXITED\n"); @@ -377,50 +388,60 @@ static int help_server(char* dest_path, int defaultPort) { return returncode; } - /* CALLBACK FUNCTIONS */ /* Initialize the Winsock */ -void __cdecl htsshow_init(t_hts_callbackarg *carg) { +void __cdecl htsshow_init(t_hts_callbackarg * carg) { } -void __cdecl htsshow_uninit(t_hts_callbackarg *carg) { +void __cdecl htsshow_uninit(t_hts_callbackarg * carg) { } -int __cdecl htsshow_start(t_hts_callbackarg *carg, httrackp* opt) { - return 1; +int __cdecl htsshow_start(t_hts_callbackarg * carg, httrackp * opt) { + return 1; } -int __cdecl htsshow_chopt(t_hts_callbackarg *carg, httrackp* opt) { +int __cdecl htsshow_chopt(t_hts_callbackarg * carg, httrackp * opt) { return htsshow_start(carg, opt); } -int __cdecl htsshow_end(t_hts_callbackarg *carg, httrackp* opt) { - return 1; +int __cdecl htsshow_end(t_hts_callbackarg * carg, httrackp * opt) { + return 1; } -int __cdecl htsshow_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) { +int __cdecl htsshow_preprocesshtml(t_hts_callbackarg * carg, httrackp * opt, + char **html, int *len, + const char *url_address, + const char *url_file) { return 1; } -int __cdecl htsshow_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) { +int __cdecl htsshow_postprocesshtml(t_hts_callbackarg * carg, httrackp * opt, + char **html, int *len, + const char *url_address, + const char *url_file) { return 1; } -int __cdecl htsshow_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_address,const char* url_file) { +int __cdecl htsshow_checkhtml(t_hts_callbackarg * carg, httrackp * opt, + char *html, int len, const char *url_address, + const char *url_file) { return 1; } -int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time, hts_stat_struct* stats) { // appelé à chaque boucle de HTTrack - static TStamp prev_mytime=0; /* ok */ - static t_InpInfo SInfo; /* ok */ +int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * back, int back_max, int back_index, int lien_n, int lien_tot, int stat_time, hts_stat_struct * stats) { // appelé à chaque boucle de HTTrack + static TStamp prev_mytime = 0; /* ok */ + static t_InpInfo SInfo; /* ok */ + // TStamp mytime; - long int rate=0; + long int rate = 0; + // - int stat_written=-1; - int stat_updated=-1; - int stat_errors=-1; - int stat_warnings=-1; - int stat_infos=-1; - int nbk=-1; - LLint nb=-1; - int stat_nsocket=-1; - LLint stat_bytes=-1; - LLint stat_bytes_recv=-1; - int irate=-1; + int stat_written = -1; + int stat_updated = -1; + int stat_errors = -1; + int stat_warnings = -1; + int stat_infos = -1; + int nbk = -1; + LLint nb = -1; + int stat_nsocket = -1; + LLint stat_bytes = -1; + LLint stat_bytes_recv = -1; + int irate = -1; + // char st[256]; @@ -432,45 +453,57 @@ int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back webhttrack_lock(); if (stats) { - stat_written=stats->stat_files; - stat_updated=stats->stat_updated_files; - stat_errors=stats->stat_errors; - stat_warnings=stats->stat_warnings; - stat_infos=stats->stat_infos; - nbk=stats->nbk; - stat_nsocket=stats->stat_nsocket; - irate=(int)stats->rate; - nb=stats->nb; - stat_bytes=stats->nb; - stat_bytes_recv=stats->HTS_TOTAL_RECV; + stat_written = stats->stat_files; + stat_updated = stats->stat_updated_files; + stat_errors = stats->stat_errors; + stat_warnings = stats->stat_warnings; + stat_infos = stats->stat_infos; + nbk = stats->nbk; + stat_nsocket = stats->stat_nsocket; + irate = (int) stats->rate; + nb = stats->nb; + stat_bytes = stats->nb; + stat_bytes_recv = stats->HTS_TOTAL_RECV; } - - mytime=mtime_local(); - if ((stat_time>0) && (stat_bytes_recv>0)) - rate=(int)(stat_bytes_recv/stat_time); + + mytime = mtime_local(); + if ((stat_time > 0) && (stat_bytes_recv > 0)) + rate = (int) (stat_bytes_recv / stat_time); else - rate=0; // pas d'infos - + rate = 0; // pas d'infos + /* Infos */ - if (stat_bytes>=0) SInfo.stat_bytes=stat_bytes; // bytes - if (stat_time>=0) SInfo.stat_time=stat_time; // time - if (lien_tot>=0) SInfo.lien_tot=lien_tot; // nb liens - if (lien_n>=0) SInfo.lien_n=lien_n; // scanned - SInfo.stat_nsocket=stat_nsocket; // socks - if (rate>0) SInfo.rate=rate; // rate - if (irate>=0) SInfo.irate=irate; // irate - if (SInfo.irate<0) SInfo.irate=SInfo.rate; - if (nbk>=0) SInfo.stat_back=nbk; - if (stat_written>=0) SInfo.stat_written=stat_written; - if (stat_updated>=0) SInfo.stat_updated=stat_updated; - if (stat_errors>=0) SInfo.stat_errors=stat_errors; - if (stat_warnings>=0) SInfo.stat_warnings=stat_warnings; - if (stat_infos>=0) SInfo.stat_infos=stat_infos; - - - st[0]='\0'; - qsec2str(st,stat_time); - + if (stat_bytes >= 0) + SInfo.stat_bytes = stat_bytes; // bytes + if (stat_time >= 0) + SInfo.stat_time = stat_time; // time + if (lien_tot >= 0) + SInfo.lien_tot = lien_tot; // nb liens + if (lien_n >= 0) + SInfo.lien_n = lien_n; // scanned + SInfo.stat_nsocket = stat_nsocket; // socks + if (rate > 0) + SInfo.rate = rate; // rate + if (irate >= 0) + SInfo.irate = irate; // irate + if (SInfo.irate < 0) + SInfo.irate = SInfo.rate; + if (nbk >= 0) + SInfo.stat_back = nbk; + if (stat_written >= 0) + SInfo.stat_written = stat_written; + if (stat_updated >= 0) + SInfo.stat_updated = stat_updated; + if (stat_errors >= 0) + SInfo.stat_errors = stat_errors; + if (stat_warnings >= 0) + SInfo.stat_warnings = stat_warnings; + if (stat_infos >= 0) + SInfo.stat_infos = stat_infos; + + st[0] = '\0'; + qsec2str(st, stat_time); + /* Set keys */ smallserver_setkeyint("info.stat_bytes", SInfo.stat_bytes); smallserver_setkeyint("info.stat_time", SInfo.stat_time); @@ -487,131 +520,148 @@ int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back smallserver_setkeyint("info.stat_infos", SInfo.stat_infos); /* */ smallserver_setkey("info.stat_time_str", st); - - if ( ((mytime - prev_mytime)>100) || ((mytime - prev_mytime)<0) ) { - prev_mytime=mytime; - - + + if (((mytime - prev_mytime) > 100) || ((mytime - prev_mytime) < 0)) { + prev_mytime = mytime; + // parcourir registre des liens - if (back_index>=0 && back_max > 0) { // seulement si index passé - int j,k; - int index=0; - int ok=0; // idem - int l; // idem + if (back_index >= 0 && back_max > 0) { // seulement si index passé + int j, k; + int index = 0; + int ok = 0; // idem + int l; // idem + // t_StatsBuffer StatsBuffer[NStatsBuffer]; - + { int i; - for(i=0;i<NStatsBuffer;i++) { - strcpybuff(StatsBuffer[i].state,""); - strcpybuff(StatsBuffer[i].name,""); - strcpybuff(StatsBuffer[i].file,""); - strcpybuff(StatsBuffer[i].url_sav,""); - StatsBuffer[i].back=0; - StatsBuffer[i].size=0; - StatsBuffer[i].sizetot=0; + + for(i = 0; i < NStatsBuffer; i++) { + strcpybuff(StatsBuffer[i].state, ""); + strcpybuff(StatsBuffer[i].name, ""); + strcpybuff(StatsBuffer[i].file, ""); + strcpybuff(StatsBuffer[i].url_sav, ""); + StatsBuffer[i].back = 0; + StatsBuffer[i].size = 0; + StatsBuffer[i].sizetot = 0; } } - for(k=0;k<2;k++) { // 0: lien en cours 1: autres liens - for(j=0;(j<3) && (index<NStatsBuffer);j++) { // passe de priorité + for(k = 0; k < 2; k++) { // 0: lien en cours 1: autres liens + for(j = 0; (j < 3) && (index < NStatsBuffer); j++) { // passe de priorité int _i; - for(_i=0+k;(_i< max(back_max*k,1) ) && (index<NStatsBuffer);_i++) { // no lien - int i=(back_index+_i)%back_max; // commencer par le "premier" (l'actuel) - if (back[i].status>=0) { // signifie "lien actif" + + for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) { // no lien + int i = (back_index + _i) % back_max; // commencer par le "premier" (l'actuel) + + if (back[i].status >= 0) { // signifie "lien actif" // int ok=0; // OPTI - ok=0; - switch(j) { - case 0: // prioritaire - if ((back[i].status>0) && (back[i].status<99)) { - strcpybuff(StatsBuffer[index].state,"receive"); ok=1; + ok = 0; + switch (j) { + case 0: // prioritaire + if ((back[i].status > 0) && (back[i].status < 99)) { + strcpybuff(StatsBuffer[index].state, "receive"); + ok = 1; } break; case 1: - if (back[i].status==STATUS_WAIT_HEADERS) { - strcpybuff(StatsBuffer[index].state,"request"); ok=1; - } - else if (back[i].status==STATUS_CONNECTING) { - strcpybuff(StatsBuffer[index].state,"connect"); ok=1; - } - else if (back[i].status==STATUS_WAIT_DNS) { - strcpybuff(StatsBuffer[index].state,"search"); ok=1; - } - else if (back[i].status==STATUS_FTP_TRANSFER) { // ohh le beau ftp - char proto[] = "ftp"; - if (back[i].url_adr[0]) { - char* ep = strchr(back[i].url_adr, ':'); - char* eps = strchr(back[i].url_adr, '/'); - int count; - if (ep != NULL && ep < eps && (count = (int) (ep - back[i].url_adr) ) < 4) { - proto[0] = '\0'; - strncat(proto, back[i].url_adr, count); - } - } - sprintf(StatsBuffer[index].state,"%s: %s",proto,back[i].info); ok=1; - } - break; - default: - if (back[i].status==STATUS_READY) { // prêt - if ((back[i].r.statuscode==HTTP_OK)) { - strcpybuff(StatsBuffer[index].state,"ready"); ok=1; + if (back[i].status == STATUS_WAIT_HEADERS) { + strcpybuff(StatsBuffer[index].state, "request"); + ok = 1; + } else if (back[i].status == STATUS_CONNECTING) { + strcpybuff(StatsBuffer[index].state, "connect"); + ok = 1; + } else if (back[i].status == STATUS_WAIT_DNS) { + strcpybuff(StatsBuffer[index].state, "search"); + ok = 1; + } else if (back[i].status == STATUS_FTP_TRANSFER) { // ohh le beau ftp + char proto[] = "ftp"; + + if (back[i].url_adr[0]) { + char *ep = strchr(back[i].url_adr, ':'); + char *eps = strchr(back[i].url_adr, '/'); + int count; + + if (ep != NULL && ep < eps + && (count = (int) (ep - back[i].url_adr)) < 4) { + proto[0] = '\0'; + strncat(proto, back[i].url_adr, count); + } } - else if ((back[i].r.statuscode>=100) && (back[i].r.statuscode<=599)) { - char tempo[256]; tempo[0]='\0'; - infostatuscode(tempo,back[i].r.statuscode); - strcpybuff(StatsBuffer[index].state,tempo); ok=1; - } - else { - strcpybuff(StatsBuffer[index].state,"error"); ok=1; + sprintf(StatsBuffer[index].state, "%s: %s", proto, + back[i].info); + ok = 1; + } + break; + default: + if (back[i].status == STATUS_READY) { // prêt + if ((back[i].r.statuscode == HTTP_OK)) { + strcpybuff(StatsBuffer[index].state, "ready"); + ok = 1; + } else if ((back[i].r.statuscode >= 100) + && (back[i].r.statuscode <= 599)) { + char tempo[256]; + + tempo[0] = '\0'; + infostatuscode(tempo, back[i].r.statuscode); + strcpybuff(StatsBuffer[index].state, tempo); + ok = 1; + } else { + strcpybuff(StatsBuffer[index].state, "error"); + ok = 1; } } break; } - + if (ok) { - char s[HTS_URLMAXSIZE*2]; + char s[HTS_URLMAXSIZE * 2]; + // - StatsBuffer[index].back=i; // index pour + d'infos + StatsBuffer[index].back = i; // index pour + d'infos // - s[0]='\0'; - strcpybuff(StatsBuffer[index].url_sav,back[i].url_sav); // pour cancel - if (strcmp(back[i].url_adr,"file://")) - strcatbuff(s,back[i].url_adr); + s[0] = '\0'; + strcpybuff(StatsBuffer[index].url_sav, back[i].url_sav); // pour cancel + if (strcmp(back[i].url_adr, "file://")) + strcatbuff(s, back[i].url_adr); else - strcatbuff(s,"localhost"); - if (back[i].url_fil[0]!='/') - strcatbuff(s,"/"); - strcatbuff(s,back[i].url_fil); - - StatsBuffer[index].file[0]='\0'; + strcatbuff(s, "localhost"); + if (back[i].url_fil[0] != '/') + strcatbuff(s, "/"); + strcatbuff(s, back[i].url_fil); + + StatsBuffer[index].file[0] = '\0'; { - char* a=strrchr(s,'/'); + char *a = strrchr(s, '/'); + if (a) { - strncatbuff(StatsBuffer[index].file,a,200); - *a='\0'; + strncatbuff(StatsBuffer[index].file, a, 200); + *a = '\0'; } } - - if ((l = (int) strlen(s))<MAX_LEN_INPROGRESS) - strcpybuff(StatsBuffer[index].name,s); + + if ((l = (int) strlen(s)) < MAX_LEN_INPROGRESS) + strcpybuff(StatsBuffer[index].name, s); else { // couper - StatsBuffer[index].name[0]='\0'; - strncatbuff(StatsBuffer[index].name,s,MAX_LEN_INPROGRESS/2-2); - strcatbuff(StatsBuffer[index].name,"..."); - strcatbuff(StatsBuffer[index].name,s+l-MAX_LEN_INPROGRESS/2+2); + StatsBuffer[index].name[0] = '\0'; + strncatbuff(StatsBuffer[index].name, s, + MAX_LEN_INPROGRESS / 2 - 2); + strcatbuff(StatsBuffer[index].name, "..."); + strcatbuff(StatsBuffer[index].name, + s + l - MAX_LEN_INPROGRESS / 2 + 2); } - - if (back[i].r.totalsize>0) { // taille prédéfinie - StatsBuffer[index].sizetot=back[i].r.totalsize; - StatsBuffer[index].size=back[i].r.size; - } else { // pas de taille prédéfinie - if (back[i].status==STATUS_READY) { // prêt - StatsBuffer[index].sizetot=back[i].r.size; - StatsBuffer[index].size=back[i].r.size; + + if (back[i].r.totalsize > 0) { // taille prédéfinie + StatsBuffer[index].sizetot = back[i].r.totalsize; + StatsBuffer[index].size = back[i].r.size; + } else { // pas de taille prédéfinie + if (back[i].status == STATUS_READY) { // prêt + StatsBuffer[index].sizetot = back[i].r.size; + StatsBuffer[index].size = back[i].r.size; } else { - StatsBuffer[index].sizetot=8192; - StatsBuffer[index].size=(back[i].r.size % 8192); + StatsBuffer[index].sizetot = 8192; + StatsBuffer[index].size = (back[i].r.size % 8192); } } index++; @@ -623,20 +673,23 @@ int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back /* Display current job */ { - int parsing=0; + int parsing = 0; + if (commandEndRequested) - smallserver_setkey("info.currentjob", "finishing pending transfers - Select [Cancel] to stop now!"); - else if (!(parsing=hts_is_parsing(opt, -1))) + smallserver_setkey("info.currentjob", + "finishing pending transfers - Select [Cancel] to stop now!"); + else if (!(parsing = hts_is_parsing(opt, -1))) smallserver_setkey("info.currentjob", "receiving files"); else { char tmp[1024]; + tmp[0] = '\0'; - switch(hts_is_testing(opt)) { + switch (hts_is_testing(opt)) { case 0: - sprintf(tmp, "parsing HTML file (%d%%)",parsing); + sprintf(tmp, "parsing HTML file (%d%%)", parsing); break; case 1: - sprintf(tmp, "parsing HTML file: testing links (%d%%)",parsing); + sprintf(tmp, "parsing HTML file: testing links (%d%%)", parsing); break; case 2: sprintf(tmp, "purging files"); @@ -658,72 +711,102 @@ int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back /* Display background jobs */ { int i; - for(i=0;i<NStatsBuffer;i++) { + + for(i = 0; i < NStatsBuffer; i++) { if (strnotempty(StatsBuffer[i].state)) { - strc_int2bytes2 strc; + strc_int2bytes2 strc; + smallserver_setkeyarr("info.state[", i, "]", StatsBuffer[i].state); smallserver_setkeyarr("info.name[", i, "]", StatsBuffer[i].name); smallserver_setkeyarr("info.file[", i, "]", StatsBuffer[i].file); - smallserver_setkeyarr("info.size[", i, "]", int2bytes(&strc,StatsBuffer[i].size)); - smallserver_setkeyarr("info.sizetot[", i, "]", int2bytes(&strc,StatsBuffer[i].sizetot)); - smallserver_setkeyarr("info.url_adr[", i, "]", StatsBuffer[i].url_adr); - smallserver_setkeyarr("info.url_fil[", i, "]", StatsBuffer[i].url_fil); - smallserver_setkeyarr("info.url_sav[", i, "]", StatsBuffer[i].url_sav); + smallserver_setkeyarr("info.size[", i, "]", + int2bytes(&strc, StatsBuffer[i].size)); + smallserver_setkeyarr("info.sizetot[", i, "]", + int2bytes(&strc, StatsBuffer[i].sizetot)); + smallserver_setkeyarr("info.url_adr[", i, "]", + StatsBuffer[i].url_adr); + smallserver_setkeyarr("info.url_fil[", i, "]", + StatsBuffer[i].url_fil); + smallserver_setkeyarr("info.url_sav[", i, "]", + StatsBuffer[i].url_sav); } } } + } - } - } - + /* UnLock */ webhttrack_release(); - + return 1; } -const char* __cdecl htsshow_query(t_hts_callbackarg *carg, httrackp *opt, const char* question) { - static char s[]=""; /* ok */ +const char *__cdecl htsshow_query(t_hts_callbackarg * carg, httrackp * opt, + const char *question) { + static char s[] = ""; /* ok */ + return s; } -const char* __cdecl htsshow_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question) { - static char s[]=""; /* ok */ +const char *__cdecl htsshow_query2(t_hts_callbackarg * carg, httrackp * opt, + const char *question) { + static char s[] = ""; /* ok */ + return s; } -const char* __cdecl htsshow_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question) { - static char s[]=""; /* ok */ +const char *__cdecl htsshow_query3(t_hts_callbackarg * carg, httrackp * opt, + const char *question) { + static char s[] = ""; /* ok */ + return s; } -int __cdecl htsshow_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status) { +int __cdecl htsshow_check(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, int status) { return -1; } -int __cdecl htsshow_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status) { +int __cdecl htsshow_check_mime(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + const char *mime, int status) { return -1; } -void __cdecl htsshow_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile) { +void __cdecl htsshow_pause(t_hts_callbackarg * carg, httrackp * opt, + const char *lockfile) { } -void __cdecl htsshow_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file) { +void __cdecl htsshow_filesave(t_hts_callbackarg * carg, httrackp * opt, + const char *file) { } -void __cdecl htsshow_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* fil, const char* save, int is_new, int is_modified,int not_updated) { +void __cdecl htsshow_filesave2(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + const char *save, int is_new, int is_modified, + int not_updated) { } -int __cdecl htsshow_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link) { +int __cdecl htsshow_linkdetected(t_hts_callbackarg * carg, httrackp * opt, + char *link) { return 1; } -int __cdecl htsshow_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag) { +int __cdecl htsshow_linkdetected2(t_hts_callbackarg * carg, httrackp * opt, + char *link, const char *start_tag) { return 1; } -int __cdecl htsshow_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, lien_back* back) { +int __cdecl htsshow_xfrstatus(t_hts_callbackarg * carg, httrackp * opt, + lien_back * back) { return 1; } -int __cdecl htsshow_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save) { +int __cdecl htsshow_savename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, const char *fil_complete, + const char *referer_adr, const char *referer_fil, + char *save) { return 1; } -int __cdecl htsshow_sendheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing) { +int __cdecl htsshow_sendheader(t_hts_callbackarg * carg, httrackp * opt, + char *buff, const char *adr, const char *fil, + const char *referer_adr, const char *referer_fil, + htsblk * outgoing) { return 1; } -int __cdecl htsshow_receiveheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming) { +int __cdecl htsshow_receiveheader(t_hts_callbackarg * carg, httrackp * opt, + char *buff, const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, htsblk * incoming) { return 1; } - - diff --git a/src/htsweb.h b/src/htsweb.h index 6a47622..c60d8eb 100644 --- a/src/htsweb.h +++ b/src/htsweb.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: webhttrack.c routines */ /* Author: Xavier Roche */ @@ -47,16 +44,16 @@ typedef struct t_StatsBuffer { char name[1024]; char file[1024]; char state[256]; - char url_sav[HTS_URLMAXSIZE*2]; // pour cancel - char url_adr[HTS_URLMAXSIZE*2]; - char url_fil[HTS_URLMAXSIZE*2]; + char url_sav[HTS_URLMAXSIZE * 2]; // pour cancel + char url_adr[HTS_URLMAXSIZE * 2]; + char url_fil[HTS_URLMAXSIZE * 2]; LLint size; LLint sizetot; int offset; // int back; // - int actived; // pour disabled + int actived; // pour disabled } t_StatsBuffer; typedef struct t_InpInfo { @@ -80,32 +77,66 @@ typedef struct t_InpInfo { } t_InpInfo; // wrappers -void __cdecl htsshow_init(t_hts_callbackarg *carg); -void __cdecl htsshow_uninit(t_hts_callbackarg *carg); -int __cdecl htsshow_start(t_hts_callbackarg *carg, httrackp* opt); -int __cdecl htsshow_chopt(t_hts_callbackarg *carg, httrackp* opt); -int __cdecl htsshow_end(t_hts_callbackarg *carg, httrackp* opt); -int __cdecl htsshow_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file); -int __cdecl htsshow_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file); -int __cdecl htsshow_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_address,const char* url_file); -int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time,hts_stat_struct* stats); -const char* __cdecl htsshow_query(t_hts_callbackarg *carg, httrackp *opt, const char* question); -const char* __cdecl htsshow_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question); -const char* __cdecl htsshow_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question); -int __cdecl htsshow_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status); -int __cdecl htsshow_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status); -void __cdecl htsshow_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile); -void __cdecl htsshow_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file); -void __cdecl htsshow_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* fil, const char* save, int is_new, int is_modified, int not_updated); -int __cdecl htsshow_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link); -int __cdecl htsshow_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag); -int __cdecl htsshow_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, lien_back* back); -int __cdecl htsshow_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save); -int __cdecl htsshow_sendheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing); -int __cdecl htsshow_receiveheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming); - +void __cdecl htsshow_init(t_hts_callbackarg * carg); +void __cdecl htsshow_uninit(t_hts_callbackarg * carg); +int __cdecl htsshow_start(t_hts_callbackarg * carg, httrackp * opt); +int __cdecl htsshow_chopt(t_hts_callbackarg * carg, httrackp * opt); +int __cdecl htsshow_end(t_hts_callbackarg * carg, httrackp * opt); +int __cdecl htsshow_preprocesshtml(t_hts_callbackarg * carg, httrackp * opt, + char **html, int *len, + const char *url_address, + const char *url_file); +int __cdecl htsshow_postprocesshtml(t_hts_callbackarg * carg, httrackp * opt, + char **html, int *len, + const char *url_address, + const char *url_file); +int __cdecl htsshow_checkhtml(t_hts_callbackarg * carg, httrackp * opt, + char *html, int len, const char *url_address, + const char *url_file); +int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, + lien_back * back, int back_max, int back_index, + int lien_n, int lien_tot, int stat_time, + hts_stat_struct * stats); +const char *__cdecl htsshow_query(t_hts_callbackarg * carg, httrackp * opt, + const char *question); +const char *__cdecl htsshow_query2(t_hts_callbackarg * carg, httrackp * opt, + const char *question); +const char *__cdecl htsshow_query3(t_hts_callbackarg * carg, httrackp * opt, + const char *question); +int __cdecl htsshow_check(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, int status); +int __cdecl htsshow_check_mime(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + const char *mime, int status); +void __cdecl htsshow_pause(t_hts_callbackarg * carg, httrackp * opt, + const char *lockfile); +void __cdecl htsshow_filesave(t_hts_callbackarg * carg, httrackp * opt, + const char *file); +void __cdecl htsshow_filesave2(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + const char *save, int is_new, int is_modified, + int not_updated); +int __cdecl htsshow_linkdetected(t_hts_callbackarg * carg, httrackp * opt, + char *link); +int __cdecl htsshow_linkdetected2(t_hts_callbackarg * carg, httrackp * opt, + char *link, const char *start_tag); +int __cdecl htsshow_xfrstatus(t_hts_callbackarg * carg, httrackp * opt, + lien_back * back); +int __cdecl htsshow_savename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, const char *fil_complete, + const char *referer_adr, const char *referer_fil, + char *save); +int __cdecl htsshow_sendheader(t_hts_callbackarg * carg, httrackp * opt, + char *buff, const char *adr, const char *fil, + const char *referer_adr, const char *referer_fil, + htsblk * outgoing); +int __cdecl htsshow_receiveheader(t_hts_callbackarg * carg, httrackp * opt, + char *buff, const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, htsblk * incoming); + int main(int argc, char **argv); -void webhttrack_main(char* cmd); +void webhttrack_main(char *cmd); void webhttrack_lock(void); void webhttrack_release(void); diff --git a/src/htswizard.c b/src/htswizard.c index 3bc8691..b4ceaeb 100644 --- a/src/htswizard.c +++ b/src/htswizard.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* wizard system (accept/refuse links) */ @@ -64,31 +61,29 @@ Please visit our Website: http://www.httrack.com } while(0) typedef struct htspair_t { - char *tag; - char *attr; + char *tag; + char *attr; } htspair_t; /* "embedded" */ htspair_t hts_detect_embed[] = { - { "img", "src" }, - { "link", "href" }, + {"img", "src"}, + {"link", "href"}, - /* embedded script hack */ - { "script", ".src" }, + /* embedded script hack */ + {"script", ".src"}, - /* style */ - { "style", "import" }, + /* style */ + {"style", "import"}, - { NULL, NULL } + {NULL, NULL} }; /* Internal */ -static int hts_acceptlink_(httrackp* opt, - int ptr,int lien_tot,lien_url** liens, - char* adr,char* fil, - char* tag, char* attribute, - int* set_prio_to, - int* just_test_it); +static int hts_acceptlink_(httrackp * opt, int ptr, int lien_tot, + lien_url ** liens, char *adr, char *fil, char *tag, + char *attribute, int *set_prio_to, + int *just_test_it); /* httrackp opt bloc d'options @@ -110,99 +105,94 @@ retour: -1 pas d'avis */ -int hts_acceptlink(httrackp* opt, - int ptr,int lien_tot,lien_url** liens, - char* adr,char* fil, - char* tag, char* attribute, - int* set_prio_to, - int* just_test_it) -{ - int forbidden_url = hts_acceptlink_(opt, ptr, lien_tot, liens, - adr, fil, tag, attribute, set_prio_to, just_test_it); - int prev_prio = set_prio_to ? *set_prio_to : 0; - - // -------------------- PHASE 6 -------------------- - { - int test_url = RUN_CALLBACK3(opt, check_link, adr, fil, forbidden_url); - if (test_url != -1) { - forbidden_url = test_url; - if (set_prio_to) - *set_prio_to = prev_prio; - } - } - - return forbidden_url; +int hts_acceptlink(httrackp * opt, int ptr, int lien_tot, lien_url ** liens, + char *adr, char *fil, char *tag, char *attribute, + int *set_prio_to, int *just_test_it) { + int forbidden_url = hts_acceptlink_(opt, ptr, lien_tot, liens, + adr, fil, tag, attribute, set_prio_to, + just_test_it); + int prev_prio = set_prio_to ? *set_prio_to : 0; + + // -------------------- PHASE 6 -------------------- + { + int test_url = RUN_CALLBACK3(opt, check_link, adr, fil, forbidden_url); + + if (test_url != -1) { + forbidden_url = test_url; + if (set_prio_to) + *set_prio_to = prev_prio; + } + } + + return forbidden_url; } static int cmp_token(const char *tag, const char *cmp) { - int p; - return (strncasecmp(tag, cmp, ( p = (int) strlen(cmp) ) ) == 0 - && !isalnum((unsigned char) tag[p])); + int p; + + return (strncasecmp(tag, cmp, (p = (int) strlen(cmp))) == 0 + && !isalnum((unsigned char) tag[p])); } -static int hts_acceptlink_(httrackp* opt, - int ptr,int lien_tot,lien_url** liens, - char* adr,char* fil, - char* tag, char* attribute, - int* set_prio_to, - int* just_test_it) -{ - int forbidden_url=-1; +static int hts_acceptlink_(httrackp * opt, int ptr, int lien_tot, + lien_url ** liens, char *adr, char *fil, char *tag, + char *attribute, int *set_prio_to, + int *just_test_it) { + int forbidden_url = -1; int meme_adresse; - int embedded_triggered = 0; + int embedded_triggered = 0; + #define _FILTERS (*opt->filters.filters) #define _FILTERS_PTR (opt->filters.filptr) #define _ROBOTS ((robots_wizard*)opt->robotsptr) - int may_set_prio_to=0; + int may_set_prio_to = 0; // -------------------- PHASE 0 -------------------- /* Infos */ - hts_log_print(opt, LOG_DEBUG, "wizard test begins: %s%s",adr,fil); - + hts_log_print(opt, LOG_DEBUG, "wizard test begins: %s%s", adr, fil); + /* Already exists? Then, we know that we knew that this link had to be known */ - if (adr[0] != '\0' - && fil[0] != '\0' - && opt->hash != NULL - && hash_read(opt->hash, adr, fil, 1, opt->urlhack) >= 0 - ) { - return 0; /* Yokai */ + if (adr[0] != '\0' && fil[0] != '\0' && opt->hash != NULL + && hash_read(opt->hash, adr, fil, 1, opt->urlhack) >= 0) { + return 0; /* Yokai */ } - // -------------------- PRELUDE OF PHASE 3-BIS -------------------- - /* Built-in known tags (<img src=..>, ..) */ - if (forbidden_url != 0 && opt->nearlink && tag != NULL && attribute != NULL) { - int i; - for(i = 0 ; hts_detect_embed[i].tag != NULL ; i++) { - if (cmp_token(tag, hts_detect_embed[i].tag) - && cmp_token(attribute, hts_detect_embed[i].attr) - ) - { - embedded_triggered = 1; - break; - } - } - } + /* Built-in known tags (<img src=..>, ..) */ + if (forbidden_url != 0 && opt->nearlink && tag != NULL && attribute != NULL) { + int i; + for(i = 0; hts_detect_embed[i].tag != NULL; i++) { + if (cmp_token(tag, hts_detect_embed[i].tag) + && cmp_token(attribute, hts_detect_embed[i].attr) + ) { + embedded_triggered = 1; + break; + } + } + } // -------------------- PHASE 1 -------------------- /* Doit-on traiter les non html? */ - if ((opt->getmode & 2)==0) { // non on ne doit pas - if (!ishtml(opt,fil)) { // non il ne faut pas + if ((opt->getmode & 2) == 0) { // non on ne doit pas + if (!ishtml(opt, fil)) { // non il ne faut pas //adr[0]='\0'; // ne pas traiter ce lien, pas traiter - forbidden_url=1; // interdire récupération du lien - hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s",adr,fil); - + forbidden_url = 1; // interdire récupération du lien + hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s", adr, + fil); + } } - + /* Niveau 1: ne pas parser suivant! */ - if (ptr>0) { - if ( ( liens[ptr]->depth <= 0 ) || ( liens[ptr]->depth <= 1 && !embedded_triggered ) ) { - forbidden_url=1; // interdire récupération du lien - hts_log_print(opt, LOG_DEBUG, "file from too far level ignored at %s : %s",adr,fil); + if (ptr > 0) { + if ((liens[ptr]->depth <= 0) + || (liens[ptr]->depth <= 1 && !embedded_triggered)) { + forbidden_url = 1; // interdire récupération du lien + hts_log_print(opt, LOG_DEBUG, + "file from too far level ignored at %s : %s", adr, fil); } } @@ -210,263 +200,295 @@ static int hts_acceptlink_(httrackp* opt, if (forbidden_url == 1) { return forbidden_url; } - // -------------------- PHASE 2 -------------------- // ------------------------------------------------------ // doit-on traiter ce lien?.. vérifier droits de déplacement - meme_adresse=strfield2(adr,urladr); - if (meme_adresse) - hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s=%s",adr,urladr); + meme_adresse = strfield2(adr, urladr); + if (meme_adresse) + hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s=%s", adr, urladr); else - hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s!=%s",adr,urladr); - if (meme_adresse) { // même adresse - { // tester interdiction de descendre + hts_log_print(opt, LOG_DEBUG, "Compare addresses: %s!=%s", adr, urladr); + if (meme_adresse) { // même adresse + { // tester interdiction de descendre // MODIFIE : en cas de remontée puis de redescente, il se pouvait qu'on ne puisse pas atteindre certains fichiers // problème: si un fichier est virtuellement accessible via une page mais dont le lien est sur une autre *uniquement*.. - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - char BIGSTK tempo2[HTS_URLMAXSIZE*2]; + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + char BIGSTK tempo2[HTS_URLMAXSIZE * 2]; + tempo[0] = tempo2[0] = '\0'; - + // note (up/down): on calcule à partir du lien primaire, ET du lien précédent. // ex: si on descend 2 fois on peut remonter 1 fois - - if (lienrelatif(tempo,fil,liens[liens[ptr]->premier]->fil)==0) { - if (lienrelatif(tempo2,fil,liens[ptr]->fil)==0) { - hts_log_print(opt, LOG_DEBUG, "build relative links to test: %s %s (with %s and %s)",tempo,tempo2,liens[liens[ptr]->premier]->fil,liens[ptr]->fil); - + + if (lienrelatif(tempo, fil, liens[liens[ptr]->premier]->fil) == 0) { + if (lienrelatif(tempo2, fil, liens[ptr]->fil) == 0) { + hts_log_print(opt, LOG_DEBUG, + "build relative links to test: %s %s (with %s and %s)", + tempo, tempo2, liens[liens[ptr]->premier]->fil, + liens[ptr]->fil); + // si vient de primary, ne pas tester lienrelatif avec (car host "différent") /*if (liens[liens[ptr]->premier] == 0) { // vient de primary - } - */ - + } + */ + // NEW: finalement OK, sauf pour les moved repérés par link_import // PROBLEME : annulé a cause d'un lien éventuel isolé accepté..qui entrainerait un miroir - + // (test même niveau (NOUVEAU à cause de certains problèmes de filtres non intégrés)) // NEW - if ( - (tempo[0] != '\0' && tempo[1] != '\0' && strchr(tempo+1,'/') == 0) - || - (tempo2[0] != '\0' && tempo2[1] != '\0' && strchr(tempo2+1,'/') == 0) + if ((tempo[0] != '\0' && tempo[1] != '\0' + && strchr(tempo + 1, '/') == 0) + || (tempo2[0] != '\0' && tempo2[1] != '\0' + && strchr(tempo2 + 1, '/') == 0) ) { - if (!liens[ptr]->link_import) { // ne résulte pas d'un 'moved' - forbidden_url=0; - hts_log_print(opt, LOG_DEBUG, "same level link authorized: %s%s",adr,fil); + if (!liens[ptr]->link_import) { // ne résulte pas d'un 'moved' + forbidden_url = 0; + hts_log_print(opt, LOG_DEBUG, "same level link authorized: %s%s", + adr, fil); } } - // down - if ( (strncmp(tempo,"../",3)) || (strncmp(tempo2,"../",3))) { // pas montée sinon ne nbous concerne pas - int test1,test2; - if (!strncmp(tempo,"../",3)) - test1=0; + if ((strncmp(tempo, "../", 3)) || (strncmp(tempo2, "../", 3))) { // pas montée sinon ne nbous concerne pas + int test1, test2; + + if (!strncmp(tempo, "../", 3)) + test1 = 0; else - test1 = (strchr(tempo +((*tempo =='/')?1:0),'/')!=NULL); - if (!strncmp(tempo2,"../",3)) - test2=0; + test1 = (strchr(tempo + ((*tempo == '/') ? 1 : 0), '/') != NULL); + if (!strncmp(tempo2, "../", 3)) + test2 = 0; else - test2 = (strchr(tempo2+((*tempo2=='/')?1:0),'/')!=NULL); - if ( (test1) && (test2) ) { // on ne peut que descendre - if ((opt->seeker & 1)==0) { // interdiction de descendre - forbidden_url=1; - hts_log_print(opt, LOG_DEBUG, "lower link canceled: %s%s",adr,fil); - } else { // autorisé à priori - NEW - if (!liens[ptr]->link_import) { // ne résulte pas d'un 'moved' - forbidden_url=0; - hts_log_print(opt, LOG_DEBUG, "lower link authorized: %s%s",adr,fil); + test2 = + (strchr(tempo2 + ((*tempo2 == '/') ? 1 : 0), '/') != NULL); + if ((test1) && (test2)) { // on ne peut que descendre + if ((opt->seeker & 1) == 0) { // interdiction de descendre + forbidden_url = 1; + hts_log_print(opt, LOG_DEBUG, "lower link canceled: %s%s", adr, + fil); + } else { // autorisé à priori - NEW + if (!liens[ptr]->link_import) { // ne résulte pas d'un 'moved' + forbidden_url = 0; + hts_log_print(opt, LOG_DEBUG, "lower link authorized: %s%s", + adr, fil); } } - } else if ( (test1) || (test2) ) { // on peut descendre pour accéder au lien - if ((opt->seeker & 1)!=0) { // on peut descendre - NEW - if (!liens[ptr]->link_import) { // ne résulte pas d'un 'moved' - forbidden_url=0; - hts_log_print(opt, LOG_DEBUG, "lower link authorized: %s%s",adr,fil); + } else if ((test1) || (test2)) { // on peut descendre pour accéder au lien + if ((opt->seeker & 1) != 0) { // on peut descendre - NEW + if (!liens[ptr]->link_import) { // ne résulte pas d'un 'moved' + forbidden_url = 0; + hts_log_print(opt, LOG_DEBUG, "lower link authorized: %s%s", + adr, fil); } } } } - - + // up - if ( (!strncmp(tempo,"../",3)) && (!strncmp(tempo2,"../",3)) ) { // impossible sans monter - if ((opt->seeker & 2)==0) { // interdiction de monter - forbidden_url=1; - hts_log_print(opt, LOG_DEBUG, "upper link canceled: %s%s",adr,fil); - } else { // autorisé à monter - NEW + if ((!strncmp(tempo, "../", 3)) && (!strncmp(tempo2, "../", 3))) { // impossible sans monter + if ((opt->seeker & 2) == 0) { // interdiction de monter + forbidden_url = 1; + hts_log_print(opt, LOG_DEBUG, "upper link canceled: %s%s", adr, + fil); + } else { // autorisé à monter - NEW if (!liens[ptr]->link_import) { // ne résulte pas d'un 'moved' - forbidden_url=0; - hts_log_print(opt, LOG_DEBUG, "upper link authorized: %s%s",adr,fil); + forbidden_url = 0; + hts_log_print(opt, LOG_DEBUG, "upper link authorized: %s%s", + adr, fil); } } - } else if ( (!strncmp(tempo,"../",3)) || (!strncmp(tempo2,"../",3)) ) { // Possible en montant - if ((opt->seeker & 2)!=0) { // autorisé à monter - NEW + } else if ((!strncmp(tempo, "../", 3)) || (!strncmp(tempo2, "../", 3))) { // Possible en montant + if ((opt->seeker & 2) != 0) { // autorisé à monter - NEW if (!liens[ptr]->link_import) { // ne résulte pas d'un 'moved' - forbidden_url=0; - hts_log_print(opt, LOG_DEBUG, "upper link authorized: %s%s",adr,fil); + forbidden_url = 0; + hts_log_print(opt, LOG_DEBUG, "upper link authorized: %s%s", + adr, fil); } - } // sinon autorisé en descente + } // sinon autorisé en descente } - - + } else { - hts_log_print(opt, LOG_ERROR, "Error building relative link %s and %s",fil,liens[ptr]->fil); + hts_log_print(opt, LOG_ERROR, + "Error building relative link %s and %s", fil, + liens[ptr]->fil); } } else { - hts_log_print(opt, LOG_ERROR, "Error building relative link %s and %s",fil,liens[liens[ptr]->premier]->fil); + hts_log_print(opt, LOG_ERROR, "Error building relative link %s and %s", + fil, liens[liens[ptr]->premier]->fil); } - - } // tester interdiction de descendre? - - { // tester interdiction de monter - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - char BIGSTK tempo2[HTS_URLMAXSIZE*2]; - if (lienrelatif(tempo,fil,liens[liens[ptr]->premier]->fil)==0) { - if (lienrelatif(tempo2,fil,liens[ptr]->fil)==0) { + + } // tester interdiction de descendre? + + { // tester interdiction de monter + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + char BIGSTK tempo2[HTS_URLMAXSIZE * 2]; + + if (lienrelatif(tempo, fil, liens[liens[ptr]->premier]->fil) == 0) { + if (lienrelatif(tempo2, fil, liens[ptr]->fil) == 0) { } else { - hts_log_print(opt, LOG_ERROR, "Error building relative link %s and %s",fil,liens[ptr]->fil); + hts_log_print(opt, LOG_ERROR, + "Error building relative link %s and %s", fil, + liens[ptr]->fil); } } else { - hts_log_print(opt, LOG_ERROR, "Error building relative link %s and %s",fil,liens[liens[ptr]->premier]->fil); - + hts_log_print(opt, LOG_ERROR, "Error building relative link %s and %s", + fil, liens[liens[ptr]->premier]->fil); + } - } // fin tester interdiction de monter - - } else { // adresse différente, sortir? - + } // fin tester interdiction de monter + + } else { // adresse différente, sortir? + //if (!opt->wizard) { // mode non wizard // doit-on traiter ce lien?.. vérifier droits de sortie - switch((opt->travel & 255)) { - case 0: - if (!opt->wizard) // mode non wizard - forbidden_url=1; - break; // interdicton de sortir au dela de l'adresse - case 1: { // sortie sur le même dom.xxx - size_t i = strlen(adr)-1; - size_t j = strlen(urladr)-1; - if ((i>0) && (j>0)) { - while( (i>0) && (adr[i]!='.')) i--; - while( (j>0) && (urladr[j]!='.')) j--; - if ((i>0) && (j>0)) { - i--; j--; - while( (i>0) && (adr[i]!='.')) i--; - while( (j>0) && (urladr[j]!='.')) j--; - } - } - if ((i>0) && (j>0)) { - if (!strfield2(adr+i,urladr+j)) { // != - if (!opt->wizard) { // mode non wizard - //printf("refused: %s\n",adr); - forbidden_url=1; // pas même domaine - hts_log_print(opt, LOG_DEBUG, "foreign domain link canceled: %s%s",adr,fil); - } - - } else { - if (opt->wizard) { // mode wizard - forbidden_url=0; // même domaine - hts_log_print(opt, LOG_DEBUG, "same domain link authorized: %s%s",adr,fil); + switch ((opt->travel & 255)) { + case 0: + if (!opt->wizard) // mode non wizard + forbidden_url = 1; + break; // interdicton de sortir au dela de l'adresse + case 1:{ // sortie sur le même dom.xxx + size_t i = strlen(adr) - 1; + size_t j = strlen(urladr) - 1; + + if ((i > 0) && (j > 0)) { + while((i > 0) && (adr[i] != '.')) + i--; + while((j > 0) && (urladr[j] != '.')) + j--; + if ((i > 0) && (j > 0)) { + i--; + j--; + while((i > 0) && (adr[i] != '.')) + i--; + while((j > 0) && (urladr[j] != '.')) + j--; } } - - } else - forbidden_url=1; - } - break; - case 2: { // sortie sur le même .xxx - size_t i = strlen(adr)-1; - size_t j = strlen(urladr)-1; - while( (i>0) && (adr[i]!='.')) i--; - while( (j>0) && (urladr[j]!='.')) j--; - if ((i>0) && (j>0)) { - if (!strfield2(adr+i,urladr+j)) { // !- - if (!opt->wizard) { // mode non wizard - //printf("refused: %s\n",adr); - forbidden_url=1; // pas même .xx - hts_log_print(opt, LOG_DEBUG, "foreign location link canceled: %s%s",adr,fil); + if ((i > 0) && (j > 0)) { + if (!strfield2(adr + i, urladr + j)) { // != + if (!opt->wizard) { // mode non wizard + //printf("refused: %s\n",adr); + forbidden_url = 1; // pas même domaine + hts_log_print(opt, LOG_DEBUG, + "foreign domain link canceled: %s%s", adr, fil); + } + + } else { + if (opt->wizard) { // mode wizard + forbidden_url = 0; // même domaine + hts_log_print(opt, LOG_DEBUG, "same domain link authorized: %s%s", + adr, fil); + } } - } else { - if (opt->wizard) { // mode wizard - forbidden_url=0; // même domaine - hts_log_print(opt, LOG_DEBUG, "same location link authorized: %s%s",adr,fil); + + } else + forbidden_url = 1; + } + break; + case 2:{ // sortie sur le même .xxx + size_t i = strlen(adr) - 1; + size_t j = strlen(urladr) - 1; + + while((i > 0) && (adr[i] != '.')) + i--; + while((j > 0) && (urladr[j] != '.')) + j--; + if ((i > 0) && (j > 0)) { + if (!strfield2(adr + i, urladr + j)) { // !- + if (!opt->wizard) { // mode non wizard + //printf("refused: %s\n",adr); + forbidden_url = 1; // pas même .xx + hts_log_print(opt, LOG_DEBUG, + "foreign location link canceled: %s%s", adr, fil); + } + } else { + if (opt->wizard) { // mode wizard + forbidden_url = 0; // même domaine + hts_log_print(opt, LOG_DEBUG, + "same location link authorized: %s%s", adr, fil); + } } - } - } else forbidden_url=1; - } + } else + forbidden_url = 1; + } break; - case 7: // everywhere!! - if (opt->wizard) { // mode wizard - forbidden_url=0; + case 7: // everywhere!! + if (opt->wizard) { // mode wizard + forbidden_url = 0; break; } - } // switch - + } // switch + // ANCIENNE POS -- récupérer les liens à côtés d'un lien (nearlink) - - } // fin test adresse identique/différente + + } // fin test adresse identique/différente // -------------------- PHASE 3 -------------------- // récupérer les liens à côtés d'un lien (nearlink) (nvelle pos) if (forbidden_url != 0 && opt->nearlink) { - if (!ishtml(opt,fil)) { // non html + if (!ishtml(opt, fil)) { // non html //printf("ok %s%s\n",ad,fil); - forbidden_url=0; // autoriser - may_set_prio_to=1+1; // set prio to 1 (parse but skip urls) if near is the winner - hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s",adr,fil); + forbidden_url = 0; // autoriser + may_set_prio_to = 1 + 1; // set prio to 1 (parse but skip urls) if near is the winner + hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s", adr, fil); } } - // -------------------- PHASE 3-BIS -------------------- - /* Built-in known tags (<img src=..>, ..) */ - if (forbidden_url != 0 && embedded_triggered) { - forbidden_url=0; // autoriser - may_set_prio_to=1+1; // set prio to 1 (parse but skip urls) if near is the winner - hts_log_print(opt, LOG_DEBUG, "near link authorized (friendly tag): %s%s",adr,fil); - } - + /* Built-in known tags (<img src=..>, ..) */ + if (forbidden_url != 0 && embedded_triggered) { + forbidden_url = 0; // autoriser + may_set_prio_to = 1 + 1; // set prio to 1 (parse but skip urls) if near is the winner + hts_log_print(opt, LOG_DEBUG, "near link authorized (friendly tag): %s%s", + adr, fil); + } // -------------------- PHASE 4 -------------------- - + // ------------------------------------------------------ // Si wizard, il se peut qu'on autorise ou qu'on interdise // un lien spécial avant même de tester sa position, sa hiérarchie etc. // peut court-circuiter le forbidden_url précédent - if (opt->wizard) { // le wizard entre en action.. + if (opt->wizard) { // le wizard entre en action.. // - int question=1; // poser une question - int force_mirror=0; // pour mirror links - int filters_answer=0; // décision prise par les filtres - char BIGSTK l[HTS_URLMAXSIZE*2]; - char BIGSTK lfull[HTS_URLMAXSIZE*2]; - - if (forbidden_url!=-1) question=0; // pas de question, résolu - + int question = 1; // poser une question + int force_mirror = 0; // pour mirror links + int filters_answer = 0; // décision prise par les filtres + char BIGSTK l[HTS_URLMAXSIZE * 2]; + char BIGSTK lfull[HTS_URLMAXSIZE * 2]; + + if (forbidden_url != -1) + question = 0; // pas de question, résolu + // former URL complète du lien actuel - strcpybuff(l,jump_identification(adr)); - if (*fil!='/') strcatbuff(l,"/"); - strcatbuff(l,fil); + strcpybuff(l, jump_identification(adr)); + if (*fil != '/') + strcatbuff(l, "/"); + strcatbuff(l, fil); // full version (http://foo:bar@www.foo.com/bar.html) if (!link_has_authority(adr)) - strcpybuff(lfull,"http://"); + strcpybuff(lfull, "http://"); else - lfull[0]='\0'; - strcatbuff(lfull,adr); - if (*fil!='/') strcatbuff(lfull,"/"); - strcatbuff(lfull,fil); - + lfull[0] = '\0'; + strcatbuff(lfull, adr); + if (*fil != '/') + strcatbuff(lfull, "/"); + strcatbuff(lfull, fil); + // tester filters (URLs autorisées ou interdites explicitement) - + // si lien primaire on saute le joker, on est pas lémur - if (ptr==0) { // lien primaire, autoriser - question=1; // la question sera résolue automatiquement - forbidden_url=0; - may_set_prio_to=0; // clear may-set flag + if (ptr == 0) { // lien primaire, autoriser + question = 1; // la question sera résolue automatiquement + forbidden_url = 0; + may_set_prio_to = 0; // clear may-set flag } else { // eternal depth first // vérifier récursivité extérieure - if (opt->extdepth>0) { - if ( /*question && */ (ptr>0) && (!force_mirror)) { + if (opt->extdepth > 0) { + if ( /*question && */ (ptr > 0) && (!force_mirror)) { // well, this is kinda a hak // we don't want to mirror EVERYTHING, and we have to decide where to stop // there is no way yet to tag "external" links, and therefore links that are @@ -475,320 +497,365 @@ static int hts_acceptlink_(httrackp* opt, if (liens[ptr]->depth > opt->extdepth) { // *set_prio_to = opt->extdepth + 1; *set_prio_to = 1 + (opt->extdepth); - may_set_prio_to=0; // clear may-set flag - forbidden_url=0; // autorisé - question=0; // résolution auto + may_set_prio_to = 0; // clear may-set flag + forbidden_url = 0; // autorisé + question = 0; // résolution auto if (question) { - hts_log_print(opt, LOG_DEBUG, "(wizard) ambiguous link accepted (external depth): link %s at %s%s",l,urladr,urlfil); + hts_log_print(opt, LOG_DEBUG, + "(wizard) ambiguous link accepted (external depth): link %s at %s%s", + l, urladr, urlfil); } else { - hts_log_print(opt, LOG_DEBUG, "(wizard) forced to accept link (external depth): link %s at %s%s",l,urladr,urlfil); + hts_log_print(opt, LOG_DEBUG, + "(wizard) forced to accept link (external depth): link %s at %s%s", + l, urladr, urlfil); } - + } } - } - + } // filters { int jok; - char* mdepth=""; + char *mdepth = ""; + // filters, 0=sait pas 1=ok -1=interdit { - int jokDepth1=0,jokDepth2=0; - int jok1=0,jok2=0; - jok1 = fa_strjoker(/*url*/0, _FILTERS,*_FILTERS_PTR,lfull,NULL,NULL,&jokDepth1); - jok2 = fa_strjoker(/*url*/0, _FILTERS,*_FILTERS_PTR,l, NULL,NULL,&jokDepth2); + int jokDepth1 = 0, jokDepth2 = 0; + int jok1 = 0, jok2 = 0; + + jok1 = + fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, NULL, NULL, + &jokDepth1); + jok2 = + fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, l, NULL, NULL, + &jokDepth2); if (jok2 == 0) { // #2 doesn't know - jok = jok1; // then, use #1 + jok = jok1; // then, use #1 mdepth = _FILTERS[jokDepth1]; - } else if (jok1 == 0) { // #1 doesn't know - jok = jok2; // then, use #2 + } else if (jok1 == 0) { // #1 doesn't know + jok = jok2; // then, use #2 mdepth = _FILTERS[jokDepth2]; - } else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary + } else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary jok = jok1; mdepth = _FILTERS[jokDepth1]; - } else { // #2 matching rule is "after" #1, then it is prioritary + } else { // #2 matching rule is "after" #1, then it is prioritary jok = jok2; mdepth = _FILTERS[jokDepth2]; } } - - if (jok == 1) { // autorisé - filters_answer=1; // décision prise par les filtres - question=0; // ne pas poser de question, autorisé - forbidden_url=0; // URL autorisée - may_set_prio_to=0; // clear may-set flag - hts_log_print(opt, LOG_DEBUG, "(wizard) explicit authorized (%s) link: link %s at %s%s",mdepth,l,urladr,urlfil); - } else if (jok == -1) { // forbidden - filters_answer=1; // décision prise par les filtres - question=0; // ne pas poser de question: - forbidden_url=1; // URL interdite - hts_log_print(opt, LOG_DEBUG, "(wizard) explicit forbidden (%s) link: link %s at %s%s",mdepth,l,urladr,urlfil); - } // sinon on touche à rien + + if (jok == 1) { // autorisé + filters_answer = 1; // décision prise par les filtres + question = 0; // ne pas poser de question, autorisé + forbidden_url = 0; // URL autorisée + may_set_prio_to = 0; // clear may-set flag + hts_log_print(opt, LOG_DEBUG, + "(wizard) explicit authorized (%s) link: link %s at %s%s", + mdepth, l, urladr, urlfil); + } else if (jok == -1) { // forbidden + filters_answer = 1; // décision prise par les filtres + question = 0; // ne pas poser de question: + forbidden_url = 1; // URL interdite + hts_log_print(opt, LOG_DEBUG, + "(wizard) explicit forbidden (%s) link: link %s at %s%s", + mdepth, l, urladr, urlfil); + } // sinon on touche à rien } } - + // vérifier mode mirror links if (question) { - if (opt->mirror_first_page) { // mode mirror links - if (liens[ptr]->precedent==0) { // parent=primary! - forbidden_url=0; // autorisé - may_set_prio_to=0; // clear may-set flag - question=1; // résolution auto - force_mirror=5; // mirror (5) - hts_log_print(opt, LOG_DEBUG, "(wizard) explicit mirror link: link %s at %s%s",l,urladr,urlfil); + if (opt->mirror_first_page) { // mode mirror links + if (liens[ptr]->precedent == 0) { // parent=primary! + forbidden_url = 0; // autorisé + may_set_prio_to = 0; // clear may-set flag + question = 1; // résolution auto + force_mirror = 5; // mirror (5) + hts_log_print(opt, LOG_DEBUG, + "(wizard) explicit mirror link: link %s at %s%s", l, + urladr, urlfil); } } } - // on doit poser la question.. peut on la poser? // (oui je sais quel preuve de délicatesse, merci merci) - if ((question) && (ptr>0) && (!force_mirror)) { - if (opt->wizard==2) { // éliminer tous les liens non répertoriés comme autorisés (ou inconnus) - question=0; - forbidden_url=1; - hts_log_print(opt, LOG_DEBUG, "(wizard) ambiguous forbidden link: link %s at %s%s",l,urladr,urlfil); + if ((question) && (ptr > 0) && (!force_mirror)) { + if (opt->wizard == 2) { // éliminer tous les liens non répertoriés comme autorisés (ou inconnus) + question = 0; + forbidden_url = 1; + hts_log_print(opt, LOG_DEBUG, + "(wizard) ambiguous forbidden link: link %s at %s%s", l, + urladr, urlfil); } } - // vérifier robots.txt if (opt->robots) { - int r = checkrobots(_ROBOTS,adr,fil); - if (r == -1) { // interdiction + int r = checkrobots(_ROBOTS, adr, fil); + + if (r == -1) { // interdiction #if DEBUG_ROBOTS - printf("robots.txt forbidden: %s%s\n",adr,fil); + printf("robots.txt forbidden: %s%s\n", adr, fil); #endif // question résolue, par les filtres, et mode robot non strict - if ((!question) && (filters_answer) && (opt->robots == 1) && (forbidden_url!=1)) { - r=0; // annuler interdiction des robots + if ((!question) && (filters_answer) && (opt->robots == 1) + && (forbidden_url != 1)) { + r = 0; // annuler interdiction des robots if (!forbidden_url) { - hts_log_print(opt, LOG_DEBUG, "Warning link followed against robots.txt: link %s at %s%s",l,adr,fil); + hts_log_print(opt, LOG_DEBUG, + "Warning link followed against robots.txt: link %s at %s%s", + l, adr, fil); } } - if (r == -1) { // interdire - forbidden_url=1; - question=0; - hts_log_print(opt, LOG_DEBUG, "(robots.txt) forbidden link: link %s at %s%s",l,adr,fil); + if (r == -1) { // interdire + forbidden_url = 1; + question = 0; + hts_log_print(opt, LOG_DEBUG, + "(robots.txt) forbidden link: link %s at %s%s", l, adr, + fil); } } } - + if (!question) { if (!forbidden_url) { - hts_log_print(opt, LOG_DEBUG, "(wizard) shared foreign domain link: link %s at %s%s",l,urladr,urlfil); + hts_log_print(opt, LOG_DEBUG, + "(wizard) shared foreign domain link: link %s at %s%s", l, + urladr, urlfil); } else { - hts_log_print(opt, LOG_DEBUG, "(wizard) cancelled foreign domain link: link %s at %s%s",l,urladr,urlfil); + hts_log_print(opt, LOG_DEBUG, + "(wizard) cancelled foreign domain link: link %s at %s%s", + l, urladr, urlfil); } #if BDEBUG==3 - printf("at %s in %s, wizard says: url %s ",urladr,urlfil,l); - if (forbidden_url) printf("cancelled"); else printf(">SHARED<"); + printf("at %s in %s, wizard says: url %s ", urladr, urlfil, l); + if (forbidden_url) + printf("cancelled"); + else + printf(">SHARED<"); printf("\n"); -#endif +#endif } /* en cas de question, ou lien primaire (enregistrer autorisations) */ - if (question || (ptr==0)) { - const char* s; - int n=0; - + if (question || (ptr == 0)) { + const char *s; + int n = 0; + // si primaire (plus bas) alors ... - if ((ptr!=0) && (force_mirror==0)) { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - tempo[0]='\0'; - strcatbuff(tempo,adr); - strcatbuff(tempo,fil); + if ((ptr != 0) && (force_mirror == 0)) { + char BIGSTK tempo[HTS_URLMAXSIZE * 2]; + + tempo[0] = '\0'; + strcatbuff(tempo, adr); + strcatbuff(tempo, fil); s = RUN_CALLBACK1(opt, query3, tempo); - if (strnotempty(s)==0) // entrée - n=0; - else if (isdigit((unsigned char)*s)) - sscanf(s,"%d",&n); - else { - switch(*s) { - case '*': n=-1; break; - case '!': n=-999; { + if (strnotempty(s) == 0) // entrée + n = 0; + else if (isdigit((unsigned char) *s)) + sscanf(s, "%d", &n); + else { + switch (*s) { + case '*': + n = -1; + break; + case '!': + n = -999; { /*char *a; - int i; - a=copie_de_adr-128; - if (a<r.adr) a=r.adr; - for(i=0;i<256;i++) { - if (a==copie_de_adr) printf("\nHERE:\n"); - printf("%c",*a++); - } - printf("\n\n"); - */ - } - break; - default: n=-999; printf("What did you say?\n"); break; - - } + int i; + a=copie_de_adr-128; + if (a<r.adr) a=r.adr; + for(i=0;i<256;i++) { + if (a==copie_de_adr) printf("\nHERE:\n"); + printf("%c",*a++); + } + printf("\n\n"); + */ + } + break; + default: + n = -999; + printf("What did you say?\n"); + break; + } + } io_flush; - } else { // lien primaire: autoriser répertoire entier + } else { // lien primaire: autoriser répertoire entier if (!force_mirror) { - if ((opt->seeker & 1)==0) { // interdiction de descendre - n=7; + if ((opt->seeker & 1) == 0) { // interdiction de descendre + n = 7; } else { - n=5; // autoriser miroir répertoires descendants (lien primaire) + n = 5; // autoriser miroir répertoires descendants (lien primaire) } - } else // forcer valeur (sub-wizard) - n=force_mirror; + } else // forcer valeur (sub-wizard) + n = force_mirror; } - + /* sanity check - reallocate filters HERE */ if ((*_FILTERS_PTR) + 1 >= opt->maxfilter) { opt->maxfilter += HTS_FILTERSINC; if (filters_init(&_FILTERS, opt->maxfilter, HTS_FILTERSINC) == 0) { - printf("PANIC! : Too many filters : >%d [%d]\n", (*_FILTERS_PTR),__LINE__); + printf("PANIC! : Too many filters : >%d [%d]\n", (*_FILTERS_PTR), + __LINE__); fflush(stdout); - hts_log_print(opt, LOG_PANIC, "Too many filters, giving up..(>%d)", (*_FILTERS_PTR) ); - hts_log_print(opt, LOG_INFO, "To avoid that: use #F option for more filters (example: -#F5000)"); - assertf("too many filters - giving up" == NULL); // wild.. + hts_log_print(opt, LOG_PANIC, "Too many filters, giving up..(>%d)", + (*_FILTERS_PTR)); + hts_log_print(opt, LOG_INFO, + "To avoid that: use #F option for more filters (example: -#F5000)"); + assertf("too many filters - giving up" == NULL); // wild.. } } - // here we have enough room for a new filter if necessary - switch(n) { - case -1: // sauter tout le reste - forbidden_url=1; - opt->wizard=2; // sauter tout le reste + switch (n) { + case -1: // sauter tout le reste + forbidden_url = 1; + opt->wizard = 2; // sauter tout le reste break; - case 0: // interdire les mêmes liens: adr/fil - forbidden_url=1; - HT_INSERT_FILTERS0; // insérer en 0 - strcpybuff(_FILTERS[0],"-"); - strcatbuff(_FILTERS[0],jump_identification(adr)); - if (*fil!='/') strcatbuff(_FILTERS[0],"/"); - strcatbuff(_FILTERS[0],fil); + case 0: // interdire les mêmes liens: adr/fil + forbidden_url = 1; + HT_INSERT_FILTERS0; // insérer en 0 + strcpybuff(_FILTERS[0], "-"); + strcatbuff(_FILTERS[0], jump_identification(adr)); + if (*fil != '/') + strcatbuff(_FILTERS[0], "/"); + strcatbuff(_FILTERS[0], fil); break; - - case 1: // éliminer répertoire entier et sous rép: adr/path/ * - forbidden_url=1; + + case 1: // éliminer répertoire entier et sous rép: adr/path/ * + forbidden_url = 1; { - size_t i = strlen(fil)-1; - while((fil[i]!='/') && (i>0)) i--; - if (fil[i]=='/') { - HT_INSERT_FILTERS0; // insérer en 0 - strcpybuff(_FILTERS[0],"-"); - strcatbuff(_FILTERS[0],jump_identification(adr)); - if (*fil!='/') strcatbuff(_FILTERS[0],"/"); - strncatbuff(_FILTERS[0] ,fil,i); - if (_FILTERS[0][strlen(_FILTERS[0])-1]!='/') - strcatbuff(_FILTERS[0],"/"); - strcatbuff(_FILTERS[0],"*"); + size_t i = strlen(fil) - 1; + + while((fil[i] != '/') && (i > 0)) + i--; + if (fil[i] == '/') { + HT_INSERT_FILTERS0; // insérer en 0 + strcpybuff(_FILTERS[0], "-"); + strcatbuff(_FILTERS[0], jump_identification(adr)); + if (*fil != '/') + strcatbuff(_FILTERS[0], "/"); + strncatbuff(_FILTERS[0], fil, i); + if (_FILTERS[0][strlen(_FILTERS[0]) - 1] != '/') + strcatbuff(_FILTERS[0], "/"); + strcatbuff(_FILTERS[0], "*"); } - } - + } + // ** ... break; - - case 2: // adresse adr* - forbidden_url=1; - HT_INSERT_FILTERS0; // insérer en 0 - strcpybuff(_FILTERS[0],"-"); - strcatbuff(_FILTERS[0],jump_identification(adr)); - strcatbuff(_FILTERS[0],"*"); + + case 2: // adresse adr* + forbidden_url = 1; + HT_INSERT_FILTERS0; // insérer en 0 + strcpybuff(_FILTERS[0], "-"); + strcatbuff(_FILTERS[0], jump_identification(adr)); + strcatbuff(_FILTERS[0], "*"); break; - - case 3: // ** A FAIRE - forbidden_url=1; + + case 3: // ** A FAIRE + forbidden_url = 1; /* - { - int i=strlen(adr)-1; - while((adr[i]!='/') && (i>0)) i--; - if (i>0) { - - } - - }*/ - + { + int i=strlen(adr)-1; + while((adr[i]!='/') && (i>0)) i--; + if (i>0) { + + } + + } */ + break; // - case 4: // same link + case 4: // same link // PAS BESOIN!! /*HT_INSERT_FILTERS0; // insérer en 0 - strcpybuff(_FILTERS[0],"+"); - strcatbuff(_FILTERS[0],adr); - if (*fil!='/') strcatbuff(_FILTERS[0],"/"); - strcatbuff(_FILTERS[0],fil);*/ - - + strcpybuff(_FILTERS[0],"+"); + strcatbuff(_FILTERS[0],adr); + if (*fil!='/') strcatbuff(_FILTERS[0],"/"); + strcatbuff(_FILTERS[0],fil); */ + // étant donné le renversement wizard/primary filter (les primary autorisent up/down ET interdisent) // il faut éviter d'un lien isolé effectue un miroir total.. - - *set_prio_to = 0+1; // niveau de récursion=0 (pas de miroir) - + + *set_prio_to = 0 + 1; // niveau de récursion=0 (pas de miroir) + break; - - case 5: // autoriser répertoire entier et fils - if ((opt->seeker & 2)==0) { // interdiction de monter - size_t i = strlen(fil)-1; - while((fil[i]!='/') && (i>0)) i--; - if (fil[i]=='/') { - HT_INSERT_FILTERS0; // insérer en 0 - strcpybuff(_FILTERS[0],"+"); - strcatbuff(_FILTERS[0],jump_identification(adr)); - if (*fil!='/') strcatbuff(_FILTERS[0],"/"); - strncatbuff(_FILTERS[0],fil,i+1); - strcatbuff(_FILTERS[0],"*"); + + case 5: // autoriser répertoire entier et fils + if ((opt->seeker & 2) == 0) { // interdiction de monter + size_t i = strlen(fil) - 1; + + while((fil[i] != '/') && (i > 0)) + i--; + if (fil[i] == '/') { + HT_INSERT_FILTERS0; // insérer en 0 + strcpybuff(_FILTERS[0], "+"); + strcatbuff(_FILTERS[0], jump_identification(adr)); + if (*fil != '/') + strcatbuff(_FILTERS[0], "/"); + strncatbuff(_FILTERS[0], fil, i + 1); + strcatbuff(_FILTERS[0], "*"); } - } else { // autoriser domaine alors!! - HT_INSERT_FILTERS0; // insérer en 0 strcpybuff(filters[filptr],"+"); - strcpybuff(_FILTERS[0],"+"); - strcatbuff(_FILTERS[0],jump_identification(adr)); - strcatbuff(_FILTERS[0],"*"); + } else { // autoriser domaine alors!! + HT_INSERT_FILTERS0; // insérer en 0 strcpybuff(filters[filptr],"+"); + strcpybuff(_FILTERS[0], "+"); + strcatbuff(_FILTERS[0], jump_identification(adr)); + strcatbuff(_FILTERS[0], "*"); } break; - - case 6: // same domain - HT_INSERT_FILTERS0; // insérer en 0 strcpybuff(filters[filptr],"+"); - strcpybuff(_FILTERS[0],"+"); - strcatbuff(_FILTERS[0],jump_identification(adr)); - strcatbuff(_FILTERS[0],"*"); + + case 6: // same domain + HT_INSERT_FILTERS0; // insérer en 0 strcpybuff(filters[filptr],"+"); + strcpybuff(_FILTERS[0], "+"); + strcatbuff(_FILTERS[0], jump_identification(adr)); + strcatbuff(_FILTERS[0], "*"); break; // - case 7: // autoriser ce répertoire + case 7: // autoriser ce répertoire { - size_t i = strlen(fil)-1; - while((fil[i]!='/') && (i>0)) i--; - if (fil[i]=='/') { - HT_INSERT_FILTERS0; // insérer en 0 - strcpybuff(_FILTERS[0],"+"); - strcatbuff(_FILTERS[0],jump_identification(adr)); - if (*fil!='/') strcatbuff(_FILTERS[0],"/"); - strncatbuff(_FILTERS[0],fil,i+1); - strcatbuff(_FILTERS[0],"*[file]"); + size_t i = strlen(fil) - 1; + + while((fil[i] != '/') && (i > 0)) + i--; + if (fil[i] == '/') { + HT_INSERT_FILTERS0; // insérer en 0 + strcpybuff(_FILTERS[0], "+"); + strcatbuff(_FILTERS[0], jump_identification(adr)); + if (*fil != '/') + strcatbuff(_FILTERS[0], "/"); + strncatbuff(_FILTERS[0], fil, i + 1); + strcatbuff(_FILTERS[0], "*[file]"); } } - + break; - - case 50: // on fait rien + + case 50: // on fait rien break; - } // switch - - } // test du wizard sur l'url - } // fin du test wizard.. + } // switch + + } // test du wizard sur l'url + } // fin du test wizard.. // -------------------- PHASE 5 -------------------- // lien non autorisé, peut-on juste le tester? if (just_test_it) { - if (forbidden_url==1) { - if (opt->travel&256) { // tester tout de même - if (strfield(adr,"ftp://")==0 + if (forbidden_url == 1) { + if (opt->travel & 256) { // tester tout de même + if (strfield(adr, "ftp://") == 0 #if HTS_USEMMS - && strfield(adr,"mms://")==0 + && strfield(adr, "mms://") == 0 #endif - ) { // PAS ftp! - forbidden_url=1; // oui oui toujours interdit (note: sert à rien car ==1 mais c pour comprendre) - *just_test_it=1; // mais on teste - hts_log_print(opt, LOG_DEBUG, "Testing link %s%s",adr,fil); + ) { // PAS ftp! + forbidden_url = 1; // oui oui toujours interdit (note: sert à rien car ==1 mais c pour comprendre) + *just_test_it = 1; // mais on teste + hts_log_print(opt, LOG_DEBUG, "Testing link %s%s", adr, fil); } } } //adr[0]='\0'; // cancel } - // -------------------- FINAL PHASE -------------------- // Test if the "Near" test won if (may_set_prio_to && forbidden_url == 0) { @@ -801,38 +868,42 @@ static int hts_acceptlink_(httrackp* opt, #undef _ROBOTS } -int hts_acceptmime(httrackp* opt, - int ptr,int lien_tot,lien_url** liens, - char* adr,char* fil, - char* mime) -{ +int hts_acceptmime(httrackp * opt, int ptr, int lien_tot, lien_url ** liens, + char *adr, char *fil, char *mime) { #define _FILTERS (*opt->filters.filters) #define _FILTERS_PTR (opt->filters.filptr) #define _ROBOTS ((robots_wizard*)opt->robotsptr) int forbidden_url = -1; - char* mdepth=""; + char *mdepth = ""; int jokDepth = 0; int jok = 0; /* Authorized ? */ - jok = fa_strjoker(/*mime*/1, _FILTERS, *_FILTERS_PTR, mime, NULL, NULL, &jokDepth); + jok = + fa_strjoker( /*mime */ 1, _FILTERS, *_FILTERS_PTR, mime, NULL, NULL, + &jokDepth); if (jok != 0) { mdepth = _FILTERS[jokDepth]; - if (jok == 1) { // autorisé - forbidden_url=0; // URL autorisée - hts_log_print(opt, LOG_DEBUG, "(wizard) explicit authorized (%s) link %s%s: mime '%s'",mdepth,adr,fil,mime); - } else if (jok == -1) { // forbidden - forbidden_url=1; // URL interdite - hts_log_print(opt, LOG_DEBUG, "(wizard) explicit forbidden (%s) link %s%s: mime '%s'",mdepth,adr,fil,mime); - } // sinon on touche à rien + if (jok == 1) { // autorisé + forbidden_url = 0; // URL autorisée + hts_log_print(opt, LOG_DEBUG, + "(wizard) explicit authorized (%s) link %s%s: mime '%s'", + mdepth, adr, fil, mime); + } else if (jok == -1) { // forbidden + forbidden_url = 1; // URL interdite + hts_log_print(opt, LOG_DEBUG, + "(wizard) explicit forbidden (%s) link %s%s: mime '%s'", + mdepth, adr, fil, mime); + } // sinon on touche à rien } /* userdef test */ - { - int test_url = RUN_CALLBACK4(opt, check_mime, adr, fil, mime, forbidden_url); - if (test_url!=-1) { - forbidden_url=test_url; + { + int test_url = + RUN_CALLBACK4(opt, check_mime, adr, fil, mime, forbidden_url); + if (test_url != -1) { + forbidden_url = test_url; } - } + } return forbidden_url; #undef _FILTERS #undef _FILTERS_PTR @@ -840,66 +911,77 @@ int hts_acceptmime(httrackp* opt, } // tester taille -int hts_testlinksize(httrackp* opt, - char* adr,char* fil, - LLint size) { - int jok=0; - if (size>=0) { - char BIGSTK l[HTS_URLMAXSIZE*2]; - char BIGSTK lfull[HTS_URLMAXSIZE*2]; - if (size>=0) { - LLint sz=size; - int size_flag=0; - +int hts_testlinksize(httrackp * opt, char *adr, char *fil, LLint size) { + int jok = 0; + + if (size >= 0) { + char BIGSTK l[HTS_URLMAXSIZE * 2]; + char BIGSTK lfull[HTS_URLMAXSIZE * 2]; + + if (size >= 0) { + LLint sz = size; + int size_flag = 0; + // former URL complète du lien actuel - strcpybuff(l,jump_identification(adr)); - if (*fil!='/') strcatbuff(l,"/"); - strcatbuff(l,fil); + strcpybuff(l, jump_identification(adr)); + if (*fil != '/') + strcatbuff(l, "/"); + strcatbuff(l, fil); // if (!link_has_authority(adr)) - strcpybuff(lfull,"http://"); + strcpybuff(lfull, "http://"); else - lfull[0]='\0'; - strcatbuff(lfull,adr); - if (*fil!='/') strcatbuff(l,"/"); - strcatbuff(lfull,fil); - + lfull[0] = '\0'; + strcatbuff(lfull, adr); + if (*fil != '/') + strcatbuff(l, "/"); + strcatbuff(lfull, fil); + // filters, 0=sait pas 1=ok -1=interdit { - int jokDepth1=0,jokDepth2=0; - int jok1=0,jok2=0; - LLint sz1=size,sz2=size; - int size_flag1=0,size_flag2=0; - jok1 = fa_strjoker(/*url*/0, *opt->filters.filters,*opt->filters.filptr,lfull,&sz1,&size_flag1,&jokDepth1); - jok2 = fa_strjoker(/*url*/0, *opt->filters.filters,*opt->filters.filptr,l, &sz2,&size_flag2,&jokDepth2); - if (jok2 == 0) { // #2 doesn't know - jok = jok1; // then, use #1 + int jokDepth1 = 0, jokDepth2 = 0; + int jok1 = 0, jok2 = 0; + LLint sz1 = size, sz2 = size; + int size_flag1 = 0, size_flag2 = 0; + + jok1 = + fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr, + lfull, &sz1, &size_flag1, &jokDepth1); + jok2 = + fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr, + l, &sz2, &size_flag2, &jokDepth2); + if (jok2 == 0) { // #2 doesn't know + jok = jok1; // then, use #1 sz = sz1; size_flag = size_flag1; - } else if (jok1 == 0) { // #1 doesn't know - jok = jok2; // then, use #2 + } else if (jok1 == 0) { // #1 doesn't know + jok = jok2; // then, use #2 sz = sz2; size_flag = size_flag2; - } else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary + } else if (jokDepth1 >= jokDepth2) { // #1 matching rule is "after" #2, then it is prioritary jok = jok1; sz = sz1; size_flag = size_flag1; - } else { // #2 matching rule is "after" #1, then it is prioritary + } else { // #2 matching rule is "after" #1, then it is prioritary jok = jok2; sz = sz2; size_flag = size_flag2; - } + } } - // log - if (jok==1) { - hts_log_print(opt, LOG_DEBUG, "File confirmed (size test): %s%s ("LLintP")",adr,fil,(LLint)(size)); - } else if (jok==-1) { + if (jok == 1) { + hts_log_print(opt, LOG_DEBUG, + "File confirmed (size test): %s%s (" LLintP ")", adr, fil, + (LLint) (size)); + } else if (jok == -1) { if (size_flag) { /* interdit à cause de la taille */ - hts_log_print(opt, LOG_DEBUG, "File cancelled due to its size: %s%s ("LLintP", limit: "LLintP")",adr,fil,(LLint)(size),(LLint)(sz)); + hts_log_print(opt, LOG_DEBUG, + "File cancelled due to its size: %s%s (" LLintP + ", limit: " LLintP ")", adr, fil, (LLint) (size), + (LLint) (sz)); } else { - jok=1; + jok = 1; } } } @@ -911,4 +993,3 @@ int hts_testlinksize(httrackp* opt, #undef urlfil #undef HT_INSERT_FILTERS0 - diff --git a/src/htswizard.h b/src/htswizard.h index 665a147..29c271f 100644 --- a/src/htswizard.h +++ b/src/htswizard.h @@ -17,14 +17,12 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ @@ -34,9 +32,8 @@ Please visit our Website: http://www.httrack.com /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - #ifndef HTSWIZARD_DEFH -#define HTSWIZARD_DEFH +#define HTSWIZARD_DEFH /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE @@ -53,19 +50,12 @@ typedef struct httrackp httrackp; typedef struct lien_url lien_url; #endif -int hts_acceptlink(httrackp* opt, - int ptr,int lien_tot,lien_url** liens, - char* adr,char* fil, - char* tag, char* attribute, - int* set_prio_to_0, - int* just_test_it); -int hts_testlinksize(httrackp* opt, - char* adr,char* fil, - LLint size); -int hts_acceptmime(httrackp* opt, - int ptr,int lien_tot,lien_url** liens, - char* adr,char* fil, - char* mime); +int hts_acceptlink(httrackp * opt, int ptr, int lien_tot, lien_url ** liens, + char *adr, char *fil, char *tag, char *attribute, + int *set_prio_to_0, int *just_test_it); +int hts_testlinksize(httrackp * opt, char *adr, char *fil, LLint size); +int hts_acceptmime(httrackp * opt, int ptr, int lien_tot, lien_url ** liens, + char *adr, char *fil, char *mime); #endif #endif diff --git a/src/htswrap.c b/src/htswrap.c index b6cebbf..59920e0 100644 --- a/src/htswrap.c +++ b/src/htswrap.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* wrapper system (for shell */ @@ -43,19 +40,20 @@ Please visit our Website: http://www.httrack.com #include "htsinthash.h" #include "htslib.h" -HTSEXT_API int htswrap_init(void) { // LEGACY +HTSEXT_API int htswrap_init(void) { // LEGACY return 1; } -HTSEXT_API int htswrap_free(void) { // LEGACY +HTSEXT_API int htswrap_free(void) { // LEGACY return 1; } -HTSEXT_API int htswrap_add(httrackp *opt, const char* name,void* fct) { - return hts_set_callback((t_hts_htmlcheck_callbacks*)opt->callbacks_fun, name, fct); +HTSEXT_API int htswrap_add(httrackp * opt, const char *name, void *fct) { + return hts_set_callback((t_hts_htmlcheck_callbacks *) opt->callbacks_fun, + name, fct); } -HTSEXT_API uintptr_t htswrap_read(httrackp *opt, const char* name) { - return (uintptr_t) hts_get_callback((t_hts_htmlcheck_callbacks*)opt->callbacks_fun, name); +HTSEXT_API uintptr_t htswrap_read(httrackp * opt, const char *name) { + return (uintptr_t) hts_get_callback((t_hts_htmlcheck_callbacks *) opt-> + callbacks_fun, name); } - diff --git a/src/htswrap.h b/src/htswrap.h index 59a8f29..96f5173 100644 --- a/src/htswrap.h +++ b/src/htswrap.h @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: httrack.c subroutines: */ /* wrapper system (for shell */ @@ -36,7 +33,7 @@ Please visit our Website: http://www.httrack.com /* ------------------------------------------------------------ */ #ifndef HTSWRAP_DEFH -#define HTSWRAP_DEFH +#define HTSWRAP_DEFH /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE @@ -50,11 +47,11 @@ Please visit our Website: http://www.httrack.com typedef struct httrackp httrackp; #endif -HTSEXT_API int htswrap_init(void); // LEGACY -HTSEXT_API int htswrap_free(void); // LEGACY +HTSEXT_API int htswrap_init(void); // LEGACY +HTSEXT_API int htswrap_free(void); // LEGACY -HTSEXT_API int htswrap_add(httrackp *opt, const char* name, void* fct); -HTSEXT_API uintptr_t htswrap_read(httrackp *opt, const char* name); +HTSEXT_API int htswrap_add(httrackp * opt, const char *name, void *fct); +HTSEXT_API uintptr_t htswrap_read(httrackp * opt, const char *name); #endif diff --git a/src/htszlib.c b/src/htszlib.c index d313d89..f4a30b4 100644 --- a/src/htszlib.c +++ b/src/htszlib.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Unpacking subroutines using Jean-loup Gailly's Zlib */ /* for http compressed data */ @@ -55,30 +52,35 @@ Please visit our Website: http://www.httrack.com Return value: size of the new file, or -1 if an error occured */ /* Note: utf-8 */ -int hts_zunpack(char* filename,char* newfile) { +int hts_zunpack(char *filename, char *newfile) { int ret = -1; - char catbuff[CATBUFF_SIZE]; + char catbuff[CATBUFF_SIZE]; + if (gz_is_available && filename && newfile) { if (filename[0] && newfile[0]) { // not: NOT an UTF-8 filename gzFile gz = gzopen(filename, "rb"); + if (gz) { - FILE*const fpout = FOPEN(fconv(catbuff, newfile), "wb"); - int size=0; + FILE *const fpout = FOPEN(fconv(catbuff, newfile), "wb"); + int size = 0; + if (fpout) { int nr; + do { char BIGSTK buff[1024]; - nr=gzread (gz, buff, 1024); - if (nr>0) { - size+=nr; - if (fwrite(buff,1,nr,fpout) != nr) - nr=size=-1; + + nr = gzread(gz, buff, 1024); + if (nr > 0) { + size += nr; + if (fwrite(buff, 1, nr, fpout) != nr) + nr = size = -1; } - } while(nr>0); + } while(nr > 0); fclose(fpout); } else - size=-1; + size = -1; gzclose(gz); ret = (int) size; } @@ -87,47 +89,41 @@ int hts_zunpack(char* filename,char* newfile) { return ret; } -int hts_extract_meta(const char* path) { - char catbuff[CATBUFF_SIZE]; - unzFile zFile = unzOpen(fconcat(catbuff,path,"hts-cache/new.zip")); - zipFile zFileOut = zipOpen(fconcat(catbuff,path,"hts-cache/meta.zip"), 0); +int hts_extract_meta(const char *path) { + char catbuff[CATBUFF_SIZE]; + unzFile zFile = unzOpen(fconcat(catbuff, path, "hts-cache/new.zip")); + zipFile zFileOut = zipOpen(fconcat(catbuff, path, "hts-cache/meta.zip"), 0); + if (zFile != NULL && zFileOut != NULL) { if (unzGoToFirstFile(zFile) == Z_OK) { zip_fileinfo fi; unz_file_info ufi; char BIGSTK filename[HTS_URLMAXSIZE * 4]; char BIGSTK comment[8192]; - memset(comment, 0, sizeof(comment)); // for truncated reads + + memset(comment, 0, sizeof(comment)); // for truncated reads memset(&fi, 0, sizeof(fi)); memset(&ufi, 0, sizeof(ufi)); - do { + do { int readSizeHeader; + filename[0] = '\0'; comment[0] = '\0'; - + if (unzOpenCurrentFile(zFile) == Z_OK) { - if ( - (readSizeHeader = unzGetLocalExtrafield(zFile, comment, sizeof(comment) - 2)) > 0 - && - unzGetCurrentFileInfo(zFile, &ufi, filename, sizeof(filename) - 2, NULL, 0, NULL, 0) == Z_OK - ) - { + if ((readSizeHeader = + unzGetLocalExtrafield(zFile, comment, sizeof(comment) - 2)) > 0 + && unzGetCurrentFileInfo(zFile, &ufi, filename, + sizeof(filename) - 2, NULL, 0, NULL, + 0) == Z_OK) { comment[readSizeHeader] = '\0'; fi.dosDate = ufi.dosDate; fi.internal_fa = ufi.internal_fa; fi.external_fa = ufi.external_fa; - if (zipOpenNewFileInZip(zFileOut, - filename, - &fi, - NULL, - 0, - NULL, - 0, - NULL, /* comment */ - Z_DEFLATED, - Z_DEFAULT_COMPRESSION) == Z_OK) - { - if (zipWriteInFileInZip(zFileOut, comment, (int) strlen(comment)) != Z_OK) { + if (zipOpenNewFileInZip(zFileOut, filename, &fi, NULL, 0, NULL, 0, NULL, /* comment */ + Z_DEFLATED, Z_DEFAULT_COMPRESSION) == Z_OK) { + if (zipWriteInFileInZip(zFileOut, comment, (int) strlen(comment)) + != Z_OK) { } if (zipCloseFileInZip(zFileOut) != Z_OK) { } @@ -135,41 +131,41 @@ int hts_extract_meta(const char* path) { } unzCloseCurrentFile(zFile); } - } while( unzGoToNextFile(zFile) == Z_OK ); + } while(unzGoToNextFile(zFile) == Z_OK); } - zipClose(zFileOut, "Meta-data extracted by HTTrack/"HTTRACK_VERSION); + zipClose(zFileOut, "Meta-data extracted by HTTrack/" HTTRACK_VERSION); unzClose(zFile); return 1; } return 0; } -const char* hts_get_zerror(int err) { - switch(err) { - case UNZ_OK: - return "no error"; - break; - case UNZ_END_OF_LIST_OF_FILE: - return "end of list of file"; - break; - case UNZ_ERRNO: - return (const char*) strerror(errno); - break; - case UNZ_PARAMERROR: - return "parameter error"; - break; - case UNZ_BADZIPFILE: - return "bad zip file"; - break; - case UNZ_INTERNALERROR: - return "internal error"; - break; - case UNZ_CRCERROR: - return "crc error"; - break; - default: - return "unknown error"; - break; +const char *hts_get_zerror(int err) { + switch (err) { + case UNZ_OK: + return "no error"; + break; + case UNZ_END_OF_LIST_OF_FILE: + return "end of list of file"; + break; + case UNZ_ERRNO: + return (const char *) strerror(errno); + break; + case UNZ_PARAMERROR: + return "parameter error"; + break; + case UNZ_BADZIPFILE: + return "bad zip file"; + break; + case UNZ_INTERNALERROR: + return "internal error"; + break; + case UNZ_CRCERROR: + return "crc error"; + break; + default: + return "unknown error"; + break; } } #endif diff --git a/src/htszlib.h b/src/htszlib.h index e597ed5..5c317b6 100644 --- a/src/htszlib.h +++ b/src/htszlib.h @@ -17,25 +17,21 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Unpacking subroutines using Jean-loup Gailly's Zlib */ /* for http compressed data */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - #ifndef HTS_DEFZLIB #define HTS_DEFZLIB @@ -51,9 +47,9 @@ Please visit our Website: http://www.httrack.com /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE extern int gz_is_available; -extern int hts_zunpack(char* filename,char* newfile); -extern int hts_extract_meta(const char* path); -extern const char* hts_get_zerror(int err); +extern int hts_zunpack(char *filename, char *newfile); +extern int hts_extract_meta(const char *path); +extern const char *hts_get_zerror(int err); #endif #endif diff --git a/src/httrack-library.h b/src/httrack-library.h index 031c7f2..f14aa2a 100644 --- a/src/httrack-library.h +++ b/src/httrack-library.h @@ -17,24 +17,20 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: HTTrack definition file for library usage */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - #ifndef HTTRACK_DEFLIB #define HTTRACK_DEFLIB @@ -51,13 +47,13 @@ typedef struct strc_int2bytes2 strc_int2bytes2; #ifndef HTS_DEF_DEFSTRUCT_hts_log_type #define HTS_DEF_DEFSTRUCT_hts_log_type typedef enum hts_log_type { - LOG_DEBUG, - LOG_INFO, - LOG_NOTICE, - LOG_WARNING, - LOG_ERROR, - LOG_PANIC, - LOG_ERRNO = 1 << 8 + LOG_DEBUG, + LOG_INFO, + LOG_NOTICE, + LOG_WARNING, + LOG_ERROR, + LOG_PANIC, + LOG_ERRNO = 1 << 8 } hts_log_type; #endif @@ -98,104 +94,110 @@ HTSEXT_API void htsthread_wait(void); /* Main functions */ HTSEXT_API int hts_main(int argc, char **argv); -HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt); +HTSEXT_API int hts_main2(int argc, char **argv, httrackp * opt); /* Options handling */ -HTSEXT_API httrackp* hts_create_opt(void); -HTSEXT_API void hts_free_opt(httrackp *opt); -HTSEXT_API void set_wrappers(httrackp *opt); // DEPRECATED - DUMMY FUNCTION -HTSEXT_API int plug_wrapper(httrackp *opt, const char *moduleName, const char* argv); +HTSEXT_API httrackp *hts_create_opt(void); +HTSEXT_API void hts_free_opt(httrackp * opt); +HTSEXT_API void set_wrappers(httrackp * opt); // DEPRECATED - DUMMY FUNCTION +HTSEXT_API int plug_wrapper(httrackp * opt, const char *moduleName, + const char *argv); /* Logging */ -HTSEXT_API int hts_log(httrackp *opt, const char* prefix, const char *msg); -HTSEXT_API void hts_log_print(httrackp *opt, int type, const char *format, ...); +HTSEXT_API int hts_log(httrackp * opt, const char *prefix, const char *msg); +HTSEXT_API void hts_log_print(httrackp * opt, int type, const char *format, + ...); /* Infos */ -HTSEXT_API const char* hts_get_version_info(httrackp *opt); -HTSEXT_API const char* hts_is_available(void); +HTSEXT_API const char *hts_get_version_info(httrackp * opt); +HTSEXT_API const char *hts_is_available(void); /* Wrapper functions */ -HTSEXT_API int htswrap_init(void); // DEPRECATED - DUMMY FUNCTION -HTSEXT_API int htswrap_free(void); // DEPRECATED - DUMMY FUNCTION -HTSEXT_API int htswrap_add(httrackp *opt, const char* name, void* fct); -HTSEXT_API unsigned long int htswrap_read(httrackp *opt, const char* name); -HTSEXT_API int htswrap_set_userdef(httrackp *opt, void *userdef); -HTSEXT_API void* htswrap_get_userdef(httrackp *opt); +HTSEXT_API int htswrap_init(void); // DEPRECATED - DUMMY FUNCTION +HTSEXT_API int htswrap_free(void); // DEPRECATED - DUMMY FUNCTION +HTSEXT_API int htswrap_add(httrackp * opt, const char *name, void *fct); +HTSEXT_API unsigned long int htswrap_read(httrackp * opt, const char *name); +HTSEXT_API int htswrap_set_userdef(httrackp * opt, void *userdef); +HTSEXT_API void *htswrap_get_userdef(httrackp * opt); /* Internal library allocators, if a different libc is being used by the client */ -HTSEXT_API char* hts_strdup(const char* string); -HTSEXT_API void* hts_malloc(size_t size); -HTSEXT_API void* hts_realloc(void* data, size_t size); -HTSEXT_API void hts_free(void* data); +HTSEXT_API char *hts_strdup(const char *string); +HTSEXT_API void *hts_malloc(size_t size); +HTSEXT_API void *hts_realloc(void *data, size_t size); +HTSEXT_API void hts_free(void *data); /* Other functions */ -HTSEXT_API int hts_resetvar(void); // DEPRECATED - DUMMY FUNCTION -HTSEXT_API int hts_buildtopindex(httrackp* opt,const char* path,const char* binpath); -HTSEXT_API const char* hts_getcategories(const char* path, int type); -HTSEXT_API const char* hts_getcategory(const char* filename); +HTSEXT_API int hts_resetvar(void); // DEPRECATED - DUMMY FUNCTION +HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path, + const char *binpath); +HTSEXT_API const char *hts_getcategories(const char *path, int type); +HTSEXT_API const char *hts_getcategory(const char *filename); /* Catch-URL */ -HTSEXT_API T_SOC catch_url_init_std(int* port_prox,char* adr_prox); -HTSEXT_API T_SOC catch_url_init(int* port,char* adr); -HTSEXT_API int catch_url(T_SOC soc,char* url,char* method,char* data); +HTSEXT_API T_SOC catch_url_init_std(int *port_prox, char *adr_prox); +HTSEXT_API T_SOC catch_url_init(int *port, char *adr); +HTSEXT_API int catch_url(T_SOC soc, char *url, char *method, char *data); /* State */ -HTSEXT_API int hts_is_parsing(httrackp *opt, int flag); -HTSEXT_API int hts_is_testing(httrackp *opt); -HTSEXT_API int hts_is_exiting(httrackp *opt); +HTSEXT_API int hts_is_parsing(httrackp * opt, int flag); +HTSEXT_API int hts_is_testing(httrackp * opt); +HTSEXT_API int hts_is_exiting(httrackp * opt); + /*HTSEXT_API int hts_setopt(httrackp* opt); DEPRECATED ; see copy_htsopt() */ -HTSEXT_API int hts_addurl(httrackp *opt, char** url); -HTSEXT_API int hts_resetaddurl(httrackp *opt); -HTSEXT_API int copy_htsopt(httrackp* from, httrackp* to); -HTSEXT_API char* hts_errmsg(httrackp *opt); -HTSEXT_API int hts_setpause(httrackp *opt, int); // pause transfer -HTSEXT_API int hts_request_stop(httrackp* opt, int force); -HTSEXT_API int hts_cancel_file_push(httrackp *opt, const char *url); -HTSEXT_API void hts_cancel_test(httrackp *opt); -HTSEXT_API void hts_cancel_parsing(httrackp *opt); -HTSEXT_API void hts_cancel_test(httrackp *opt); -HTSEXT_API void hts_cancel_parsing(httrackp *opt); +HTSEXT_API int hts_addurl(httrackp * opt, char **url); +HTSEXT_API int hts_resetaddurl(httrackp * opt); +HTSEXT_API int copy_htsopt(httrackp * from, httrackp * to); +HTSEXT_API char *hts_errmsg(httrackp * opt); +HTSEXT_API int hts_setpause(httrackp * opt, int); // pause transfer +HTSEXT_API int hts_request_stop(httrackp * opt, int force); +HTSEXT_API int hts_cancel_file_push(httrackp * opt, const char *url); +HTSEXT_API void hts_cancel_test(httrackp * opt); +HTSEXT_API void hts_cancel_parsing(httrackp * opt); +HTSEXT_API void hts_cancel_test(httrackp * opt); +HTSEXT_API void hts_cancel_parsing(httrackp * opt); /* Tools */ -HTSEXT_API int structcheck(const char* path); -HTSEXT_API int dir_exists(const char* path); -HTSEXT_API void infostatuscode(char* msg,int statuscode); +HTSEXT_API int structcheck(const char *path); +HTSEXT_API int dir_exists(const char *path); +HTSEXT_API void infostatuscode(char *msg, int statuscode); HTSEXT_API HTS_INLINE TStamp mtime_local(void); -HTSEXT_API void qsec2str(char *st,TStamp t); -HTSEXT_API char* int2char(strc_int2bytes2* strc, int n); -HTSEXT_API char* int2bytes(strc_int2bytes2* strc, LLint n); -HTSEXT_API char* int2bytessec(strc_int2bytes2* strc, long int n); -HTSEXT_API char** int2bytes2(strc_int2bytes2* strc, LLint n); -HTSEXT_API char* jump_identification(char*); -HTSEXT_API char* jump_normalized(char*); -HTSEXT_API char* jump_toport(char*); -HTSEXT_API char* fil_normalized(char* source, char* dest); -HTSEXT_API char* adr_normalized(char* source, char* dest); -HTSEXT_API char* hts_rootdir(char* file); +HTSEXT_API void qsec2str(char *st, TStamp t); +HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n); +HTSEXT_API char *int2bytes(strc_int2bytes2 * strc, LLint n); +HTSEXT_API char *int2bytessec(strc_int2bytes2 * strc, long int n); +HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n); +HTSEXT_API char *jump_identification(char *); +HTSEXT_API char *jump_normalized(char *); +HTSEXT_API char *jump_toport(char *); +HTSEXT_API char *fil_normalized(char *source, char *dest); +HTSEXT_API char *adr_normalized(char *source, char *dest); +HTSEXT_API char *hts_rootdir(char *file); /* Escaping URLs */ -HTSEXT_API void unescape_amp(char* s); -HTSEXT_API void escape_spc_url(char* s); -HTSEXT_API void escape_in_url(char* s); -HTSEXT_API void escape_uri(char* s); -HTSEXT_API void escape_uri_utf(char* s); -HTSEXT_API void escape_check_url(char* s); -HTSEXT_API char* escape_check_url_addr(char* s); -HTSEXT_API void x_escape_http(char* s,int mode); -HTSEXT_API char* unescape_http(char *catbuff, const char* s); -HTSEXT_API char* unescape_http_unharm(char *catbuff, const char* s, int no_high); -HTSEXT_API char* antislash_unescaped(char *catbuff, const char* s); -HTSEXT_API void escape_remove_control(char* s); -HTSEXT_API void get_httptype(httrackp *opt,char *s,const char *fil,int flag); -HTSEXT_API int is_knowntype(httrackp *opt,const char *fil); -HTSEXT_API int is_userknowntype(httrackp *opt,const char *fil); +HTSEXT_API void unescape_amp(char *s); +HTSEXT_API void escape_spc_url(char *s); +HTSEXT_API void escape_in_url(char *s); +HTSEXT_API void escape_uri(char *s); +HTSEXT_API void escape_uri_utf(char *s); +HTSEXT_API void escape_check_url(char *s); +HTSEXT_API char *escape_check_url_addr(char *s); +HTSEXT_API void x_escape_http(char *s, int mode); +HTSEXT_API char *unescape_http(char *catbuff, const char *s); +HTSEXT_API char *unescape_http_unharm(char *catbuff, const char *s, + int no_high); +HTSEXT_API char *antislash_unescaped(char *catbuff, const char *s); +HTSEXT_API void escape_remove_control(char *s); +HTSEXT_API void get_httptype(httrackp * opt, char *s, const char *fil, + int flag); +HTSEXT_API int is_knowntype(httrackp * opt, const char *fil); +HTSEXT_API int is_userknowntype(httrackp * opt, const char *fil); HTSEXT_API int is_dyntype(const char *fil); -HTSEXT_API char* get_ext(char *catbuff, const char *fil); +HTSEXT_API char *get_ext(char *catbuff, const char *fil); /* Ugly string tools */ -HTSEXT_API char* concat(char *catbuff,const char* a,const char* b) ; -HTSEXT_API char* fconcat(char *catbuff, const char* a, const char* b); -HTSEXT_API char* fconv(char *catbuff, const char* a); +HTSEXT_API char *concat(char *catbuff, const char *a, const char *b); +HTSEXT_API char *fconcat(char *catbuff, const char *a, const char *b); +HTSEXT_API char *fconv(char *catbuff, const char *a); /* Debugging */ HTSEXT_API void hts_debug(int level); @@ -205,7 +207,7 @@ HTSEXT_API void hts_debug(int level); #ifndef HTS_DEF_FWSTRUCT_find_handle_struct #define HTS_DEF_FWSTRUCT_find_handle_struct typedef struct find_handle_struct find_handle_struct; -typedef find_handle_struct* find_handle; +typedef find_handle_struct *find_handle; #endif #ifndef HTS_DEF_FWSTRUCT_topindex_chain @@ -213,15 +215,15 @@ typedef find_handle_struct* find_handle; typedef struct topindex_chain topindex_chain; #endif struct topindex_chain { - int level; /* sort level */ - char* category; /* category */ - char name[2048]; /* path */ - struct topindex_chain* next; /* next element */ + int level; /* sort level */ + char *category; /* category */ + char name[2048]; /* path */ + struct topindex_chain *next; /* next element */ }; -HTSEXT_API find_handle hts_findfirst(char* path); +HTSEXT_API find_handle hts_findfirst(char *path); HTSEXT_API int hts_findnext(find_handle find); HTSEXT_API int hts_findclose(find_handle find); -HTSEXT_API char* hts_findgetname(find_handle find); +HTSEXT_API char *hts_findgetname(find_handle find); HTSEXT_API int hts_findgetsize(find_handle find); HTSEXT_API int hts_findisdir(find_handle find); HTSEXT_API int hts_findisfile(find_handle find); @@ -231,27 +233,35 @@ HTSEXT_API int hts_findissystem(find_handle find); #ifndef HTS_DEF_FILEAPI #ifdef _WIN32 #define FOPEN hts_fopen_utf8 -HTSEXT_API FILE* hts_fopen_utf8(const char *path, const char *mode); +HTSEXT_API FILE *hts_fopen_utf8(const char *path, const char *mode); + #define STAT hts_stat_utf8 typedef struct _stat STRUCT_STAT; -HTSEXT_API int hts_stat_utf8(const char *path, STRUCT_STAT *buf); +HTSEXT_API int hts_stat_utf8(const char *path, STRUCT_STAT * buf); + #define UNLINK hts_unlink_utf8 HTSEXT_API int hts_unlink_utf8(const char *pathname); + #define RENAME hts_rename_utf8 HTSEXT_API int hts_rename_utf8(const char *oldpath, const char *newpath); + #define MKDIR(F) hts_mkdir_utf8(F) HTSEXT_API int hts_mkdir_utf8(const char *pathname); + #define UTIME(A,B) hts_utime_utf8(A,B) typedef struct _utimbuf STRUCT_UTIMBUF; -HTSEXT_API int hts_utime_utf8(const char *filename, const STRUCT_UTIMBUF *times); +HTSEXT_API int hts_utime_utf8(const char *filename, + const STRUCT_UTIMBUF * times); #else #define FOPEN fopen #define STAT stat typedef struct stat STRUCT_STAT; + #define UNLINK unlink #define RENAME rename #define MKDIR(F) mkdir(F, HTS_ACCESS_FOLDER) typedef struct utimbuf STRUCT_UTIMBUF; + #define UTIME(A,B) utime(A,B) #endif #define HTS_DEF_FILEAPI diff --git a/src/httrack.c b/src/httrack.c index eaea638..e149948 100644 --- a/src/httrack.c +++ b/src/httrack.c @@ -17,18 +17,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsshow.c console progress info */ /* Only used on Linux & FreeBSD versions */ @@ -51,9 +48,8 @@ Please visit our Website: http://www.httrack.com #include "htslib.h" /* Static definitions */ -static int fexist(const char* s); -static int linput(FILE* fp,char* s,int max); - +static int fexist(const char *s); +static int linput(FILE * fp, char *s, int max); // htswrap_add #include "htswrap.h" @@ -76,29 +72,69 @@ static int linput(FILE* fp,char* s,int max); #include <ctype.h> /* END specific definitions */ -static void __cdecl htsshow_init(t_hts_callbackarg *carg); -static void __cdecl htsshow_uninit(t_hts_callbackarg *carg); -static int __cdecl htsshow_start(t_hts_callbackarg *carg, httrackp* opt); -static int __cdecl htsshow_chopt(t_hts_callbackarg *carg, httrackp* opt); -static int __cdecl htsshow_end(t_hts_callbackarg *carg, httrackp* opt); -static int __cdecl htsshow_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file); -static int __cdecl htsshow_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file); -static int __cdecl htsshow_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_address,const char* url_file); -static int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time,hts_stat_struct* stats); -static const char* __cdecl htsshow_query(t_hts_callbackarg *carg, httrackp *opt, const char* question); -static const char* __cdecl htsshow_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question); -static const char* __cdecl htsshow_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question); -static int __cdecl htsshow_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status); -static int __cdecl htsshow_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status); -static void __cdecl htsshow_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile); -static void __cdecl htsshow_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file); -static void __cdecl htsshow_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* fil, const char* save, int is_new, int is_modified,int not_updated); -static int __cdecl htsshow_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link); -static int __cdecl htsshow_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag); -static int __cdecl htsshow_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, lien_back* back); -static int __cdecl htsshow_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save); -static int __cdecl htsshow_sendheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing); -static int __cdecl htsshow_receiveheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming); +static void __cdecl htsshow_init(t_hts_callbackarg * carg); +static void __cdecl htsshow_uninit(t_hts_callbackarg * carg); +static int __cdecl htsshow_start(t_hts_callbackarg * carg, httrackp * opt); +static int __cdecl htsshow_chopt(t_hts_callbackarg * carg, httrackp * opt); +static int __cdecl htsshow_end(t_hts_callbackarg * carg, httrackp * opt); +static int __cdecl htsshow_preprocesshtml(t_hts_callbackarg * carg, + httrackp * opt, char **html, int *len, + const char *url_address, + const char *url_file); +static int __cdecl htsshow_postprocesshtml(t_hts_callbackarg * carg, + httrackp * opt, char **html, + int *len, const char *url_address, + const char *url_file); +static int __cdecl htsshow_checkhtml(t_hts_callbackarg * carg, httrackp * opt, + char *html, int len, + const char *url_address, + const char *url_file); +static int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, + lien_back * back, int back_max, int back_index, + int lien_n, int lien_tot, int stat_time, + hts_stat_struct * stats); +static const char *__cdecl htsshow_query(t_hts_callbackarg * carg, + httrackp * opt, const char *question); +static const char *__cdecl htsshow_query2(t_hts_callbackarg * carg, + httrackp * opt, const char *question); +static const char *__cdecl htsshow_query3(t_hts_callbackarg * carg, + httrackp * opt, const char *question); +static int __cdecl htsshow_check(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, int status); +static int __cdecl htsshow_check_mime(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + const char *mime, int status); +static void __cdecl htsshow_pause(t_hts_callbackarg * carg, httrackp * opt, + const char *lockfile); +static void __cdecl htsshow_filesave(t_hts_callbackarg * carg, httrackp * opt, + const char *file); +static void __cdecl htsshow_filesave2(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + const char *save, int is_new, + int is_modified, int not_updated); +static int __cdecl htsshow_linkdetected(t_hts_callbackarg * carg, + httrackp * opt, char *link); +static int __cdecl htsshow_linkdetected2(t_hts_callbackarg * carg, + httrackp * opt, char *link, + const char *start_tag); +static int __cdecl htsshow_xfrstatus(t_hts_callbackarg * carg, httrackp * opt, + lien_back * back); +static int __cdecl htsshow_savename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, + const char *fil_complete, + const char *referer_adr, + const char *referer_fil, char *save); +static int __cdecl htsshow_sendheader(t_hts_callbackarg * carg, httrackp * opt, + char *buff, const char *adr, + const char *fil, const char *referer_adr, + const char *referer_fil, + htsblk * outgoing); +static int __cdecl htsshow_receiveheader(t_hts_callbackarg * carg, + httrackp * opt, char *buff, + const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, + htsblk * incoming); static void vt_clear(void); static void vt_home(void); @@ -140,13 +176,13 @@ static void vt_home(void); // #define csi(X) printf(s_csi( X )); static void vt_clear(void) { - printf("%s%s%s",VT_RESET,VT_CLRSCR,VT_GOTOXY("1","0")); + printf("%s%s%s", VT_RESET, VT_CLRSCR, VT_GOTOXY("1", "0")); } static void vt_home(void) { - printf("%s%s",VT_RESET,VT_GOTOXY("1","0")); + printf("%s%s", VT_RESET, VT_GOTOXY("1", "0")); } -// +// /* #define STYLE_STATVALUES VT_COLOR(VT_COL_TEXT_BLACK) @@ -169,15 +205,16 @@ int main(int argc, char **argv) { #ifdef _WIN32 { - WORD wVersionRequested; // requested version WinSock API + WORD wVersionRequested; // requested version WinSock API WSADATA wsadata; // Windows Sockets API data int stat; + wVersionRequested = 0x0101; - stat = WSAStartup( wVersionRequested, &wsadata ); + stat = WSAStartup(wVersionRequested, &wsadata); if (stat != 0) { printf("Winsock not found!\n"); return; - } else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) { + } else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) { printf("WINSOCK.DLL does not support version 1.1\n"); WSACleanup(); return; @@ -185,7 +222,7 @@ int main(int argc, char **argv) { } #endif - signal_handlers(); + signal_handlers(); hts_init(); opt = global_opt = hts_create_opt(); @@ -213,289 +250,293 @@ int main(int argc, char **argv) { CHAIN_FUNCTION(opt, sendhead, htsshow_sendheader, NULL); CHAIN_FUNCTION(opt, receivehead, htsshow_receiveheader, NULL); - ret = hts_main2(argc, argv, opt); + ret = hts_main2(argc, argv, opt); if (ret) { fprintf(stderr, "* %s\n", hts_errmsg(opt)); } - global_opt = NULL; - hts_free_opt(opt); - htsthread_wait(); /* wait for pending threads */ - hts_uninit(); + global_opt = NULL; + hts_free_opt(opt); + htsthread_wait(); /* wait for pending threads */ + hts_uninit(); #ifdef _WIN32 WSACleanup(); #endif - return ret; + return ret; } - /* CALLBACK FUNCTIONS */ /* Initialize the Winsock */ -static void __cdecl htsshow_init(t_hts_callbackarg *carg) { +static void __cdecl htsshow_init(t_hts_callbackarg * carg) { } -static void __cdecl htsshow_uninit(t_hts_callbackarg *carg) { +static void __cdecl htsshow_uninit(t_hts_callbackarg * carg) { } -static int __cdecl htsshow_start(t_hts_callbackarg *carg, httrackp* opt) { - use_show=0; - if (opt->verbosedisplay==2) { - use_show=1; +static int __cdecl htsshow_start(t_hts_callbackarg * carg, httrackp * opt) { + use_show = 0; + if (opt->verbosedisplay == 2) { + use_show = 1; vt_clear(); } - return 1; + return 1; } -static int __cdecl htsshow_chopt(t_hts_callbackarg *carg, httrackp* opt) { +static int __cdecl htsshow_chopt(t_hts_callbackarg * carg, httrackp * opt) { return htsshow_start(carg, opt); } -static int __cdecl htsshow_end(t_hts_callbackarg *carg, httrackp* opt) { - return 1; +static int __cdecl htsshow_end(t_hts_callbackarg * carg, httrackp * opt) { + return 1; } -static int __cdecl htsshow_preprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) { +static int __cdecl htsshow_preprocesshtml(t_hts_callbackarg * carg, + httrackp * opt, char **html, int *len, + const char *url_address, + const char *url_file) { return 1; } -static int __cdecl htsshow_postprocesshtml(t_hts_callbackarg *carg, httrackp *opt, char** html,int* len,const char* url_address,const char* url_file) { +static int __cdecl htsshow_postprocesshtml(t_hts_callbackarg * carg, + httrackp * opt, char **html, + int *len, const char *url_address, + const char *url_file) { return 1; } -static int __cdecl htsshow_checkhtml(t_hts_callbackarg *carg, httrackp *opt, char* html,int len,const char* url_address,const char* url_file) { +static int __cdecl htsshow_checkhtml(t_hts_callbackarg * carg, httrackp * opt, + char *html, int len, + const char *url_address, + const char *url_file) { return 1; } -static int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_back* back,int back_max,int back_index,int lien_n,int lien_tot,int stat_time, hts_stat_struct* stats) { // appelé à chaque boucle de HTTrack - static TStamp prev_mytime=0; /* ok */ - static t_InpInfo SInfo; /* ok */ +static int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * back, int back_max, int back_index, int lien_n, int lien_tot, int stat_time, hts_stat_struct * stats) { // appelé à chaque boucle de HTTrack + static TStamp prev_mytime = 0; /* ok */ + static t_InpInfo SInfo; /* ok */ + // TStamp mytime; - long int rate=0; + long int rate = 0; char st[256]; + // - int stat_written=-1; - int stat_updated=-1; - int stat_errors=-1; - int stat_warnings=-1; - int stat_infos=-1; - int nbk=-1; - LLint nb=-1; - int stat_nsocket=-1; - LLint stat_bytes=-1; - LLint stat_bytes_recv=-1; - int irate=-1; + int stat_written = -1; + int stat_updated = -1; + int stat_errors = -1; + int stat_warnings = -1; + int stat_infos = -1; + int nbk = -1; + LLint nb = -1; + int stat_nsocket = -1; + LLint stat_bytes = -1; + LLint stat_bytes_recv = -1; + int irate = -1; + if (stats) { - stat_written=stats->stat_files; - stat_updated=stats->stat_updated_files; - stat_errors=stats->stat_errors; - stat_warnings=stats->stat_warnings; - stat_infos=stats->stat_infos; - nbk=stats->nbk; - stat_nsocket=stats->stat_nsocket; - irate=(int)stats->rate; - nb=stats->nb; - stat_bytes=stats->nb; - stat_bytes_recv=stats->HTS_TOTAL_RECV; + stat_written = stats->stat_files; + stat_updated = stats->stat_updated_files; + stat_errors = stats->stat_errors; + stat_warnings = stats->stat_warnings; + stat_infos = stats->stat_infos; + nbk = stats->nbk; + stat_nsocket = stats->stat_nsocket; + irate = (int) stats->rate; + nb = stats->nb; + stat_bytes = stats->nb; + stat_bytes_recv = stats->HTS_TOTAL_RECV; } if (!use_show) return 1; - mytime=mtime_local(); - if ((stat_time>0) && (stat_bytes_recv>0)) - rate=(int)(stat_bytes_recv/stat_time); + mytime = mtime_local(); + if ((stat_time > 0) && (stat_bytes_recv > 0)) + rate = (int) (stat_bytes_recv / stat_time); else - rate=0; // pas d'infos + rate = 0; // pas d'infos /* Infos */ - if (stat_bytes>=0) SInfo.stat_bytes=stat_bytes; // bytes - if (stat_time>=0) SInfo.stat_time=stat_time; // time - if (lien_tot>=0) SInfo.lien_tot=lien_tot; // nb liens - if (lien_n>=0) SInfo.lien_n=lien_n; // scanned - SInfo.stat_nsocket=stat_nsocket; // socks - if (rate>0) SInfo.rate=rate; // rate - if (irate>=0) SInfo.irate=irate; // irate - if (SInfo.irate<0) SInfo.irate=SInfo.rate; - if (nbk>=0) SInfo.stat_back=nbk; - if (stat_written>=0) SInfo.stat_written=stat_written; - if (stat_updated>=0) SInfo.stat_updated=stat_updated; - if (stat_errors>=0) SInfo.stat_errors=stat_errors; - if (stat_warnings>=0) SInfo.stat_warnings=stat_warnings; - if (stat_infos>=0) SInfo.stat_infos=stat_infos; - - - if ( ((mytime - prev_mytime)>100) || ((mytime - prev_mytime)<0) ) { - strc_int2bytes2 strc, strc2, strc3; - prev_mytime=mytime; - - - st[0]='\0'; - qsec2str(st,stat_time); + if (stat_bytes >= 0) + SInfo.stat_bytes = stat_bytes; // bytes + if (stat_time >= 0) + SInfo.stat_time = stat_time; // time + if (lien_tot >= 0) + SInfo.lien_tot = lien_tot; // nb liens + if (lien_n >= 0) + SInfo.lien_n = lien_n; // scanned + SInfo.stat_nsocket = stat_nsocket; // socks + if (rate > 0) + SInfo.rate = rate; // rate + if (irate >= 0) + SInfo.irate = irate; // irate + if (SInfo.irate < 0) + SInfo.irate = SInfo.rate; + if (nbk >= 0) + SInfo.stat_back = nbk; + if (stat_written >= 0) + SInfo.stat_written = stat_written; + if (stat_updated >= 0) + SInfo.stat_updated = stat_updated; + if (stat_errors >= 0) + SInfo.stat_errors = stat_errors; + if (stat_warnings >= 0) + SInfo.stat_warnings = stat_warnings; + if (stat_infos >= 0) + SInfo.stat_infos = stat_infos; + + if (((mytime - prev_mytime) > 100) || ((mytime - prev_mytime) < 0)) { + strc_int2bytes2 strc, strc2, strc3; + + prev_mytime = mytime; + + st[0] = '\0'; + qsec2str(st, stat_time); vt_home(); - printf( - VT_GOTOXY("1","1") - VT_CLREOL - STYLE_STATTEXT "Bytes saved:" - STYLE_STATVALUES " \t%s" - "\t" - VT_CLREOL - VT_GOTOXY("40","1") - STYLE_STATTEXT "Links scanned:" - STYLE_STATVALUES " \t%d/%d (+%d)" - VT_CLREOL"\n"VT_CLREOL - VT_GOTOXY("1","2") - STYLE_STATTEXT "Time:" - " \t" - STYLE_STATVALUES "%s" - "\t" - VT_CLREOL - VT_GOTOXY("40","2") - STYLE_STATTEXT "Files written:" - " \t" - STYLE_STATVALUES "%d" - VT_CLREOL"\n"VT_CLREOL - VT_GOTOXY("1","3") - STYLE_STATTEXT "Transfer rate:" - " \t" - STYLE_STATVALUES "%s (%s)" - "\t" - VT_CLREOL - VT_GOTOXY("40","3") - STYLE_STATTEXT "Files updated:" - " \t" - STYLE_STATVALUES "%d" - VT_CLREOL"\n"VT_CLREOL - VT_GOTOXY("1","4") - STYLE_STATTEXT "Active connections:" - " \t" - STYLE_STATVALUES "%d" - "\t" - VT_CLREOL - VT_GOTOXY("40","4") - STYLE_STATTEXT "Errors:" - STYLE_STATVALUES " \t" - STYLE_STATVALUES "%d" - VT_CLREOL"\n" - STYLE_STATRESET - , - /* */ - (char*)int2bytes(&strc,SInfo.stat_bytes), - (int)lien_n,(int)SInfo.lien_tot,(int)nbk, - (char*)st, - (int)SInfo.stat_written, - (char*)int2bytessec(&strc2,SInfo.irate),(char*)int2bytessec(&strc3,SInfo.rate), - (int)SInfo.stat_updated, - (int)SInfo.stat_nsocket, - (int)SInfo.stat_errors - /* */ + printf(VT_GOTOXY("1", "1") + VT_CLREOL STYLE_STATTEXT "Bytes saved:" STYLE_STATVALUES " \t%s" "\t" + VT_CLREOL VT_GOTOXY("40", "1") + STYLE_STATTEXT "Links scanned:" STYLE_STATVALUES " \t%d/%d (+%d)" + VT_CLREOL "\n" VT_CLREOL VT_GOTOXY("1", "2") + STYLE_STATTEXT "Time:" " \t" STYLE_STATVALUES "%s" "\t" VT_CLREOL + VT_GOTOXY("40", "2") + STYLE_STATTEXT "Files written:" " \t" STYLE_STATVALUES "%d" VT_CLREOL + "\n" VT_CLREOL VT_GOTOXY("1", "3") + STYLE_STATTEXT "Transfer rate:" " \t" STYLE_STATVALUES "%s (%s)" "\t" + VT_CLREOL VT_GOTOXY("40", "3") + STYLE_STATTEXT "Files updated:" " \t" STYLE_STATVALUES "%d" VT_CLREOL + "\n" VT_CLREOL VT_GOTOXY("1", "4") + STYLE_STATTEXT "Active connections:" " \t" STYLE_STATVALUES "%d" "\t" + VT_CLREOL VT_GOTOXY("40", "4") + STYLE_STATTEXT "Errors:" STYLE_STATVALUES " \t" STYLE_STATVALUES "%d" + VT_CLREOL "\n" STYLE_STATRESET, + /* */ + (char *) int2bytes(&strc, SInfo.stat_bytes), (int) lien_n, + (int) SInfo.lien_tot, (int) nbk, (char *) st, + (int) SInfo.stat_written, (char *) int2bytessec(&strc2, SInfo.irate), + (char *) int2bytessec(&strc3, SInfo.rate), (int) SInfo.stat_updated, + (int) SInfo.stat_nsocket, (int) SInfo.stat_errors + /* */ ); - // parcourir registre des liens - if (back_index>=0) { // seulement si index passé - int j,k; - int index=0; - int ok=0; // idem - int l; // idem + if (back_index >= 0) { // seulement si index passé + int j, k; + int index = 0; + int ok = 0; // idem + int l; // idem + // t_StatsBuffer StatsBuffer[NStatsBuffer]; - + { int i; - for(i=0;i<NStatsBuffer;i++) { - strcpybuff(StatsBuffer[i].state,""); - strcpybuff(StatsBuffer[i].name,""); - strcpybuff(StatsBuffer[i].file,""); - strcpybuff(StatsBuffer[i].url_sav,""); - StatsBuffer[i].back=0; - StatsBuffer[i].size=0; - StatsBuffer[i].sizetot=0; + + for(i = 0; i < NStatsBuffer; i++) { + strcpybuff(StatsBuffer[i].state, ""); + strcpybuff(StatsBuffer[i].name, ""); + strcpybuff(StatsBuffer[i].file, ""); + strcpybuff(StatsBuffer[i].url_sav, ""); + StatsBuffer[i].back = 0; + StatsBuffer[i].size = 0; + StatsBuffer[i].sizetot = 0; } } - for(k=0;k<2;k++) { // 0: lien en cours 1: autres liens - for(j=0;(j<3) && (index<NStatsBuffer);j++) { // passe de priorité + for(k = 0; k < 2; k++) { // 0: lien en cours 1: autres liens + for(j = 0; (j < 3) && (index < NStatsBuffer); j++) { // passe de priorité int _i; - for(_i=0+k;(_i< max(back_max*k,1) ) && (index<NStatsBuffer);_i++) { // no lien - int i=(back_index+_i)%back_max; // commencer par le "premier" (l'actuel) - if (back[i].status>=0) { // signifie "lien actif" + + for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) { // no lien + int i = (back_index + _i) % back_max; // commencer par le "premier" (l'actuel) + + if (back[i].status >= 0) { // signifie "lien actif" // int ok=0; // OPTI - ok=0; - switch(j) { - case 0: // prioritaire - if ((back[i].status>0) && (back[i].status<99)) { - strcpybuff(StatsBuffer[index].state,"receive"); ok=1; + ok = 0; + switch (j) { + case 0: // prioritaire + if ((back[i].status > 0) && (back[i].status < 99)) { + strcpybuff(StatsBuffer[index].state, "receive"); + ok = 1; } break; case 1: - if (back[i].status==STATUS_WAIT_HEADERS) { - strcpybuff(StatsBuffer[index].state,"request"); ok=1; - } - else if (back[i].status==STATUS_CONNECTING) { - strcpybuff(StatsBuffer[index].state,"connect"); ok=1; - } - else if (back[i].status==STATUS_WAIT_DNS) { - strcpybuff(StatsBuffer[index].state,"search"); ok=1; - } - else if (back[i].status==STATUS_FTP_TRANSFER) { // ohh le beau ftp - sprintf(StatsBuffer[index].state,"ftp: %s",back[i].info); ok=1; + if (back[i].status == STATUS_WAIT_HEADERS) { + strcpybuff(StatsBuffer[index].state, "request"); + ok = 1; + } else if (back[i].status == STATUS_CONNECTING) { + strcpybuff(StatsBuffer[index].state, "connect"); + ok = 1; + } else if (back[i].status == STATUS_WAIT_DNS) { + strcpybuff(StatsBuffer[index].state, "search"); + ok = 1; + } else if (back[i].status == STATUS_FTP_TRANSFER) { // ohh le beau ftp + sprintf(StatsBuffer[index].state, "ftp: %s", back[i].info); + ok = 1; } break; default: - if (back[i].status==STATUS_READY) { // prêt - if ((back[i].r.statuscode==200)) { - strcpybuff(StatsBuffer[index].state,"ready"); ok=1; - } - else if ((back[i].r.statuscode>=100) && (back[i].r.statuscode<=599)) { - char tempo[256]; tempo[0]='\0'; - infostatuscode(tempo,back[i].r.statuscode); - strcpybuff(StatsBuffer[index].state,tempo); ok=1; - } - else { - strcpybuff(StatsBuffer[index].state,"error"); ok=1; + if (back[i].status == STATUS_READY) { // prêt + if ((back[i].r.statuscode == 200)) { + strcpybuff(StatsBuffer[index].state, "ready"); + ok = 1; + } else if ((back[i].r.statuscode >= 100) + && (back[i].r.statuscode <= 599)) { + char tempo[256]; + + tempo[0] = '\0'; + infostatuscode(tempo, back[i].r.statuscode); + strcpybuff(StatsBuffer[index].state, tempo); + ok = 1; + } else { + strcpybuff(StatsBuffer[index].state, "error"); + ok = 1; } } break; } - + if (ok) { - char BIGSTK s[HTS_URLMAXSIZE*2]; + char BIGSTK s[HTS_URLMAXSIZE * 2]; + // - StatsBuffer[index].back=i; // index pour + d'infos + StatsBuffer[index].back = i; // index pour + d'infos // - s[0]='\0'; - strcpybuff(StatsBuffer[index].url_sav,back[i].url_sav); // pour cancel - if (strcmp(back[i].url_adr,"file://")) - strcatbuff(s,back[i].url_adr); + s[0] = '\0'; + strcpybuff(StatsBuffer[index].url_sav, back[i].url_sav); // pour cancel + if (strcmp(back[i].url_adr, "file://")) + strcatbuff(s, back[i].url_adr); else - strcatbuff(s,"localhost"); - if (back[i].url_fil[0]!='/') - strcatbuff(s,"/"); - strcatbuff(s,back[i].url_fil); - - StatsBuffer[index].file[0]='\0'; + strcatbuff(s, "localhost"); + if (back[i].url_fil[0] != '/') + strcatbuff(s, "/"); + strcatbuff(s, back[i].url_fil); + + StatsBuffer[index].file[0] = '\0'; { - char* a=strrchr(s,'/'); + char *a = strrchr(s, '/'); + if (a) { - strncatbuff(StatsBuffer[index].file,a,200); - *a='\0'; + strncatbuff(StatsBuffer[index].file, a, 200); + *a = '\0'; } } - - if ((l = (int) strlen(s))<MAX_LEN_INPROGRESS) - strcpybuff(StatsBuffer[index].name,s); + + if ((l = (int) strlen(s)) < MAX_LEN_INPROGRESS) + strcpybuff(StatsBuffer[index].name, s); else { // couper - StatsBuffer[index].name[0]='\0'; - strncatbuff(StatsBuffer[index].name,s,MAX_LEN_INPROGRESS/2-2); - strcatbuff(StatsBuffer[index].name,"..."); - strcatbuff(StatsBuffer[index].name,s+l-MAX_LEN_INPROGRESS/2+2); + StatsBuffer[index].name[0] = '\0'; + strncatbuff(StatsBuffer[index].name, s, + MAX_LEN_INPROGRESS / 2 - 2); + strcatbuff(StatsBuffer[index].name, "..."); + strcatbuff(StatsBuffer[index].name, + s + l - MAX_LEN_INPROGRESS / 2 + 2); } - - if (back[i].r.totalsize>=0) { // taille prédéfinie - StatsBuffer[index].sizetot=back[i].r.totalsize; - StatsBuffer[index].size=back[i].r.size; - } else { // pas de taille prédéfinie - if (back[i].status==STATUS_READY) { // prêt - StatsBuffer[index].sizetot=back[i].r.size; - StatsBuffer[index].size=back[i].r.size; + + if (back[i].r.totalsize >= 0) { // taille prédéfinie + StatsBuffer[index].sizetot = back[i].r.totalsize; + StatsBuffer[index].size = back[i].r.size; + } else { // pas de taille prédéfinie + if (back[i].status == STATUS_READY) { // prêt + StatsBuffer[index].sizetot = back[i].r.size; + StatsBuffer[index].size = back[i].r.size; } else { - StatsBuffer[index].sizetot=8192; - StatsBuffer[index].size=(back[i].r.size % 8192); + StatsBuffer[index].sizetot = 8192; + StatsBuffer[index].size = (back[i].r.size % 8192); } } index++; @@ -506,21 +547,22 @@ static int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_bac } /* LF */ - printf("%s\n",VT_CLREOL); + printf("%s\n", VT_CLREOL); /* Display current job */ { - int parsing=0; + int parsing = 0; + printf("Current job: "); - if (!(parsing=hts_is_parsing(opt, -1))) + if (!(parsing = hts_is_parsing(opt, -1))) printf("receiving files"); else { - switch(hts_is_testing(opt)) { + switch (hts_is_testing(opt)) { case 0: - printf("parsing HTML file (%d%%)",parsing); + printf("parsing HTML file (%d%%)", parsing); break; case 1: - printf("parsing HTML file: testing links (%d%%)",parsing); + printf("parsing HTML file: testing links (%d%%)", parsing); break; case 2: printf("purging files"); @@ -536,65 +578,66 @@ static int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_bac break; } } - printf("%s\n",VT_CLREOL); + printf("%s\n", VT_CLREOL); } - + /* Display background jobs */ { int i; - for(i=0;i<NStatsBuffer;i++) { + + for(i = 0; i < NStatsBuffer; i++) { if (strnotempty(StatsBuffer[i].state)) { - printf(VT_CLREOL" %s - \t%s%s \t%s / \t%s", - StatsBuffer[i].state, - StatsBuffer[i].name, - StatsBuffer[i].file, - int2bytes(&strc,StatsBuffer[i].size), - int2bytes(&strc2,StatsBuffer[i].sizetot) + printf(VT_CLREOL " %s - \t%s%s \t%s / \t%s", StatsBuffer[i].state, + StatsBuffer[i].name, StatsBuffer[i].file, int2bytes(&strc, + StatsBuffer + [i]. + size), + int2bytes(&strc2, StatsBuffer[i].sizetot) ); } - printf("%s\n",VT_CLREOL); + printf("%s\n", VT_CLREOL); } } - } } - - return 1; } -static const char* __cdecl htsshow_query(t_hts_callbackarg *carg, httrackp *opt, const char* question) { - static char s[12]=""; /* ok */ +static const char *__cdecl htsshow_query(t_hts_callbackarg * carg, + httrackp * opt, const char *question) { + static char s[12] = ""; /* ok */ + printf("%s\nPress <Y><Enter> to confirm, <N><Enter> to abort\n", question); io_flush; - linput(stdin,s,4); + linput(stdin, s, 4); return s; } -static const char* __cdecl htsshow_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question) { - static char s[12]=""; /* ok */ +static const char *__cdecl htsshow_query2(t_hts_callbackarg * carg, + httrackp * opt, + const char *question) { + static char s[12] = ""; /* ok */ + printf("%s\nPress <Y><Enter> to confirm, <N><Enter> to abort\n", question); io_flush; - linput(stdin,s,4); + linput(stdin, s, 4); return s; } -static const char* __cdecl htsshow_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question) { - static char line[256]; /* ok */ - printf( - "\n" - "A link, %s, is located beyond this mirror scope.\n" - "What should I do? (type in the choice + enter)\n\n" - "* Ignore all further links and do not ask any more questions\n" - "0 Ignore this link (default if empty entry)\n" - "1 Ignore directory and lower structures\n" - "2 Ignore all domain\n" - "\n" - "4 Get only this page/link, but not links inside this page\n" - "5 Mirror this link (useful)\n" - "6 Mirror all links located on the same domain as this link\n" - "\n", - question); +static const char *__cdecl htsshow_query3(t_hts_callbackarg * carg, + httrackp * opt, + const char *question) { + static char line[256]; /* ok */ + + printf("\n" "A link, %s, is located beyond this mirror scope.\n" + "What should I do? (type in the choice + enter)\n\n" + "* Ignore all further links and do not ask any more questions\n" + "0 Ignore this link (default if empty entry)\n" + "1 Ignore directory and lower structures\n" "2 Ignore all domain\n" + "\n" "4 Get only this page/link, but not links inside this page\n" + "5 Mirror this link (useful)\n" + "6 Mirror all links located on the same domain as this link\n" "\n", + question); do { printf(">> "); io_flush; @@ -603,45 +646,70 @@ static const char* __cdecl htsshow_query3(t_hts_callbackarg *carg, httrackp *opt printf("ok..\n"); return line; } -static int __cdecl htsshow_check(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,int status) { +static int __cdecl htsshow_check(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, int status) { return -1; } -static int __cdecl htsshow_check_mime(t_hts_callbackarg *carg, httrackp *opt, const char* adr,const char* fil,const char* mime,int status) { +static int __cdecl htsshow_check_mime(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + const char *mime, int status) { return -1; } -static void __cdecl htsshow_pause(t_hts_callbackarg *carg, httrackp *opt, const char* lockfile) { - while (fexist(lockfile)) { +static void __cdecl htsshow_pause(t_hts_callbackarg * carg, httrackp * opt, + const char *lockfile) { + while(fexist(lockfile)) { Sleep(1000); } } -static void __cdecl htsshow_filesave(t_hts_callbackarg *carg, httrackp *opt, const char* file) { +static void __cdecl htsshow_filesave(t_hts_callbackarg * carg, httrackp * opt, + const char *file) { } -static void __cdecl htsshow_filesave2(t_hts_callbackarg *carg, httrackp *opt, const char* adr, const char* fil, const char* save, int is_new, int is_modified,int not_updated) { +static void __cdecl htsshow_filesave2(t_hts_callbackarg * carg, httrackp * opt, + const char *adr, const char *fil, + const char *save, int is_new, + int is_modified, int not_updated) { } -static int __cdecl htsshow_linkdetected(t_hts_callbackarg *carg, httrackp *opt, char* link) { +static int __cdecl htsshow_linkdetected(t_hts_callbackarg * carg, + httrackp * opt, char *link) { return 1; } -static int __cdecl htsshow_linkdetected2(t_hts_callbackarg *carg, httrackp *opt, char* link, const char* start_tag) { +static int __cdecl htsshow_linkdetected2(t_hts_callbackarg * carg, + httrackp * opt, char *link, + const char *start_tag) { return 1; } -static int __cdecl htsshow_xfrstatus(t_hts_callbackarg *carg, httrackp *opt, lien_back* back) { +static int __cdecl htsshow_xfrstatus(t_hts_callbackarg * carg, httrackp * opt, + lien_back * back) { return 1; } -static int __cdecl htsshow_savename(t_hts_callbackarg *carg, httrackp *opt, const char* adr_complete,const char* fil_complete,const char* referer_adr,const char* referer_fil,char* save) { +static int __cdecl htsshow_savename(t_hts_callbackarg * carg, httrackp * opt, + const char *adr_complete, + const char *fil_complete, + const char *referer_adr, + const char *referer_fil, char *save) { return 1; } -static int __cdecl htsshow_sendheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing) { +static int __cdecl htsshow_sendheader(t_hts_callbackarg * carg, httrackp * opt, + char *buff, const char *adr, + const char *fil, const char *referer_adr, + const char *referer_fil, + htsblk * outgoing) { return 1; } -static int __cdecl htsshow_receiveheader(t_hts_callbackarg *carg, httrackp *opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming) { +static int __cdecl htsshow_receiveheader(t_hts_callbackarg * carg, + httrackp * opt, char *buff, + const char *adr, const char *fil, + const char *referer_adr, + const char *referer_fil, + htsblk * incoming) { return 1; } /* *** Various functions *** */ - -static int fexist(const char* s) { +static int fexist(const char *s) { struct stat st; + memset(&st, 0, sizeof(st)); if (stat(s, &st) == 0) { if (S_ISREG(st.st_mode)) { @@ -649,68 +717,78 @@ static int fexist(const char* s) { } } return 0; -} +} -static int linput(FILE* fp,char* s,int max) { +static int linput(FILE * fp, char *s, int max) { int c; - int j=0; + int j = 0; + do { - c=fgetc(fp); - if (c!=EOF) { - switch(c) { - case 13: break; // sauter CR - case 10: c=-1; break; - case 9: case 12: break; // sauter ces caractères - default: s[j++]=(char) c; break; + c = fgetc(fp); + if (c != EOF) { + switch (c) { + case 13: + break; // sauter CR + case 10: + c = -1; + break; + case 9: + case 12: + break; // sauter ces caractères + default: + s[j++] = (char) c; + break; } } - } while((c!=-1) && (c!=EOF) && (j<(max-1))); - s[j]='\0'; + } while((c != -1) && (c != EOF) && (j < (max - 1))); + s[j] = '\0'; return j; } - // routines de détournement de SIGHUP & co (Unix) // -static void sig_ignore( int code ) { // ignorer signal +static void sig_ignore(int code) { // ignorer signal } -static void sig_term( int code ) { // quitter brutalement - fprintf(stderr,"\nProgram terminated (signal %d)\n",code); +static void sig_term(int code) { // quitter brutalement + fprintf(stderr, "\nProgram terminated (signal %d)\n", code); exit(0); } -static void sig_finish( int code ) { // finir et quitter - signal(code,sig_term); // quitter si encore - if (global_opt != NULL) { - global_opt->state.exit_xh=1; - } - fprintf(stderr,"\nExit requested to engine (signal %d)\n",code); +static void sig_finish(int code) { // finir et quitter + signal(code, sig_term); // quitter si encore + if (global_opt != NULL) { + global_opt->state.exit_xh = 1; + } + fprintf(stderr, "\nExit requested to engine (signal %d)\n", code); } #ifdef _WIN32 #if 0 -static void sig_ask( int code ) { // demander +static void sig_ask(int code) { // demander char s[256]; - signal(code,sig_term); // quitter si encore + + signal(code, sig_term); // quitter si encore printf("\nQuit program/Interrupt/Cancel? (Q/I/C) "); fflush(stdout); - scanf("%s",s); - if ( (s[0]=='y') || (s[0]=='Y') || (s[0]=='o') || (s[0]=='O') || (s[0]=='q') || (s[0]=='Q')) - exit(0); // quitter - else if ( (s[0]=='i') || (s[0]=='I') ) { + scanf("%s", s); + if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O') + || (s[0] == 'q') || (s[0] == 'Q')) + exit(0); // quitter + else if ((s[0] == 'i') || (s[0] == 'I')) { if (global_opt != NULL) { // ask for stop - global_opt->state.stop=1; + global_opt->state.stop = 1; } } - signal(code,sig_ask); // remettre signal + signal(code, sig_ask); // remettre signal } #endif #else static void sig_doback(int blind); -static void sig_back( int code ) { // ignorer et mettre en backing +static void sig_back(int code) { // ignorer et mettre en backing if (global_opt != NULL && !global_opt->background_on_suspend) { - signal( SIGTSTP , SIG_DFL); // ^Z - printf("\nInterrupting the program.\n"); fflush(stdout); + signal(SIGTSTP, SIG_DFL); // ^Z + printf("\nInterrupting the program.\n"); + fflush(stdout); kill(getpid(), SIGTSTP); } else { // Background the process. @@ -718,76 +796,81 @@ static void sig_back( int code ) { // ignorer et mettre en backing sig_doback(0); } } + #if 0 -static void sig_ask( int code ) { // demander +static void sig_ask(int code) { // demander char s[256]; - signal(code,sig_term); // quitter si encore - printf("\nQuit program/Interrupt/Background/bLind background/Cancel? (Q/I/B/L/C) "); + + signal(code, sig_term); // quitter si encore + printf + ("\nQuit program/Interrupt/Background/bLind background/Cancel? (Q/I/B/L/C) "); fflush(stdout); - scanf("%s",s); - if ( (s[0]=='y') || (s[0]=='Y') || (s[0]=='o') || (s[0]=='O') || (s[0]=='q') || (s[0]=='Q')) - exit(0); // quitter - else if ( (s[0]=='b') || (s[0]=='B') || (s[0]=='a') || (s[0]=='A') ) - sig_doback(0); // arrière plan - else if ( (s[0]=='l') || (s[0]=='L') ) - sig_doback(1); // arrière plan - else if ( (s[0]=='i') || (s[0]=='I') ) { + scanf("%s", s); + if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O') + || (s[0] == 'q') || (s[0] == 'Q')) + exit(0); // quitter + else if ((s[0] == 'b') || (s[0] == 'B') || (s[0] == 'a') || (s[0] == 'A')) + sig_doback(0); // arrière plan + else if ((s[0] == 'l') || (s[0] == 'L')) + sig_doback(1); // arrière plan + else if ((s[0] == 'i') || (s[0] == 'I')) { if (global_opt != NULL) { // ask for stop printf("finishing pending transfers.. please wait\n"); - global_opt->state.stop=1; + global_opt->state.stop = 1; } - signal(code,sig_ask); // remettre signal - } - else { + signal(code, sig_ask); // remettre signal + } else { printf("cancel..\n"); - signal(code,sig_ask); // remettre signal + signal(code, sig_ask); // remettre signal } } #endif -static void sig_brpipe( int code ) { // treat if necessary +static void sig_brpipe(int code) { // treat if necessary signal(code, sig_brpipe); } -static void sig_doback(int blind) { // mettre en backing - int out=-1; +static void sig_doback(int blind) { // mettre en backing + int out = -1; + // - printf("\nMoving into background to complete the mirror...\n"); fflush(stdout); + printf("\nMoving into background to complete the mirror...\n"); + fflush(stdout); - if (global_opt != NULL) { - // suppress logging and asking lousy questions - global_opt->quiet=1; - global_opt->verbosedisplay=0; + if (global_opt != NULL) { + // suppress logging and asking lousy questions + global_opt->quiet = 1; + global_opt->verbosedisplay = 0; } if (!blind) - out = open("hts-nohup.out",O_CREAT|O_WRONLY,S_IRUSR|S_IWUSR); + out = open("hts-nohup.out", O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); if (out == -1) - out = open("/dev/null",O_WRONLY,S_IRUSR|S_IWUSR); + out = open("/dev/null", O_WRONLY, S_IRUSR | S_IWUSR); dup2(out, 0); dup2(out, 1); dup2(out, 2); // switch (fork()) { - case 0: + case 0: break; case -1: - fprintf(stderr,"Error: can not fork process\n"); + fprintf(stderr, "Error: can not fork process\n"); break; - default: // pere + default: // pere _exit(0); - break; - } + break; + } } #endif -static void sig_leave( int code ) { +static void sig_leave(int code) { if (global_opt != NULL && global_opt->state._hts_in_mirror) { - signal(code, sig_term); // quitter si encore + signal(code, sig_term); // quitter si encore printf("\n** Finishing pending transfers.. press again ^C to quit.\n"); if (global_opt != NULL) { // ask for stop hts_log_print(global_opt, LOG_ERROR, "Exit requested by shell or user"); - global_opt->state.stop=1; + global_opt->state.stop = 1; } } else { sig_term(code); @@ -797,26 +880,26 @@ static void sig_leave( int code ) { static void signal_handlers(void) { #ifdef _WIN32 #ifndef _WIN32_WCE -#if 0 /* BUG366763 */ - signal( SIGINT , sig_ask ); // ^C +#if 0 /* BUG366763 */ + signal(SIGINT, sig_ask); // ^C #else - signal( SIGINT , sig_leave ); // ^C + signal(SIGINT, sig_leave); // ^C #endif - signal( SIGTERM , sig_finish ); // kill <process> + signal(SIGTERM, sig_finish); // kill <process> #endif #else -#if 0 /* BUG366763 */ - signal( SIGHUP , sig_back ); // close window +#if 0 /* BUG366763 */ + signal(SIGHUP, sig_back); // close window #endif - signal( SIGTSTP , sig_back ); // ^Z - signal( SIGTERM , sig_finish ); // kill <process> -#if 0 /* BUG366763 */ - signal( SIGINT , sig_ask ); // ^C + signal(SIGTSTP, sig_back); // ^Z + signal(SIGTERM, sig_finish); // kill <process> +#if 0 /* BUG366763 */ + signal(SIGINT, sig_ask); // ^C #else - signal( SIGINT , sig_leave ); // ^C + signal(SIGINT, sig_leave); // ^C #endif - signal( SIGPIPE , sig_brpipe ); // broken pipe (write into non-opened socket) - signal( SIGCHLD , sig_ignore ); // child change status + signal(SIGPIPE, sig_brpipe); // broken pipe (write into non-opened socket) + signal(SIGCHLD, sig_ignore); // child change status #endif } diff --git a/src/httrack.h b/src/httrack.h index ec4984e..05fd8e2 100644 --- a/src/httrack.h +++ b/src/httrack.h @@ -17,25 +17,22 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - Important notes: - We hereby ask people using this source NOT to use it in purpose of grabbing emails addresses, or collecting any other private information on persons. This would disgrace our work, and spoil the many hours we spent on it. - Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: htsshow.c console progress info */ /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ #ifndef HTSTOOLS_DEFH -#define HTSTOOLS_DEFH +#define HTSTOOLS_DEFH #include "htsglobal.h" #include "htscore.h" @@ -48,16 +45,16 @@ struct t_StatsBuffer { char name[1024]; char file[1024]; char state[256]; - char BIGSTK url_sav[HTS_URLMAXSIZE*2]; // pour cancel - char BIGSTK url_adr[HTS_URLMAXSIZE*2]; - char BIGSTK url_fil[HTS_URLMAXSIZE*2]; + char BIGSTK url_sav[HTS_URLMAXSIZE * 2]; // pour cancel + char BIGSTK url_adr[HTS_URLMAXSIZE * 2]; + char BIGSTK url_fil[HTS_URLMAXSIZE * 2]; LLint size; LLint sizetot; int offset; // int back; // - int actived; // pour disabled + int actived; // pour disabled }; #ifndef HTS_DEF_FWSTRUCT_t_InpInfo @@ -90,13 +87,14 @@ int main(int argc, char **argv); /* */ // Engine internal variables -typedef void (* htsErrorCallback)(char* msg, char* file, int line); +typedef void (*htsErrorCallback) (char *msg, char *file, int line); extern HTSEXT_API htsErrorCallback htsCallbackErr; extern HTSEXT_API int htsMemoryFastXfr; + /* */ extern HTSEXT_API hts_stat_struct HTS_STAT; extern int _DEBUG_HEAD; -extern FILE* ioinfo; +extern FILE *ioinfo; // from htsbase.h @@ -187,8 +185,9 @@ extern FILE* ioinfo; } while(0) // emergency log -typedef void (*t_abortLog)(char* msg, char* file, int line); +typedef void (*t_abortLog) (char *msg, char *file, int line); extern HTSEXT_API t_abortLog abortLog__; + #define abortLog(a) abortLog__(a, __FILE__, __LINE__) #define abortLogFmt(a) do { \ FILE* fp = fopen("CRASH.TXT", "wb"); \ @@ -17,7 +17,7 @@ /* #include "config.h" */ -#include <string.h> /* for memcpy() */ +#include <string.h> /* for memcpy() */ #include "md5.h" static void byteReverse(unsigned char *buf, unsigned longs); @@ -30,69 +30,66 @@ static void byteReverse(unsigned char *buf, unsigned longs); b ^= a; \ a ^= b; \ } while(0) -static void byteReverse(unsigned char *buf, unsigned longs) -{ - /*uint32 t;*/ +static void byteReverse(unsigned char *buf, unsigned longs) { + /*uint32 t; */ do { /* - t = (uint32) ((unsigned) buf[3] << 8 | buf[2]) << 16 | - ((unsigned) buf[1] << 8 | buf[0]); - *(uint32 *) buf = t; - */ + t = (uint32) ((unsigned) buf[3] << 8 | buf[2]) << 16 | + ((unsigned) buf[1] << 8 | buf[0]); + *(uint32 *) buf = t; + */ byteSwap(buf[0], buf[3]); byteSwap(buf[1], buf[2]); buf += 4; - } while (--longs); + } while(--longs); } /* * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious * initialization constants. */ -void MD5Init(struct MD5Context *ctx, int brokenEndian) -{ +void MD5Init(struct MD5Context *ctx, int brokenEndian) { ctx->buf[0] = 0x67452301; ctx->buf[1] = 0xefcdab89; ctx->buf[2] = 0x98badcfe; ctx->buf[3] = 0x10325476; - + ctx->bits[0] = 0; ctx->bits[1] = 0; - - /*#ifdef WORDS_BIGENDIAN*/ + + /*#ifdef WORDS_BIGENDIAN */ if (brokenEndian) { ctx->doByteReverse = 0; } else { ctx->doByteReverse = 1; } /*#else - ctx->doByteReverse = 0; - #endif - */ + ctx->doByteReverse = 0; + #endif + */ } /* * Update context to reflect the concatenation of another buffer full * of bytes. */ -void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len) -{ +void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len) { uint32 t; - + /* Update bitcount */ - + t = ctx->bits[0]; if ((ctx->bits[0] = t + ((uint32) len << 3)) < t) - ctx->bits[1]++; /* Carry from low to high */ + ctx->bits[1]++; /* Carry from low to high */ ctx->bits[1] += len >> 29; - - t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ - + + t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ + /* Handle any leading odd-sized chunks */ - + if (t) { unsigned char *p = (unsigned char *) ctx->in + t; - + t = 64 - t; if (len < t) { memcpy(p, buf, len); @@ -106,8 +103,8 @@ void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len) len -= t; } /* Process data in 64-byte chunks */ - - while (len >= 64) { + + while(len >= 64) { memcpy(ctx->in, buf, 64); if (ctx->doByteReverse) byteReverse(ctx->in, 16); @@ -115,9 +112,9 @@ void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len) buf += 64; len -= 64; } - + /* Handle any remaining bytes of data. */ - + memcpy(ctx->in, buf, len); } @@ -125,22 +122,21 @@ void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len) * Final wrapup - pad to 64-byte boundary with the bit pattern * 1 0* (64-bit count of bits processed, MSB-first) */ -void MD5Final(unsigned char digest[16], struct MD5Context *ctx) -{ +void MD5Final(unsigned char digest[16], struct MD5Context *ctx) { unsigned count; unsigned char *p; - + /* Compute number of bytes mod 64 */ count = (ctx->bits[0] >> 3) & 0x3F; - + /* Set the first char of padding to 0x80. This is safe since there is - always at least one byte free */ + always at least one byte free */ p = ctx->in + count; *p++ = 0x80; - + /* Bytes of padding needed to make 64 bytes */ count = 64 - 1 - count; - + /* Pad out to 56 mod 64 */ if (count < 8) { /* Two lots of padding: Pad the first block to 64 bytes */ @@ -148,7 +144,7 @@ void MD5Final(unsigned char digest[16], struct MD5Context *ctx) if (ctx->doByteReverse) byteReverse(ctx->in, 16); MD5Transform(ctx->buf, (uint32 *) ctx->in); - + /* Now fill the next block with 56 bytes */ memset(ctx->in, 0, 56); } else { @@ -157,19 +153,19 @@ void MD5Final(unsigned char digest[16], struct MD5Context *ctx) } if (ctx->doByteReverse) byteReverse(ctx->in, 14); - + /* Append length in bits and transform */ /* Note: see patch for PAM from Tomas Mraz */ - memcpy((uint32 *)ctx->in + 14, ctx->bits, 2*sizeof(uint32)); + memcpy((uint32 *) ctx->in + 14, ctx->bits, 2 * sizeof(uint32)); /*((uint32 *) ctx->in)[14] = ctx->bits[0]; - ((uint32 *) ctx->in)[15] = ctx->bits[1]; - */ - + ((uint32 *) ctx->in)[15] = ctx->bits[1]; + */ + MD5Transform(ctx->buf, (uint32 *) ctx->in); if (ctx->doByteReverse) byteReverse((unsigned char *) ctx->buf, 4); memcpy(digest, ctx->buf, 16); - memset(ctx, 0, sizeof(ctx)); /* In case it's sensitive */ + memset(ctx, 0, sizeof(ctx)); /* In case it's sensitive */ } /* The four core functions - F1 is optimized somewhat */ @@ -189,15 +185,14 @@ void MD5Final(unsigned char digest[16], struct MD5Context *ctx) * reflect the addition of 16 longwords of new data. MD5Update blocks * the data and converts bytes into longwords for this routine. */ -void MD5Transform(uint32 buf[4], uint32 const in[16]) -{ +void MD5Transform(uint32 buf[4], uint32 const in[16]) { register uint32 a, b, c, d; - + a = buf[0]; b = buf[1]; c = buf[2]; d = buf[3]; - + MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7); MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12); MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17); @@ -214,7 +209,7 @@ void MD5Transform(uint32 buf[4], uint32 const in[16]) MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12); MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17); MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22); - + MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5); MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9); MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14); @@ -231,7 +226,7 @@ void MD5Transform(uint32 buf[4], uint32 const in[16]) MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9); MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14); MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20); - + MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4); MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11); MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16); @@ -248,7 +243,7 @@ void MD5Transform(uint32 buf[4], uint32 const in[16]) MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11); MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16); MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23); - + MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6); MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10); MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15); @@ -265,10 +260,9 @@ void MD5Transform(uint32 buf[4], uint32 const in[16]) MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10); MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15); MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21); - + buf[0] += a; buf[1] += b; buf[2] += c; buf[3] += d; } - diff --git a/src/proxy/main.c b/src/proxy/main.c index bb81810..8bce9bb 100644 --- a/src/proxy/main.c +++ b/src/proxy/main.c @@ -40,45 +40,47 @@ Please visit our Website: http://www.httrack.com #ifndef _WIN32 #include <signal.h> -static void sig_brpipe( int code ) { +static void sig_brpipe(int code) { /* ignore */ } #endif -static int scanHostPort(const char* str, char *host, int *port) { - char* pos = strrchr(str, ':'); - if (pos != NULL) { - int n = (int) ( pos - str ); - if (n < 256) { - host[0] = '\0'; - strncat(host, str, n); - if (sscanf(pos + 1, "%d", port) == 1) { - return 1; - } - } - } - return 0; +static int scanHostPort(const char *str, char *host, int *port) { + char *pos = strrchr(str, ':'); + + if (pos != NULL) { + int n = (int) (pos - str); + + if (n < 256) { + host[0] = '\0'; + strncat(host, str, n); + if (sscanf(pos + 1, "%d", port) == 1) { + return 1; + } + } + } + return 0; } -int main(int argc, char* argv[]) -{ +int main(int argc, char *argv[]) { int i; int ret = 0; - int proxyPort = 0, icpPort = 0; - char proxyAddr[256 + 1], icpAddr[256 + 1]; - PT_Indexes index; + int proxyPort = 0, icpPort = 0; + char proxyAddr[256 + 1], icpAddr[256 + 1]; + PT_Indexes index; #ifdef _WIN32 { - WORD wVersionRequested; // requested version WinSock API + WORD wVersionRequested; // requested version WinSock API WSADATA wsadata; // Windows Sockets API data int stat; + wVersionRequested = 0x0101; - stat = WSAStartup( wVersionRequested, &wsadata ); + stat = WSAStartup(wVersionRequested, &wsadata); if (stat != 0) { fprintf(stderr, "Winsock not found!\n"); return -1; - } else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) { + } else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) { fprintf(stderr, "WINSOCK.DLL does not support version 1.1\n"); WSACleanup(); return -1; @@ -86,93 +88,103 @@ int main(int argc, char* argv[]) } #endif - /* Args */ - printf("ProxyTrack %s, build proxies upon HTTrack Website Copier Archives\n", PROXYTRACK_VERSION); - printf("Copyright (C) Xavier Roche and other contributors\n"); - printf("\n"); - printf("This program is free software; you can redistribute it and/or\n"); - printf("modify it under the terms of the GNU General Public License\n"); - printf("as published by the Free Software Foundation; either version 3\n"); - printf("of the License, or any later version.\n"); - printf("\n"); - printf("*** This version is a development release ***\n"); - printf("\n"); - if (argc < 3 - || ( - strcmp(argv[1], "--convert") != 0 - && - ( - !scanHostPort(argv[1], proxyAddr, &proxyPort) - || !scanHostPort(argv[2], icpAddr, &icpPort) - ) - ) - ) - { - fprintf(stderr, "proxy mode:\n"); - fprintf(stderr, "usage: %s <proxy-addr:proxy-port> <ICP-addr:ICP-port> [ ( <new.zip path> | <new.ndx path> | <archive.arc path> | --list <file-list> ) ..]\n", argv[0]); - fprintf(stderr, "\texample:%s proxy:8080 localhost:3130 /home/archives/www-archive-01.zip /home/old-archives/www-archive-02.ndx\n", argv[0]); - fprintf(stderr, "convert mode:\n"); - fprintf(stderr, "usage: %s --convert <archive-output-path> [ ( <new.zip path> | <new.ndx path> | <archive.arc path> | --list <file-list> ) ..]\n", argv[0]); - fprintf(stderr, "\texample:%s proxy:8080 localhost:3130 /home/archives/www-archive-01.zip /home/old-archives/www-archive-02.ndx\n", argv[0]); + /* Args */ + printf("ProxyTrack %s, build proxies upon HTTrack Website Copier Archives\n", + PROXYTRACK_VERSION); + printf("Copyright (C) Xavier Roche and other contributors\n"); + printf("\n"); + printf("This program is free software; you can redistribute it and/or\n"); + printf("modify it under the terms of the GNU General Public License\n"); + printf("as published by the Free Software Foundation; either version 3\n"); + printf("of the License, or any later version.\n"); + printf("\n"); + printf("*** This version is a development release ***\n"); + printf("\n"); + if (argc < 3 + || (strcmp(argv[1], "--convert") != 0 + && (!scanHostPort(argv[1], proxyAddr, &proxyPort) + || !scanHostPort(argv[2], icpAddr, &icpPort) + ) + ) + ) { + fprintf(stderr, "proxy mode:\n"); + fprintf(stderr, + "usage: %s <proxy-addr:proxy-port> <ICP-addr:ICP-port> [ ( <new.zip path> | <new.ndx path> | <archive.arc path> | --list <file-list> ) ..]\n", + argv[0]); + fprintf(stderr, + "\texample:%s proxy:8080 localhost:3130 /home/archives/www-archive-01.zip /home/old-archives/www-archive-02.ndx\n", + argv[0]); + fprintf(stderr, "convert mode:\n"); + fprintf(stderr, + "usage: %s --convert <archive-output-path> [ ( <new.zip path> | <new.ndx path> | <archive.arc path> | --list <file-list> ) ..]\n", + argv[0]); + fprintf(stderr, + "\texample:%s proxy:8080 localhost:3130 /home/archives/www-archive-01.zip /home/old-archives/www-archive-02.ndx\n", + argv[0]); return 1; } - index = PT_New(); - for(i = 3 ; i < argc ; i++) { - if (argv[i][0] == '-') { - if (strcmp(argv[i], "--list") == 0) { - if (i + 1 < argc) { - char line[256 + 1]; - FILE *fp = fopen(argv[++i], "rb"); - if (fp == NULL) { - fprintf(stderr, "error: could not process list %s\n", argv[i]); - exit(1); - } - while(linput(fp, line, 256)) { - int itemsAdded = PT_AddIndex(index, line); - if (itemsAdded > 0) { - fprintf(stderr, "processed: %s (%d items added)\n", line, itemsAdded); - } else if (itemsAdded == 0) { - fprintf(stderr, "processed: %s (no items added)\n", line); - } else { - fprintf(stderr, "error: could not process %s\n", line); - } - } - fclose(fp); - } - } else { - fprintf(stderr, "* bad arg %s\n", argv[i]); - exit(1); - } - } else { - int itemsAdded = PT_AddIndex(index, argv[i]); - if (itemsAdded > 0) { - fprintf(stderr, "processed: %s (%d items added)\n", argv[i], itemsAdded); - } else if (itemsAdded == 0) { - fprintf(stderr, "processed: %s (no items added)\n", argv[i]); - } else { - fprintf(stderr, "error: could not process %s\n", argv[i]); - } - } - } - - /* sigpipe */ + index = PT_New(); + for(i = 3; i < argc; i++) { + if (argv[i][0] == '-') { + if (strcmp(argv[i], "--list") == 0) { + if (i + 1 < argc) { + char line[256 + 1]; + FILE *fp = fopen(argv[++i], "rb"); + + if (fp == NULL) { + fprintf(stderr, "error: could not process list %s\n", argv[i]); + exit(1); + } + while(linput(fp, line, 256)) { + int itemsAdded = PT_AddIndex(index, line); + + if (itemsAdded > 0) { + fprintf(stderr, "processed: %s (%d items added)\n", line, + itemsAdded); + } else if (itemsAdded == 0) { + fprintf(stderr, "processed: %s (no items added)\n", line); + } else { + fprintf(stderr, "error: could not process %s\n", line); + } + } + fclose(fp); + } + } else { + fprintf(stderr, "* bad arg %s\n", argv[i]); + exit(1); + } + } else { + int itemsAdded = PT_AddIndex(index, argv[i]); + + if (itemsAdded > 0) { + fprintf(stderr, "processed: %s (%d items added)\n", argv[i], + itemsAdded); + } else if (itemsAdded == 0) { + fprintf(stderr, "processed: %s (no items added)\n", argv[i]); + } else { + fprintf(stderr, "error: could not process %s\n", argv[i]); + } + } + } + + /* sigpipe */ #ifndef _WIN32 - signal( SIGPIPE , sig_brpipe ); // broken pipe (write into non-opened socket) + signal(SIGPIPE, sig_brpipe); // broken pipe (write into non-opened socket) #endif /* Go */ - if (strcmp(argv[1], "--convert") != 0) { - ret = proxytrack_main(proxyAddr, proxyPort, icpAddr, icpPort, index); - } else { - if ((ret = PT_SaveCache(index, argv[2])) == 0) { - fprintf(stderr, "processed: '%s'\n", argv[2]); - } else { - fprintf(stderr, "error: could not save '%s'\n", argv[2]); - } - } - - /* Wipe */ - PT_Delete(index); + if (strcmp(argv[1], "--convert") != 0) { + ret = proxytrack_main(proxyAddr, proxyPort, icpAddr, icpPort, index); + } else { + if ((ret = PT_SaveCache(index, argv[2])) == 0) { + fprintf(stderr, "processed: '%s'\n", argv[2]); + } else { + fprintf(stderr, "error: could not save '%s'\n", argv[2]); + } + } + + /* Wipe */ + PT_Delete(index); #ifdef _WIN32 WSACleanup(); @@ -180,4 +192,3 @@ int main(int argc, char* argv[]) return ret; } - diff --git a/src/proxy/proxystrings.h b/src/proxy/proxystrings.h index 23456a7..09d0f9f 100755 --- a/src/proxy/proxystrings.h +++ b/src/proxy/proxystrings.h @@ -20,7 +20,6 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. Please visit our Website: http://www.httrack.com */ - /* ------------------------------------------------------------ */ /* File: Strings */ /* Author: Xavier Roche */ @@ -29,89 +28,93 @@ Please visit our Website: http://www.httrack.com // Strings a bit safer than static buffers #ifndef HTS_PROXYSTRINGS_DEFSTATIC -#define HTS_PROXYSTRINGS_DEFSTATIC +#define HTS_PROXYSTRINGS_DEFSTATIC #include "htsstrings.h" - /* Tools */ static int ehexh(char c) { - if ((c>='0') && (c<='9')) return c-'0'; - if ((c>='a') && (c<='f')) c-=('a'-'A'); - if ((c>='A') && (c<='F')) return (c-'A'+10); + if ((c >= '0') && (c <= '9')) + return c - '0'; + if ((c >= 'a') && (c <= 'f')) + c -= ('a' - 'A'); + if ((c >= 'A') && (c <= 'F')) + return (c - 'A' + 10); return 0; } -static int ehex(const char* s) { - return 16*ehexh(*s)+ehexh(*(s+1)); +static int ehex(const char *s) { + return 16 * ehexh(*s) + ehexh(*(s + 1)); } -static void unescapehttp(const char* s, String* tempo) { +static void unescapehttp(const char *s, String * tempo) { int i; - for (i = 0; s[i] != '\0' ; i++) { - if (s[i]=='%' && s[i+1]=='%') { + + for(i = 0; s[i] != '\0'; i++) { + if (s[i] == '%' && s[i + 1] == '%') { i++; StringAddchar(*tempo, '%'); - } else if (s[i]=='%') { + } else if (s[i] == '%') { char hc; + i++; - hc = (char) ehex(s+i); + hc = (char) ehex(s + i); StringAddchar(*tempo, (char) hc); - i++; // sauter 2 caractères finalement - } - else if (s[i]=='+') { + i++; // sauter 2 caractères finalement + } else if (s[i] == '+') { StringAddchar(*tempo, ' '); - } - else + } else StringAddchar(*tempo, s[i]); } } -static void escapexml(const char* s, String* tempo) { +static void escapexml(const char *s, String * tempo) { int i; - for (i=0 ; s[i] != '\0' ; i++) { + + for(i = 0; s[i] != '\0'; i++) { if (s[i] == '&') StringCat(*tempo, "&"); - else if (s[i] == '<') + else if (s[i] == '<') StringCat(*tempo, "<"); - else if (s[i] == '>') + else if (s[i] == '>') StringCat(*tempo, ">"); - else if (s[i] == '\"') + else if (s[i] == '\"') StringCat(*tempo, """); else StringAddchar(*tempo, s[i]); } } -static char* concat(char *catbuff,const char* a,const char* b) { - if (a != NULL && a[0] != '\0') { - strcpy(catbuff, a); - } else { - catbuff[0] = '\0'; - } - if (b != NULL && b[0] != '\0') { - strcat(catbuff, b); - } +static char *concat(char *catbuff, const char *a, const char *b) { + if (a != NULL && a[0] != '\0') { + strcpy(catbuff, a); + } else { + catbuff[0] = '\0'; + } + if (b != NULL && b[0] != '\0') { + strcat(catbuff, b); + } return catbuff; } -static char* __fconv(char* a) { +static char *__fconv(char *a) { #ifdef WIN32 int i; - for(i = 0 ; a[i] != 0 ; i++) - if (a[i] == '/') // Unix-to-DOS style + + for(i = 0; a[i] != 0; i++) + if (a[i] == '/') // Unix-to-DOS style a[i] = '\\'; #endif return a; } -static char* fconcat(char *catbuff, const char* a, const char* b) { - return __fconv(concat(catbuff,a,b)); +static char *fconcat(char *catbuff, const char *a, const char *b) { + return __fconv(concat(catbuff, a, b)); } -static char* fconv(char *catbuff, const char* a) { - return __fconv(concat(catbuff,a,"")); +static char *fconv(char *catbuff, const char *a) { + return __fconv(concat(catbuff, a, "")); } #endif diff --git a/src/proxy/proxytrack.c b/src/proxy/proxytrack.c index ded2e13..e78f457 100644 --- a/src/proxy/proxytrack.c +++ b/src/proxy/proxytrack.c @@ -25,7 +25,6 @@ Please visit our Website: http://www.httrack.com /* Author: Xavier Roche */ /* ------------------------------------------------------------ */ - /* /\/\/\/\/\/\/\/\/\/\/\/\/\ PENDING WORK /\/\/\/\/\/\/\/\/\/\/\/\/\ @@ -45,7 +44,6 @@ Aim: Building a sub-proxy to be linked with other top level proxies (such as Squ Basic design: Classical HTTP/1.0 proxy server, with ICP server support Internal data design: HTTrack cache indexing in fast hashtables, with 'pluggable' design (add/removal of caches on-the-fly) - Index structure organization: ----------------------------- @@ -59,7 +57,6 @@ Indexes are hashtables with URL (STRING) -> INTEGER lookup. URL -----> CIL Ask for index ID URL -----> Index[ID] Ask for index properties (ZIP cache index) - Lookup of an entry: ------------------- @@ -70,7 +67,6 @@ Else return FAILURE EndIf - Fetching of an entry: --------------------- @@ -83,7 +79,6 @@ If ID is valid Then EndIf EndIf - Removal of index N: ------------------- @@ -94,7 +89,6 @@ For all entries in Index[N] Done Delete Index[N] - Adding of index N: ------------------ @@ -146,65 +140,81 @@ Remark: If no cache newer than the added one is found, all entries can be added /* threads */ #ifdef _WIN32 -#include <process.h> /* _beginthread, _endthread */ +#include <process.h> /* _beginthread, _endthread */ #else #include <pthread.h> #endif /* External references */ // htsErrorCallback htsCallbackErr = NULL; -int htsMemoryFastXfr = 1; /* fast xfr by default */ -void abortLog__fnc(char* msg, char* file, int line); -void abortLog__fnc(char* msg, char* file, int line) { - FILE* fp = fopen("CRASH.TXT", "wb"); - if (!fp) fp = fopen("/tmp/CRASH.TXT", "wb"); - if (!fp) fp = fopen("C:\\CRASH.TXT", "wb"); - if (!fp) fp = fopen("CRASH.TXT", "wb"); +int htsMemoryFastXfr = 1; /* fast xfr by default */ +void abortLog__fnc(char *msg, char *file, int line); +void abortLog__fnc(char *msg, char *file, int line) { + FILE *fp = fopen("CRASH.TXT", "wb"); + + if (!fp) + fp = fopen("/tmp/CRASH.TXT", "wb"); + if (!fp) + fp = fopen("C:\\CRASH.TXT", "wb"); + if (!fp) + fp = fopen("CRASH.TXT", "wb"); if (fp) { - fprintf(fp, "HTTrack " HTTRACK_VERSIONID " closed at '%s', line %d\r\n", file, line); + fprintf(fp, "HTTrack " HTTRACK_VERSIONID " closed at '%s', line %d\r\n", + file, line); fprintf(fp, "Reason:\r\n%s\r\n", msg); fflush(fp); fclose(fp); } } + // HTSEXT_API t_abortLog abortLog__ = abortLog__fnc; /* avoid VC++ inlining */ #define webhttrack_lock(A) do{}while(0) /* Static definitions */ -static int linputsoc(T_SOC soc, char* s, int max) { +static int linputsoc(T_SOC soc, char *s, int max) { int c; - int j=0; + int j = 0; + do { unsigned char ch; + if (recv(soc, &ch, 1, 0) == 1) { c = ch; } else { c = EOF; } - if (c!=EOF) { - switch(c) { - case 13: break; // sauter CR - case 10: c=-1; break; - case 9: case 12: break; // sauter ces caractères - default: s[j++]=(char) c; break; + if (c != EOF) { + switch (c) { + case 13: + break; // sauter CR + case 10: + c = -1; + break; + case 9: + case 12: + break; // sauter ces caractères + default: + s[j++] = (char) c; + break; } } - } while((c!=-1) && (c!=EOF) && (j<(max-1))); - s[j]='\0'; + } while((c != -1) && (c != EOF) && (j < (max - 1))); + s[j] = '\0'; return j; } static int check_readinput_t(T_SOC soc, int timeout) { if (soc != INVALID_SOCKET) { - fd_set fds; // poll structures + fd_set fds; // poll structures struct timeval tv; // structure for select + FD_ZERO(&fds); - FD_SET(soc,&fds); - tv.tv_sec=timeout; - tv.tv_usec=0; - select((int)(soc + 1),&fds,NULL,NULL,&tv); - if (FD_ISSET(soc,&fds)) + FD_SET(soc, &fds); + tv.tv_sec = timeout; + tv.tv_usec = 0; + select((int) (soc + 1), &fds, NULL, NULL, &tv); + if (FD_ISSET(soc, &fds)) return 1; else return 0; @@ -212,102 +222,107 @@ static int check_readinput_t(T_SOC soc, int timeout) { return 0; } -static int linputsoc_t(T_SOC soc, char* s, int max, int timeout) { +static int linputsoc_t(T_SOC soc, char *s, int max, int timeout) { if (check_readinput_t(soc, timeout)) { return linputsoc(soc, s, max); } return -1; } -static int gethost(const char* hostname, SOCaddr *server, size_t server_size) { +static int gethost(const char *hostname, SOCaddr * server, size_t server_size) { if (hostname != NULL && *hostname != '\0') { #if HTS_INET6==0 - /* - ipV4 resolver - */ - t_hostent* hp=gethostbyname(hostname); - if (hp!=NULL) { - if (hp->h_length) { - SOCaddr_copyaddr(*server, server_size, hp->h_addr_list[0], hp->h_length); - return 1; - } - } + /* + ipV4 resolver + */ + t_hostent *hp = gethostbyname(hostname); + + if (hp != NULL) { + if (hp->h_length) { + SOCaddr_copyaddr(*server, server_size, hp->h_addr_list[0], + hp->h_length); + return 1; + } + } #else - /* - ipV6 resolver - */ - struct addrinfo* res = NULL; - struct addrinfo hints; - memset(&hints, 0, sizeof(hints)); + /* + ipV6 resolver + */ + struct addrinfo *res = NULL; + struct addrinfo hints; + + memset(&hints, 0, sizeof(hints)); #if 0 - if (IPV6_resolver == 1) // V4 only (for bogus V6 entries) - hints.ai_family = PF_INET; - else if (IPV6_resolver == 2) // V6 only (for testing V6 only) - hints.ai_family = PF_INET6; - else + if (IPV6_resolver == 1) // V4 only (for bogus V6 entries) + hints.ai_family = PF_INET; + else if (IPV6_resolver == 2) // V6 only (for testing V6 only) + hints.ai_family = PF_INET6; + else #endif - hints.ai_family = PF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - hints.ai_protocol = IPPROTO_TCP; - if (getaddrinfo(hostname, NULL, &hints, &res) == 0) { - if (res) { - if ( (res->ai_addr) && (res->ai_addrlen) ) { - SOCaddr_copyaddr(*server, server_size, res->ai_addr, res->ai_addrlen); - freeaddrinfo(res); - return 1; - } - } - } - if (res) { - freeaddrinfo(res); - } - + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = IPPROTO_TCP; + if (getaddrinfo(hostname, NULL, &hints, &res) == 0) { + if (res) { + if ((res->ai_addr) && (res->ai_addrlen)) { + SOCaddr_copyaddr(*server, server_size, res->ai_addr, res->ai_addrlen); + freeaddrinfo(res); + return 1; + } + } + } + if (res) { + freeaddrinfo(res); + } #endif - } - return 0; + } + return 0; } -static String getip(SOCaddr *server, int serverLen) { - String s = STRING_EMPTY; +static String getip(SOCaddr * server, int serverLen) { + String s = STRING_EMPTY; + #if HTS_INET6==0 - unsigned int sizeMax = sizeof("999.999.999.999:65535"); + unsigned int sizeMax = sizeof("999.999.999.999:65535"); #else - unsigned int sizeMax = sizeof("ffff:ffff:ffff:ffff:ffff:ffff:ffff:65535"); + unsigned int sizeMax = sizeof("ffff:ffff:ffff:ffff:ffff:ffff:ffff:65535"); #endif - char * dotted = malloc(sizeMax + 1); - unsigned short port = ntohs(SOCaddr_sinport(*server)); - if (dotted == NULL) { - proxytrack_print_log(CRITICAL, "memory exhausted"); - return s; - } - SOCaddr_inetntoa(dotted, sizeMax, *server, serverLen); - sprintf(dotted + strlen(dotted), ":%d", port); - StringAttach(&s, &dotted); - return s; + char *dotted = malloc(sizeMax + 1); + unsigned short port = ntohs(SOCaddr_sinport(*server)); + + if (dotted == NULL) { + proxytrack_print_log(CRITICAL, "memory exhausted"); + return s; + } + SOCaddr_inetntoa(dotted, sizeMax, *server, serverLen); + sprintf(dotted + strlen(dotted), ":%d", port); + StringAttach(&s, &dotted); + return s; } +static T_SOC smallserver_init(const char *adr, int port, int family) { + SOCaddr server; + size_t server_size = sizeof(server); -static T_SOC smallserver_init(const char* adr, int port, int family) { - SOCaddr server; - size_t server_size = sizeof(server); + memset(&server, 0, sizeof(server)); + SOCaddr_initany(server, server_size); + if (gethost(adr, &server, server_size)) { // host name + T_SOC soc = INVALID_SOCKET; - memset(&server, 0, sizeof(server)); - SOCaddr_initany(server, server_size); - if (gethost(adr, &server, server_size)) { // host name - T_SOC soc = INVALID_SOCKET; - if ( (soc = (T_SOC) socket(SOCaddr_sinfamily(server), family, 0)) != INVALID_SOCKET) { + if ((soc = + (T_SOC) socket(SOCaddr_sinfamily(server), family, + 0)) != INVALID_SOCKET) { SOCaddr_initport(server, port); - if ( bind(soc,(struct sockaddr*) &server, (int)server_size) == 0 ) { - if (family != SOCK_STREAM - || listen(soc, 10) >=0 ) { - return soc; + if (bind(soc, (struct sockaddr *) &server, (int) server_size) == 0) { + if (family != SOCK_STREAM || listen(soc, 10) >= 0) { + return soc; } else { #ifdef _WIN32 closesocket(soc); #else close(soc); #endif - soc=INVALID_SOCKET; + soc = INVALID_SOCKET; } } else { #ifdef _WIN32 @@ -315,7 +330,7 @@ static T_SOC smallserver_init(const char* adr, int port, int family) { #else close(soc); #endif - soc=INVALID_SOCKET; + soc = INVALID_SOCKET; } } } @@ -323,26 +338,26 @@ static T_SOC smallserver_init(const char* adr, int port, int family) { } static int proxytrack_start(PT_Indexes indexes, T_SOC soc, T_SOC socICP); -int proxytrack_main(char* proxyAddr, int proxyPort, - char* icpAddr, int icpPort, - PT_Indexes index) { +int proxytrack_main(char *proxyAddr, int proxyPort, char *icpAddr, int icpPort, + PT_Indexes index) { int returncode = 0; T_SOC soc = smallserver_init(proxyAddr, proxyPort, SOCK_STREAM); T_SOC socICP = smallserver_init(proxyAddr, icpPort, SOCK_DGRAM); - if (soc != INVALID_SOCKET - && socICP != INVALID_SOCKET) - { - char url[HTS_URLMAXSIZE*2]; + + if (soc != INVALID_SOCKET && socICP != INVALID_SOCKET) { + char url[HTS_URLMAXSIZE * 2]; char method[32]; char data[32768]; - url[0]=method[0]=data[0]='\0'; + + url[0] = method[0] = data[0] = '\0'; // printf("HTTP Proxy installed on %s:%d/\n", proxyAddr, proxyPort); printf("ICP Proxy installed on %s:%d/\n", icpAddr, icpPort); #ifndef _WIN32 { pid_t pid = getpid(); - printf("PID=%d\n", (int)pid); + + printf("PID=%d\n", (int) pid); } #endif fflush(stdout); @@ -350,7 +365,9 @@ int proxytrack_main(char* proxyAddr, int proxyPort, // if (!proxytrack_start(index, soc, socICP)) { int last_errno = errno; - fprintf(stderr, "Unable to create the server: %s\n", strerror(last_errno)); + + fprintf(stderr, "Unable to create the server: %s\n", + strerror(last_errno)); #ifdef _WIN32 closesocket(soc); #else @@ -363,7 +380,9 @@ int proxytrack_main(char* proxyAddr, int proxyPort, } } else { int last_errno = errno; - fprintf(stderr, "Unable to initialize a temporary server : %s\n", strerror(last_errno)); + + fprintf(stderr, "Unable to initialize a temporary server : %s\n", + strerror(last_errno)); returncode = 1; } printf("EXITED\n"); @@ -372,1020 +391,1104 @@ int proxytrack_main(char* proxyAddr, int proxyPort, return returncode; } -static const char* GetHttpMessage(int statuscode) { +static const char *GetHttpMessage(int statuscode) { // Erreurs HTTP, selon RFC - switch( statuscode) { - case 100: return "Continue"; break; - case 101: return "Switching Protocols"; break; - case 200: return "OK"; break; - case 201: return "Created"; break; - case 202: return "Accepted"; break; - case 203: return "Non-Authoritative Information"; break; - case 204: return "No Content"; break; - case 205: return "Reset Content"; break; - case 206: return "Partial Content"; break; - case 207: return "Multi-Status"; break; - case 300: return "Multiple Choices"; break; - case 301: return "Moved Permanently"; break; - case 302: return "Moved Temporarily"; break; - case 303: return "See Other"; break; - case 304: return "Not Modified"; break; - case 305: return "Use Proxy"; break; - case 306: return "Undefined 306 error"; break; - case 307: return "Temporary Redirect"; break; - case 400: return "Bad Request"; break; - case 401: return "Unauthorized"; break; - case 402: return "Payment Required"; break; - case 403: return "Forbidden"; break; - case 404: return "Not Found"; break; - case 405: return "Method Not Allowed"; break; - case 406: return "Not Acceptable"; break; - case 407: return "Proxy Authentication Required"; break; - case 408: return "Request Time-out"; break; - case 409: return "Conflict"; break; - case 410: return "Gone"; break; - case 411: return "Length Required"; break; - case 412: return "Precondition Failed"; break; - case 413: return "Request Entity Too Large"; break; - case 414: return "Request-URI Too Large"; break; - case 415: return "Unsupported Media Type"; break; - case 416: return "Requested Range Not Satisfiable"; break; - case 417: return "Expectation Failed"; break; - case 500: return "Internal Server Error"; break; - case 501: return "Not Implemented"; break; - case 502: return "Bad Gateway"; break; - case 503: return "Service Unavailable"; break; - case 504: return "Gateway Time-out"; break; - case 505: return "HTTP Version Not Supported"; break; - default: return "Unknown HTTP Error"; break; - } + switch (statuscode) { + case 100: + return "Continue"; + break; + case 101: + return "Switching Protocols"; + break; + case 200: + return "OK"; + break; + case 201: + return "Created"; + break; + case 202: + return "Accepted"; + break; + case 203: + return "Non-Authoritative Information"; + break; + case 204: + return "No Content"; + break; + case 205: + return "Reset Content"; + break; + case 206: + return "Partial Content"; + break; + case 207: + return "Multi-Status"; + break; + case 300: + return "Multiple Choices"; + break; + case 301: + return "Moved Permanently"; + break; + case 302: + return "Moved Temporarily"; + break; + case 303: + return "See Other"; + break; + case 304: + return "Not Modified"; + break; + case 305: + return "Use Proxy"; + break; + case 306: + return "Undefined 306 error"; + break; + case 307: + return "Temporary Redirect"; + break; + case 400: + return "Bad Request"; + break; + case 401: + return "Unauthorized"; + break; + case 402: + return "Payment Required"; + break; + case 403: + return "Forbidden"; + break; + case 404: + return "Not Found"; + break; + case 405: + return "Method Not Allowed"; + break; + case 406: + return "Not Acceptable"; + break; + case 407: + return "Proxy Authentication Required"; + break; + case 408: + return "Request Time-out"; + break; + case 409: + return "Conflict"; + break; + case 410: + return "Gone"; + break; + case 411: + return "Length Required"; + break; + case 412: + return "Precondition Failed"; + break; + case 413: + return "Request Entity Too Large"; + break; + case 414: + return "Request-URI Too Large"; + break; + case 415: + return "Unsupported Media Type"; + break; + case 416: + return "Requested Range Not Satisfiable"; + break; + case 417: + return "Expectation Failed"; + break; + case 500: + return "Internal Server Error"; + break; + case 501: + return "Not Implemented"; + break; + case 502: + return "Bad Gateway"; + break; + case 503: + return "Service Unavailable"; + break; + case 504: + return "Gateway Time-out"; + break; + case 505: + return "HTTP Version Not Supported"; + break; + default: + return "Unknown HTTP Error"; + break; + } } #ifndef NO_WEBDAV -static void proxytrack_add_DAV_Item(String *item, String *buff, - const char* filename, - size_t size, - time_t timestamp, - const char* mime, - int isDir, - int isRoot, - int isDefault) -{ - struct tm * timetm; - if (timestamp == (time_t) 0 || timestamp == (time_t) -1) { - timestamp = time(NULL); - } - if ((timetm = gmtime(×tamp)) != NULL) { - char tms[256 + 1]; - const char * name; - strftime(tms, 256, "%a, %d %b %Y %H:%M:%S GMT", timetm); /* Sun, 18 Sep 2005 11:45:45 GMT */ - - if (mime == NULL || *mime == 0) - mime = "application/octet-stream"; - - StringLength(*buff) = 0; - escapexml(filename, buff); - - name = strrchr(StringBuff(*buff), '/'); - if (name != NULL) - name++; - if (name == NULL || *name == 0) { - if (strcmp(mime, "text/html") == 0) - name = "Default Document for the Folder.html"; - else - name = "Default Document for the Folder"; - } - - StringRoom(*item, 1024); - sprintf(StringBuffRW(*item), - "<response xmlns=\"DAV:\">\r\n" - "<href>/webdav%s%s</href>\r\n" - "<propstat>\r\n" - "<prop>\r\n" - "<displayname>%s</displayname>\r\n" - "<iscollection>%d</iscollection>\r\n" - "<haschildren>%d</haschildren>\r\n" - "<isfolder>%d</isfolder>\r\n" - "<resourcetype>%s</resourcetype>\r\n" - "<creationdate>%d-%02d-%02dT%02d:%02d:%02dZ</creationdate>\r\n" - "<getlastmodified>%s</getlastmodified>\r\n" - "<supportedlock></supportedlock>\r\n" - "<lockdiscovery/>\r\n" - "<getcontenttype>%s</getcontenttype>\r\n" - "<getcontentlength>%d</getcontentlength>\r\n" - "<isroot>%d</isroot>\r\n" - "</prop>\r\n" - "<status>HTTP/1.1 200 OK</status>\r\n" - "</propstat>\r\n" - "</response>\r\n", - /* */ - ( StringBuff(*buff)[0] == '/' ) ? "" : "/", StringBuff(*buff), - name, - isDir ? 1 : 0, - isDir ? 1 : 0, - isDir ? 1 : 0, - isDir ? "<collection/>" : "", - timetm->tm_year + 1900, timetm->tm_mon + 1, timetm->tm_mday, timetm->tm_hour, timetm->tm_min, timetm->tm_sec, - tms, - isDir ? "httpd/unix-directory" : mime, - (int)size, - isRoot ? 1 : 0 - ); - StringLength(*item) = (int) strlen(StringBuff(*item)); - } +static void proxytrack_add_DAV_Item(String * item, String * buff, + const char *filename, size_t size, + time_t timestamp, const char *mime, + int isDir, int isRoot, int isDefault) { + struct tm *timetm; + + if (timestamp == (time_t) 0 || timestamp == (time_t) - 1) { + timestamp = time(NULL); + } + if ((timetm = gmtime(×tamp)) != NULL) { + char tms[256 + 1]; + const char *name; + + strftime(tms, 256, "%a, %d %b %Y %H:%M:%S GMT", timetm); /* Sun, 18 Sep 2005 11:45:45 GMT */ + + if (mime == NULL || *mime == 0) + mime = "application/octet-stream"; + + StringLength(*buff) = 0; + escapexml(filename, buff); + + name = strrchr(StringBuff(*buff), '/'); + if (name != NULL) + name++; + if (name == NULL || *name == 0) { + if (strcmp(mime, "text/html") == 0) + name = "Default Document for the Folder.html"; + else + name = "Default Document for the Folder"; + } + + StringRoom(*item, 1024); + sprintf(StringBuffRW(*item), + "<response xmlns=\"DAV:\">\r\n" "<href>/webdav%s%s</href>\r\n" + "<propstat>\r\n" "<prop>\r\n" "<displayname>%s</displayname>\r\n" + "<iscollection>%d</iscollection>\r\n" + "<haschildren>%d</haschildren>\r\n" "<isfolder>%d</isfolder>\r\n" + "<resourcetype>%s</resourcetype>\r\n" + "<creationdate>%d-%02d-%02dT%02d:%02d:%02dZ</creationdate>\r\n" + "<getlastmodified>%s</getlastmodified>\r\n" + "<supportedlock></supportedlock>\r\n" "<lockdiscovery/>\r\n" + "<getcontenttype>%s</getcontenttype>\r\n" + "<getcontentlength>%d</getcontentlength>\r\n" + "<isroot>%d</isroot>\r\n" "</prop>\r\n" + "<status>HTTP/1.1 200 OK</status>\r\n" "</propstat>\r\n" + "</response>\r\n", + /* */ + (StringBuff(*buff)[0] == '/') ? "" : "/", StringBuff(*buff), name, + isDir ? 1 : 0, isDir ? 1 : 0, isDir ? 1 : 0, + isDir ? "<collection/>" : "", timetm->tm_year + 1900, + timetm->tm_mon + 1, timetm->tm_mday, timetm->tm_hour, + timetm->tm_min, timetm->tm_sec, tms, + isDir ? "httpd/unix-directory" : mime, (int) size, isRoot ? 1 : 0); + StringLength(*item) = (int) strlen(StringBuff(*item)); + } } /* Convert a RFC822 time to time_t */ -static time_t get_time_rfc822(const char* s) { +static time_t get_time_rfc822(const char *s) { struct tm result; + /* */ - char months[]="jan feb mar apr may jun jul aug sep oct nov dec"; + char months[] = "jan feb mar apr may jun jul aug sep oct nov dec"; char str[256]; - char* a; - int i; + char *a; + int i; + /* */ - int result_mm=-1; - int result_dd=-1; - int result_n1=-1; - int result_n2=-1; - int result_n3=-1; - int result_n4=-1; + int result_mm = -1; + int result_dd = -1; + int result_n1 = -1; + int result_n2 = -1; + int result_n3 = -1; + int result_n4 = -1; + /* */ if ((int) strlen(s) > 200) - return (time_t)0; - for(i = 0 ; s[i] != 0 ; i++) { - if (s[i] >= 'A' && s[i] <= 'Z') - str[i] = s[i] + ('a' - 'A'); - else - str[i] = s[i]; - } - str[i] = 0; + return (time_t) 0; + for(i = 0; s[i] != 0; i++) { + if (s[i] >= 'A' && s[i] <= 'Z') + str[i] = s[i] + ('a' - 'A'); + else + str[i] = s[i]; + } + str[i] = 0; /* éliminer :,- */ - while( (a=strchr(str,'-')) ) *a=' '; - while( (a=strchr(str,':')) ) *a=' '; - while( (a=strchr(str,',')) ) *a=' '; + while((a = strchr(str, '-'))) + *a = ' '; + while((a = strchr(str, ':'))) + *a = ' '; + while((a = strchr(str, ','))) + *a = ' '; /* tokeniser */ - a=str; + a = str; while(*a) { - char *first,*last; + char *first, *last; char tok[256]; + /* découper mot */ - while(*a==' ') a++; /* sauter espaces */ - first=a; - while((*a) && (*a!=' ')) a++; - last=a; - tok[0]='\0'; - if (first!=last) { - char* pos; - strncat(tok,first,(int) (last - first)); + while(*a == ' ') + a++; /* sauter espaces */ + first = a; + while((*a) && (*a != ' ')) + a++; + last = a; + tok[0] = '\0'; + if (first != last) { + char *pos; + + strncat(tok, first, (int) (last - first)); /* analyser */ - if ( (pos=strstr(months,tok)) ) { /* month always in letters */ - result_mm=((int) (pos - months))/4; + if ((pos = strstr(months, tok))) { /* month always in letters */ + result_mm = ((int) (pos - months)) / 4; } else { int number; - if (sscanf(tok,"%d",&number) == 1) { /* number token */ - if (result_dd<0) /* day always first number */ - result_dd=number; - else if (result_n1<0) - result_n1=number; - else if (result_n2<0) - result_n2=number; - else if (result_n3<0) - result_n3=number; - else if (result_n4<0) - result_n4=number; - } /* sinon, bruit de fond(+1GMT for exampel) */ + + if (sscanf(tok, "%d", &number) == 1) { /* number token */ + if (result_dd < 0) /* day always first number */ + result_dd = number; + else if (result_n1 < 0) + result_n1 = number; + else if (result_n2 < 0) + result_n2 = number; + else if (result_n3 < 0) + result_n3 = number; + else if (result_n4 < 0) + result_n4 = number; + } /* sinon, bruit de fond(+1GMT for exampel) */ } } } - if ((result_n1>=0) && (result_mm>=0) && (result_dd>=0) && (result_n2>=0) && (result_n3>=0) && (result_n4>=0)) { - if (result_n4>=1000) { /* Sun Nov 6 08:49:37 1994 */ - result.tm_year=result_n4-1900; - result.tm_hour=result_n1; - result.tm_min=result_n2; - result.tm_sec=max(result_n3,0); - } else { /* Sun, 06 Nov 1994 08:49:37 GMT or Sunday, 06-Nov-94 08:49:37 GMT */ - result.tm_hour=result_n2; - result.tm_min=result_n3; - result.tm_sec=max(result_n4,0); - if (result_n1<=50) /* 00 means 2000 */ - result.tm_year=result_n1+100; - else if (result_n1<1000) /* 99 means 1999 */ - result.tm_year=result_n1; - else /* 2000 */ - result.tm_year=result_n1-1900; + if ((result_n1 >= 0) && (result_mm >= 0) && (result_dd >= 0) + && (result_n2 >= 0) && (result_n3 >= 0) && (result_n4 >= 0)) { + if (result_n4 >= 1000) { /* Sun Nov 6 08:49:37 1994 */ + result.tm_year = result_n4 - 1900; + result.tm_hour = result_n1; + result.tm_min = result_n2; + result.tm_sec = max(result_n3, 0); + } else { /* Sun, 06 Nov 1994 08:49:37 GMT or Sunday, 06-Nov-94 08:49:37 GMT */ + result.tm_hour = result_n2; + result.tm_min = result_n3; + result.tm_sec = max(result_n4, 0); + if (result_n1 <= 50) /* 00 means 2000 */ + result.tm_year = result_n1 + 100; + else if (result_n1 < 1000) /* 99 means 1999 */ + result.tm_year = result_n1; + else /* 2000 */ + result.tm_year = result_n1 - 1900; } - result.tm_isdst=0; /* assume GMT */ - result.tm_yday=-1; /* don't know */ - result.tm_wday=-1; /* don't know */ - result.tm_mon=result_mm; - result.tm_mday=result_dd; + result.tm_isdst = 0; /* assume GMT */ + result.tm_yday = -1; /* don't know */ + result.tm_wday = -1; /* don't know */ + result.tm_mon = result_mm; + result.tm_mday = result_dd; return mktime(&result); } return (time_t) 0; } -static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes, const char * urlFull, int depth) { - const char * file = jump_protocol_and_auth(urlFull); - if ( (file = strchr(file, '/')) == NULL) - return NULL; - - if (strncmp(file, "/webdav", 7) != 0) { - PT_Element elt = PT_ElementNew(); - elt->statuscode = 405; - strcpy(elt->msg, "Method Not Allowed"); - return elt; - } - - /* Skip /webdav */ - file += 7; - - /* */ - { - PT_Element elt = PT_ElementNew(); - int i, isDir; - String url = STRING_EMPTY; - String response = STRING_EMPTY; - String item = STRING_EMPTY; - String itemUrl = STRING_EMPTY; - String buff = STRING_EMPTY; - StringClear(response); - StringClear(item); - StringClear(itemUrl); - StringClear(buff); - - /* Canonize URL */ - StringCopy(url, file + ((file[0] == '/') ? 1 : 0)); - if (StringLength(url) > 0) { - if (StringBuff(url)[StringLength(url) - 1] == '/') { - StringBuffRW(url)[StringLength(url) - 1] = '\0'; - StringLength(url)--; - } - } - - /* Form response */ - StringRoom(response, 1024); - sprintf(StringBuffRW(response), - "<?xml version=\"1.0\" encoding=\"utf-8\"?>\r\n" - "<multistatus xmlns=\"DAV:\">\r\n"); - StringLength(response) = (int) strlen(StringBuff(response)); - /* */ - - /* Root */ - StringLength(item) = 0; - proxytrack_add_DAV_Item(&item, &buff, - StringBuff(url), /*size*/0, /*timestamp*/(time_t) 0, /*mime*/NULL, /*isDir*/1, /*isRoot*/1, /*isDefault*/0); - StringMemcat(response, StringBuff(item), StringLength(item)); - - /* Childrens (Depth > 0) */ - if (depth > 0) { - time_t timestampRep = (time_t) -1; - const char * prefix = StringBuff(url); - unsigned int prefixLen = (unsigned int) strlen(prefix); - char ** list = PT_Enumerate(indexes, prefix, 0); - if (list != NULL) { - for(isDir = 1 ; isDir >= 0 ; isDir--) { - for(i = 0 ; list[i] != NULL ; i++) { - const char * thisUrl = list[i]; - const char * mimeType = "application/octet-stream"; - unsigned int thisUrlLen = (unsigned int) strlen(thisUrl); - int thisIsDir = (thisUrl[thisUrlLen - 1] == '/') ? 1 : 0; - - /* Item URL */ - StringRoom(itemUrl, thisUrlLen + prefixLen + sizeof("/webdav/") + 1); - StringClear(itemUrl); - sprintf(StringBuffRW(itemUrl), "/%s/%s", prefix, thisUrl); - if (!thisIsDir) - StringLength(itemUrl) = (int) strlen(StringBuff(itemUrl)); - else - StringLength(itemUrl) = (int) strlen(StringBuff(itemUrl)) - 1; - StringBuffRW(itemUrl)[StringLength(itemUrl)] = '\0'; - - if (thisIsDir == isDir) { - size_t size = 0; - time_t timestamp = (time_t) 0; - PT_Element file = NULL; - - /* Item stats */ - if (!isDir) { - file = PT_ReadIndex(indexes, StringBuff(itemUrl) + 1, FETCH_HEADERS); - if (file != NULL && file->statuscode == HTTP_OK ) { - size = file->size; - if (file->lastmodified) { - timestamp = get_time_rfc822(file->lastmodified); - } - if (timestamp == (time_t) 0) { - if (timestampRep == (time_t) -1) { - timestampRep = 0; - if (file->indexId != -1) { - timestampRep = PT_Index_Timestamp(PT_GetIndex(indexes, file->indexId)); - } - } - timestamp = timestampRep; - } - if (file->contenttype) { - mimeType = file->contenttype; - } - } - } - - /* Add item */ - StringLength(item) = 0; - proxytrack_add_DAV_Item(&item, &buff, - StringBuff(itemUrl), size, timestamp, mimeType, isDir, /*isRoot*/0, /*isDefault*/(thisUrlLen == 0)); - StringMemcat(response, StringBuff(item), StringLength(item)); - - /* Wipe element */ - if (file != NULL) - PT_Element_Delete(&file); - } - } - } - PT_Enumerate_Delete(&list); - } /* items != NULL */ - } /* Depth > 0 */ - - /* End of responses */ - StringCat(response, - "</multistatus>\r\n" - ); - - StringFree(item); - StringFree(itemUrl); - StringFree(url); - StringFree(buff); - - elt->size = StringLength(response); - elt->adr = StringAcquire(&response); - elt->statuscode = 207; /* Multi-Status */ - strcpy(elt->charset, "utf-8"); - strcpy(elt->contenttype, "text/xml"); - strcpy(elt->msg, "Multi-Status"); - StringFree(response); - - fprintf(stderr, "RESPONSE:\n%s\n", elt->adr); - - return elt; - } - return NULL; +static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes, + const char *urlFull, + int depth) { + const char *file = jump_protocol_and_auth(urlFull); + + if ((file = strchr(file, '/')) == NULL) + return NULL; + + if (strncmp(file, "/webdav", 7) != 0) { + PT_Element elt = PT_ElementNew(); + + elt->statuscode = 405; + strcpy(elt->msg, "Method Not Allowed"); + return elt; + } + + /* Skip /webdav */ + file += 7; + + /* */ + { + PT_Element elt = PT_ElementNew(); + int i, isDir; + String url = STRING_EMPTY; + String response = STRING_EMPTY; + String item = STRING_EMPTY; + String itemUrl = STRING_EMPTY; + String buff = STRING_EMPTY; + + StringClear(response); + StringClear(item); + StringClear(itemUrl); + StringClear(buff); + + /* Canonize URL */ + StringCopy(url, file + ((file[0] == '/') ? 1 : 0)); + if (StringLength(url) > 0) { + if (StringBuff(url)[StringLength(url) - 1] == '/') { + StringBuffRW(url)[StringLength(url) - 1] = '\0'; + StringLength(url)--; + } + } + + /* Form response */ + StringRoom(response, 1024); + sprintf(StringBuffRW(response), + "<?xml version=\"1.0\" encoding=\"utf-8\"?>\r\n" + "<multistatus xmlns=\"DAV:\">\r\n"); + StringLength(response) = (int) strlen(StringBuff(response)); + /* */ + + /* Root */ + StringLength(item) = 0; + proxytrack_add_DAV_Item(&item, &buff, StringBuff(url), /*size */ 0, + /*timestamp */ (time_t) 0, /*mime */ NULL, /*isDir */ 1, /*isRoot */ 1, +/*isDefault */ 0); + StringMemcat(response, StringBuff(item), StringLength(item)); + + /* Childrens (Depth > 0) */ + if (depth > 0) { + time_t timestampRep = (time_t) - 1; + const char *prefix = StringBuff(url); + unsigned int prefixLen = (unsigned int) strlen(prefix); + char **list = PT_Enumerate(indexes, prefix, 0); + + if (list != NULL) { + for(isDir = 1; isDir >= 0; isDir--) { + for(i = 0; list[i] != NULL; i++) { + const char *thisUrl = list[i]; + const char *mimeType = "application/octet-stream"; + unsigned int thisUrlLen = (unsigned int) strlen(thisUrl); + int thisIsDir = (thisUrl[thisUrlLen - 1] == '/') ? 1 : 0; + + /* Item URL */ + StringRoom(itemUrl, + thisUrlLen + prefixLen + sizeof("/webdav/") + 1); + StringClear(itemUrl); + sprintf(StringBuffRW(itemUrl), "/%s/%s", prefix, thisUrl); + if (!thisIsDir) + StringLength(itemUrl) = (int) strlen(StringBuff(itemUrl)); + else + StringLength(itemUrl) = (int) strlen(StringBuff(itemUrl)) - 1; + StringBuffRW(itemUrl)[StringLength(itemUrl)] = '\0'; + + if (thisIsDir == isDir) { + size_t size = 0; + time_t timestamp = (time_t) 0; + PT_Element file = NULL; + + /* Item stats */ + if (!isDir) { + file = + PT_ReadIndex(indexes, StringBuff(itemUrl) + 1, FETCH_HEADERS); + if (file != NULL && file->statuscode == HTTP_OK) { + size = file->size; + if (file->lastmodified) { + timestamp = get_time_rfc822(file->lastmodified); + } + if (timestamp == (time_t) 0) { + if (timestampRep == (time_t) - 1) { + timestampRep = 0; + if (file->indexId != -1) { + timestampRep = + PT_Index_Timestamp(PT_GetIndex + (indexes, file->indexId)); + } + } + timestamp = timestampRep; + } + if (file->contenttype) { + mimeType = file->contenttype; + } + } + } + + /* Add item */ + StringLength(item) = 0; + proxytrack_add_DAV_Item(&item, &buff, StringBuff(itemUrl), size, + timestamp, mimeType, isDir, /*isRoot */ 0, + /*isDefault */ (thisUrlLen == 0)); + StringMemcat(response, StringBuff(item), StringLength(item)); + + /* Wipe element */ + if (file != NULL) + PT_Element_Delete(&file); + } + } + } + PT_Enumerate_Delete(&list); + } /* items != NULL */ + } + + /* Depth > 0 */ + /* End of responses */ + StringCat(response, "</multistatus>\r\n"); + + StringFree(item); + StringFree(itemUrl); + StringFree(url); + StringFree(buff); + + elt->size = StringLength(response); + elt->adr = StringAcquire(&response); + elt->statuscode = 207; /* Multi-Status */ + strcpy(elt->charset, "utf-8"); + strcpy(elt->contenttype, "text/xml"); + strcpy(elt->msg, "Multi-Status"); + StringFree(response); + + fprintf(stderr, "RESPONSE:\n%s\n", elt->adr); + + return elt; + } + return NULL; } #endif -static PT_Element proxytrack_process_HTTP_List(PT_Indexes indexes, const char * url) { - char ** list = PT_Enumerate(indexes, url, 0); - if (list != NULL) { - PT_Element elt = PT_ElementNew(); - int i, isDir; - String html = STRING_EMPTY; - StringClear(html); - StringCat(html, - "<html>" - PROXYTRACK_COMMENT_HEADER - DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES - "<head>\r\n" - "<title>ProxyTrack " PROXYTRACK_VERSION " Catalog</title>" - "</head>\r\n" - "<body>\r\n" - "<h3>Directory index:</h3><br />" - "<br />" - "<hr>" - "<tt>[DIR] <a href=\"..\">..</a></tt><br />" - ); - for(isDir = 1 ; isDir >= 0 ; isDir--) { - for(i = 0 ; list[i] != NULL ; i++) { - char * thisUrl = list[i]; - unsigned int thisUrlLen = (unsigned int) strlen(thisUrl); - int thisIsDir = (thisUrl[thisUrlLen - 1] == '/') ? 1 : 0; - if (thisIsDir == isDir) { - if (isDir) - StringCat(html, "<tt>[DIR] "); - else - StringCat(html, "<tt> "); - StringCat(html, "<a href=\""); - if (isDir) { - StringCat(html, "http://proxytrack/"); - } - StringCat(html, url); - StringCat(html, list[i]); - StringCat(html, "\">"); - StringCat(html, list[i]); - StringCat(html, "</a></tt><br />"); - } - } - } - StringCat(html, - "</body>" - "</html>"); - PT_Enumerate_Delete(&list); - elt->size = StringLength(html); - elt->adr = StringAcquire(&html); - elt->statuscode = HTTP_OK; - strcpy(elt->charset, "iso-8859-1"); - strcpy(elt->contenttype, "text/html"); - strcpy(elt->msg, "OK"); - StringFree(html); - return elt; - } - return NULL; +static PT_Element proxytrack_process_HTTP_List(PT_Indexes indexes, + const char *url) { + char **list = PT_Enumerate(indexes, url, 0); + + if (list != NULL) { + PT_Element elt = PT_ElementNew(); + int i, isDir; + String html = STRING_EMPTY; + + StringClear(html); + StringCat(html, + "<html>" PROXYTRACK_COMMENT_HEADER + DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES "<head>\r\n" + "<title>ProxyTrack " PROXYTRACK_VERSION " Catalog</title>" + "</head>\r\n" "<body>\r\n" "<h3>Directory index:</h3><br />" + "<br />" "<hr>" "<tt>[DIR] <a href=\"..\">..</a></tt><br />"); + for(isDir = 1; isDir >= 0; isDir--) { + for(i = 0; list[i] != NULL; i++) { + char *thisUrl = list[i]; + unsigned int thisUrlLen = (unsigned int) strlen(thisUrl); + int thisIsDir = (thisUrl[thisUrlLen - 1] == '/') ? 1 : 0; + + if (thisIsDir == isDir) { + if (isDir) + StringCat(html, "<tt>[DIR] "); + else + StringCat(html, "<tt> "); + StringCat(html, "<a href=\""); + if (isDir) { + StringCat(html, "http://proxytrack/"); + } + StringCat(html, url); + StringCat(html, list[i]); + StringCat(html, "\">"); + StringCat(html, list[i]); + StringCat(html, "</a></tt><br />"); + } + } + } + StringCat(html, "</body>" "</html>"); + PT_Enumerate_Delete(&list); + elt->size = StringLength(html); + elt->adr = StringAcquire(&html); + elt->statuscode = HTTP_OK; + strcpy(elt->charset, "iso-8859-1"); + strcpy(elt->contenttype, "text/html"); + strcpy(elt->msg, "OK"); + StringFree(html); + return elt; + } + return NULL; } static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) { - int timeout=30; - int buffer_size = 32768; - char * buffer = (char*)malloc(buffer_size); - int line1Size = 1024; - char * line1 = (char*)malloc(line1Size); - int lineSize = 8192; - char * line = (char*)malloc(lineSize); - int length = 0; - int keepAlive = 1; - - String url = STRING_EMPTY; - String urlRedirect = STRING_EMPTY; - String headers = STRING_EMPTY; - String output = STRING_EMPTY; - String host = STRING_EMPTY; - String localhost = STRING_EMPTY; + int timeout = 30; + int buffer_size = 32768; + char *buffer = (char *) malloc(buffer_size); + int line1Size = 1024; + char *line1 = (char *) malloc(line1Size); + int lineSize = 8192; + char *line = (char *) malloc(lineSize); + int length = 0; + int keepAlive = 1; + + String url = STRING_EMPTY; + String urlRedirect = STRING_EMPTY; + String headers = STRING_EMPTY; + String output = STRING_EMPTY; + String host = STRING_EMPTY; + String localhost = STRING_EMPTY; + #ifndef NO_WEBDAV - String davHeaders = STRING_EMPTY; - String davRequest = STRING_EMPTY; + String davHeaders = STRING_EMPTY; + String davRequest = STRING_EMPTY; #endif - StringRoom(localhost, 256); - if (gethostname(StringBuffRW(localhost), (int) StringCapacity(localhost) - 1) == 0) { - StringLength(localhost) = (int) strlen(StringBuff(localhost)); - } else { - StringCopy(localhost, "localhost"); - } + StringRoom(localhost, 256); + if (gethostname(StringBuffRW(localhost), (int) StringCapacity(localhost) - 1) + == 0) { + StringLength(localhost) = (int) strlen(StringBuff(localhost)); + } else { + StringCopy(localhost, "localhost"); + } #ifdef _DEBUG - Sleep(1000); + Sleep(1000); #endif - if (buffer == NULL || line == NULL || line1 == NULL) { - proxytrack_print_log(CRITICAL, "proxytrack_process_HTTP:memory exhausted"); + if (buffer == NULL || line == NULL || line1 == NULL) { + proxytrack_print_log(CRITICAL, "proxytrack_process_HTTP:memory exhausted"); #ifdef _WIN32 - closesocket(soc_c); + closesocket(soc_c); #else - close(soc_c); + close(soc_c); #endif - return ; - } - - do { - const char* msgError = NULL; - int msgCode = 0; - PT_Element element = NULL; - char* command; - char* proto; - char* surl; - int directHit = 0; - int headRequest = 0; - int listRequest = 0; + return; + } + + do { + const char *msgError = NULL; + int msgCode = 0; + PT_Element element = NULL; + char *command; + char *proto; + char *surl; + int directHit = 0; + int headRequest = 0; + int listRequest = 0; + #ifndef NO_WEBDAV - int davDepth = 0; + int davDepth = 0; #endif - /* Clear context */ - line[0] = line1[0] = '\0'; - buffer[0] = '\0'; - command = line1; - StringClear(url); - StringClear(urlRedirect); - StringClear(headers); - StringClear(output); - StringClear(host); + /* Clear context */ + line[0] = line1[0] = '\0'; + buffer[0] = '\0'; + command = line1; + StringClear(url); + StringClear(urlRedirect); + StringClear(headers); + StringClear(output); + StringClear(host); #ifndef NO_WEBDAV - StringClear(davHeaders); - StringClear(davRequest); + StringClear(davHeaders); + StringClear(davRequest); #endif - /* line1: "GET http://www.example.com/ HTTP/1.0" */ - if (linputsoc_t(soc_c, line1, line1Size - 2, timeout) > 0 - && ( surl = strchr(line1, ' ') ) - && !(*surl = '\0') - && ++surl - && (proto = strchr(surl, ' ')) && !(*proto = '\0') && ++proto) - { - /* Flush headers */ - while(linputsoc_t(soc_c, line, lineSize - 2, timeout) > 0 - && line[0] != 0) - { - int p; - if ((p = strfield(line, "Content-length:"))!=0) { - if (sscanf(line+p, "%d", &length) != 1) { - msgCode = 500; - msgError = "Bad HTTP Content-Length Field"; - keepAlive = 0; - length = 0; - } - } else if (strcasecmp(line, "Connection: close") == 0) { - keepAlive = 0; - } else if (strcasecmp(line, "Connection: keep-alive") == 0) { - keepAlive = 1; - } else if ((p = strfield(line, "Host:"))) { - char* chost = line + p; - if (*chost == ' ') - chost++; - StringCopy(host, chost); - } + /* line1: "GET http://www.example.com/ HTTP/1.0" */ + if (linputsoc_t(soc_c, line1, line1Size - 2, timeout) > 0 + && (surl = strchr(line1, ' ')) + && !(*surl = '\0') + && ++surl && (proto = strchr(surl, ' ')) && !(*proto = '\0') && ++proto) { + /* Flush headers */ + while(linputsoc_t(soc_c, line, lineSize - 2, timeout) > 0 && line[0] != 0) { + int p; + + if ((p = strfield(line, "Content-length:")) != 0) { + if (sscanf(line + p, "%d", &length) != 1) { + msgCode = 500; + msgError = "Bad HTTP Content-Length Field"; + keepAlive = 0; + length = 0; + } + } else if (strcasecmp(line, "Connection: close") == 0) { + keepAlive = 0; + } else if (strcasecmp(line, "Connection: keep-alive") == 0) { + keepAlive = 1; + } else if ((p = strfield(line, "Host:"))) { + char *chost = line + p; + + if (*chost == ' ') + chost++; + StringCopy(host, chost); + } #ifndef NO_WEBDAV - else if ((p = strfield(line, "Depth: "))) { - char* depth = line + p; - if (sscanf(depth, "%d", &davDepth) != 1) { - davDepth = 0; - } - } + else if ((p = strfield(line, "Depth: "))) { + char *depth = line + p; + + if (sscanf(depth, "%d", &davDepth) != 1) { + davDepth = 0; + } + } #endif - } + } - /* Flush body */ + /* Flush body */ #ifndef NO_WEBDAV - if (length > 0) { - if (length < 32768) { - StringRoom(davRequest, length + 1); - if (recv(soc_c, StringBuffRW(davRequest), length, 0) == length) { - StringBuffRW(davRequest)[length] = 0; - } else { - msgCode = 500; - msgError = "Posted Data Read Error"; - keepAlive = 0; - } - } else { - msgCode = 500; - msgError = "Posted Data Too Large"; - keepAlive = 0; - } - } + if (length > 0) { + if (length < 32768) { + StringRoom(davRequest, length + 1); + if (recv(soc_c, StringBuffRW(davRequest), length, 0) == length) { + StringBuffRW(davRequest)[length] = 0; + } else { + msgCode = 500; + msgError = "Posted Data Read Error"; + keepAlive = 0; + } + } else { + msgCode = 500; + msgError = "Posted Data Too Large"; + keepAlive = 0; + } + } #endif - /* Switch protocol ID */ - if (strcasecmp(command, "post") == 0) { + /* Switch protocol ID */ + if (strcasecmp(command, "post") == 0) { #ifndef NO_WEBDAV - msgCode = 404; + msgCode = 404; #else - msgCode = 501; - keepAlive = 0; + msgCode = 501; + keepAlive = 0; #endif - msgError = "Proxy Error (POST Request Forbidden)"; - } - else if (strcasecmp(command, "get") == 0) { - headRequest = 0; - } - else if (strcasecmp(command, "head") == 0) { - headRequest = 1; - } + msgError = "Proxy Error (POST Request Forbidden)"; + } else if (strcasecmp(command, "get") == 0) { + headRequest = 0; + } else if (strcasecmp(command, "head") == 0) { + headRequest = 1; + } #ifndef NO_WEBDAV - else if (strcasecmp(command, "options") == 0) { - const char * options = "GET, HEAD, OPTIONS, POST, PROPFIND, TRACE" - ", MKCOL, DELETE, PUT"; /* Not supported */ - msgCode = HTTP_OK; - StringRoom(headers, 8192); - sprintf(StringBuffRW(headers), - "HTTP/1.1 %d %s\r\n" - "DAV: 1, 2\r\n" - "MS-Author-Via: DAV\r\n" - "Cache-Control: private\r\n" - "Allow: %s\r\n", - msgCode, GetHttpMessage(msgCode), options); - StringLength(headers) = (int) strlen(StringBuff(headers)); - } - else if (strcasecmp(command, "propfind") == 0) { - if (davDepth > 1) { - msgCode = 403; - msgError = "DAV Depth Limit Forbidden"; - } else { - fprintf(stderr, "DEBUG: DAV-DATA=<%s>\n", StringBuff(davRequest)); - listRequest = 2; /* propfind */ - } - } - else if (strcasecmp(command, "mkcol") == 0 - || strcasecmp(command, "delete") == 0 - || strcasecmp(command, "put") == 0 - || strcasecmp(command, "proppatch") == 0 - || strcasecmp(command, "lock") == 0 - || strcasecmp(command, "unlock") == 0 - || strcasecmp(command, "copy") == 0 - || strcasecmp(command, "trace") == 0) - { - msgCode = 403; - msgError = "Method Forbidden"; - } + else if (strcasecmp(command, "options") == 0) { + const char *options = "GET, HEAD, OPTIONS, POST, PROPFIND, TRACE" ", MKCOL, DELETE, PUT"; /* Not supported */ + + msgCode = HTTP_OK; + StringRoom(headers, 8192); + sprintf(StringBuffRW(headers), + "HTTP/1.1 %d %s\r\n" "DAV: 1, 2\r\n" "MS-Author-Via: DAV\r\n" + "Cache-Control: private\r\n" "Allow: %s\r\n", msgCode, + GetHttpMessage(msgCode), options); + StringLength(headers) = (int) strlen(StringBuff(headers)); + } else if (strcasecmp(command, "propfind") == 0) { + if (davDepth > 1) { + msgCode = 403; + msgError = "DAV Depth Limit Forbidden"; + } else { + fprintf(stderr, "DEBUG: DAV-DATA=<%s>\n", StringBuff(davRequest)); + listRequest = 2; /* propfind */ + } + } else if (strcasecmp(command, "mkcol") == 0 + || strcasecmp(command, "delete") == 0 + || strcasecmp(command, "put") == 0 + || strcasecmp(command, "proppatch") == 0 + || strcasecmp(command, "lock") == 0 + || strcasecmp(command, "unlock") == 0 + || strcasecmp(command, "copy") == 0 + || strcasecmp(command, "trace") == 0) { + msgCode = 403; + msgError = "Method Forbidden"; + } #endif - else { - msgCode = 501; - msgError = "Proxy Error (Unsupported or Unknown HTTP Command Request)"; - keepAlive = 0; - } - if (strcasecmp(proto, "http/1.1") == 0) { - keepAlive = 1; - } else if (strcasecmp(proto, "http/1.0") == 0) { - keepAlive = 0; - } else { - msgCode = 505; - msgError = "Proxy Error (Unknown HTTP Version)"; - keepAlive = 0; - } - - /* Post-process request */ - if (link_has_authority(surl)) { - if (strncasecmp(surl, "http://proxytrack/", sizeof("http://proxytrack/") - 1) == 0) { - directHit = 1; /* Another direct hit hack */ - } - StringCopy(url, surl); - } else { - if (StringLength(host) > 0) { - /* Direct hit */ - if ( + else { + msgCode = 501; + msgError = "Proxy Error (Unsupported or Unknown HTTP Command Request)"; + keepAlive = 0; + } + if (strcasecmp(proto, "http/1.1") == 0) { + keepAlive = 1; + } else if (strcasecmp(proto, "http/1.0") == 0) { + keepAlive = 0; + } else { + msgCode = 505; + msgError = "Proxy Error (Unknown HTTP Version)"; + keepAlive = 0; + } + + /* Post-process request */ + if (link_has_authority(surl)) { + if (strncasecmp + (surl, "http://proxytrack/", + sizeof("http://proxytrack/") - 1) == 0) { + directHit = 1; /* Another direct hit hack */ + } + StringCopy(url, surl); + } else { + if (StringLength(host) > 0) { + /* Direct hit */ + if ( #ifndef NO_WEBDAV - listRequest != 2 - && + listRequest != 2 && #endif - strncasecmp(StringBuff(host), StringBuff(localhost), StringLength(localhost)) == 0 - && - (StringBuff(host)[StringLength(localhost)] == '\0' - || StringBuff(host)[StringLength(localhost)] == ':') - && surl[0] == '/' - ) - { - const char * toHit = surl + 1; - if (strncmp(toHit, "webdav/", 7) == 0) { - toHit += 7; - } - /* Direct hit */ - directHit = 1; - StringCopy(url, ""); - if (!link_has_authority(toHit)) - StringCat(url, "http://"); - StringCat(url, toHit); - } else if (strncasecmp(surl, "/proxytrack/", sizeof("/proxytrack/") - 1) == 0) { - const char * toHit = surl + sizeof("/proxytrack/") - 1; - /* Direct hit */ - directHit = 1; - StringCopy(url, ""); - if (!link_has_authority(toHit)) - StringCat(url, "http://"); - StringCat(url, toHit); - } else { - /* Transparent proxy */ - StringCopy(url, "http://"); - StringCat(url, StringBuff(host)); - StringCat(url, surl); - } - } else { - msgCode = 500; - msgError = "Transparent Proxy Error ('Host' HTTP Request Header Field Missing)"; - keepAlive = 0; - } - } - - /* Response */ - if (msgCode == 0) { - if (listRequest == 1) { - element = proxytrack_process_HTTP_List(indexes, StringBuff(url)); - } + strncasecmp(StringBuff(host), StringBuff(localhost), + StringLength(localhost)) == 0 + && (StringBuff(host)[StringLength(localhost)] == '\0' + || StringBuff(host)[StringLength(localhost)] == ':') + && surl[0] == '/') { + const char *toHit = surl + 1; + + if (strncmp(toHit, "webdav/", 7) == 0) { + toHit += 7; + } + /* Direct hit */ + directHit = 1; + StringCopy(url, ""); + if (!link_has_authority(toHit)) + StringCat(url, "http://"); + StringCat(url, toHit); + } else + if (strncasecmp(surl, "/proxytrack/", sizeof("/proxytrack/") - 1) == + 0) { + const char *toHit = surl + sizeof("/proxytrack/") - 1; + + /* Direct hit */ + directHit = 1; + StringCopy(url, ""); + if (!link_has_authority(toHit)) + StringCat(url, "http://"); + StringCat(url, toHit); + } else { + /* Transparent proxy */ + StringCopy(url, "http://"); + StringCat(url, StringBuff(host)); + StringCat(url, surl); + } + } else { + msgCode = 500; + msgError = + "Transparent Proxy Error ('Host' HTTP Request Header Field Missing)"; + keepAlive = 0; + } + } + + /* Response */ + if (msgCode == 0) { + if (listRequest == 1) { + element = proxytrack_process_HTTP_List(indexes, StringBuff(url)); + } #ifndef NO_WEBDAV - else if (listRequest == 2) { - if ((element = proxytrack_process_DAV_Request(indexes, StringBuff(url), davDepth)) != NULL) { - msgCode = element->statuscode; - StringRoom(davHeaders, 1024); - sprintf(StringBuffRW(davHeaders), - "DAV: 1, 2\r\n" - "MS-Author-Via: DAV\r\n" - "Cache-Control: private\r\n"); - StringLength(davHeaders) = (int) strlen(StringBuff(davHeaders)); - } - } + else if (listRequest == 2) { + if ((element = + proxytrack_process_DAV_Request(indexes, StringBuff(url), + davDepth)) != NULL) { + msgCode = element->statuscode; + StringRoom(davHeaders, 1024); + sprintf(StringBuffRW(davHeaders), + "DAV: 1, 2\r\n" "MS-Author-Via: DAV\r\n" + "Cache-Control: private\r\n"); + StringLength(davHeaders) = (int) strlen(StringBuff(davHeaders)); + } + } #endif - else { - element = PT_ReadIndex(indexes, StringBuff(url), FETCH_BODY); - } - if (element == NULL + else { + element = PT_ReadIndex(indexes, StringBuff(url), FETCH_BODY); + } + if (element == NULL #ifndef NO_WEBDAV - && listRequest == 2 + && listRequest == 2 #endif - && StringLength(url) > 0 - && StringBuff(url)[StringLength(url) - 1] == '/' - ) - { - element = PT_Index_HTML_BuildRootInfo(indexes); - if (element != NULL) { - element->statuscode = 404; /* HTML page, but in error */ - } - } - if (element != NULL) { - msgCode = element->statuscode; - StringRoom(headers, 8192); - sprintf(StringBuffRW(headers), - "HTTP/1.1 %d %s\r\n" + && StringLength(url) > 0 + && StringBuff(url)[StringLength(url) - 1] == '/') { + element = PT_Index_HTML_BuildRootInfo(indexes); + if (element != NULL) { + element->statuscode = 404; /* HTML page, but in error */ + } + } + if (element != NULL) { + msgCode = element->statuscode; + StringRoom(headers, 8192); + sprintf(StringBuffRW(headers), "HTTP/1.1 %d %s\r\n" #ifndef NO_WEBDAV - "%s" + "%s" #endif - "Content-Type: %s%s%s%s\r\n" - "%s%s%s" - "%s%s%s" - "%s%s%s", - /* */ - msgCode, - element->msg, + "Content-Type: %s%s%s%s\r\n" "%s%s%s" "%s%s%s" "%s%s%s", + /* */ + msgCode, element->msg, #ifndef NO_WEBDAV - /* DAV */ - StringBuff(davHeaders), + /* DAV */ + StringBuff(davHeaders), #endif - /* Content-type: foo; [ charset=bar ] */ - element->contenttype, - ( ( element->charset[0]) ? "; charset=\"" : ""), - element->charset, - ( ( element->charset[0]) ? "\"" : ""), - /* location */ - ( ( element->location != NULL && element->location[0]) ? "Location: " : ""), - ( ( element->location != NULL && element->location[0]) ? element->location : ""), - ( ( element->location != NULL && element->location[0]) ? "\r\n" : ""), - /* last-modified */ - ( ( element->lastmodified[0]) ? "Last-Modified: " : ""), - ( ( element->lastmodified[0]) ? element->lastmodified : ""), - ( ( element->lastmodified[0]) ? "\r\n" : ""), - /* etag */ - ( ( element->etag[0]) ? "ETag: " : ""), - ( ( element->etag[0]) ? element->etag : ""), - ( ( element->etag[0]) ? "\r\n" : "") - ); - StringLength(headers) = (int) strlen(StringBuff(headers)); - } else { - /* No query string, no ending / : check the the <url>/ page */ - if (StringLength(url) > 0 && StringBuff(url)[StringLength(url) - 1] != '/' && strchr(StringBuff(url), '?') == NULL) { - StringCopy(urlRedirect, StringBuff(url)); - StringCat(urlRedirect, "/"); - if (PT_LookupIndex(indexes, StringBuff(urlRedirect))) { - msgCode = 301; /* Moved Permanently */ - StringRoom(headers, 8192); - sprintf(StringBuffRW(headers), - "HTTP/1.1 %d %s\r\n" - "Content-Type: text/html\r\n" - "Location: %s\r\n", - /* */ - msgCode, - GetHttpMessage(msgCode), - StringBuff(urlRedirect) - ); - StringLength(headers) = (int) strlen(StringBuff(headers)); - /* */ - StringRoom(output, 1024 + sizeof(PROXYTRACK_COMMENT_HEADER) + sizeof(DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES)); - sprintf(StringBuffRW(output), - "<html>" - PROXYTRACK_COMMENT_HEADER - DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES - "<head>" - "<title>ProxyTrack - Page has moved</title>" - "</head>\r\n" - "<body>" - "<h3>The correct location is:</h3><br />" - "<b><a href=\"%s\">%s</a></b><br />" - "<br />" - "<br />\r\n" - "<i>Generated by ProxyTrack " PROXYTRACK_VERSION ", (C) Xavier Roche and other contributors</i>" - "\r\n" - "</body>" - "</header>", - StringBuff(urlRedirect), - StringBuff(urlRedirect)); - StringLength(output) = (int) strlen(StringBuff(output)); - } - } - if (msgCode == 0) { - msgCode = 404; - msgError = "Not Found in this cache"; - } - } - } - } else { - msgCode = 500; - msgError = "Server Error"; - keepAlive = 0; - } - if (StringLength(headers) == 0) { - if (msgCode == 0) { - msgCode = 500; - msgError = "Internal Proxy Error"; - } else if (msgError == NULL) { - msgError = GetHttpMessage(msgCode); - } - StringRoom(headers, 256); - sprintf(StringBuffRW(headers), - "HTTP/1.1 %d %s\r\n" - "Content-type: text/html\r\n", - msgCode, - msgError); - StringLength(headers) = (int) strlen(StringBuff(headers)); - StringRoom(output, 1024 + sizeof(PROXYTRACK_COMMENT_HEADER) + sizeof(DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES)); - sprintf(StringBuffRW(output), - "<html>" - PROXYTRACK_COMMENT_HEADER - DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES - "<head>" - "<title>ProxyTrack - HTTP Proxy Error %d</title>" - "</head>\r\n" - "<body>" - "<h3>A proxy error has occured while processing the request.</h3><br />" - "<b>Error HTTP %d: <i>%s</i></b><br />" - "<br />" - "<br />\r\n" - "<i>Generated by ProxyTrack " PROXYTRACK_VERSION ", (C) Xavier Roche and other contributors</i>" - "\r\n" - "</body>" - "</html>", - msgCode, - msgCode, - msgError); - StringLength(output) = (int) strlen(StringBuff(output)); - } - { - char tmp[20 + 1]; /* 2^64 = 18446744073709551616 */ - size_t dataSize = 0; - if (!headRequest) { - dataSize = StringLength(output); - if (dataSize == 0 && element != NULL) { - dataSize = element->size; - } - } - sprintf(tmp, "%d", (int) dataSize); - StringCat(headers, "Content-length: "); - StringCat(headers, tmp); - StringCat(headers, "\r\n"); - } - if (keepAlive) { - StringCat(headers, - "Connection: Keep-Alive\r\n" - "Proxy-Connection: Keep-Alive\r\n"); - } else { - StringCat(headers, - "Connection: Close\r\n" - "Proxy-Connection: Close\r\n"); - } - if (msgCode != 500) - StringCat(headers, "X-Cache: HIT from "); - else - StringCat(headers, "X-Cache: MISS from "); - StringCat(headers, StringBuff(localhost)); - StringCat(headers, "\r\n"); - - /* Logging */ - { - const char * contentType = "text/html"; - size_t size = StringLength(output) ? StringLength(output) : ( element ? element->size : 0 ); - /* */ - String ip = STRING_EMPTY; - SOCaddr serverClient; - SOClen lenServerClient = (int) sizeof(serverClient); - memset(&serverClient, 0, sizeof(serverClient)); - if (getsockname(soc_c, (struct sockaddr*) &serverClient, &lenServerClient) == 0) { - ip = getip(&serverClient, lenServerClient); - } else { - StringCopy(ip, "unknown"); - } - if (element != NULL && element->contenttype[0] != '\0') { - contentType = element->contenttype; - } - proxytrack_print_log(LOG, "HTTP %s %d %d %s %s %s", StringBuff(ip), msgCode, (int)size, command, StringBuff(url), contentType); - StringFree(ip); - } - - /* Send reply */ - StringCat(headers, "Server: ProxyTrack " PROXYTRACK_VERSION " (HTTrack " HTTRACK_VERSIONID ")\r\n"); - StringCat(headers, "\r\n"); /* Headers separator */ - if (send(soc_c, StringBuff(headers), (int)StringLength(headers), 0) != StringLength(headers) - || ( !headRequest && StringLength(output) > 0 && send(soc_c, StringBuff(output), (int)StringLength(output), 0) != StringLength(output)) - || ( !headRequest && StringLength(output) == 0 && element != NULL && element->adr != NULL && send(soc_c, element->adr, (int)element->size, 0) != element->size) - ) - { - keepAlive = 0; /* Error, abort connection */ - } - PT_Element_Delete(&element); - - /* Shutdown (FIN) and wait until confirmed */ - if (!keepAlive) { - char c; + /* Content-type: foo; [ charset=bar ] */ + element->contenttype, + ((element->charset[0]) ? "; charset=\"" : ""), + element->charset, ((element->charset[0]) ? "\"" : ""), + /* location */ + ((element->location != NULL + && element->location[0]) ? "Location: " : ""), + ((element->location != NULL + && element->location[0]) ? element->location : ""), + ((element->location != NULL + && element->location[0]) ? "\r\n" : ""), + /* last-modified */ + ((element->lastmodified[0]) ? "Last-Modified: " : ""), + ((element->lastmodified[0]) ? element->lastmodified : ""), + ((element->lastmodified[0]) ? "\r\n" : ""), + /* etag */ + ((element->etag[0]) ? "ETag: " : ""), + ((element->etag[0]) ? element->etag : ""), + ((element->etag[0]) ? "\r\n" : "") + ); + StringLength(headers) = (int) strlen(StringBuff(headers)); + } else { + /* No query string, no ending / : check the the <url>/ page */ + if (StringLength(url) > 0 + && StringBuff(url)[StringLength(url) - 1] != '/' + && strchr(StringBuff(url), '?') == NULL) { + StringCopy(urlRedirect, StringBuff(url)); + StringCat(urlRedirect, "/"); + if (PT_LookupIndex(indexes, StringBuff(urlRedirect))) { + msgCode = 301; /* Moved Permanently */ + StringRoom(headers, 8192); + sprintf(StringBuffRW(headers), + "HTTP/1.1 %d %s\r\n" "Content-Type: text/html\r\n" + "Location: %s\r\n", + /* */ + msgCode, GetHttpMessage(msgCode), StringBuff(urlRedirect) + ); + StringLength(headers) = (int) strlen(StringBuff(headers)); + /* */ + StringRoom(output, + 1024 + sizeof(PROXYTRACK_COMMENT_HEADER) + + sizeof(DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES)); + sprintf(StringBuffRW(output), + "<html>" PROXYTRACK_COMMENT_HEADER + DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES "<head>" + "<title>ProxyTrack - Page has moved</title>" "</head>\r\n" + "<body>" "<h3>The correct location is:</h3><br />" + "<b><a href=\"%s\">%s</a></b><br />" "<br />" "<br />\r\n" + "<i>Generated by ProxyTrack " PROXYTRACK_VERSION + ", (C) Xavier Roche and other contributors</i>" "\r\n" + "</body>" "</header>", StringBuff(urlRedirect), + StringBuff(urlRedirect)); + StringLength(output) = (int) strlen(StringBuff(output)); + } + } + if (msgCode == 0) { + msgCode = 404; + msgError = "Not Found in this cache"; + } + } + } + } else { + msgCode = 500; + msgError = "Server Error"; + keepAlive = 0; + } + if (StringLength(headers) == 0) { + if (msgCode == 0) { + msgCode = 500; + msgError = "Internal Proxy Error"; + } else if (msgError == NULL) { + msgError = GetHttpMessage(msgCode); + } + StringRoom(headers, 256); + sprintf(StringBuffRW(headers), + "HTTP/1.1 %d %s\r\n" "Content-type: text/html\r\n", msgCode, + msgError); + StringLength(headers) = (int) strlen(StringBuff(headers)); + StringRoom(output, + 1024 + sizeof(PROXYTRACK_COMMENT_HEADER) + + sizeof(DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES)); + sprintf(StringBuffRW(output), + "<html>" PROXYTRACK_COMMENT_HEADER + DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES "<head>" + "<title>ProxyTrack - HTTP Proxy Error %d</title>" "</head>\r\n" + "<body>" + "<h3>A proxy error has occured while processing the request.</h3><br />" + "<b>Error HTTP %d: <i>%s</i></b><br />" "<br />" "<br />\r\n" + "<i>Generated by ProxyTrack " PROXYTRACK_VERSION + ", (C) Xavier Roche and other contributors</i>" "\r\n" "</body>" + "</html>", msgCode, msgCode, msgError); + StringLength(output) = (int) strlen(StringBuff(output)); + } + { + char tmp[20 + 1]; /* 2^64 = 18446744073709551616 */ + size_t dataSize = 0; + + if (!headRequest) { + dataSize = StringLength(output); + if (dataSize == 0 && element != NULL) { + dataSize = element->size; + } + } + sprintf(tmp, "%d", (int) dataSize); + StringCat(headers, "Content-length: "); + StringCat(headers, tmp); + StringCat(headers, "\r\n"); + } + if (keepAlive) { + StringCat(headers, + "Connection: Keep-Alive\r\n" + "Proxy-Connection: Keep-Alive\r\n"); + } else { + StringCat(headers, "Connection: Close\r\n" "Proxy-Connection: Close\r\n"); + } + if (msgCode != 500) + StringCat(headers, "X-Cache: HIT from "); + else + StringCat(headers, "X-Cache: MISS from "); + StringCat(headers, StringBuff(localhost)); + StringCat(headers, "\r\n"); + + /* Logging */ + { + const char *contentType = "text/html"; + size_t size = + StringLength(output) ? StringLength(output) : (element ? element-> + size : 0); + /* */ + String ip = STRING_EMPTY; + SOCaddr serverClient; + SOClen lenServerClient = (int) sizeof(serverClient); + + memset(&serverClient, 0, sizeof(serverClient)); + if (getsockname + (soc_c, (struct sockaddr *) &serverClient, &lenServerClient) == 0) { + ip = getip(&serverClient, lenServerClient); + } else { + StringCopy(ip, "unknown"); + } + if (element != NULL && element->contenttype[0] != '\0') { + contentType = element->contenttype; + } + proxytrack_print_log(LOG, "HTTP %s %d %d %s %s %s", StringBuff(ip), + msgCode, (int) size, command, StringBuff(url), + contentType); + StringFree(ip); + } + + /* Send reply */ + StringCat(headers, + "Server: ProxyTrack " PROXYTRACK_VERSION " (HTTrack " + HTTRACK_VERSIONID ")\r\n"); + StringCat(headers, "\r\n"); /* Headers separator */ + if (send(soc_c, StringBuff(headers), (int) StringLength(headers), 0) != + StringLength(headers) + || (!headRequest && StringLength(output) > 0 + && send(soc_c, StringBuff(output), (int) StringLength(output), + 0) != StringLength(output)) + || (!headRequest && StringLength(output) == 0 && element != NULL + && element->adr != NULL + && send(soc_c, element->adr, (int) element->size, + 0) != element->size) + ) { + keepAlive = 0; /* Error, abort connection */ + } + PT_Element_Delete(&element); + + /* Shutdown (FIN) and wait until confirmed */ + if (!keepAlive) { + char c; + #ifdef _WIN32 - shutdown(soc_c, SD_SEND); + shutdown(soc_c, SD_SEND); #else - shutdown(soc_c, 1); + shutdown(soc_c, 1); #endif - while(recv(soc_c, ((char*)&c), 1, 0) > 0); - } - } while(keepAlive); + while(recv(soc_c, ((char *) &c), 1, 0) > 0) ; + } + } while(keepAlive); #ifdef _WIN32 - closesocket(soc_c); + closesocket(soc_c); #else - close(soc_c); + close(soc_c); #endif - StringFree(url); - StringFree(urlRedirect); - StringFree(headers); - StringFree(output); - StringFree(host); - StringFree(localhost); + StringFree(url); + StringFree(urlRedirect); + StringFree(headers); + StringFree(output); + StringFree(host); + StringFree(localhost); #ifndef NO_WEBDAV - StringFree(davHeaders); - StringFree(davRequest); + StringFree(davHeaders); + StringFree(davRequest); #endif - if (buffer) - free(buffer); - if (line) - free(line); - if (line1) - free(line1); + if (buffer) + free(buffer); + if (line) + free(line); + if (line1) + free(line1); } /* Generic threaded function start */ -static int startThread(void (*funct)(void*), void* param) -{ - if (param != NULL) { +static int startThread(void (*funct) (void *), void *param) { + if (param != NULL) { #ifdef _WIN32 - if (_beginthread(funct, 0, param) == -1) { - free(param); - return 0; - } - return 1; + if (_beginthread(funct, 0, param) == -1) { + free(param); + return 0; + } + return 1; #else - pthread_t handle = 0; - int retcode; - retcode = pthread_create(&handle, NULL, funct, param); - if (retcode != 0) { /* error */ - free(param); - return 0; - } else { - /* detach the thread from the main process so that is can be independent */ - pthread_detach(handle); - return 1; - } + pthread_t handle = 0; + int retcode; + + retcode = pthread_create(&handle, NULL, funct, param); + if (retcode != 0) { /* error */ + free(param); + return 0; + } else { + /* detach the thread from the main process so that is can be independent */ + pthread_detach(handle); + return 1; + } #endif - } else { - return 0; - } + } else { + return 0; + } } /* Generic socket/index structure */ typedef struct proxytrack_process_th_p { - T_SOC soc_c; - PT_Indexes indexes; - void (*process)(PT_Indexes indexes, T_SOC soc_c); + T_SOC soc_c; + PT_Indexes indexes; + void (*process) (PT_Indexes indexes, T_SOC soc_c); } proxytrack_process_th_p; /* Generic socket/index function stub */ -static void proxytrack_process_th(void* param_) { - proxytrack_process_th_p *param = (proxytrack_process_th_p *) param_; - T_SOC soc_c = param->soc_c; - PT_Indexes indexes = param->indexes; - void (*process)(PT_Indexes indexes, T_SOC soc_c) = param->process; - free(param); - process(indexes, soc_c); - return ; +static void proxytrack_process_th(void *param_) { + proxytrack_process_th_p *param = (proxytrack_process_th_p *) param_; + T_SOC soc_c = param->soc_c; + PT_Indexes indexes = param->indexes; + void (*process) (PT_Indexes indexes, T_SOC soc_c) = param->process; + + free(param); + process(indexes, soc_c); + return; } /* Process generic socket/index operation */ -static int proxytrack_process_generic(void (*process)(PT_Indexes indexes, T_SOC soc_c), - PT_Indexes indexes, T_SOC soc_c) -{ - proxytrack_process_th_p *param = calloc(sizeof(proxytrack_process_th_p), 1); - if (param != NULL) { - param->soc_c = soc_c; - param->indexes = indexes; - param->process = process; - return startThread(proxytrack_process_th, param); - } else { - proxytrack_print_log(CRITICAL, "proxytrack_process_generic:Memory exhausted"); - return 0; - } - return 0; +static int +proxytrack_process_generic(void (*process) (PT_Indexes indexes, T_SOC soc_c), + PT_Indexes indexes, T_SOC soc_c) { + proxytrack_process_th_p *param = calloc(sizeof(proxytrack_process_th_p), 1); + + if (param != NULL) { + param->soc_c = soc_c; + param->indexes = indexes; + param->process = process; + return startThread(proxytrack_process_th, param); + } else { + proxytrack_print_log(CRITICAL, + "proxytrack_process_generic:Memory exhausted"); + return 0; + } + return 0; } /* Process HTTP proxy requests */ static int proxytrack_process_HTTP_threaded(PT_Indexes indexes, T_SOC soc) { - return proxytrack_process_generic(proxytrack_process_HTTP, indexes, soc); + return proxytrack_process_generic(proxytrack_process_HTTP, indexes, soc); } /* HTTP Server */ static int proxytrack_start_HTTP(PT_Indexes indexes, T_SOC soc) { - while(soc != INVALID_SOCKET) { + while(soc != INVALID_SOCKET) { T_SOC soc_c; - if ( (soc_c = (T_SOC) accept(soc, NULL, NULL)) != INVALID_SOCKET) { - if (!proxytrack_process_HTTP_threaded(indexes, soc_c)) { - proxytrack_print_log(CRITICAL, "proxytrack_start_HTTP::Can not fork a thread"); - } - } - } - if (soc != INVALID_SOCKET) { + + if ((soc_c = (T_SOC) accept(soc, NULL, NULL)) != INVALID_SOCKET) { + if (!proxytrack_process_HTTP_threaded(indexes, soc_c)) { + proxytrack_print_log(CRITICAL, + "proxytrack_start_HTTP::Can not fork a thread"); + } + } + } + if (soc != INVALID_SOCKET) { #ifdef _WIN32 - closesocket(soc); + closesocket(soc); #else - close(soc); + close(soc); #endif - } - return 1; + } + return 1; } /* Network order is big endian */ @@ -1403,191 +1506,208 @@ static int proxytrack_start_HTTP(PT_Indexes indexes, T_SOC soc) { WRITE_NET16(((unsigned char*)buffer) + 2, ( ((unsigned int)(value)) ) & 0xffff); \ } while(0) -static int ICP_reply(struct sockaddr * clientAddr, - int clientAddrLen, - T_SOC soc, - /* */ - unsigned char Opcode, - unsigned char Version, - unsigned short Message_Length, - unsigned int Request_Number, - unsigned int Options, - unsigned int Option_Data, - unsigned int Sender_Host_Address, - unsigned char *Message - ) -{ - int ret = 0; - unsigned long int BufferSize; - unsigned char * buffer; - if (Message_Length == 0 && Message != NULL) /* We have to get the message size */ - Message_Length = (unsigned int) strlen(Message) + 1; /* NULL terminated */ - BufferSize = 20 + Message_Length; - buffer = malloc(BufferSize); - if (buffer != NULL) { - WRITE_NET8(&buffer[0], Opcode); - WRITE_NET8(&buffer[1], Version); - WRITE_NET16(&buffer[2], Message_Length); - WRITE_NET32(&buffer[4], Request_Number); - WRITE_NET32(&buffer[8], Options); - WRITE_NET32(&buffer[12], Option_Data); - WRITE_NET32(&buffer[16], Sender_Host_Address); - if (Message != NULL && Message_Length > 0) { - memcpy(buffer + 20, Message, Message_Length); - } - if (sendto(soc, buffer, BufferSize, 0, clientAddr, clientAddrLen) == BufferSize) { - ret = 1; - } - free(buffer); - } - return ret; +static int ICP_reply(struct sockaddr *clientAddr, int clientAddrLen, T_SOC soc, + /* */ + unsigned char Opcode, unsigned char Version, + unsigned short Message_Length, unsigned int Request_Number, + unsigned int Options, unsigned int Option_Data, + unsigned int Sender_Host_Address, unsigned char *Message) { + int ret = 0; + unsigned long int BufferSize; + unsigned char *buffer; + + if (Message_Length == 0 && Message != NULL) /* We have to get the message size */ + Message_Length = (unsigned int) strlen(Message) + 1; /* NULL terminated */ + BufferSize = 20 + Message_Length; + buffer = malloc(BufferSize); + if (buffer != NULL) { + WRITE_NET8(&buffer[0], Opcode); + WRITE_NET8(&buffer[1], Version); + WRITE_NET16(&buffer[2], Message_Length); + WRITE_NET32(&buffer[4], Request_Number); + WRITE_NET32(&buffer[8], Options); + WRITE_NET32(&buffer[12], Option_Data); + WRITE_NET32(&buffer[16], Sender_Host_Address); + if (Message != NULL && Message_Length > 0) { + memcpy(buffer + 20, Message, Message_Length); + } + if (sendto(soc, buffer, BufferSize, 0, clientAddr, clientAddrLen) == + BufferSize) { + ret = 1; + } + free(buffer); + } + return ret; } /* ICP Server */ static int proxytrack_start_ICP(PT_Indexes indexes, T_SOC soc) { - /* "ICP messages MUST not exceed 16,384 octets in length." (RFC2186) */ - int bufferSize = 16384; - unsigned char * buffer = (unsigned char*) malloc(bufferSize + 1); - if (buffer == NULL) { - proxytrack_print_log(CRITICAL, "proxytrack_start_ICP:memory exhausted"); + /* "ICP messages MUST not exceed 16,384 octets in length." (RFC2186) */ + int bufferSize = 16384; + unsigned char *buffer = (unsigned char *) malloc(bufferSize + 1); + + if (buffer == NULL) { + proxytrack_print_log(CRITICAL, "proxytrack_start_ICP:memory exhausted"); #ifdef _WIN32 - closesocket(soc); + closesocket(soc); #else - close(soc); + close(soc); #endif - return -1; - } - while(soc != INVALID_SOCKET) { - struct sockaddr clientAddr; - SOClen clientAddrLen = sizeof(struct sockaddr); - int n; - memset(&clientAddr, 0, sizeof(clientAddr)); - n = recvfrom(soc, (char*)buffer, bufferSize, 0, &clientAddr, &clientAddrLen); - if (n != -1) { - const char * LogRequest = "ERROR"; - const char * LogReply = "ERROR"; - unsigned char * UrlRequest = NULL; - if (n >= 20) { - enum { - ICP_OP_MIN = 0, - ICP_OP_INVALID = 0, - ICP_OP_QUERY = 1, - ICP_OP_HIT = 2, - ICP_OP_MISS = 3, - ICP_OP_ERR = 4, - ICP_OP_SECHO = 10, - ICP_OP_DECHO = 11, - ICP_OP_MISS_NOFETCH = 21, - ICP_OP_DENIED = 22, - ICP_OP_HIT_OBJ = 23, - ICP_OP_MAX = ICP_OP_HIT_OBJ - }; - unsigned char Opcode = buffer[0]; - unsigned char Version = buffer[1]; - unsigned short Message_Length = READ_NET16(&buffer[2]); - unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */ - unsigned int Options = READ_NET32(&buffer[8]); - unsigned int Option_Data = READ_NET32(&buffer[12]); /* ICP_FLAG_SRC_RTT */ - unsigned int Sender_Host_Address = READ_NET32(&buffer[16]); /* ignored */ - unsigned char* Payload = &buffer[20]; - buffer[bufferSize] = '\0'; /* Ensure payload is NULL terminated */ - if (Message_Length <= bufferSize - 20) { - if (Opcode >= ICP_OP_MIN && Opcode <= ICP_OP_MAX) { - if (Version == 2) { - switch(Opcode) { - case ICP_OP_QUERY: - { - unsigned int UrlRequestSize; - UrlRequest = &Payload[4]; - UrlRequestSize = (unsigned int)strlen((char*)UrlRequest); - LogRequest = "ICP_OP_QUERY"; - if (indexes == NULL) { - ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_DENIED, Version, 0, Request_Number, 0, 0, 0, UrlRequest); - LogReply = "ICP_OP_DENIED"; - } else if (PT_LookupIndex(indexes, UrlRequest)) { - ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_HIT, Version, 0, Request_Number, 0, 0, 0, UrlRequest); - LogReply = "ICP_OP_HIT"; - } else { - if (UrlRequestSize > 0 && UrlRequest[UrlRequestSize - 1] != '/' && strchr(UrlRequest, '?') == NULL) { - char * UrlRedirect = malloc(UrlRequestSize + 1 + 1); - if (UrlRedirect != NULL) { - sprintf(UrlRedirect, "%s/", UrlRequest); - if (PT_LookupIndex(indexes, UrlRedirect)) { /* We'll generate a redirect */ - ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_HIT, Version, 0, Request_Number, 0, 0, 0, UrlRequest); - LogReply = "ICP_OP_HIT"; - free(UrlRedirect); - break; - } - free(UrlRedirect); - } - } - /* We won't retrive the cache MISS online, no way! */ - ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_MISS_NOFETCH, Version, 0, Request_Number, 0, 0, 0, UrlRequest); - LogReply = "ICP_OP_MISS_NOFETCH"; - } - } - break; - case ICP_OP_SECHO: - { - UrlRequest = &Payload[4]; - LogRequest = "ICP_OP_QUERY"; - LogReply = "ICP_OP_QUERY"; - ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_SECHO, Version, 0, Request_Number, 0, 0, 0, UrlRequest); - } - break; - default: - LogRequest = "NOTIMPLEMENTED"; - LogReply = "ICP_OP_ERR"; - ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_ERR, Version, 0, Request_Number, 0, 0, 0, NULL); - break; - } - } else { - ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_ERR, 2, 0, Request_Number, 0, 0, 0, NULL); - } - } /* Ignored (RFC2186) */ - } else { - ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_ERR, Version, 0, Request_Number, 0, 0, 0, NULL); - } - } - - /* Logging */ - { - String ip = STRING_EMPTY; - SOCaddr serverClient; - int lenServerClient = (int) sizeof(serverClient); - SOCaddr_copyaddr(serverClient, lenServerClient, &clientAddr, clientAddrLen); - if (lenServerClient > 0) { - ip = getip(&serverClient, lenServerClient); - } else { - StringCopy(ip, "unknown"); - } - proxytrack_print_log(LOG, "ICP %s %s/%s %s", StringBuff(ip), LogRequest, LogReply, (UrlRequest ? UrlRequest : "-") ); - StringFree(ip); - } - - } - } - if (soc != INVALID_SOCKET) { + return -1; + } + while(soc != INVALID_SOCKET) { + struct sockaddr clientAddr; + SOClen clientAddrLen = sizeof(struct sockaddr); + int n; + + memset(&clientAddr, 0, sizeof(clientAddr)); + n = + recvfrom(soc, (char *) buffer, bufferSize, 0, &clientAddr, + &clientAddrLen); + if (n != -1) { + const char *LogRequest = "ERROR"; + const char *LogReply = "ERROR"; + unsigned char *UrlRequest = NULL; + + if (n >= 20) { + enum { + ICP_OP_MIN = 0, + ICP_OP_INVALID = 0, + ICP_OP_QUERY = 1, + ICP_OP_HIT = 2, + ICP_OP_MISS = 3, + ICP_OP_ERR = 4, + ICP_OP_SECHO = 10, + ICP_OP_DECHO = 11, + ICP_OP_MISS_NOFETCH = 21, + ICP_OP_DENIED = 22, + ICP_OP_HIT_OBJ = 23, + ICP_OP_MAX = ICP_OP_HIT_OBJ + }; + unsigned char Opcode = buffer[0]; + unsigned char Version = buffer[1]; + unsigned short Message_Length = READ_NET16(&buffer[2]); + unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */ + unsigned int Options = READ_NET32(&buffer[8]); + unsigned int Option_Data = READ_NET32(&buffer[12]); /* ICP_FLAG_SRC_RTT */ + unsigned int Sender_Host_Address = READ_NET32(&buffer[16]); /* ignored */ + unsigned char *Payload = &buffer[20]; + + buffer[bufferSize] = '\0'; /* Ensure payload is NULL terminated */ + if (Message_Length <= bufferSize - 20) { + if (Opcode >= ICP_OP_MIN && Opcode <= ICP_OP_MAX) { + if (Version == 2) { + switch (Opcode) { + case ICP_OP_QUERY: + { + unsigned int UrlRequestSize; + + UrlRequest = &Payload[4]; + UrlRequestSize = (unsigned int) strlen((char *) UrlRequest); + LogRequest = "ICP_OP_QUERY"; + if (indexes == NULL) { + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_DENIED, + Version, 0, Request_Number, 0, 0, 0, UrlRequest); + LogReply = "ICP_OP_DENIED"; + } else if (PT_LookupIndex(indexes, UrlRequest)) { + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_HIT, + Version, 0, Request_Number, 0, 0, 0, UrlRequest); + LogReply = "ICP_OP_HIT"; + } else { + if (UrlRequestSize > 0 + && UrlRequest[UrlRequestSize - 1] != '/' + && strchr(UrlRequest, '?') == NULL) { + char *UrlRedirect = malloc(UrlRequestSize + 1 + 1); + + if (UrlRedirect != NULL) { + sprintf(UrlRedirect, "%s/", UrlRequest); + if (PT_LookupIndex(indexes, UrlRedirect)) { /* We'll generate a redirect */ + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_HIT, + Version, 0, Request_Number, 0, 0, 0, + UrlRequest); + LogReply = "ICP_OP_HIT"; + free(UrlRedirect); + break; + } + free(UrlRedirect); + } + } + /* We won't retrive the cache MISS online, no way! */ + ICP_reply(&clientAddr, clientAddrLen, soc, + ICP_OP_MISS_NOFETCH, Version, 0, Request_Number, + 0, 0, 0, UrlRequest); + LogReply = "ICP_OP_MISS_NOFETCH"; + } + } + break; + case ICP_OP_SECHO: + { + UrlRequest = &Payload[4]; + LogRequest = "ICP_OP_QUERY"; + LogReply = "ICP_OP_QUERY"; + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_SECHO, + Version, 0, Request_Number, 0, 0, 0, UrlRequest); + } + break; + default: + LogRequest = "NOTIMPLEMENTED"; + LogReply = "ICP_OP_ERR"; + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_ERR, Version, + 0, Request_Number, 0, 0, 0, NULL); + break; + } + } else { + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_ERR, 2, 0, + Request_Number, 0, 0, 0, NULL); + } + } /* Ignored (RFC2186) */ + } else { + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_ERR, Version, 0, + Request_Number, 0, 0, 0, NULL); + } + } + + /* Logging */ + { + String ip = STRING_EMPTY; + SOCaddr serverClient; + int lenServerClient = (int) sizeof(serverClient); + + SOCaddr_copyaddr(serverClient, lenServerClient, &clientAddr, + clientAddrLen); + if (lenServerClient > 0) { + ip = getip(&serverClient, lenServerClient); + } else { + StringCopy(ip, "unknown"); + } + proxytrack_print_log(LOG, "ICP %s %s/%s %s", StringBuff(ip), LogRequest, + LogReply, (UrlRequest ? UrlRequest : "-")); + StringFree(ip); + } + + } + } + if (soc != INVALID_SOCKET) { #ifdef _WIN32 - closesocket(soc); + closesocket(soc); #else - close(soc); + close(soc); #endif - } - free(buffer); - return 1; + } + free(buffer); + return 1; } static int proxytrack_start(PT_Indexes indexes, T_SOC soc, T_SOC socICP) { - int ret = 1; - if (proxytrack_process_generic(proxytrack_start_ICP, indexes, socICP)) { - //if (!proxytrack_process_generic(proxytrack_start_HTTP, indexes, soc)) - if (!proxytrack_start_HTTP(indexes, soc)) { - ret = 0; - } - } else { - ret = 0; - } - return ret; -} + int ret = 1; + if (proxytrack_process_generic(proxytrack_start_ICP, indexes, socICP)) { + //if (!proxytrack_process_generic(proxytrack_start_HTTP, indexes, soc)) + if (!proxytrack_start_HTTP(indexes, soc)) { + ret = 0; + } + } else { + ret = 0; + } + return ret; +} diff --git a/src/proxy/proxytrack.h b/src/proxy/proxytrack.h index ae5af27..6152919 100644 --- a/src/proxy/proxytrack.h +++ b/src/proxy/proxytrack.h @@ -53,9 +53,8 @@ Please visit our Website: http://www.httrack.com /* generic */ -int proxytrack_main(char* proxyAddr, int proxyPort, - char* icpAddr, int icpPort, - PT_Indexes index); +int proxytrack_main(char *proxyAddr, int proxyPort, char *icpAddr, int icpPort, + PT_Indexes index); /* Spaces: CR,LF,TAB,FF */ #define is_space(c) ( ((c)==' ') || ((c)=='\"') || ((c)==10) || ((c)==13) || ((c)==9) || ((c)==12) || ((c)==11) || ((c)=='\'') ) @@ -71,6 +70,7 @@ static void proxytrack_print_log(const char *severity, const char *format, ...) const int error = errno; FILE *const fp = stderr; va_list args; + fprintf(fp, " * %s: ", severity); va_start(args, format); (void) vfprintf(fp, format, args); @@ -110,90 +110,105 @@ static void proxytrack_print_log(const char *severity, const char *format, ...) "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ "<!-- End Disable IE Friendly HTTP Error Messages -->\r\n" -static char* gethomedir(void) { - char* home = getenv( "HOME" ); +static char *gethomedir(void) { + char *home = getenv("HOME"); + if (home) return home; else return "."; } -static int linput(FILE* fp,char* s,int max) { +static int linput(FILE * fp, char *s, int max) { int c; - int j=0; + int j = 0; + do { - c=fgetc(fp); - if (c!=EOF) { - switch(c) { - case 13: break; // sauter CR - case 10: c=-1; break; - case 0: case 9: case 12: break; // sauter ces caractères - default: s[j++]=(char) c; break; + c = fgetc(fp); + if (c != EOF) { + switch (c) { + case 13: + break; // sauter CR + case 10: + c = -1; + break; + case 0: + case 9: + case 12: + break; // sauter ces caractères + default: + s[j++] = (char) c; + break; } } - } while((c!=-1) && (c!=EOF) && (j<(max-1))); - s[j]='\0'; + } while((c != -1) && (c != EOF) && (j < (max - 1))); + s[j] = '\0'; return j; } -static int link_has_authority(const char* lien) { - const char* a=lien; - if (isalpha((const unsigned char)*a)) { +static int link_has_authority(const char *lien) { + const char *a = lien; + + if (isalpha((const unsigned char) *a)) { // Skip scheme? - while (isalpha((const unsigned char)*a)) + while(isalpha((const unsigned char) *a)) a++; if (*a == ':') a++; else return 0; } - if (strncmp(a,"//",2) == 0) + if (strncmp(a, "//", 2) == 0) return 1; return 0; } -static const char* jump_protocol(const char* source) { +static const char *jump_protocol(const char *source) { int p; + // scheme // "Comparisons of scheme names MUST be case-insensitive" (RFC2616) - if ((p = strfield(source,"http:"))) - source+=p; - else if ((p = strfield(source,"ftp:"))) - source+=p; - else if ((p = strfield(source,"https:"))) - source+=p; - else if ((p = strfield(source,"file:"))) - source+=p; - else if ((p = strfield(source,"mms:"))) - source+=p; + if ((p = strfield(source, "http:"))) + source += p; + else if ((p = strfield(source, "ftp:"))) + source += p; + else if ((p = strfield(source, "https:"))) + source += p; + else if ((p = strfield(source, "file:"))) + source += p; + else if ((p = strfield(source, "mms:"))) + source += p; // net_path - if (strncmp(source,"//",2)==0) - source+=2; + if (strncmp(source, "//", 2) == 0) + source += 2; return source; } -static const char* strrchr_limit(const char* s, char c, const char* limit) { +static const char *strrchr_limit(const char *s, char c, const char *limit) { if (limit == NULL) { - char* p = strrchr(s, c); - return p?(p+1):NULL; + char *p = strrchr(s, c); + + return p ? (p + 1) : NULL; } else { - char *a=NULL, *p; + char *a = NULL, *p; + for(;;) { - p=strchr((a)?a:s, c); + p = strchr((a) ? a : s, c); if ((p >= limit) || (p == NULL)) return a; - a=p+1; + a = p + 1; } } } -static const char* jump_protocol_and_auth(const char* source) { - const char *a,*trytofind; +static const char *jump_protocol_and_auth(const char *source) { + const char *a, *trytofind; + if (strcmp(source, "file://") == 0) - return source; + return source; a = jump_protocol(source); - trytofind = strrchr_limit(a, '@', strchr(a,'/')); - return (trytofind != NULL)?trytofind:a; + trytofind = strrchr_limit(a, '@', strchr(a, '/')); + return (trytofind != NULL) ? trytofind : a; } #ifndef min @@ -202,27 +217,29 @@ static const char* jump_protocol_and_auth(const char* source) { #ifndef max #define max(a,b) ((a)>(b)?(a):(b)) #endif -static int linput_trim(FILE* fp,char* s,int max) { - int rlen=0; - char* ls=(char*) malloc(max+2); - s[0]='\0'; +static int linput_trim(FILE * fp, char *s, int max) { + int rlen = 0; + char *ls = (char *) malloc(max + 2); + + s[0] = '\0'; if (ls) { - char* a; + char *a; + // lire ligne - rlen=linput(fp,ls,max); + rlen = linput(fp, ls, max); if (rlen) { // sauter espaces et tabs en fin - while( (rlen>0) && is_realspace(ls[max(rlen-1,0)]) ) - ls[--rlen]='\0'; + while((rlen > 0) && is_realspace(ls[max(rlen - 1, 0)])) + ls[--rlen] = '\0'; // sauter espaces en début - a=ls; - while((rlen>0) && ((*a==' ') || (*a=='\t'))) { + a = ls; + while((rlen > 0) && ((*a == ' ') || (*a == '\t'))) { a++; rlen--; } - if (rlen>0) { - memcpy(s,a,rlen); // can copy \0 chars - s[rlen]='\0'; + if (rlen > 0) { + memcpy(s, a, rlen); // can copy \0 chars + s[rlen] = '\0'; } } // @@ -234,8 +251,9 @@ static int linput_trim(FILE* fp,char* s,int max) { #ifndef S_ISREG #define S_ISREG(m) ((m) & _S_IFREG) #endif -static int fexist(char* s) { +static int fexist(char *s) { struct stat st; + memset(&st, 0, sizeof(st)); if (stat(s, &st) == 0) { if (S_ISREG(st.st_mode)) { @@ -243,137 +261,155 @@ static int fexist(char* s) { } } return 0; -} +} /* convertir une chaine en temps */ -static void set_lowcase(char* s) { +static void set_lowcase(char *s) { int i; - for(i=0;i<(int) strlen(s);i++) - if ((s[i]>='A') && (s[i]<='Z')) - s[i]+=('a'-'A'); + + for(i = 0; i < (int) strlen(s); i++) + if ((s[i] >= 'A') && (s[i] <= 'Z')) + s[i] += ('a' - 'A'); } -static struct tm* convert_time_rfc822(struct tm *result,const char* s) { - char months[]="jan feb mar apr may jun jul aug sep oct nov dec"; +static struct tm *convert_time_rfc822(struct tm *result, const char *s) { + char months[] = "jan feb mar apr may jun jul aug sep oct nov dec"; char str[256]; - char* a; + char *a; + /* */ - int result_mm=-1; - int result_dd=-1; - int result_n1=-1; - int result_n2=-1; - int result_n3=-1; - int result_n4=-1; + int result_mm = -1; + int result_dd = -1; + int result_n1 = -1; + int result_n2 = -1; + int result_n3 = -1; + int result_n4 = -1; + /* */ if ((int) strlen(s) > 200) return NULL; - strcpy(str,s); + strcpy(str, s); set_lowcase(str); /* éliminer :,- */ - while( (a=strchr(str,'-')) ) *a=' '; - while( (a=strchr(str,':')) ) *a=' '; - while( (a=strchr(str,',')) ) *a=' '; + while((a = strchr(str, '-'))) + *a = ' '; + while((a = strchr(str, ':'))) + *a = ' '; + while((a = strchr(str, ','))) + *a = ' '; /* tokeniser */ - a=str; + a = str; while(*a) { char *first, *last; char tok[256]; + /* découper mot */ - while(*a==' ') a++; /* sauter espaces */ - first=a; - while((*a) && (*a!=' ')) a++; - last=a; - tok[0]='\0'; - if (first!=last) { - char* pos; - strncat(tok,first,(int) (last - first)); + while(*a == ' ') + a++; /* sauter espaces */ + first = a; + while((*a) && (*a != ' ')) + a++; + last = a; + tok[0] = '\0'; + if (first != last) { + char *pos; + + strncat(tok, first, (int) (last - first)); /* analyser */ - if ( (pos=strstr(months,tok)) ) { /* month always in letters */ - result_mm=((int) (pos - months))/4; + if ((pos = strstr(months, tok))) { /* month always in letters */ + result_mm = ((int) (pos - months)) / 4; } else { int number; - if (sscanf(tok,"%d",&number) == 1) { /* number token */ - if (result_dd<0) /* day always first number */ - result_dd=number; - else if (result_n1<0) - result_n1=number; - else if (result_n2<0) - result_n2=number; - else if (result_n3<0) - result_n3=number; - else if (result_n4<0) - result_n4=number; - } /* sinon, bruit de fond(+1GMT for exampel) */ + + if (sscanf(tok, "%d", &number) == 1) { /* number token */ + if (result_dd < 0) /* day always first number */ + result_dd = number; + else if (result_n1 < 0) + result_n1 = number; + else if (result_n2 < 0) + result_n2 = number; + else if (result_n3 < 0) + result_n3 = number; + else if (result_n4 < 0) + result_n4 = number; + } /* sinon, bruit de fond(+1GMT for exampel) */ } } } - if ((result_n1>=0) && (result_mm>=0) && (result_dd>=0) && (result_n2>=0) && (result_n3>=0) && (result_n4>=0)) { - if (result_n4>=1000) { /* Sun Nov 6 08:49:37 1994 */ - result->tm_year=result_n4-1900; - result->tm_hour=result_n1; - result->tm_min=result_n2; - result->tm_sec=max(result_n3,0); - } else { /* Sun, 06 Nov 1994 08:49:37 GMT or Sunday, 06-Nov-94 08:49:37 GMT */ - result->tm_hour=result_n2; - result->tm_min=result_n3; - result->tm_sec=max(result_n4,0); - if (result_n1<=50) /* 00 means 2000 */ - result->tm_year=result_n1+100; - else if (result_n1<1000) /* 99 means 1999 */ - result->tm_year=result_n1; - else /* 2000 */ - result->tm_year=result_n1-1900; + if ((result_n1 >= 0) && (result_mm >= 0) && (result_dd >= 0) + && (result_n2 >= 0) && (result_n3 >= 0) && (result_n4 >= 0)) { + if (result_n4 >= 1000) { /* Sun Nov 6 08:49:37 1994 */ + result->tm_year = result_n4 - 1900; + result->tm_hour = result_n1; + result->tm_min = result_n2; + result->tm_sec = max(result_n3, 0); + } else { /* Sun, 06 Nov 1994 08:49:37 GMT or Sunday, 06-Nov-94 08:49:37 GMT */ + result->tm_hour = result_n2; + result->tm_min = result_n3; + result->tm_sec = max(result_n4, 0); + if (result_n1 <= 50) /* 00 means 2000 */ + result->tm_year = result_n1 + 100; + else if (result_n1 < 1000) /* 99 means 1999 */ + result->tm_year = result_n1; + else /* 2000 */ + result->tm_year = result_n1 - 1900; } - result->tm_isdst=0; /* assume GMT */ - result->tm_yday=-1; /* don't know */ - result->tm_wday=-1; /* don't know */ - result->tm_mon=result_mm; - result->tm_mday=result_dd; + result->tm_isdst = 0; /* assume GMT */ + result->tm_yday = -1; /* don't know */ + result->tm_wday = -1; /* don't know */ + result->tm_mon = result_mm; + result->tm_mday = result_dd; return result; } return NULL; } static struct tm PT_GetTime(time_t t) { - struct tm tmbuf; + struct tm tmbuf; + #ifdef _WIN32 - struct tm * tm = gmtime(&t); + struct tm *tm = gmtime(&t); #else - struct tm * tm = gmtime_r(&t, &tmbuf); + struct tm *tm = gmtime_r(&t, &tmbuf); #endif - if (tm != NULL) - return *tm; - else { - memset(&tmbuf, 0, sizeof(tmbuf)); - return tmbuf; - } + if (tm != NULL) + return *tm; + else { + memset(&tmbuf, 0, sizeof(tmbuf)); + return tmbuf; + } } -static int set_filetime(const char* file, struct tm* tm_time) { +static int set_filetime(const char *file, struct tm *tm_time) { struct utimbuf tim; + #ifndef HTS_DO_NOT_USE_FTIME struct timeb B; + memset(&B, 0, sizeof(B)); - B.timezone=0; - ftime( &B ); - tim.actime = tim.modtime = mktime(tm_time) - B.timezone*60; + B.timezone = 0; + ftime(&B); + tim.actime = tim.modtime = mktime(tm_time) - B.timezone * 60; #else // bogus time (GMT/local).. - tim.actime=tim.modtime=mktime(tm_time); + tim.actime = tim.modtime = mktime(tm_time); #endif return utime(file, &tim); } -static int set_filetime_time_t(const char* file, time_t t) { - if (t != (time_t) 0 && t != (time_t) -1) { - struct tm tm = PT_GetTime(t); - return set_filetime(file, &tm); - } - return -1; +static int set_filetime_time_t(const char *file, time_t t) { + if (t != (time_t) 0 && t != (time_t) - 1) { + struct tm tm = PT_GetTime(t); + + return set_filetime(file, &tm); + } + return -1; } -static int set_filetime_rfc822(const char* file, const char* date) { - struct tm buffer; - struct tm* tm_s = convert_time_rfc822(&buffer,date); +static int set_filetime_rfc822(const char *file, const char *date) { + struct tm buffer; + struct tm *tm_s = convert_time_rfc822(&buffer, date); + if (tm_s) { - return set_filetime(file,tm_s); - } else return -1; + return set_filetime(file, tm_s); + } else + return -1; } #endif diff --git a/src/proxy/store.c b/src/proxy/store.c index e5d0c8f..2f2b92c 100644 --- a/src/proxy/store.c +++ b/src/proxy/store.c @@ -34,7 +34,7 @@ Please visit our Website: http://www.httrack.com /* Locking */ #ifdef _WIN32 -#include <process.h> /* _beginthread, _endthread */ +#include <process.h> /* _beginthread, _endthread */ #else #include <pthread.h> #endif @@ -57,49 +57,52 @@ Please visit our Website: http://www.httrack.com /* Unlocked functions */ -static int PT_LookupCache__New_u(PT_Index index, const char* url); -static PT_Element PT_ReadCache__New_u(PT_Index index, const char* url, int flags); +static int PT_LookupCache__New_u(PT_Index index, const char *url); +static PT_Element PT_ReadCache__New_u(PT_Index index, const char *url, + int flags); -static int PT_LookupCache__Old_u(PT_Index index, const char* url); -static PT_Element PT_ReadCache__Old_u(PT_Index index, const char* url, int flags); +static int PT_LookupCache__Old_u(PT_Index index, const char *url); +static PT_Element PT_ReadCache__Old_u(PT_Index index, const char *url, + int flags); -static int PT_LookupCache__Arc_u(PT_Index index, const char* url); -static PT_Element PT_ReadCache__Arc_u(PT_Index index, const char* url, int flags); +static int PT_LookupCache__Arc_u(PT_Index index, const char *url); +static PT_Element PT_ReadCache__Arc_u(PT_Index index, const char *url, + int flags); /* Locking */ #ifdef _WIN32 -void MutexInit(PT_Mutex *pMutex) { - *pMutex = CreateMutex(NULL,FALSE,NULL); +void MutexInit(PT_Mutex * pMutex) { + *pMutex = CreateMutex(NULL, FALSE, NULL); } -void MutexLock(PT_Mutex *pMutex) { - WaitForSingleObject(*pMutex, INFINITE); +void MutexLock(PT_Mutex * pMutex) { + WaitForSingleObject(*pMutex, INFINITE); } -void MutexUnlock(PT_Mutex *pMutex) { - ReleaseMutex(*pMutex); +void MutexUnlock(PT_Mutex * pMutex) { + ReleaseMutex(*pMutex); } -void MutexFree(PT_Mutex *pMutex) { - CloseHandle(*pMutex); - *pMutex = NULL; +void MutexFree(PT_Mutex * pMutex) { + CloseHandle(*pMutex); + *pMutex = NULL; } #else -void MutexInit(PT_Mutex *pMutex) { - (void) pthread_mutex_init(pMutex, 0); +void MutexInit(PT_Mutex * pMutex) { + (void) pthread_mutex_init(pMutex, 0); } -void MutexLock(PT_Mutex *pMutex) { - pthread_mutex_lock(pMutex); +void MutexLock(PT_Mutex * pMutex) { + pthread_mutex_lock(pMutex); } -void MutexUnlock(PT_Mutex *pMutex) { - pthread_mutex_unlock(pMutex); +void MutexUnlock(PT_Mutex * pMutex) { + pthread_mutex_unlock(pMutex); } -void MutexFree(PT_Mutex *pMutex) { - pthread_mutex_destroy(pMutex); +void MutexFree(PT_Mutex * pMutex) { + pthread_mutex_destroy(pMutex); } #endif @@ -116,47 +119,48 @@ typedef struct _PT_Index__Arc *PT_Index__Arc; typedef struct _PT_Index_Functions *PT_Index_Functions; enum { - PT_CACHE_UNDEFINED = -1, - PT_CACHE_MIN = 0, - PT_CACHE__NEW = PT_CACHE_MIN, - PT_CACHE__OLD, - PT_CACHE__ARC, - PT_CACHE_MAX = PT_CACHE__ARC + PT_CACHE_UNDEFINED = -1, + PT_CACHE_MIN = 0, + PT_CACHE__NEW = PT_CACHE_MIN, + PT_CACHE__OLD, + PT_CACHE__ARC, + PT_CACHE_MAX = PT_CACHE__ARC }; static int PT_LoadCache__New(PT_Index index, const char *filename); -static void PT_Index_Delete__New(PT_Index *pindex); -static PT_Element PT_ReadCache__New(PT_Index index, const char* url, int flags); -static int PT_LookupCache__New(PT_Index index, const char* url); +static void PT_Index_Delete__New(PT_Index * pindex); +static PT_Element PT_ReadCache__New(PT_Index index, const char *url, int flags); +static int PT_LookupCache__New(PT_Index index, const char *url); static int PT_SaveCache__New(PT_Indexes indexes, const char *filename); -/**/ -static int PT_LoadCache__Old(PT_Index index, const char *filename); -static void PT_Index_Delete__Old(PT_Index *pindex); -static PT_Element PT_ReadCache__Old(PT_Index index, const char* url, int flags); -static int PT_LookupCache__Old(PT_Index index, const char* url); -/**/ -static int PT_LoadCache__Arc(PT_Index index, const char *filename); -static void PT_Index_Delete__Arc(PT_Index *pindex); -static PT_Element PT_ReadCache__Arc(PT_Index index, const char* url, int flags); -static int PT_LookupCache__Arc(PT_Index index, const char* url); + /**/ static int PT_LoadCache__Old(PT_Index index, const char *filename); +static void PT_Index_Delete__Old(PT_Index * pindex); +static PT_Element PT_ReadCache__Old(PT_Index index, const char *url, int flags); +static int PT_LookupCache__Old(PT_Index index, const char *url); + /**/ static int PT_LoadCache__Arc(PT_Index index, const char *filename); +static void PT_Index_Delete__Arc(PT_Index * pindex); +static PT_Element PT_ReadCache__Arc(PT_Index index, const char *url, int flags); +static int PT_LookupCache__Arc(PT_Index index, const char *url); static int PT_SaveCache__Arc(PT_Indexes indexes, const char *filename); struct _PT_Index_Functions { - /* Mandatory services */ - int (*PT_LoadCache)(PT_Index index, const char *filename); - void (*PT_Index_Delete)(PT_Index *pindex); - PT_Element (*PT_ReadCache)(PT_Index index, const char* url, int flags); - int (*PT_LookupCache)(PT_Index index, const char* url); - - /* Optional services */ - int (*PT_SaveCache)(PT_Indexes indexes, const char *filename); + /* Mandatory services */ + int (*PT_LoadCache) (PT_Index index, const char *filename); + void (*PT_Index_Delete) (PT_Index * pindex); + PT_Element(*PT_ReadCache) (PT_Index index, const char *url, int flags); + int (*PT_LookupCache) (PT_Index index, const char *url); + + /* Optional services */ + int (*PT_SaveCache) (PT_Indexes indexes, const char *filename); }; static _PT_Index_Functions _IndexFuncts[] = { - { PT_LoadCache__New, PT_Index_Delete__New, PT_ReadCache__New, PT_LookupCache__New, PT_SaveCache__New }, - { PT_LoadCache__Old, PT_Index_Delete__Old, PT_ReadCache__Old, PT_LookupCache__Old, NULL }, - { PT_LoadCache__Arc, PT_Index_Delete__Arc, PT_ReadCache__Arc, PT_LookupCache__Arc, PT_SaveCache__Arc }, - { NULL, NULL, NULL, NULL } + {PT_LoadCache__New, PT_Index_Delete__New, PT_ReadCache__New, + PT_LookupCache__New, PT_SaveCache__New}, + {PT_LoadCache__Old, PT_Index_Delete__Old, PT_ReadCache__Old, + PT_LookupCache__Old, NULL}, + {PT_LoadCache__Arc, PT_Index_Delete__Arc, PT_ReadCache__Arc, + PT_LookupCache__Arc, PT_SaveCache__Arc}, + {NULL, NULL, NULL, NULL} }; #define PT_INDEX_COMMON_STRUCTURE \ @@ -165,90 +169,91 @@ static _PT_Index_Functions _IndexFuncts[] = { char startUrl[1024] struct _PT_Index__New { - PT_INDEX_COMMON_STRUCTURE; - char path[1024]; /* either empty, or must include ending / */ - int fixedPath; - int safeCache; - unzFile zFile; - PT_Mutex zFileLock; + PT_INDEX_COMMON_STRUCTURE; + char path[1024]; /* either empty, or must include ending / */ + int fixedPath; + int safeCache; + unzFile zFile; + PT_Mutex zFileLock; }; struct _PT_Index__Old { - PT_INDEX_COMMON_STRUCTURE; - char filenameDat[1024]; - char filenameNdx[1024]; - FILE *dat,*ndx; - PT_Mutex fileLock; - int version; - char lastmodified[1024]; - char path[1024]; /* either empty, or must include ending / */ - int fixedPath; - int safeCache; + PT_INDEX_COMMON_STRUCTURE; + char filenameDat[1024]; + char filenameNdx[1024]; + FILE *dat, *ndx; + PT_Mutex fileLock; + int version; + char lastmodified[1024]; + char path[1024]; /* either empty, or must include ending / */ + int fixedPath; + int safeCache; }; struct _PT_Index__Arc { - PT_INDEX_COMMON_STRUCTURE; - FILE *file; - PT_Mutex fileLock; - int version; - char lastmodified[1024]; - char line[2048]; - char filenameIndexBuff[2048]; + PT_INDEX_COMMON_STRUCTURE; + FILE *file; + PT_Mutex fileLock; + int version; + char lastmodified[1024]; + char line[2048]; + char filenameIndexBuff[2048]; }; struct _PT_Index { - int type; - union { - _PT_Index__New formatNew; - _PT_Index__Old formatOld; - _PT_Index__Arc formatArc; - struct { - PT_INDEX_COMMON_STRUCTURE; - } common; - } slots; + int type; + union { + _PT_Index__New formatNew; + _PT_Index__Old formatOld; + _PT_Index__Arc formatArc; + struct { + PT_INDEX_COMMON_STRUCTURE; + } common; + } slots; }; struct _PT_Indexes { - inthash cil; - struct _PT_Index **index; - int index_size; + inthash cil; + struct _PT_Index **index; + int index_size; }; struct _PT_CacheItem { - time_t lastUsed; - size_t size; - void* data; + time_t lastUsed; + size_t size; + void *data; }; struct _PT_Cache { - inthash index; - size_t maxSize; - size_t totalSize; - int count; + inthash index; + size_t maxSize; + size_t totalSize; + int count; }; PT_Indexes PT_New(void) { - PT_Indexes index = (PT_Indexes) calloc(sizeof(_PT_Indexes), 1); - index->cil = inthash_new(127); - index->index_size = 0; - index->index = NULL; - return index; + PT_Indexes index = (PT_Indexes) calloc(sizeof(_PT_Indexes), 1); + + index->cil = inthash_new(127); + index->index_size = 0; + index->index = NULL; + return index; } void PT_Delete(PT_Indexes index) { - if (index != NULL) { - inthash_delete(&index->cil); - free(index); - } + if (index != NULL) { + inthash_delete(&index->cil); + free(index); + } } int PT_RemoveIndex(PT_Indexes index, int indexId) { - return 0; + return 0; } #define assertf(exp) -static int binput(char* buff,char* s,int max) { +static int binput(char *buff, char *s, int max) { int count = 0; int destCount = 0; @@ -257,7 +262,7 @@ static int binput(char* buff,char* s,int max) { if (buff[count] != '\r') { s[destCount++] = buff[count]; } - count++; + count++; } s[destCount] = '\0'; @@ -265,529 +270,573 @@ static int binput(char* buff,char* s,int max) { return count + 1; } -static time_t file_timestamp(const char* file) { +static time_t file_timestamp(const char *file) { struct stat buf; + if (stat(file, &buf) == 0) { time_t tt = buf.st_mtime; - if (tt != (time_t) 0 && tt != (time_t) -1) { - return tt; - } + + if (tt != (time_t) 0 && tt != (time_t) - 1) { + return tt; + } } return (time_t) 0; } -static int PT_Index_Check__(PT_Index index, const char* file, int line) { - if (index == NULL) - return 0; - if (index->type >= PT_CACHE_MIN && index->type <= PT_CACHE_MAX) - return 1; - proxytrack_print_log(CRITICAL, "index corrupted in memory at %s:%d", file, line); - return 0; +static int PT_Index_Check__(PT_Index index, const char *file, int line) { + if (index == NULL) + return 0; + if (index->type >= PT_CACHE_MIN && index->type <= PT_CACHE_MAX) + return 1; + proxytrack_print_log(CRITICAL, "index corrupted in memory at %s:%d", file, + line); + return 0; } -#define SAFE_INDEX(index) PT_Index_Check__(index, __FILE__, __LINE__) +#define SAFE_INDEX(index) PT_Index_Check__(index, __FILE__, __LINE__) /* ------------------------------------------------------------ */ /* Generic cache dispatch */ /* ------------------------------------------------------------ */ -void PT_Index_Delete(PT_Index *pindex) { - if (pindex != NULL && (*pindex) != NULL) { - PT_Index index = *pindex; - if (SAFE_INDEX(index)) { - _IndexFuncts[index->type].PT_Index_Delete(pindex); - } - free(index); - *pindex = NULL; - } -} - -static void PT_Index_Delete__New(PT_Index *pindex) { - if (pindex != NULL && (*pindex) != NULL) { - PT_Index__New index = &(*pindex)->slots.formatNew; - if (index->zFile != NULL) { - unzClose(index->zFile); - index->zFile = NULL; - } - if (index->hash != NULL) { - inthash_delete(&index->hash); - index->hash = NULL; - } - MutexFree(&index->zFileLock); - } -} - -static void PT_Index_Delete__Old(PT_Index *pindex) { - if (pindex != NULL && (*pindex) != NULL) { - PT_Index__Old index = &(*pindex)->slots.formatOld; - if (index->dat != NULL) { - fclose(index->dat); - } - if (index->ndx != NULL) { - fclose(index->ndx); - } - if (index->hash != NULL) { - inthash_delete(&index->hash); - index->hash = NULL; - } - MutexFree(&index->fileLock); - } -} - -static void PT_Index_Delete__Arc(PT_Index *pindex) { - if (pindex != NULL && (*pindex) != NULL) { - PT_Index__Arc index = &(*pindex)->slots.formatArc; - if (index->file != NULL) { - fclose(index->file); - } - MutexFree(&index->fileLock); - } +void PT_Index_Delete(PT_Index * pindex) { + if (pindex != NULL && (*pindex) != NULL) { + PT_Index index = *pindex; + + if (SAFE_INDEX(index)) { + _IndexFuncts[index->type].PT_Index_Delete(pindex); + } + free(index); + *pindex = NULL; + } +} + +static void PT_Index_Delete__New(PT_Index * pindex) { + if (pindex != NULL && (*pindex) != NULL) { + PT_Index__New index = &(*pindex)->slots.formatNew; + + if (index->zFile != NULL) { + unzClose(index->zFile); + index->zFile = NULL; + } + if (index->hash != NULL) { + inthash_delete(&index->hash); + index->hash = NULL; + } + MutexFree(&index->zFileLock); + } +} + +static void PT_Index_Delete__Old(PT_Index * pindex) { + if (pindex != NULL && (*pindex) != NULL) { + PT_Index__Old index = &(*pindex)->slots.formatOld; + + if (index->dat != NULL) { + fclose(index->dat); + } + if (index->ndx != NULL) { + fclose(index->ndx); + } + if (index->hash != NULL) { + inthash_delete(&index->hash); + index->hash = NULL; + } + MutexFree(&index->fileLock); + } +} + +static void PT_Index_Delete__Arc(PT_Index * pindex) { + if (pindex != NULL && (*pindex) != NULL) { + PT_Index__Arc index = &(*pindex)->slots.formatArc; + + if (index->file != NULL) { + fclose(index->file); + } + MutexFree(&index->fileLock); + } } int PT_AddIndex(PT_Indexes indexes, const char *path) { - PT_Index index = PT_LoadCache(path); - if (index != NULL) { - int ret = PT_IndexMerge(indexes, &index); - if (index != NULL) { - PT_Index_Delete(&index); - } - return ret; - } - return -1; + PT_Index index = PT_LoadCache(path); + + if (index != NULL) { + int ret = PT_IndexMerge(indexes, &index); + + if (index != NULL) { + PT_Index_Delete(&index); + } + return ret; + } + return -1; } PT_Element PT_Index_HTML_BuildRootInfo(PT_Indexes indexes) { - if (indexes != NULL) { - PT_Element elt = PT_ElementNew(); - int i; - String html = STRING_EMPTY; - StringClear(html); - StringCat(html, - "<html>" - PROXYTRACK_COMMENT_HEADER - DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES - "<head>\r\n" - "<title>ProxyTrack " PROXYTRACK_VERSION " Catalog</title>" - "</head>\r\n" - "<body>\r\n" - "<h3>Available sites in this cache:</h3><br />" - "<br />" - ); - StringCat(html, "<ul>\r\n"); - for(i = 0 ; i < indexes->index_size ; i++) { - if (indexes->index[i] != NULL - && indexes->index[i]->slots.common.startUrl[0] != '\0') - { - const char * url = indexes->index[i]->slots.common.startUrl; - StringCat(html, "<li>\r\n"); - StringCat(html, "<a href=\""); - StringCat(html, url); - StringCat(html, "\">"); - StringCat(html, url); - StringCat(html, "</a>\r\n"); - StringCat(html, "</li>\r\n"); - } - } - StringCat(html, "</ul>\r\n"); - StringCat(html, "</body></html>\r\n"); - elt->size = StringLength(html); - elt->adr = StringAcquire(&html); - elt->statuscode = HTTP_OK; - strcpy(elt->charset, "iso-8859-1"); - strcpy(elt->contenttype, "text/html"); - strcpy(elt->msg, "OK"); - StringFree(html); - return elt; - } - return NULL; -} - -static char* strchr_stop(char* str, char c, char stop) { - for( ; *str != 0 && *str != stop && *str != c ; str++); - if (*str == c) - return str; - return NULL; -} - -char ** PT_Enumerate(PT_Indexes indexes, const char *url, int subtree) { - // should be cached! - if (indexes != NULL && indexes->cil != NULL) { - unsigned int urlSize; - String list = STRING_EMPTY; - String listindexes = STRING_EMPTY; - String subitem = STRING_EMPTY; - unsigned int listCount = 0; - struct_inthash_enum en = inthash_enum_new(indexes->cil); - inthash_chain* chain; - inthash hdupes = NULL; - if (!subtree) - hdupes= inthash_new(127); - StringClear(list); - StringClear(listindexes); - StringClear(subitem); - if (strncmp(url, "http://", 7) == 0) - url += 7; - urlSize = (unsigned int) strlen(url); - while((chain = inthash_enum_next(&en))) { - long int index = (long int)chain->value.intg; - if (urlSize == 0 || strncmp(chain->name, url, urlSize) == 0) { - if (index >= 0 && index < indexes->index_size) { - char * item = chain->name + urlSize; - if (*item == '/') - item++; - { - char * pos = subtree ? 0 : strchr_stop(item, '/', '?'); - unsigned int len = pos ? (unsigned int)( pos - item ) : (unsigned int)strlen(item); - if (len > 0 /* default document */ || *item == 0) { - int isFolder = ( item[len] == '/' ); - StringClear(subitem); - if (len > 0) - StringMemcat(subitem, item, len); - if (len == 0 || !inthash_exists(hdupes, StringBuff(subitem))) { - char* ptr = NULL; - ptr += StringLength(list); - if (len > 0) - StringCat(list, StringBuff(subitem)); - if (isFolder) - StringCat(list, "/"); - StringMemcat(list, "\0", 1); /* NULL terminated strings */ - StringMemcat(listindexes, &ptr, sizeof(ptr)); - listCount++; - inthash_write(hdupes, StringBuff(subitem), 0); - } - } - } - } else { - proxytrack_print_log(CRITICAL, "PT_Enumerate:Corrupted central index locator"); - } - } - } - StringFree(subitem); - inthash_delete(&hdupes); - if (listCount > 0) { - unsigned int i; - void* blk; - char *nullPointer = NULL; - char* startStrings; - /* NULL terminated index */ - StringMemcat(listindexes, &nullPointer, sizeof(nullPointer)); - /* start of all strings (index) */ - startStrings = nullPointer + StringLength(listindexes); - /* copy list of URLs after indexes */ - StringMemcat(listindexes, StringBuff(list), StringLength(list)); - /* ---- no reallocation beyond this point (fixed addresses) ---- */ - /* start of all strings (pointer) */ - startStrings = (startStrings - nullPointer) + StringBuffRW(listindexes); - /* transform indexes into references */ - for(i = 0 ; i < listCount ; i++) { - char *ptr = NULL; - unsigned int ndx; - memcpy(&ptr, &StringBuff(listindexes)[i*sizeof(char*)], sizeof(char*)); - ndx = (unsigned int) (ptr - nullPointer); - ptr = startStrings + ndx; - memcpy(&StringBuffRW(listindexes)[i*sizeof(char*)], &ptr, sizeof(char*)); - } - blk = StringAcquire(&listindexes); - StringFree(list); - StringFree(listindexes); - return (char **)blk; - } - } - return NULL; + if (indexes != NULL) { + PT_Element elt = PT_ElementNew(); + int i; + String html = STRING_EMPTY; + + StringClear(html); + StringCat(html, + "<html>" PROXYTRACK_COMMENT_HEADER + DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES "<head>\r\n" + "<title>ProxyTrack " PROXYTRACK_VERSION " Catalog</title>" + "</head>\r\n" "<body>\r\n" + "<h3>Available sites in this cache:</h3><br />" "<br />"); + StringCat(html, "<ul>\r\n"); + for(i = 0; i < indexes->index_size; i++) { + if (indexes->index[i] != NULL + && indexes->index[i]->slots.common.startUrl[0] != '\0') { + const char *url = indexes->index[i]->slots.common.startUrl; + + StringCat(html, "<li>\r\n"); + StringCat(html, "<a href=\""); + StringCat(html, url); + StringCat(html, "\">"); + StringCat(html, url); + StringCat(html, "</a>\r\n"); + StringCat(html, "</li>\r\n"); + } + } + StringCat(html, "</ul>\r\n"); + StringCat(html, "</body></html>\r\n"); + elt->size = StringLength(html); + elt->adr = StringAcquire(&html); + elt->statuscode = HTTP_OK; + strcpy(elt->charset, "iso-8859-1"); + strcpy(elt->contenttype, "text/html"); + strcpy(elt->msg, "OK"); + StringFree(html); + return elt; + } + return NULL; +} + +static char *strchr_stop(char *str, char c, char stop) { + for(; *str != 0 && *str != stop && *str != c; str++) ; + if (*str == c) + return str; + return NULL; +} + +char **PT_Enumerate(PT_Indexes indexes, const char *url, int subtree) { + // should be cached! + if (indexes != NULL && indexes->cil != NULL) { + unsigned int urlSize; + String list = STRING_EMPTY; + String listindexes = STRING_EMPTY; + String subitem = STRING_EMPTY; + unsigned int listCount = 0; + struct_inthash_enum en = inthash_enum_new(indexes->cil); + inthash_chain *chain; + inthash hdupes = NULL; + + if (!subtree) + hdupes = inthash_new(127); + StringClear(list); + StringClear(listindexes); + StringClear(subitem); + if (strncmp(url, "http://", 7) == 0) + url += 7; + urlSize = (unsigned int) strlen(url); + while((chain = inthash_enum_next(&en))) { + long int index = (long int) chain->value.intg; + + if (urlSize == 0 || strncmp(chain->name, url, urlSize) == 0) { + if (index >= 0 && index < indexes->index_size) { + char *item = chain->name + urlSize; + + if (*item == '/') + item++; + { + char *pos = subtree ? 0 : strchr_stop(item, '/', '?'); + unsigned int len = + pos ? (unsigned int) (pos - item) : (unsigned int) strlen(item); + if (len > 0 /* default document */ || *item == 0) { + int isFolder = (item[len] == '/'); + + StringClear(subitem); + if (len > 0) + StringMemcat(subitem, item, len); + if (len == 0 || !inthash_exists(hdupes, StringBuff(subitem))) { + char *ptr = NULL; + + ptr += StringLength(list); + if (len > 0) + StringCat(list, StringBuff(subitem)); + if (isFolder) + StringCat(list, "/"); + StringMemcat(list, "\0", 1); /* NULL terminated strings */ + StringMemcat(listindexes, &ptr, sizeof(ptr)); + listCount++; + inthash_write(hdupes, StringBuff(subitem), 0); + } + } + } + } else { + proxytrack_print_log(CRITICAL, + "PT_Enumerate:Corrupted central index locator"); + } + } + } + StringFree(subitem); + inthash_delete(&hdupes); + if (listCount > 0) { + unsigned int i; + void *blk; + char *nullPointer = NULL; + char *startStrings; + + /* NULL terminated index */ + StringMemcat(listindexes, &nullPointer, sizeof(nullPointer)); + /* start of all strings (index) */ + startStrings = nullPointer + StringLength(listindexes); + /* copy list of URLs after indexes */ + StringMemcat(listindexes, StringBuff(list), StringLength(list)); + /* ---- no reallocation beyond this point (fixed addresses) ---- */ + /* start of all strings (pointer) */ + startStrings = (startStrings - nullPointer) + StringBuffRW(listindexes); + /* transform indexes into references */ + for(i = 0; i < listCount; i++) { + char *ptr = NULL; + unsigned int ndx; + + memcpy(&ptr, &StringBuff(listindexes)[i * sizeof(char *)], + sizeof(char *)); + ndx = (unsigned int) (ptr - nullPointer); + ptr = startStrings + ndx; + memcpy(&StringBuffRW(listindexes)[i * sizeof(char *)], &ptr, + sizeof(char *)); + } + blk = StringAcquire(&listindexes); + StringFree(list); + StringFree(listindexes); + return (char **) blk; + } + } + return NULL; } void PT_Enumerate_Delete(char ***plist) { - if (plist != NULL && *plist != NULL) { - free(*plist); - *plist = NULL; - } + if (plist != NULL && *plist != NULL) { + free(*plist); + *plist = NULL; + } } static int PT_GetType(const char *filename) { - char * dot = strrchr(filename, '.'); - if (dot != NULL) { - if (strcasecmp(dot, ".zip") == 0) { - return PT_CACHE__NEW; - } else if (strcasecmp(dot, ".ndx") == 0 || strcasecmp(dot, ".dat") == 0) { - return PT_CACHE__OLD; - } else if (strcasecmp(dot, ".arc") == 0) { - return PT_CACHE__ARC; - } - } - return PT_CACHE_UNDEFINED; + char *dot = strrchr(filename, '.'); + + if (dot != NULL) { + if (strcasecmp(dot, ".zip") == 0) { + return PT_CACHE__NEW; + } else if (strcasecmp(dot, ".ndx") == 0 || strcasecmp(dot, ".dat") == 0) { + return PT_CACHE__OLD; + } else if (strcasecmp(dot, ".arc") == 0) { + return PT_CACHE__ARC; + } + } + return PT_CACHE_UNDEFINED; } PT_Index PT_LoadCache(const char *filename) { - int type = PT_GetType(filename); - if (type != PT_CACHE_UNDEFINED) { - PT_Index index = calloc(sizeof(_PT_Index), 1); - if (index != NULL) { - index->type = type; - index->slots.common.timestamp = (time_t) time(NULL); - index->slots.common.startUrl[0] = '\0'; - index->slots.common.hash = inthash_new(8191); - if (!_IndexFuncts[type].PT_LoadCache(index, filename)) { - proxytrack_print_log(DEBUG, "reading httrack cache (format #%d) %s : error", type, filename ); - free(index); - index = NULL; - return NULL; - } else { - proxytrack_print_log(DEBUG, "reading httrack cache (format #%d) %s : success", type, filename ); - } - /* default starting URL is the first hash entry */ - if (index->slots.common.startUrl[0] == '\0') { - struct_inthash_enum en = inthash_enum_new(index->slots.common.hash); - inthash_chain* chain; - chain = inthash_enum_next(&en); - if (chain != NULL - && strstr(chain->name, "/robots.txt") != NULL) - { - chain = inthash_enum_next(&en); - } - if (chain != NULL) { - if (!link_has_authority(chain->name)) - strcat(index->slots.common.startUrl, "http://"); - strcat(index->slots.common.startUrl, chain->name); - } - } - } - return index; - } - return NULL; -} - - -static long int filesize(const char* filename) { + int type = PT_GetType(filename); + + if (type != PT_CACHE_UNDEFINED) { + PT_Index index = calloc(sizeof(_PT_Index), 1); + + if (index != NULL) { + index->type = type; + index->slots.common.timestamp = (time_t) time(NULL); + index->slots.common.startUrl[0] = '\0'; + index->slots.common.hash = inthash_new(8191); + if (!_IndexFuncts[type].PT_LoadCache(index, filename)) { + proxytrack_print_log(DEBUG, + "reading httrack cache (format #%d) %s : error", + type, filename); + free(index); + index = NULL; + return NULL; + } else { + proxytrack_print_log(DEBUG, + "reading httrack cache (format #%d) %s : success", + type, filename); + } + /* default starting URL is the first hash entry */ + if (index->slots.common.startUrl[0] == '\0') { + struct_inthash_enum en = inthash_enum_new(index->slots.common.hash); + inthash_chain *chain; + + chain = inthash_enum_next(&en); + if (chain != NULL && strstr(chain->name, "/robots.txt") != NULL) { + chain = inthash_enum_next(&en); + } + if (chain != NULL) { + if (!link_has_authority(chain->name)) + strcat(index->slots.common.startUrl, "http://"); + strcat(index->slots.common.startUrl, chain->name); + } + } + } + return index; + } + return NULL; +} + +static long int filesize(const char *filename) { struct stat st; + memset(&st, 0, sizeof(st)); if (stat(filename, &st) == 0) { - return (long int)st.st_size; + return (long int) st.st_size; } return -1; -} +} -int PT_LookupCache(PT_Index index, const char* url) { - if (index != NULL && SAFE_INDEX(index)) { - return _IndexFuncts[index->type].PT_LookupCache(index, url); - } - return 0; +int PT_LookupCache(PT_Index index, const char *url) { + if (index != NULL && SAFE_INDEX(index)) { + return _IndexFuncts[index->type].PT_LookupCache(index, url); + } + return 0; } int PT_SaveCache(PT_Indexes indexes, const char *filename) { - int type = PT_GetType(filename); - if (type != PT_CACHE_UNDEFINED) { - if (_IndexFuncts[type].PT_SaveCache != NULL) { - int ret = _IndexFuncts[type].PT_SaveCache(indexes, filename); - if (ret == 0) { - (void) set_filetime_time_t(filename, PT_GetTimeIndex(indexes)); - return 0; - } - } - } - return -1; -} - -int PT_EnumCache(PT_Indexes indexes, int (*callback)(void *, const char *url, PT_Element), void *arg) { - if (indexes != NULL && indexes->cil != NULL) { - struct_inthash_enum en = inthash_enum_new(indexes->cil); - inthash_chain* chain; - while((chain = inthash_enum_next(&en))) { - const long int index_id = (long int)chain->value.intg; - const char *const url = chain->name; - if (index_id >= 0 && index_id <= indexes->index_size) { - PT_Element item = PT_ReadCache(indexes->index[index_id], url, FETCH_HEADERS | FETCH_BODY); - if (item != NULL) { - int ret = callback(arg, url, item); - PT_Element_Delete(&item); - if (ret != 0) - return ret; - } - } else { - proxytrack_print_log(CRITICAL, "PT_ReadCache:Corrupted central index locator"); - return -1; - } - } - } - return 0; + int type = PT_GetType(filename); + + if (type != PT_CACHE_UNDEFINED) { + if (_IndexFuncts[type].PT_SaveCache != NULL) { + int ret = _IndexFuncts[type].PT_SaveCache(indexes, filename); + + if (ret == 0) { + (void) set_filetime_time_t(filename, PT_GetTimeIndex(indexes)); + return 0; + } + } + } + return -1; +} + +int PT_EnumCache(PT_Indexes indexes, + int (*callback) (void *, const char *url, PT_Element), + void *arg) { + if (indexes != NULL && indexes->cil != NULL) { + struct_inthash_enum en = inthash_enum_new(indexes->cil); + inthash_chain *chain; + + while((chain = inthash_enum_next(&en))) { + const long int index_id = (long int) chain->value.intg; + const char *const url = chain->name; + + if (index_id >= 0 && index_id <= indexes->index_size) { + PT_Element item = + PT_ReadCache(indexes->index[index_id], url, + FETCH_HEADERS | FETCH_BODY); + if (item != NULL) { + int ret = callback(arg, url, item); + + PT_Element_Delete(&item); + if (ret != 0) + return ret; + } + } else { + proxytrack_print_log(CRITICAL, + "PT_ReadCache:Corrupted central index locator"); + return -1; + } + } + } + return 0; } time_t PT_Index_Timestamp(PT_Index index) { - return index->slots.common.timestamp; -} - -static int PT_LookupCache__New(PT_Index index, const char* url) { - int retCode; - MutexLock(&index->slots.formatNew.zFileLock); - { - retCode = PT_LookupCache__New_u(index, url); - } - MutexUnlock(&index->slots.formatNew.zFileLock); - return retCode; -} - -static int PT_LookupCache__New_u(PT_Index index_, const char* url) { - if (index_ != NULL) { - PT_Index__New index = &index_->slots.formatNew; - if (index->hash != NULL && index->zFile != NULL && url != NULL && *url != 0) { - int hash_pos_return; - if (strncmp(url, "http://", 7) == 0) - url += 7; - hash_pos_return = inthash_read(index->hash, url, NULL); - if (hash_pos_return) - return 1; - } - } - return 0; -} - -int PT_IndexMerge(PT_Indexes indexes, PT_Index *pindex) -{ - if (pindex != NULL && *pindex != NULL && (*pindex)->slots.common.hash != NULL - && indexes != NULL) - { - PT_Index index = *pindex; - struct_inthash_enum en = inthash_enum_new(index->slots.common.hash); - inthash_chain* chain; - int index_id = indexes->index_size++; - int nMerged = 0; - if ((indexes->index = realloc(indexes->index, sizeof(struct _PT_Index)*indexes->index_size)) != NULL) { - indexes->index[index_id] = index; - *pindex = NULL; - while((chain = inthash_enum_next(&en)) != NULL) { - const char * url = chain->name; - if (url != NULL && url[0] != '\0') { - intptr_t previous_index_id = 0; - if (inthash_read(indexes->cil, url, &previous_index_id)) { - if (previous_index_id >= 0 && previous_index_id < indexes->index_size) { - if (indexes->index[previous_index_id]->slots.common.timestamp > index->slots.common.timestamp) // existing entry is newer - break; - } else { - proxytrack_print_log(CRITICAL, "PT_IndexMerge:Corrupted central index locator"); - } - } - inthash_write(indexes->cil, chain->name, index_id); - nMerged++; - } - } - } else { - proxytrack_print_log(CRITICAL, "PT_IndexMerge:Memory exhausted"); - } - return nMerged; - } - return -1; -} - -void PT_Element_Delete(PT_Element *pentry) { - if (pentry != NULL) { - PT_Element entry = *pentry; - if (entry != NULL) { - if (entry->adr != NULL) { - free(entry->adr); - entry->adr = NULL; - } - if (entry->headers != NULL) { - free(entry->headers); - entry->headers = NULL; - } - if (entry->location != NULL) { - free(entry->location); - entry->location = NULL; - } - free(entry); - } - *pentry = NULL; - } -} - -PT_Element PT_ReadIndex(PT_Indexes indexes, const char* url, int flags) -{ - if (indexes != NULL) - { - intptr_t index_id; - if (strncmp(url, "http://", 7) == 0) - url += 7; - if (inthash_read(indexes->cil, url, &index_id)) { - if (index_id >= 0 && index_id <= indexes->index_size) { - PT_Element item = PT_ReadCache(indexes->index[index_id], url, flags); - if (item != NULL) { - item->indexId = (int) index_id; - return item; - } - } else { - proxytrack_print_log(CRITICAL, "PT_ReadCache:Corrupted central index locator"); - } - } - } - return NULL; -} - -int PT_LookupIndex(PT_Indexes indexes, const char* url) { - if (indexes != NULL) - { - intptr_t index_id; - if (strncmp(url, "http://", 7) == 0) - url += 7; - if (inthash_read(indexes->cil, url, &index_id)) { - if (index_id >= 0 && index_id <= indexes->index_size) { - return 1; - } else { - proxytrack_print_log(CRITICAL, "PT_ReadCache:Corrupted central index locator"); - } - } - } - return 0; + return index->slots.common.timestamp; +} + +static int PT_LookupCache__New(PT_Index index, const char *url) { + int retCode; + + MutexLock(&index->slots.formatNew.zFileLock); + { + retCode = PT_LookupCache__New_u(index, url); + } + MutexUnlock(&index->slots.formatNew.zFileLock); + return retCode; +} + +static int PT_LookupCache__New_u(PT_Index index_, const char *url) { + if (index_ != NULL) { + PT_Index__New index = &index_->slots.formatNew; + + if (index->hash != NULL && index->zFile != NULL && url != NULL && *url != 0) { + int hash_pos_return; + + if (strncmp(url, "http://", 7) == 0) + url += 7; + hash_pos_return = inthash_read(index->hash, url, NULL); + if (hash_pos_return) + return 1; + } + } + return 0; +} + +int PT_IndexMerge(PT_Indexes indexes, PT_Index * pindex) { + if (pindex != NULL && *pindex != NULL && (*pindex)->slots.common.hash != NULL + && indexes != NULL) { + PT_Index index = *pindex; + struct_inthash_enum en = inthash_enum_new(index->slots.common.hash); + inthash_chain *chain; + int index_id = indexes->index_size++; + int nMerged = 0; + + if ((indexes->index = + realloc(indexes->index, + sizeof(struct _PT_Index) * indexes->index_size)) != NULL) { + indexes->index[index_id] = index; + *pindex = NULL; + while((chain = inthash_enum_next(&en)) != NULL) { + const char *url = chain->name; + + if (url != NULL && url[0] != '\0') { + intptr_t previous_index_id = 0; + + if (inthash_read(indexes->cil, url, &previous_index_id)) { + if (previous_index_id >= 0 + && previous_index_id < indexes->index_size) { + if (indexes->index[previous_index_id]->slots.common.timestamp > index->slots.common.timestamp) // existing entry is newer + break; + } else { + proxytrack_print_log(CRITICAL, + "PT_IndexMerge:Corrupted central index locator"); + } + } + inthash_write(indexes->cil, chain->name, index_id); + nMerged++; + } + } + } else { + proxytrack_print_log(CRITICAL, "PT_IndexMerge:Memory exhausted"); + } + return nMerged; + } + return -1; +} + +void PT_Element_Delete(PT_Element * pentry) { + if (pentry != NULL) { + PT_Element entry = *pentry; + + if (entry != NULL) { + if (entry->adr != NULL) { + free(entry->adr); + entry->adr = NULL; + } + if (entry->headers != NULL) { + free(entry->headers); + entry->headers = NULL; + } + if (entry->location != NULL) { + free(entry->location); + entry->location = NULL; + } + free(entry); + } + *pentry = NULL; + } +} + +PT_Element PT_ReadIndex(PT_Indexes indexes, const char *url, int flags) { + if (indexes != NULL) { + intptr_t index_id; + + if (strncmp(url, "http://", 7) == 0) + url += 7; + if (inthash_read(indexes->cil, url, &index_id)) { + if (index_id >= 0 && index_id <= indexes->index_size) { + PT_Element item = PT_ReadCache(indexes->index[index_id], url, flags); + + if (item != NULL) { + item->indexId = (int) index_id; + return item; + } + } else { + proxytrack_print_log(CRITICAL, + "PT_ReadCache:Corrupted central index locator"); + } + } + } + return NULL; +} + +int PT_LookupIndex(PT_Indexes indexes, const char *url) { + if (indexes != NULL) { + intptr_t index_id; + + if (strncmp(url, "http://", 7) == 0) + url += 7; + if (inthash_read(indexes->cil, url, &index_id)) { + if (index_id >= 0 && index_id <= indexes->index_size) { + return 1; + } else { + proxytrack_print_log(CRITICAL, + "PT_ReadCache:Corrupted central index locator"); + } + } + } + return 0; } time_t PT_GetTimeIndex(PT_Indexes indexes) { - if (indexes != NULL && indexes->index_size > 0) - { - int i; - time_t maxt = indexes->index[0]->slots.common.timestamp; - for(i = 1 ; i < indexes->index_size ; i++) { - const time_t currt = indexes->index[i]->slots.common.timestamp; - if (currt > maxt) { - maxt = currt; - } - } - return maxt; - } - return (time_t) -1; + if (indexes != NULL && indexes->index_size > 0) { + int i; + time_t maxt = indexes->index[0]->slots.common.timestamp; + + for(i = 1; i < indexes->index_size; i++) { + const time_t currt = indexes->index[i]->slots.common.timestamp; + + if (currt > maxt) { + maxt = currt; + } + } + return maxt; + } + return (time_t) - 1; } PT_Index PT_GetIndex(PT_Indexes indexes, int indexId) { - if (indexes != NULL && indexId >= 0 && indexId < indexes->index_size) - { - return indexes->index[indexId]; - } - return NULL; + if (indexes != NULL && indexId >= 0 && indexId < indexes->index_size) { + return indexes->index[indexId]; + } + return NULL; } PT_Element PT_ElementNew(void) { - PT_Element r = NULL; - if ((r = calloc(sizeof(_PT_Element), 1)) == NULL) - return NULL; - r->statuscode=STATUSCODE_INVALID; - r->indexId = -1; - return r; -} + PT_Element r = NULL; -PT_Element PT_ReadCache(PT_Index index, const char* url, int flags) { - if (index != NULL && SAFE_INDEX(index)) { - return _IndexFuncts[index->type].PT_ReadCache(index, url, flags); - } - return NULL; + if ((r = calloc(sizeof(_PT_Element), 1)) == NULL) + return NULL; + r->statuscode = STATUSCODE_INVALID; + r->indexId = -1; + return r; } -static PT_Element PT_ReadCache__New(PT_Index index, const char* url, int flags) { - PT_Element retCode; - MutexLock(&index->slots.formatNew.zFileLock); - { - retCode = PT_ReadCache__New_u(index, url, flags); - } - MutexUnlock(&index->slots.formatNew.zFileLock); - return retCode; +PT_Element PT_ReadCache(PT_Index index, const char *url, int flags) { + if (index != NULL && SAFE_INDEX(index)) { + return _IndexFuncts[index->type].PT_ReadCache(index, url, flags); + } + return NULL; } +static PT_Element PT_ReadCache__New(PT_Index index, const char *url, int flags) { + PT_Element retCode; + + MutexLock(&index->slots.formatNew.zFileLock); + { + retCode = PT_ReadCache__New_u(index, url, flags); + } + MutexUnlock(&index->slots.formatNew.zFileLock); + return retCode; +} /* ------------------------------------------------------------ */ /* New HTTrack cache (new.zip) format */ @@ -825,167 +874,182 @@ static PT_Element PT_ReadCache__New(PT_Index index, const char* url, int flags) } while(0) int PT_LoadCache__New(PT_Index index_, const char *filename) { - if (index_ != NULL && filename != NULL) { - PT_Index__New index = &index_->slots.formatNew; - unzFile zFile = index->zFile = unzOpen(filename); - index->timestamp = file_timestamp(filename); - MutexInit(&index->zFileLock); - - // Opened ? - if (zFile!=NULL) { - const char * abpath; - int slashes; - inthash hashtable = index->hash; - - /* Compute base path for this index - the filename MUST be absolute! */ - for(slashes = 2, abpath = filename + (int)strlen(filename) - 1 - ; abpath > filename && ( ( *abpath != '/'&& *abpath != '\\' ) || --slashes > 0) - ; abpath--); - index->path[0] = '\0'; - if (slashes == 0 && *abpath != 0) { - int i; - strncat(index->path, filename, (int) ( abpath - filename ) + 1 ); - for(i = 0 ; index->path[i] != 0 ; i++) { - if (index->path[i] == '\\') { - index->path[i] = '/'; - } - } - } - - /* Ready directory entries */ - if (unzGoToFirstFile(zFile) == Z_OK) { - char comment[128]; - char filename[HTS_URLMAXSIZE * 4]; - int entries = 0; - int firstSeen = 0; - memset(comment, 0, sizeof(comment)); // for truncated reads - do { - int readSizeHeader = 0; - filename[0] = '\0'; - comment[0] = '\0'; - if (unzOpenCurrentFile(zFile) == Z_OK) { - if ( - (readSizeHeader = unzGetLocalExtrafield(zFile, comment, sizeof(comment) - 2)) > 0 - && - unzGetCurrentFileInfo(zFile, NULL, filename, sizeof(filename) - 2, NULL, 0, NULL, 0) == Z_OK - ) - { - long int pos = (long int) unzGetOffset(zFile); - assertf(readSizeHeader < sizeof(comment)); - comment[readSizeHeader] = '\0'; - entries++; - if (pos > 0) { - int dataincache = 0; // data in cache ? - char* filenameIndex = filename; - if (strncmp(filenameIndex, "http://", 7) == 0) { - filenameIndex += 7; - } - if (comment[0] != '\0') { - int maxLine = 2; - char* a = comment; - while(*a && maxLine-- > 0) { // parse only few first lines - char line[1024]; - line[0] = '\0'; - a+=binput(a, line, sizeof(line) - 2); - if (strncmp(line, "X-In-Cache:", 11) == 0) { - if (strcmp(line, "X-In-Cache: 1") == 0) { - dataincache = 1; - } else { - dataincache = 0; - } - break; - } - } - } - if (dataincache) - inthash_add(hashtable, filenameIndex, pos); - else - inthash_add(hashtable, filenameIndex, -pos); - - /* First link as starting URL */ - if (!firstSeen) { - if (strstr(filenameIndex, "/robots.txt") == NULL) { - firstSeen = 1; - if (!link_has_authority(filenameIndex)) - strcat(index->startUrl, "http://"); - strcat(index->startUrl, filenameIndex); - } - } - } else { - fprintf(stderr, "Corrupted cache meta entry #%d"LF, (int)entries); - } - } else { - fprintf(stderr, "Corrupted cache entry #%d"LF, (int)entries); - } - unzCloseCurrentFile(zFile); - } else { - fprintf(stderr, "Corrupted cache entry #%d"LF, (int)entries); - } - } while( unzGoToNextFile(zFile) == Z_OK ); - return 1; - } else { - inthash_delete(&index->hash); - index = NULL; - } - } else { - index = NULL; - } - } - return 0; -} - -static PT_Element PT_ReadCache__New_u(PT_Index index_, const char* url, int flags) -{ - PT_Index__New index = (PT_Index__New) &index_->slots.formatNew; - char location_default[HTS_URLMAXSIZE*2]; - char previous_save[HTS_URLMAXSIZE*2]; - char previous_save_[HTS_URLMAXSIZE*2]; - char catbuff[CATBUFF_SIZE]; + if (index_ != NULL && filename != NULL) { + PT_Index__New index = &index_->slots.formatNew; + unzFile zFile = index->zFile = unzOpen(filename); + + index->timestamp = file_timestamp(filename); + MutexInit(&index->zFileLock); + + // Opened ? + if (zFile != NULL) { + const char *abpath; + int slashes; + inthash hashtable = index->hash; + + /* Compute base path for this index - the filename MUST be absolute! */ + for(slashes = 2, abpath = filename + (int) strlen(filename) - 1; + abpath > filename && ((*abpath != '/' && *abpath != '\\') + || --slashes > 0); + abpath--) ; + index->path[0] = '\0'; + if (slashes == 0 && *abpath != 0) { + int i; + + strncat(index->path, filename, (int) (abpath - filename) + 1); + for(i = 0; index->path[i] != 0; i++) { + if (index->path[i] == '\\') { + index->path[i] = '/'; + } + } + } + + /* Ready directory entries */ + if (unzGoToFirstFile(zFile) == Z_OK) { + char comment[128]; + char filename[HTS_URLMAXSIZE * 4]; + int entries = 0; + int firstSeen = 0; + + memset(comment, 0, sizeof(comment)); // for truncated reads + do { + int readSizeHeader = 0; + + filename[0] = '\0'; + comment[0] = '\0'; + if (unzOpenCurrentFile(zFile) == Z_OK) { + if ((readSizeHeader = + unzGetLocalExtrafield(zFile, comment, sizeof(comment) - 2)) > 0 + && unzGetCurrentFileInfo(zFile, NULL, filename, + sizeof(filename) - 2, NULL, 0, NULL, + 0) == Z_OK) { + long int pos = (long int) unzGetOffset(zFile); + + assertf(readSizeHeader < sizeof(comment)); + comment[readSizeHeader] = '\0'; + entries++; + if (pos > 0) { + int dataincache = 0; // data in cache ? + char *filenameIndex = filename; + + if (strncmp(filenameIndex, "http://", 7) == 0) { + filenameIndex += 7; + } + if (comment[0] != '\0') { + int maxLine = 2; + char *a = comment; + + while(*a && maxLine-- > 0) { // parse only few first lines + char line[1024]; + + line[0] = '\0'; + a += binput(a, line, sizeof(line) - 2); + if (strncmp(line, "X-In-Cache:", 11) == 0) { + if (strcmp(line, "X-In-Cache: 1") == 0) { + dataincache = 1; + } else { + dataincache = 0; + } + break; + } + } + } + if (dataincache) + inthash_add(hashtable, filenameIndex, pos); + else + inthash_add(hashtable, filenameIndex, -pos); + + /* First link as starting URL */ + if (!firstSeen) { + if (strstr(filenameIndex, "/robots.txt") == NULL) { + firstSeen = 1; + if (!link_has_authority(filenameIndex)) + strcat(index->startUrl, "http://"); + strcat(index->startUrl, filenameIndex); + } + } + } else { + fprintf(stderr, "Corrupted cache meta entry #%d" LF, + (int) entries); + } + } else { + fprintf(stderr, "Corrupted cache entry #%d" LF, (int) entries); + } + unzCloseCurrentFile(zFile); + } else { + fprintf(stderr, "Corrupted cache entry #%d" LF, (int) entries); + } + } while(unzGoToNextFile(zFile) == Z_OK); + return 1; + } else { + inthash_delete(&index->hash); + index = NULL; + } + } else { + index = NULL; + } + } + return 0; +} + +static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url, + int flags) { + PT_Index__New index = (PT_Index__New) & index_->slots.formatNew; + char location_default[HTS_URLMAXSIZE * 2]; + char previous_save[HTS_URLMAXSIZE * 2]; + char previous_save_[HTS_URLMAXSIZE * 2]; + char catbuff[CATBUFF_SIZE]; intptr_t hash_pos; int hash_pos_return; - PT_Element r = NULL; - if (index == NULL || index->hash == NULL || index->zFile == NULL || url == NULL || *url == 0) - return NULL; - if ((r = PT_ElementNew()) == NULL) - return NULL; - location_default[0] = '\0'; - previous_save[0] = previous_save_[0] = '\0'; + PT_Element r = NULL; + + if (index == NULL || index->hash == NULL || index->zFile == NULL + || url == NULL || *url == 0) + return NULL; + if ((r = PT_ElementNew()) == NULL) + return NULL; + location_default[0] = '\0'; + previous_save[0] = previous_save_[0] = '\0'; memset(r, 0, sizeof(_PT_Element)); r->location = location_default; - strcpy(r->location, ""); - if (strncmp(url, "http://", 7) == 0) - url += 7; + strcpy(r->location, ""); + if (strncmp(url, "http://", 7) == 0) + url += 7; hash_pos_return = inthash_read(index->hash, url, &hash_pos); if (hash_pos_return) { uLong posInZip; + if (hash_pos > 0) { posInZip = (uLong) hash_pos; } else { - posInZip = (uLong) -hash_pos; + posInZip = (uLong) - hash_pos; } - if (unzSetOffset(index->zFile, posInZip) == Z_OK) { + if (unzSetOffset(index->zFile, posInZip) == Z_OK) { /* Read header (Max 8KiB) */ if (unzOpenCurrentFile(index->zFile) == Z_OK) { char headerBuff[8192 + 2]; int readSizeHeader; int totalHeader = 0; int dataincache = 0; - + /* For BIG comments */ - headerBuff[0] - = headerBuff[sizeof(headerBuff) - 1] - = headerBuff[sizeof(headerBuff) - 2] + headerBuff[0] + = headerBuff[sizeof(headerBuff) - 1] + = headerBuff[sizeof(headerBuff) - 2] = headerBuff[sizeof(headerBuff) - 3] = '\0'; - if ( (readSizeHeader = unzGetLocalExtrafield(index->zFile, headerBuff, sizeof(headerBuff) - 2)) > 0) - { + if ((readSizeHeader = + unzGetLocalExtrafield(index->zFile, headerBuff, + sizeof(headerBuff) - 2)) > 0) { int offset = 0; char line[HTS_URLMAXSIZE + 2]; int lineEof = 0; + headerBuff[readSizeHeader] = '\0'; do { - char* value; + char *value; + line[0] = '\0'; offset += binput(headerBuff + offset, line, sizeof(line) - 2); if (line[0] == '\0') { @@ -994,175 +1058,187 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char* url, int flag value = strchr(line, ':'); if (value != NULL) { *value++ = '\0'; - if (*value == ' ' || *value == '\t') value++; + if (*value == ' ' || *value == '\t') + value++; ZIP_READFIELD_INT(line, value, "X-In-Cache", dataincache); ZIP_READFIELD_INT(line, value, "X-Statuscode", r->statuscode); - ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r->msg); // msg - ZIP_READFIELD_INT(line, value, "X-Size", r->size); // size - ZIP_READFIELD_STRING(line, value, "Content-Type", r->contenttype); // contenttype - ZIP_READFIELD_STRING(line, value, "X-Charset", r->charset); // contenttype - ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified - ZIP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag - ZIP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved - ZIP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo); // Content-disposition + ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r->msg); // msg + ZIP_READFIELD_INT(line, value, "X-Size", r->size); // size + ZIP_READFIELD_STRING(line, value, "Content-Type", r->contenttype); // contenttype + ZIP_READFIELD_STRING(line, value, "X-Charset", r->charset); // contenttype + ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified + ZIP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag + ZIP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved + ZIP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo); // Content-disposition //ZIP_READFIELD_STRING(line, value, "X-Addr", ..); // Original address //ZIP_READFIELD_STRING(line, value, "X-Fil", ..); // Original URI filename - ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename - if (line[0] != '\0') { - int len = r->headers ? ((int) strlen(r->headers)) : 0; - int nlen = (int) ( strlen(line) + 2 + strlen(value) + sizeof("\r\n") + 1 ); - r->headers = realloc(r->headers, len + nlen); - r->headers[len] = '\0'; - strcat(r->headers, line); - strcat(r->headers, ": "); - strcat(r->headers, value); - strcat(r->headers, "\r\n"); - } + ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename + if (line[0] != '\0') { + int len = r->headers ? ((int) strlen(r->headers)) : 0; + int nlen = + (int) (strlen(line) + 2 + strlen(value) + sizeof("\r\n") + 1); + r->headers = realloc(r->headers, len + nlen); + r->headers[len] = '\0'; + strcat(r->headers, line); + strcat(r->headers, ": "); + strcat(r->headers, value); + strcat(r->headers, "\r\n"); + } } } while(offset < readSizeHeader && !lineEof); totalHeader = offset; - /* Previous entry */ - if (previous_save_[0] != '\0') { - int pathLen = (int) strlen(index->path); - if (pathLen > 0 && strncmp(previous_save_, index->path, pathLen) == 0) { // old (<3.40) buggy format - strcpy(previous_save, previous_save_); - } - // relative ? (hack) - else if (index->safeCache - || (previous_save_[0] != '/' // /home/foo/bar.gif - && ( !isalpha(previous_save_[0]) || previous_save_[1] != ':' ) ) // c:/home/foo/bar.gif - ) - { - index->safeCache = 1; - sprintf(previous_save, "%s%s", index->path, previous_save_); - } - // bogus format (includes buggy absolute path) - else { - /* guess previous path */ - if (index->fixedPath == 0) { - const char * start = jump_protocol_and_auth(url); - const char * end = start ? strchr(start, '/') : NULL; - int len = (int) (end - start); - if (start != NULL && end != NULL && len > 0 && len < 128) { - char piece[128 + 2]; - const char * where; - piece[0] = '\0'; - strncat(piece, start, len); - if ((where = strstr(previous_save_, piece)) != NULL) { - index->fixedPath = (int) (where - previous_save_); // offset to relative path - } - } - } - if (index->fixedPath > 0) { - int saveLen = (int) strlen(previous_save_); - if (index->fixedPath < saveLen) { - sprintf(previous_save, "%s%s", index->path, previous_save_ + index->fixedPath); - } else { - sprintf(r->msg, "Bogus fixePath prefix for %s (prefixLen=%d)", previous_save_, (int)index->fixedPath); - r->statuscode = STATUSCODE_INVALID; - } - } else { - sprintf(previous_save, "%s%s", index->path, previous_save_); - } - } - } + /* Previous entry */ + if (previous_save_[0] != '\0') { + int pathLen = (int) strlen(index->path); + + if (pathLen > 0 && strncmp(previous_save_, index->path, pathLen) == 0) { // old (<3.40) buggy format + strcpy(previous_save, previous_save_); + } + // relative ? (hack) + else if (index->safeCache || (previous_save_[0] != '/' // /home/foo/bar.gif + && (!isalpha(previous_save_[0]) || previous_save_[1] != ':')) // c:/home/foo/bar.gif + ) { + index->safeCache = 1; + sprintf(previous_save, "%s%s", index->path, previous_save_); + } + // bogus format (includes buggy absolute path) + else { + /* guess previous path */ + if (index->fixedPath == 0) { + const char *start = jump_protocol_and_auth(url); + const char *end = start ? strchr(start, '/') : NULL; + int len = (int) (end - start); + + if (start != NULL && end != NULL && len > 0 && len < 128) { + char piece[128 + 2]; + const char *where; + + piece[0] = '\0'; + strncat(piece, start, len); + if ((where = strstr(previous_save_, piece)) != NULL) { + index->fixedPath = (int) (where - previous_save_); // offset to relative path + } + } + } + if (index->fixedPath > 0) { + int saveLen = (int) strlen(previous_save_); + + if (index->fixedPath < saveLen) { + sprintf(previous_save, "%s%s", index->path, + previous_save_ + index->fixedPath); + } else { + sprintf(r->msg, "Bogus fixePath prefix for %s (prefixLen=%d)", + previous_save_, (int) index->fixedPath); + r->statuscode = STATUSCODE_INVALID; + } + } else { + sprintf(previous_save, "%s%s", index->path, previous_save_); + } + } + } /* Complete fields */ - r->adr=NULL; - if (r->statuscode != STATUSCODE_INVALID) { /* Can continue */ + r->adr = NULL; + if (r->statuscode != STATUSCODE_INVALID) { /* Can continue */ int ok = 0; - + // Court-circuit: // Peut-on stocker le fichier directement sur disque? if (ok) { if (r->msg[0] == '\0') { - strcpy(r->msg,"Cache Read Error : Unexpected error"); + strcpy(r->msg, "Cache Read Error : Unexpected error"); } - } else { // lire en mémoire - + } else { // lire en mémoire + if (!dataincache) { - /* Read in memory from cache */ - if (flags & FETCH_BODY) { - if (strnotempty(previous_save)) { - FILE* fp = fopen(fconv(catbuff,previous_save), "rb"); - if (fp != NULL) { - r->adr = (char*) malloc(r->size + 4); - if (r->adr != NULL) { - if (r->size > 0 && fread(r->adr, 1, r->size, fp) != r->size) { + /* Read in memory from cache */ + if (flags & FETCH_BODY) { + if (strnotempty(previous_save)) { + FILE *fp = fopen(fconv(catbuff, previous_save), "rb"); + + if (fp != NULL) { + r->adr = (char *) malloc(r->size + 4); + if (r->adr != NULL) { + if (r->size > 0 + && fread(r->adr, 1, r->size, fp) != r->size) { int last_errno = errno; - r->statuscode=STATUSCODE_INVALID; - sprintf(r->msg,"Read error in cache disk data: %s", strerror(last_errno)); - } - } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Read error (memory exhausted) from cache"); - } - fclose(fp); - } else { - r->statuscode=STATUSCODE_INVALID; - sprintf(r->msg, "Read error (can't open '%s') from cache", fconv(catbuff,previous_save)); - } - } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cached file name is invalid"); - } - } + + r->statuscode = STATUSCODE_INVALID; + sprintf(r->msg, "Read error in cache disk data: %s", + strerror(last_errno)); + } + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, + "Read error (memory exhausted) from cache"); + } + fclose(fp); + } else { + r->statuscode = STATUSCODE_INVALID; + sprintf(r->msg, "Read error (can't open '%s') from cache", + fconv(catbuff, previous_save)); + } + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cached file name is invalid"); + } + } } else { - // lire fichier (d'un coup) - if (flags & FETCH_BODY) { - r->adr=(char*) malloc(r->size+1); - if (r->adr!=NULL) { - if (unzReadCurrentFile(index->zFile, r->adr, (unsigned int) r->size) != r->size) { // erreur - free(r->adr); - r->adr=NULL; - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cache Read Error : Read Data"); - } else - *(r->adr+r->size)='\0'; - //printf(">%s status %d\n",back[p].r->contenttype,back[p].r->statuscode); - } else { // erreur - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cache Memory Error"); - } - } - } + // lire fichier (d'un coup) + if (flags & FETCH_BODY) { + r->adr = (char *) malloc(r->size + 1); + if (r->adr != NULL) { + if (unzReadCurrentFile(index->zFile, r->adr, (unsigned int) r->size) != r->size) { // erreur + free(r->adr); + r->adr = NULL; + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Read Data"); + } else + *(r->adr + r->size) = '\0'; + //printf(">%s status %d\n",back[p].r->contenttype,back[p].r->statuscode); + } else { // erreur + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Memory Error"); + } + } + } } - } // si save==null, ne rien charger (juste en tête) + } // si save==null, ne rien charger (juste en tête) } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cache Read Error : Read Header Data"); + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Read Header Data"); } unzCloseCurrentFile(index->zFile); } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cache Read Error : Open File"); + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Open File"); } } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cache Read Error : Bad Offset"); + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Bad Offset"); } } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"File Cache Entry Not Found"); - } - if (r->location[0] != '\0') { - r->location = strdup(r->location); - } else { - r->location = NULL; - } + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "File Cache Entry Not Found"); + } + if (r->location[0] != '\0') { + r->location = strdup(r->location); + } else { + r->location = NULL; + } return r; } static int PT_SaveCache__New_Fun(void *arg, const char *url, PT_Element element) { - zipFile zFileOut = (zipFile) arg; - char headers[8192]; + zipFile zFileOut = (zipFile) arg; + char headers[8192]; int headersSize; zip_fileinfo fi; - int zErr; - const char *url_adr = ""; - const char *url_fil = ""; + int zErr; + const char *url_adr = ""; + const char *url_fil = ""; headers[0] = '\0'; headersSize = 0; @@ -1172,14 +1248,16 @@ static int PT_SaveCache__New_Fun(void *arg, const char *url, PT_Element element) headersSize = 0; /* */ { - char* message; + char *message; + if (strlen(element->msg) < 32) { message = element->msg; } else { message = "(See X-StatusMessage)"; } /* 64 characters MAX for first line */ - sprintf(headers + headersSize, "HTTP/1.%c %d %s\r\n", '1', element->statuscode, element->msg); + sprintf(headers + headersSize, "HTTP/1.%c %d %s\r\n", '1', + element->statuscode, element->msg); } headersSize += (int) strlen(headers + headersSize); @@ -1187,22 +1265,23 @@ static int PT_SaveCache__New_Fun(void *arg, const char *url, PT_Element element) ZIP_FIELD_INT_FORCE(headers, headersSize, "X-In-Cache", 1); ZIP_FIELD_INT(headers, headersSize, "X-StatusCode", element->statuscode); ZIP_FIELD_STRING(headers, headersSize, "X-StatusMessage", element->msg); - ZIP_FIELD_INT(headers, headersSize, "X-Size", element->size); // size - ZIP_FIELD_STRING(headers, headersSize, "Content-Type", element->contenttype); // contenttype - ZIP_FIELD_STRING(headers, headersSize, "X-Charset", element->charset); // contenttype - ZIP_FIELD_STRING(headers, headersSize, "Last-Modified", element->lastmodified); // last-modified - ZIP_FIELD_STRING(headers, headersSize, "Etag", element->etag); // Etag - ZIP_FIELD_STRING(headers, headersSize, "Location", element->location); // 'location' pour moved - ZIP_FIELD_STRING(headers, headersSize, "Content-Disposition", element->cdispo); // Content-disposition - ZIP_FIELD_STRING(headers, headersSize, "X-Addr", url_adr); // Original address - ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil); // Original URI filename - ZIP_FIELD_STRING(headers, headersSize, "X-Save", ""); // Original save filename - + ZIP_FIELD_INT(headers, headersSize, "X-Size", element->size); // size + ZIP_FIELD_STRING(headers, headersSize, "Content-Type", element->contenttype); // contenttype + ZIP_FIELD_STRING(headers, headersSize, "X-Charset", element->charset); // contenttype + ZIP_FIELD_STRING(headers, headersSize, "Last-Modified", element->lastmodified); // last-modified + ZIP_FIELD_STRING(headers, headersSize, "Etag", element->etag); // Etag + ZIP_FIELD_STRING(headers, headersSize, "Location", element->location); // 'location' pour moved + ZIP_FIELD_STRING(headers, headersSize, "Content-Disposition", element->cdispo); // Content-disposition + ZIP_FIELD_STRING(headers, headersSize, "X-Addr", url_adr); // Original address + ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil); // Original URI filename + ZIP_FIELD_STRING(headers, headersSize, "X-Save", ""); // Original save filename + /* Time */ memset(&fi, 0, sizeof(fi)); if (element->lastmodified[0] != '\0') { - struct tm buffer; - struct tm* tm_s = convert_time_rfc822(&buffer, element->lastmodified); + struct tm buffer; + struct tm *tm_s = convert_time_rfc822(&buffer, element->lastmodified); + if (tm_s) { fi.tmz_date.tm_sec = (uInt) tm_s->tm_sec; fi.tmz_date.tm_min = (uInt) tm_s->tm_min; @@ -1214,333 +1293,355 @@ static int PT_SaveCache__New_Fun(void *arg, const char *url, PT_Element element) } /* Open file - NOTE: headers in "comment" */ - if ((zErr = zipOpenNewFileInZip(zFileOut, - url, - &fi, - /* - Store headers in realtime in the local file directory as extra field - In case of crash, we'll be able to recover the whole ZIP file by rescanning it - */ - headers, - (uInt) strlen(headers), - NULL, - 0, - NULL, /* comment */ - Z_DEFLATED, - Z_DEFAULT_COMPRESSION)) != Z_OK) - { + if ((zErr = zipOpenNewFileInZip(zFileOut, url, &fi, + /* + Store headers in realtime in the local file directory as extra field + In case of crash, we'll be able to recover the whole ZIP file by rescanning it + */ + headers, (uInt) strlen(headers), NULL, 0, NULL, /* comment */ + Z_DEFLATED, Z_DEFAULT_COMPRESSION)) != Z_OK) { int zip_zipOpenNewFileInZip_failed = 0; + assertf(zip_zipOpenNewFileInZip_failed); } - - /* Write data in cache */ - if (element->size > 0 && element->adr != NULL) { - if ((zErr = zipWriteInFileInZip(zFileOut, element->adr, (int) element->size)) != Z_OK) { - int zip_zipWriteInFileInZip_failed = 0; - assertf(zip_zipWriteInFileInZip_failed); - } - } - - /* Close */ - if ((zErr = zipCloseFileInZip(zFileOut)) != Z_OK) { - int zip_zipCloseFileInZip_failed = 0; + + /* Write data in cache */ + if (element->size > 0 && element->adr != NULL) { + if ((zErr = + zipWriteInFileInZip(zFileOut, element->adr, + (int) element->size)) != Z_OK) { + int zip_zipWriteInFileInZip_failed = 0; + + assertf(zip_zipWriteInFileInZip_failed); + } + } + + /* Close */ + if ((zErr = zipCloseFileInZip(zFileOut)) != Z_OK) { + int zip_zipCloseFileInZip_failed = 0; + assertf(zip_zipCloseFileInZip_failed); } /* Flush */ if ((zErr = zipFlush(zFileOut)) != 0) { int zip_zipFlush_failed = 0; + assertf(zip_zipFlush_failed); } - return 0; + return 0; } static int PT_SaveCache__New(PT_Indexes indexes, const char *filename) { - zipFile zFileOut = zipOpen(filename, 0); - if (zFileOut != NULL) { - int ret = PT_EnumCache(indexes, PT_SaveCache__New_Fun, (void *) zFileOut); - zipClose(zFileOut, "Created by HTTrack Website Copier/ProxyTrack "PROXYTRACK_VERSION); - zFileOut = NULL; - if (ret != 0) - (void) unlink(filename); - return ret; - } - return -1; + zipFile zFileOut = zipOpen(filename, 0); + + if (zFileOut != NULL) { + int ret = PT_EnumCache(indexes, PT_SaveCache__New_Fun, (void *) zFileOut); + + zipClose(zFileOut, + "Created by HTTrack Website Copier/ProxyTrack " + PROXYTRACK_VERSION); + zFileOut = NULL; + if (ret != 0) + (void) unlink(filename); + return ret; + } + return -1; } - - /* ------------------------------------------------------------ */ /* Old HTTrack cache (dat/ndx) format */ /* ------------------------------------------------------------ */ -static int cache_brstr(char* adr,char* s) { +static int cache_brstr(char *adr, char *s) { int i; int off; char buff[256 + 1]; - off=binput(adr,buff,256); - adr+=off; - sscanf(buff,"%d",&i); - if (i>0) - strncpy(s,adr,i); - *(s+i)='\0'; - off+=i; + + off = binput(adr, buff, 256); + adr += off; + sscanf(buff, "%d", &i); + if (i > 0) + strncpy(s, adr, i); + *(s + i) = '\0'; + off += i; return off; } -static void cache_rstr(FILE* fp,char* s) { +static void cache_rstr(FILE * fp, char *s) { INTsys i; - char buff[256+4]; - linput(fp,buff,256); - sscanf(buff,INTsysP,&i); - if (i < 0 || i > 32768) /* error, something nasty happened */ - i=0; - if (i>0) { - if ((int) fread(s,1,i,fp) != i) { + char buff[256 + 4]; + + linput(fp, buff, 256); + sscanf(buff, INTsysP, &i); + if (i < 0 || i > 32768) /* error, something nasty happened */ + i = 0; + if (i > 0) { + if ((int) fread(s, 1, i, fp) != i) { int fread_cache_failed = 0; + assertf(fread_cache_failed); } } - *(s+i)='\0'; + *(s + i) = '\0'; } -static char* cache_rstr_addr(FILE* fp) { +static char *cache_rstr_addr(FILE * fp) { INTsys i; - char* addr = NULL; - char buff[256+4]; - linput(fp,buff,256); - sscanf(buff,"%d",&i); - if (i < 0 || i > 32768) /* error, something nasty happened */ - i=0; + char *addr = NULL; + char buff[256 + 4]; + + linput(fp, buff, 256); + sscanf(buff, "%d", &i); + if (i < 0 || i > 32768) /* error, something nasty happened */ + i = 0; if (i > 0) { addr = malloc(i + 1); if (addr != NULL) { - if ((int) fread(addr,1,i,fp) != i) { + if ((int) fread(addr, 1, i, fp) != i) { int fread_cache_failed = 0; + assertf(fread_cache_failed); } - *(addr+i)='\0'; + *(addr + i) = '\0'; } } return addr; } -static void cache_rint(FILE* fp,int* i) { +static void cache_rint(FILE * fp, int *i) { char s[256]; - cache_rstr(fp,s); - sscanf(s,"%d",i); + + cache_rstr(fp, s); + sscanf(s, "%d", i); } -static void cache_rLLint(FILE* fp,unsigned long* i) { - int l; +static void cache_rLLint(FILE * fp, unsigned long *i) { + int l; char s[256]; - cache_rstr(fp,s); - sscanf(s,"%d",&l); - *i = (unsigned long)l; + + cache_rstr(fp, s); + sscanf(s, "%d", &l); + *i = (unsigned long) l; } static int PT_LoadCache__Old(PT_Index index_, const char *filename) { - if (index_ != NULL && filename != NULL) { - char * pos = strrchr(filename, '.'); - PT_Index__Old cache = &index_->slots.formatOld; - long int ndxSize; - cache->filenameDat[0] = '\0'; - cache->filenameNdx[0] = '\0'; - cache->path[0] = '\0'; - - { - PT_Index__Old index = cache; - const char * abpath; - int slashes; - /* -------------------- COPY OF THE __New() CODE -------------------- */ - /* Compute base path for this index - the filename MUST be absolute! */ - for(slashes = 2, abpath = filename + (int)strlen(filename) - 1 - ; abpath > filename && ( ( *abpath != '/'&& *abpath != '\\' ) || --slashes > 0) - ; abpath--); - index->path[0] = '\0'; - if (slashes == 0 && *abpath != 0) { - int i; - strncat(index->path, filename, (int) ( abpath - filename ) + 1 ); - for(i = 0 ; index->path[i] != 0 ; i++) { - if (index->path[i] == '\\') { - index->path[i] = '/'; - } - } - } - /* -------------------- END OF COPY OF THE __New() CODE -------------------- */ - } - - /* Index/data filenames */ - if (pos != NULL) { - int nLen = (int) (pos - filename); - strncat(cache->filenameDat, filename, nLen); - strncat(cache->filenameNdx, filename, nLen); - strcat(cache->filenameDat, ".dat"); - strcat(cache->filenameNdx, ".ndx"); - } - ndxSize = filesize(cache->filenameNdx); - cache->timestamp = file_timestamp(cache->filenameDat); - cache->dat = fopen(cache->filenameDat, "rb"); - cache->ndx = fopen(cache->filenameNdx, "rb"); - if (cache->dat != NULL && cache->ndx != NULL && ndxSize > 0) { - char * use = malloc(ndxSize + 1); - if (fread(use, 1, ndxSize, cache->ndx) == ndxSize) { - char firstline[256]; - char* a=use; - use[ndxSize] = '\0'; - a += cache_brstr(a, firstline); - if (strncmp(firstline,"CACHE-",6)==0) { // Nouvelle version du cache - if (strncmp(firstline,"CACHE-1.",8)==0) { // Version 1.1x - cache->version=(int)(firstline[8]-'0'); // cache 1.x - if (cache->version <= 5) { - a+=cache_brstr(a,firstline); - strcpy(cache->lastmodified,firstline); - } else { - // fprintf(opt->errlog,"Cache: version 1.%d not supported, ignoring current cache"LF,cache->version); - fclose(cache->dat); - cache->dat=NULL; - free(use); - use=NULL; - } - } else { // non supporté - // fspc(opt->errlog,"error"); fprintf(opt->errlog,"Cache: %s not supported, ignoring current cache"LF,firstline); - fclose(cache->dat); - cache->dat=NULL; - free(use); - use=NULL; - } - /* */ - } else { // Vieille version du cache - /* */ - // hts_log_print(opt, LOG_WARNING, "Cache: importing old cache format"); - cache->version=0; // cache 1.0 - strcpy(cache->lastmodified,firstline); - } - - /* Create hash table for the cache (MUCH FASTER!) */ - if (use) { - char line[HTS_URLMAXSIZE*2]; - char linepos[256]; - int pos; - int firstSeen = 0; - while ( (a!=NULL) && (a < (use + ndxSize) ) ) { - a=strchr(a+1,'\n'); /* start of line */ - if (a) { - a++; - /* read "host/file" */ - a+=binput(a,line,HTS_URLMAXSIZE); - a+=binput(a,line+strlen(line),HTS_URLMAXSIZE); - /* read position */ - a+=binput(a,linepos,200); - sscanf(linepos,"%d",&pos); - - /* Add entry */ - inthash_add(cache->hash,line,pos); - - /* First link as starting URL */ - if (!firstSeen) { - if (strstr(line, "/robots.txt") == NULL) { - PT_Index__Old index = cache; - firstSeen = 1; - if (!link_has_authority(line)) - strcat(index->startUrl, "http://"); - strcat(index->startUrl, line); - } - } - - } - } - /* Not needed anymore! */ - free(use); - use=NULL; - return 1; - } - } - } - } - return 0; -} - -static String DecodeUrl(const char * url) { - int i; - String s = STRING_EMPTY; - StringClear(s); - for(i = 0 ; url[i] != '\0' ; i++) { - if (url[i] == '+') { - StringAddchar(s, ' '); - } else if (url[i] == '%') { - if (url[i + 1] == '%') { - StringAddchar(s, '%'); - i++; - } else if (url[i + 1] != 0 && url[i + 2] != 0) { - char tmp[3]; - int codepoint = 0; - tmp[0] = url[i + 1]; - tmp[1] = url[i + 2]; - tmp[2] = 0; - if (sscanf(tmp, "%x", &codepoint) == 1) { - StringAddchar(s, (char)codepoint); - } - i += 2; - } - } else { - StringAddchar(s, url[i]); - } - } - return s; -} - -static PT_Element PT_ReadCache__Old(PT_Index index, const char* url, int flags) { - PT_Element retCode; - MutexLock(&index->slots.formatOld.fileLock); - { - retCode = PT_ReadCache__Old_u(index, url, flags); - } - MutexUnlock(&index->slots.formatOld.fileLock); - return retCode; -} - -static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char* url, int flags) { - PT_Index__Old cache = (PT_Index__Old) &index_->slots.formatOld; + if (index_ != NULL && filename != NULL) { + char *pos = strrchr(filename, '.'); + PT_Index__Old cache = &index_->slots.formatOld; + long int ndxSize; + + cache->filenameDat[0] = '\0'; + cache->filenameNdx[0] = '\0'; + cache->path[0] = '\0'; + + { + PT_Index__Old index = cache; + const char *abpath; + int slashes; + + /* -------------------- COPY OF THE __New() CODE -------------------- */ + /* Compute base path for this index - the filename MUST be absolute! */ + for(slashes = 2, abpath = filename + (int) strlen(filename) - 1; + abpath > filename && ((*abpath != '/' && *abpath != '\\') + || --slashes > 0); + abpath--) ; + index->path[0] = '\0'; + if (slashes == 0 && *abpath != 0) { + int i; + + strncat(index->path, filename, (int) (abpath - filename) + 1); + for(i = 0; index->path[i] != 0; i++) { + if (index->path[i] == '\\') { + index->path[i] = '/'; + } + } + } + /* -------------------- END OF COPY OF THE __New() CODE -------------------- */ + } + + /* Index/data filenames */ + if (pos != NULL) { + int nLen = (int) (pos - filename); + + strncat(cache->filenameDat, filename, nLen); + strncat(cache->filenameNdx, filename, nLen); + strcat(cache->filenameDat, ".dat"); + strcat(cache->filenameNdx, ".ndx"); + } + ndxSize = filesize(cache->filenameNdx); + cache->timestamp = file_timestamp(cache->filenameDat); + cache->dat = fopen(cache->filenameDat, "rb"); + cache->ndx = fopen(cache->filenameNdx, "rb"); + if (cache->dat != NULL && cache->ndx != NULL && ndxSize > 0) { + char *use = malloc(ndxSize + 1); + + if (fread(use, 1, ndxSize, cache->ndx) == ndxSize) { + char firstline[256]; + char *a = use; + + use[ndxSize] = '\0'; + a += cache_brstr(a, firstline); + if (strncmp(firstline, "CACHE-", 6) == 0) { // Nouvelle version du cache + if (strncmp(firstline, "CACHE-1.", 8) == 0) { // Version 1.1x + cache->version = (int) (firstline[8] - '0'); // cache 1.x + if (cache->version <= 5) { + a += cache_brstr(a, firstline); + strcpy(cache->lastmodified, firstline); + } else { + // fprintf(opt->errlog,"Cache: version 1.%d not supported, ignoring current cache"LF,cache->version); + fclose(cache->dat); + cache->dat = NULL; + free(use); + use = NULL; + } + } else { // non supporté + // fspc(opt->errlog,"error"); fprintf(opt->errlog,"Cache: %s not supported, ignoring current cache"LF,firstline); + fclose(cache->dat); + cache->dat = NULL; + free(use); + use = NULL; + } + /* */ + } else { // Vieille version du cache + /* */ + // hts_log_print(opt, LOG_WARNING, "Cache: importing old cache format"); + cache->version = 0; // cache 1.0 + strcpy(cache->lastmodified, firstline); + } + + /* Create hash table for the cache (MUCH FASTER!) */ + if (use) { + char line[HTS_URLMAXSIZE * 2]; + char linepos[256]; + int pos; + int firstSeen = 0; + + while((a != NULL) && (a < (use + ndxSize))) { + a = strchr(a + 1, '\n'); /* start of line */ + if (a) { + a++; + /* read "host/file" */ + a += binput(a, line, HTS_URLMAXSIZE); + a += binput(a, line + strlen(line), HTS_URLMAXSIZE); + /* read position */ + a += binput(a, linepos, 200); + sscanf(linepos, "%d", &pos); + + /* Add entry */ + inthash_add(cache->hash, line, pos); + + /* First link as starting URL */ + if (!firstSeen) { + if (strstr(line, "/robots.txt") == NULL) { + PT_Index__Old index = cache; + + firstSeen = 1; + if (!link_has_authority(line)) + strcat(index->startUrl, "http://"); + strcat(index->startUrl, line); + } + } + + } + } + /* Not needed anymore! */ + free(use); + use = NULL; + return 1; + } + } + } + } + return 0; +} + +static String DecodeUrl(const char *url) { + int i; + String s = STRING_EMPTY; + + StringClear(s); + for(i = 0; url[i] != '\0'; i++) { + if (url[i] == '+') { + StringAddchar(s, ' '); + } else if (url[i] == '%') { + if (url[i + 1] == '%') { + StringAddchar(s, '%'); + i++; + } else if (url[i + 1] != 0 && url[i + 2] != 0) { + char tmp[3]; + int codepoint = 0; + + tmp[0] = url[i + 1]; + tmp[1] = url[i + 2]; + tmp[2] = 0; + if (sscanf(tmp, "%x", &codepoint) == 1) { + StringAddchar(s, (char) codepoint); + } + i += 2; + } + } else { + StringAddchar(s, url[i]); + } + } + return s; +} + +static PT_Element PT_ReadCache__Old(PT_Index index, const char *url, int flags) { + PT_Element retCode; + + MutexLock(&index->slots.formatOld.fileLock); + { + retCode = PT_ReadCache__Old_u(index, url, flags); + } + MutexUnlock(&index->slots.formatOld.fileLock); + return retCode; +} + +static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url, + int flags) { + PT_Index__Old cache = (PT_Index__Old) & index_->slots.formatOld; intptr_t hash_pos; int hash_pos_return; - char location_default[HTS_URLMAXSIZE*2]; - char previous_save[HTS_URLMAXSIZE*2]; - char previous_save_[HTS_URLMAXSIZE*2]; + char location_default[HTS_URLMAXSIZE * 2]; + char previous_save[HTS_URLMAXSIZE * 2]; + char previous_save_[HTS_URLMAXSIZE * 2]; PT_Element r; - int ok=0; - - if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0) - return NULL; - if ((r = PT_ElementNew()) == NULL) - return NULL; - location_default[0] = '\0'; - previous_save[0] = previous_save_[0] = '\0'; + int ok = 0; + + if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0) + return NULL; + if ((r = PT_ElementNew()) == NULL) + return NULL; + location_default[0] = '\0'; + previous_save[0] = previous_save_[0] = '\0'; memset(r, 0, sizeof(_PT_Element)); r->location = location_default; - strcpy(r->location, ""); - if (strncmp(url, "http://", 7) == 0) - url += 7; - hash_pos_return=inthash_read(cache->hash, url, &hash_pos); + strcpy(r->location, ""); + if (strncmp(url, "http://", 7) == 0) + url += 7; + hash_pos_return = inthash_read(cache->hash, url, &hash_pos); if (hash_pos_return) { - int pos = (int) hash_pos; /* simply */ - - if (fseek(cache->dat, (pos>0) ? pos : (-pos), SEEK_SET) == 0) { - /* Importer cache1.0 */ - if (cache->version==0) { - OLD_htsblk old_r; - if (fread((char*) &old_r,1,sizeof(old_r),cache->dat) == sizeof(old_r)) { // lire tout (y compris statuscode etc) - int i; - String urlDecoded; - r->statuscode = old_r.statuscode; - r->size = old_r.size; // taille fichier - strcpy(r->msg, old_r.msg); - strcpy(r->contenttype, old_r.contenttype); - - /* Guess the destination filename.. this sucks, because this method is not reliable. - Yes, the old 1.0 cache format was *that* bogus. /rx */ + int pos = (int) hash_pos; /* simply */ + + if (fseek(cache->dat, (pos > 0) ? pos : (-pos), SEEK_SET) == 0) { + /* Importer cache1.0 */ + if (cache->version == 0) { + OLD_htsblk old_r; + + if (fread((char *) &old_r, 1, sizeof(old_r), cache->dat) == sizeof(old_r)) { // lire tout (y compris statuscode etc) + int i; + String urlDecoded; + + r->statuscode = old_r.statuscode; + r->size = old_r.size; // taille fichier + strcpy(r->msg, old_r.msg); + strcpy(r->contenttype, old_r.contenttype); + + /* Guess the destination filename.. this sucks, because this method is not reliable. + Yes, the old 1.0 cache format was *that* bogus. /rx */ #define FORBIDDEN_CHAR(c) (c == '~' \ || c == '\\' \ || c == ':' \ @@ -1554,203 +1655,211 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char* url, int flag || ((unsigned char) c ) <= 31 \ || ((unsigned char) c ) == 127 \ ) - urlDecoded = DecodeUrl(jump_protocol_and_auth(url)); - strcpy(previous_save_, StringBuff(urlDecoded)); - StringFree(urlDecoded); - for(i = 0 ; previous_save_[i] != '\0' && previous_save_[i] != '?' ; i++) { - if (FORBIDDEN_CHAR(previous_save_[i])) { - previous_save_[i] = '_'; - } - } - previous_save_[i] = '\0'; + urlDecoded = DecodeUrl(jump_protocol_and_auth(url)); + strcpy(previous_save_, StringBuff(urlDecoded)); + StringFree(urlDecoded); + for(i = 0; previous_save_[i] != '\0' && previous_save_[i] != '?'; i++) { + if (FORBIDDEN_CHAR(previous_save_[i])) { + previous_save_[i] = '_'; + } + } + previous_save_[i] = '\0'; #undef FORBIDDEN_CHAR - ok = 1; /* import ok */ - } - /* */ - /* Cache 1.1 */ + ok = 1; /* import ok */ + } + /* */ + /* Cache 1.1 */ } else { char check[256]; unsigned long size_read; unsigned long int size_; - check[0]='\0'; + + check[0] = '\0'; // - cache_rint(cache->dat,&r->statuscode); - cache_rLLint(cache->dat,&size_); + cache_rint(cache->dat, &r->statuscode); + cache_rLLint(cache->dat, &size_); r->size = (size_t) size_; - cache_rstr(cache->dat,r->msg); - cache_rstr(cache->dat,r->contenttype); + cache_rstr(cache->dat, r->msg); + cache_rstr(cache->dat, r->contenttype); if (cache->version >= 3) - cache_rstr(cache->dat,r->charset); - cache_rstr(cache->dat,r->lastmodified); - cache_rstr(cache->dat,r->etag); - cache_rstr(cache->dat,r->location); + cache_rstr(cache->dat, r->charset); + cache_rstr(cache->dat, r->lastmodified); + cache_rstr(cache->dat, r->etag); + cache_rstr(cache->dat, r->location); if (cache->version >= 2) - cache_rstr(cache->dat,r->cdispo); + cache_rstr(cache->dat, r->cdispo); if (cache->version >= 4) { - cache_rstr(cache->dat, previous_save_); // adr - cache_rstr(cache->dat, previous_save_); // fil + cache_rstr(cache->dat, previous_save_); // adr + cache_rstr(cache->dat, previous_save_); // fil previous_save[0] = '\0'; - cache_rstr(cache->dat, previous_save_); // save + cache_rstr(cache->dat, previous_save_); // save } if (cache->version >= 5) { r->headers = cache_rstr_addr(cache->dat); } // - cache_rstr(cache->dat,check); - if (strcmp(check,"HTS")==0) { /* intégrité OK */ - ok=1; + cache_rstr(cache->dat, check); + if (strcmp(check, "HTS") == 0) { /* intégrité OK */ + ok = 1; } - cache_rLLint(cache->dat, &size_read); /* lire size pour être sûr de la taille déclarée (réécrire) */ - if (size_read > 0) { /* si inscrite ici */ + cache_rLLint(cache->dat, &size_read); /* lire size pour être sûr de la taille déclarée (réécrire) */ + if (size_read > 0) { /* si inscrite ici */ r->size = size_read; - } else { /* pas de données directement dans le cache, fichier présent? */ - r->size = 0; + } else { /* pas de données directement dans le cache, fichier présent? */ + r->size = 0; } } - /* Check destination filename */ - - { - PT_Index__Old index = cache; - /* -------------------- COPY OF THE __New() CODE -------------------- */ - if (previous_save_[0] != '\0') { - int pathLen = (int) strlen(index->path); - if (pathLen > 0 && strncmp(previous_save_, index->path, pathLen) == 0) { // old (<3.40) buggy format - strcpy(previous_save, previous_save_); - } - // relative ? (hack) - else if (index->safeCache - || (previous_save_[0] != '/' // /home/foo/bar.gif - && ( !isalpha(previous_save_[0]) || previous_save_[1] != ':' ) ) // c:/home/foo/bar.gif - ) - { - index->safeCache = 1; - sprintf(previous_save, "%s%s", index->path, previous_save_); - } - // bogus format (includes buggy absolute path) - else { - /* guess previous path */ - if (index->fixedPath == 0) { - const char * start = jump_protocol_and_auth(url); - const char * end = start ? strchr(start, '/') : NULL; - int len = (int) (end - start); - if (start != NULL && end != NULL && len > 0 && len < 128) { - char piece[128 + 2]; - const char * where; - piece[0] = '\0'; - strncat(piece, start, len); - if ((where = strstr(previous_save_, piece)) != NULL) { - index->fixedPath = (int) (where - previous_save_); // offset to relative path - } - } - } - if (index->fixedPath > 0) { - int saveLen = (int) strlen(previous_save_); - if (index->fixedPath < saveLen) { - sprintf(previous_save, "%s%s", index->path, previous_save_ + index->fixedPath); - } else { - sprintf(r->msg, "Bogus fixePath prefix for %s (prefixLen=%d)", previous_save_, (int)index->fixedPath); - r->statuscode = STATUSCODE_INVALID; - } - } else { - sprintf(previous_save, "%s%s", index->path, previous_save_); - } - } - } - /* -------------------- END OF COPY OF THE __New() CODE -------------------- */ - } + /* Check destination filename */ + + { + PT_Index__Old index = cache; + + /* -------------------- COPY OF THE __New() CODE -------------------- */ + if (previous_save_[0] != '\0') { + int pathLen = (int) strlen(index->path); + + if (pathLen > 0 && strncmp(previous_save_, index->path, pathLen) == 0) { // old (<3.40) buggy format + strcpy(previous_save, previous_save_); + } + // relative ? (hack) + else if (index->safeCache || (previous_save_[0] != '/' // /home/foo/bar.gif + && (!isalpha(previous_save_[0]) || previous_save_[1] != ':')) // c:/home/foo/bar.gif + ) { + index->safeCache = 1; + sprintf(previous_save, "%s%s", index->path, previous_save_); + } + // bogus format (includes buggy absolute path) + else { + /* guess previous path */ + if (index->fixedPath == 0) { + const char *start = jump_protocol_and_auth(url); + const char *end = start ? strchr(start, '/') : NULL; + int len = (int) (end - start); + + if (start != NULL && end != NULL && len > 0 && len < 128) { + char piece[128 + 2]; + const char *where; + + piece[0] = '\0'; + strncat(piece, start, len); + if ((where = strstr(previous_save_, piece)) != NULL) { + index->fixedPath = (int) (where - previous_save_); // offset to relative path + } + } + } + if (index->fixedPath > 0) { + int saveLen = (int) strlen(previous_save_); + + if (index->fixedPath < saveLen) { + sprintf(previous_save, "%s%s", index->path, + previous_save_ + index->fixedPath); + } else { + sprintf(r->msg, "Bogus fixePath prefix for %s (prefixLen=%d)", + previous_save_, (int) index->fixedPath); + r->statuscode = STATUSCODE_INVALID; + } + } else { + sprintf(previous_save, "%s%s", index->path, previous_save_); + } + } + } + /* -------------------- END OF COPY OF THE __New() CODE -------------------- */ + } /* Read data */ - if (ok) { - r->adr = NULL; - if ( (r->statuscode>=0) && (r->statuscode<=999)) { - r->adr = NULL; - if (pos<0) { - if (flags & FETCH_BODY) { - FILE* fp = fopen(previous_save, "rb"); - if (fp != NULL) { - r->adr = (char*) malloc(r->size + 1); - if (r->adr != NULL) { - if (r->size > 0 && fread(r->adr, 1, r->size, fp) != r->size) { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Read error in cache disk data"); - } - r->adr[r->size] = '\0'; - } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Read error (memory exhausted) from cache"); - } - fclose(fp); - } else { - r->statuscode = STATUSCODE_INVALID; - strcpy(r->msg, "Previous cache file not found (2)"); - } - } - } else { - // lire fichier (d'un coup) - if (flags & FETCH_BODY) { - r->adr=(char*) malloc(r->size + 1); - if (r->adr!=NULL) { - if (fread(r->adr, 1, r->size,cache->dat) != r->size) { // erreur - free(r->adr); - r->adr=NULL; - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cache Read Error : Read Data"); - } else - r->adr[r->size] = '\0'; - } else { // erreur - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cache Memory Error"); - } - } - } - } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cache Read Error : Bad Data"); + if (ok) { + r->adr = NULL; + if ((r->statuscode >= 0) && (r->statuscode <= 999)) { + r->adr = NULL; + if (pos < 0) { + if (flags & FETCH_BODY) { + FILE *fp = fopen(previous_save, "rb"); + + if (fp != NULL) { + r->adr = (char *) malloc(r->size + 1); + if (r->adr != NULL) { + if (r->size > 0 && fread(r->adr, 1, r->size, fp) != r->size) { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Read error in cache disk data"); + } + r->adr[r->size] = '\0'; + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Read error (memory exhausted) from cache"); + } + fclose(fp); + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Previous cache file not found (2)"); + } + } + } else { + // lire fichier (d'un coup) + if (flags & FETCH_BODY) { + r->adr = (char *) malloc(r->size + 1); + if (r->adr != NULL) { + if (fread(r->adr, 1, r->size, cache->dat) != r->size) { // erreur + free(r->adr); + r->adr = NULL; + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Read Data"); + } else + r->adr[r->size] = '\0'; + } else { // erreur + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Memory Error"); + } + } + } + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Bad Data"); } - } else { // erreur - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cache Read Error : Read Header"); + } else { // erreur + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Read Header"); } } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Cache Read Error : Seek Failed"); + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Seek Failed"); } } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"File Cache Entry Not Found"); - } - if (r->location[0] != '\0') { - r->location = strdup(r->location); - } else { - r->location = NULL; - } + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "File Cache Entry Not Found"); + } + if (r->location[0] != '\0') { + r->location = strdup(r->location); + } else { + r->location = NULL; + } return r; } -static int PT_LookupCache__Old(PT_Index index, const char* url) { - int retCode; - MutexLock(&index->slots.formatOld.fileLock); - { - retCode = PT_LookupCache__Old_u(index, url); - } - MutexUnlock(&index->slots.formatOld.fileLock); - return retCode; -} +static int PT_LookupCache__Old(PT_Index index, const char *url) { + int retCode; -static int PT_LookupCache__Old_u(PT_Index index_, const char* url) { - if (index_ != NULL) { - PT_Index__New cache = (PT_Index__New) &index_->slots.formatNew; - if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0) - return 0; - if (strncmp(url, "http://", 7) == 0) - url += 7; - if (inthash_read(cache->hash, url, NULL)) - return 1; - } - return 0; + MutexLock(&index->slots.formatOld.fileLock); + { + retCode = PT_LookupCache__Old_u(index, url); + } + MutexUnlock(&index->slots.formatOld.fileLock); + return retCode; } +static int PT_LookupCache__Old_u(PT_Index index_, const char *url) { + if (index_ != NULL) { + PT_Index__New cache = (PT_Index__New) & index_->slots.formatNew; + + if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0) + return 0; + if (strncmp(url, "http://", 7) == 0) + url += 7; + if (inthash_read(cache->hash, url, NULL)) + return 1; + } + return 0; +} /* ------------------------------------------------------------ */ /* Internet Archive Arc 1.0 (arc) format */ @@ -1760,199 +1869,221 @@ static int PT_LookupCache__Old_u(PT_Index index_, const char* url) { #define ARC_SP ' ' -static const char* getArcField(const char *line, int pos) { - int i; - for(i = 0 ; line[i] != '\0' && pos > 0 ; i++) { - if (line[i] == ARC_SP) - pos--; - } - if (pos == 0) - return &line[i]; - return NULL; -} - -static char* copyArcField(const char *line, int npos, char *dest, int destMax) { - const char *pos; - if ((pos = getArcField(line, npos)) != NULL) { - int i; - for(i = 0 ; pos[i] != '\0' && pos[i] != ARC_SP && ( --destMax ) > 0; i++) { - dest[i] = pos[i]; - } - dest[i] = 0; - return dest; - } - dest[0] = 0; - return NULL; +static const char *getArcField(const char *line, int pos) { + int i; + + for(i = 0; line[i] != '\0' && pos > 0; i++) { + if (line[i] == ARC_SP) + pos--; + } + if (pos == 0) + return &line[i]; + return NULL; +} + +static char *copyArcField(const char *line, int npos, char *dest, int destMax) { + const char *pos; + + if ((pos = getArcField(line, npos)) != NULL) { + int i; + + for(i = 0; pos[i] != '\0' && pos[i] != ARC_SP && (--destMax) > 0; i++) { + dest[i] = pos[i]; + } + dest[i] = 0; + return dest; + } + dest[0] = 0; + return NULL; } static int getArcLength(const char *line) { - const char *pos; - if ((pos = getArcField(line, 9)) != NULL - || (pos = getArcField(line, 4)) != NULL - || (pos = getArcField(line, 2)) != NULL - ) { - int length; - if (sscanf(pos, "%d", &length) == 1) { - return length; - } - } - return -1; -} - -static int skipArcNl(FILE* file) { - if (fgetc(file) == 0x0a) { - return 0; - } - return -1; -} - -static int skipArcData(FILE* file, const char *line) { - int jump = getArcLength(line); - if (jump != -1) { - if (fseek(file, jump, SEEK_CUR) == 0 /* && skipArcNl(file) == 0 */) { - return 0; - } - } - return -1; + const char *pos; + + if ((pos = getArcField(line, 9)) != NULL + || (pos = getArcField(line, 4)) != NULL + || (pos = getArcField(line, 2)) != NULL) { + int length; + + if (sscanf(pos, "%d", &length) == 1) { + return length; + } + } + return -1; +} + +static int skipArcNl(FILE * file) { + if (fgetc(file) == 0x0a) { + return 0; + } + return -1; +} + +static int skipArcData(FILE * file, const char *line) { + int jump = getArcLength(line); + + if (jump != -1) { + if (fseek(file, jump, SEEK_CUR) == 0 /* && skipArcNl(file) == 0 */ ) { + return 0; + } + } + return -1; } static int getDigit(const char digit) { - return (int) ( digit - '0' ); + return (int) (digit - '0'); } -static int getDigit2(const char * const pos) { - return getDigit(pos[0])*10 + getDigit(pos[1]); +static int getDigit2(const char *const pos) { + return getDigit(pos[0]) * 10 + getDigit(pos[1]); } -static int getDigit4(const char * const pos) { - return getDigit(pos[0])*1000 + getDigit(pos[1])*100 + getDigit(pos[2])*10 + getDigit(pos[3]); +static int getDigit4(const char *const pos) { + return getDigit(pos[0]) * 1000 + getDigit(pos[1]) * 100 + + getDigit(pos[2]) * 10 + getDigit(pos[3]); } -static time_t getGMT(struct tm *tm) { /* hey, time_t is local! */ - time_t t = mktime(tm); - if (t != (time_t) -1 && t != (time_t) 0) { +static time_t getGMT(struct tm *tm) { /* hey, time_t is local! */ + time_t t = mktime(tm); + + if (t != (time_t) - 1 && t != (time_t) 0) { /* BSD does not have static "timezone" declared */ #if (defined(BSD) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__FreeBSD_kernel__)) - time_t now = time(NULL); - time_t timezone = - localtime(&now)->tm_gmtoff; + time_t now = time(NULL); + time_t timezone = -localtime(&now)->tm_gmtoff; #endif - return (time_t) (t - timezone); - } - return (time_t) -1; -} - -static time_t getArcTimestamp(const char * const line) { - const char *pos; - if ((pos = getArcField(line, 2)) != NULL) { - int i; - /* date == YYYYMMDDhhmmss (Greenwich Mean Time) */ - /* example: 20050405154029 */ - for(i = 0 ; pos[i] >= '0' && pos[i] <= '9' ; i++); - if (i == 14) { - struct tm tm; - memset(&tm, 0, sizeof(tm)); - tm.tm_year = getDigit4(pos + 0) - 1900; /* current year minus 1900 */ - tm.tm_mon = getDigit2(pos + 4) - 1; /* 0 – 11 */ - tm.tm_mday = getDigit2(pos + 6); /* 1 – 31 */ - tm.tm_hour = getDigit2(pos + 8); /* 0 – 23 */ - tm.tm_min = getDigit2(pos + 10); /* 0 – 59 */ - tm.tm_sec = getDigit2(pos + 12); /* 0 – 59 */ - tm.tm_isdst = 0; - return getGMT(&tm); - } - } - return (time_t) -1; + return (time_t) (t - timezone); + } + return (time_t) - 1; +} + +static time_t getArcTimestamp(const char *const line) { + const char *pos; + + if ((pos = getArcField(line, 2)) != NULL) { + int i; + + /* date == YYYYMMDDhhmmss (Greenwich Mean Time) */ + /* example: 20050405154029 */ + for(i = 0; pos[i] >= '0' && pos[i] <= '9'; i++) ; + if (i == 14) { + struct tm tm; + + memset(&tm, 0, sizeof(tm)); + tm.tm_year = getDigit4(pos + 0) - 1900; /* current year minus 1900 */ + tm.tm_mon = getDigit2(pos + 4) - 1; /* 0 – 11 */ + tm.tm_mday = getDigit2(pos + 6); /* 1 – 31 */ + tm.tm_hour = getDigit2(pos + 8); /* 0 – 23 */ + tm.tm_min = getDigit2(pos + 10); /* 0 – 59 */ + tm.tm_sec = getDigit2(pos + 12); /* 0 – 59 */ + tm.tm_isdst = 0; + return getGMT(&tm); + } + } + return (time_t) - 1; } static int readArcURLRecord(PT_Index__Arc index) { - index->line[0] = '\0'; - if (linput(index->file, index->line, sizeof(index->line) - 1)) { - return 0; - } - return -1; + index->line[0] = '\0'; + if (linput(index->file, index->line, sizeof(index->line) - 1)) { + return 0; + } + return -1; } #define str_begins(str, sstr) ( strncmp(str, sstr, sizeof(sstr) - 1) == 0 ) static int PT_CompatibleScheme(const char *url) { - return (str_begins(url, "http:") - || str_begins(url, "https:") - || str_begins(url, "ftp:") - || str_begins(url, "file:")); + return (str_begins(url, "http:") + || str_begins(url, "https:") + || str_begins(url, "ftp:") + || str_begins(url, "file:")); } int PT_LoadCache__Arc(PT_Index index_, const char *filename) { - if (index_ != NULL && filename != NULL) { - PT_Index__Arc index = &index_->slots.formatArc; - index->timestamp = file_timestamp(filename); - MutexInit(&index->fileLock); - index->file = fopen(filename, "rb"); - - // Opened ? - if (index->file != NULL) { - inthash hashtable = index->hash; - if (readArcURLRecord(index) == 0) { - int entries = 0; - /* Read first line */ - if (strncmp(index->line, "filedesc://", sizeof("filedesc://") - 1) != 0) { - fprintf(stderr, "Unexpected bad signature #%s"LF, index->line); - fclose(index->file); - index->file = NULL; - return 0; - } - /* Timestamp */ - index->timestamp = getArcTimestamp(index->line); - /* Skip first entry */ - if (skipArcData(index->file, index->line) != 0 || skipArcNl(index->file) != 0) { - fprintf(stderr, "Unexpected bad data offset size first entry"LF); - fclose(index->file); - index->file = NULL; - return 0; - } - /* Read all meta-entries (not data) */ - while(!feof(index->file)) { - unsigned long int fpos = ftell(index->file); - if (skipArcNl(index->file) == 0 && readArcURLRecord(index) == 0) { - int length = getArcLength(index->line); - if (length >= 0) { - const char * filenameIndex = copyArcField(index->line, 0, - index->filenameIndexBuff, sizeof(index->filenameIndexBuff) - 1); /* can not be NULL */ - if (strncmp(filenameIndex, "http://", 7) == 0) { - filenameIndex += 7; - } - if (*filenameIndex != 0) { - if (skipArcData(index->file, index->line) != 0) { - fprintf(stderr, "Corrupted cache data entry #%d (truncated file?), aborting read"LF, (int)entries); - } - /*fprintf(stdout, "adding %s [%d]\n", filenameIndex, (int)fpos);*/ - if (PT_CompatibleScheme(index->filenameIndexBuff)) { - inthash_add(hashtable, filenameIndex, fpos); /* position of meta-data */ - entries++; - } - } else { - fprintf(stderr, "Corrupted cache meta entry #%d"LF, (int)entries); - } - } else { - fprintf(stderr, "Corrupted cache meta entry #%d, aborting read"LF, (int)entries); - break ; - } - } else { - break ; - } - } - - /* OK */ - return 1; - } else { - fprintf(stderr, "Bad file (empty ?)"LF); - } - } else { - fprintf(stderr, "Unable to open file"LF); - index = NULL; - } - } else { - fprintf(stderr, "Bad arguments"LF); - } - return 0; + if (index_ != NULL && filename != NULL) { + PT_Index__Arc index = &index_->slots.formatArc; + + index->timestamp = file_timestamp(filename); + MutexInit(&index->fileLock); + index->file = fopen(filename, "rb"); + + // Opened ? + if (index->file != NULL) { + inthash hashtable = index->hash; + + if (readArcURLRecord(index) == 0) { + int entries = 0; + + /* Read first line */ + if (strncmp(index->line, "filedesc://", sizeof("filedesc://") - 1) != 0) { + fprintf(stderr, "Unexpected bad signature #%s" LF, index->line); + fclose(index->file); + index->file = NULL; + return 0; + } + /* Timestamp */ + index->timestamp = getArcTimestamp(index->line); + /* Skip first entry */ + if (skipArcData(index->file, index->line) != 0 + || skipArcNl(index->file) != 0) { + fprintf(stderr, "Unexpected bad data offset size first entry" LF); + fclose(index->file); + index->file = NULL; + return 0; + } + /* Read all meta-entries (not data) */ + while(!feof(index->file)) { + unsigned long int fpos = ftell(index->file); + + if (skipArcNl(index->file) == 0 && readArcURLRecord(index) == 0) { + int length = getArcLength(index->line); + + if (length >= 0) { + const char *filenameIndex = copyArcField(index->line, 0, + index->filenameIndexBuff, sizeof(index->filenameIndexBuff) - 1); /* can not be NULL */ + + if (strncmp(filenameIndex, "http://", 7) == 0) { + filenameIndex += 7; + } + if (*filenameIndex != 0) { + if (skipArcData(index->file, index->line) != 0) { + fprintf(stderr, + "Corrupted cache data entry #%d (truncated file?), aborting read" + LF, (int) entries); + } + /*fprintf(stdout, "adding %s [%d]\n", filenameIndex, (int)fpos); */ + if (PT_CompatibleScheme(index->filenameIndexBuff)) { + inthash_add(hashtable, filenameIndex, fpos); /* position of meta-data */ + entries++; + } + } else { + fprintf(stderr, "Corrupted cache meta entry #%d" LF, + (int) entries); + } + } else { + fprintf(stderr, + "Corrupted cache meta entry #%d, aborting read" LF, + (int) entries); + break; + } + } else { + break; + } + } + + /* OK */ + return 1; + } else { + fprintf(stderr, "Bad file (empty ?)" LF); + } + } else { + fprintf(stderr, "Unable to open file" LF); + index = NULL; + } + } else { + fprintf(stderr, "Bad arguments" LF); + } + return 0; } #define HTTP_READFIELD_STRING(line, value, refline, refvalue) do { \ @@ -1970,278 +2101,287 @@ int PT_LoadCache__Arc(PT_Index index_, const char *filename) { } \ } while(0) -static PT_Element PT_ReadCache__Arc(PT_Index index, const char* url, int flags) { - PT_Element retCode; - MutexLock(&index->slots.formatArc.fileLock); - { - retCode = PT_ReadCache__Arc_u(index, url, flags); - } - MutexUnlock(&index->slots.formatArc.fileLock); - return retCode; +static PT_Element PT_ReadCache__Arc(PT_Index index, const char *url, int flags) { + PT_Element retCode; + + MutexLock(&index->slots.formatArc.fileLock); + { + retCode = PT_ReadCache__Arc_u(index, url, flags); + } + MutexUnlock(&index->slots.formatArc.fileLock); + return retCode; } -static PT_Element PT_ReadCache__Arc_u(PT_Index index_, const char* url, int flags) -{ - PT_Index__Arc index = (PT_Index__Arc) &index_->slots.formatArc; - char location_default[HTS_URLMAXSIZE*2]; +static PT_Element PT_ReadCache__Arc_u(PT_Index index_, const char *url, + int flags) { + PT_Index__Arc index = (PT_Index__Arc) & index_->slots.formatArc; + char location_default[HTS_URLMAXSIZE * 2]; intptr_t hash_pos; int hash_pos_return; - PT_Element r = NULL; - if (index == NULL || index->hash == NULL || url == NULL || *url == 0) - return NULL; - if ((r = PT_ElementNew()) == NULL) - return NULL; - location_default[0] = '\0'; + PT_Element r = NULL; + + if (index == NULL || index->hash == NULL || url == NULL || *url == 0) + return NULL; + if ((r = PT_ElementNew()) == NULL) + return NULL; + location_default[0] = '\0'; memset(r, 0, sizeof(_PT_Element)); r->location = location_default; - strcpy(r->location, ""); - if (strncmp(url, "http://", 7) == 0) - url += 7; + strcpy(r->location, ""); + if (strncmp(url, "http://", 7) == 0) + url += 7; hash_pos_return = inthash_read(index->hash, url, &hash_pos); - if (hash_pos_return) { - if (fseek(index->file, (long)hash_pos, SEEK_SET) == 0) { - if (skipArcNl(index->file) == 0 && readArcURLRecord(index) == 0) { - long int fposMeta = ftell(index->file); - int dataLength = getArcLength(index->line); - const char *pos; - - /* Read HTTP headers */ - /* HTTP/1.1 404 Not Found */ - if (linput(index->file, index->line, sizeof(index->line) - 1)) { - if ((pos = getArcField(index->line, 1)) != NULL) { - if (sscanf(pos, "%d", &r->statuscode) != 1) { - r->statuscode = STATUSCODE_INVALID; - } - } - if ((pos = getArcField(index->line, 2)) != NULL) { - r->msg[0] = '\0'; - strncat(r->msg, pos, sizeof(pos) - 1); - } - while (linput(index->file, index->line, sizeof(index->line) - 1) && index->line[0] != '\0') { - char* const line = index->line; - char* value = strchr(line, ':'); - if (value != NULL) { - *value = '\0'; - for( value++ ; *value == ' ' || *value == '\t' ; value++); - HTTP_READFIELD_INT(line, value, "Content-Length", r->size); // size - HTTP_READFIELD_STRING(line, value, "Content-Type", r->contenttype); // contenttype - HTTP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified - HTTP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag - HTTP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved - HTTP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo); // Content-disposition - if (line[0] != '\0') { - int len = r->headers ? ((int) strlen(r->headers)) : 0; - int nlen = (int) ( strlen(line) + 2 + strlen(value) + sizeof("\r\n") + 1 ); - r->headers = realloc(r->headers, len + nlen); - r->headers[len] = '\0'; - strcat(r->headers, line); - strcat(r->headers, ": "); - strcat(r->headers, value); - strcat(r->headers, "\r\n"); - } - } - } - - /* FIXME charset */ - if (r->contenttype[0] != '\0') { - char *pos = strchr(r->contenttype, ';'); - if (pos != NULL) { - /*char *chs = strchr(pos, "charset=");*/ - /*HTTP_READFIELD_STRING(line, value, "X-Charset", r->charset);*/ - *pos = 0; - if ((pos = strchr(r->contenttype, ' ')) != NULL) { - *pos = 0; - } - } - } - - /* Read data */ - if (r->statuscode != STATUSCODE_INVALID) { /* Can continue */ - if (flags & FETCH_BODY) { - long int fposCurrent = ftell(index->file); - long int metaSize = fposCurrent - fposMeta; - long int fetchSize = (long int) r->size; - if (fetchSize <= 0) { - fetchSize = dataLength - metaSize; - } else if (fetchSize > dataLength - metaSize) { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg, "Cache Read Error : Truncated Data"); - } - r->size = 0; - if (r->statuscode != STATUSCODE_INVALID) { - r->adr = (char*) malloc(fetchSize); - if (r->adr != NULL) { - if (fetchSize > 0 && ( r->size = (int) fread(r->adr, 1, fetchSize, index->file) ) != fetchSize) { + if (hash_pos_return) { + if (fseek(index->file, (long) hash_pos, SEEK_SET) == 0) { + if (skipArcNl(index->file) == 0 && readArcURLRecord(index) == 0) { + long int fposMeta = ftell(index->file); + int dataLength = getArcLength(index->line); + const char *pos; + + /* Read HTTP headers */ + /* HTTP/1.1 404 Not Found */ + if (linput(index->file, index->line, sizeof(index->line) - 1)) { + if ((pos = getArcField(index->line, 1)) != NULL) { + if (sscanf(pos, "%d", &r->statuscode) != 1) { + r->statuscode = STATUSCODE_INVALID; + } + } + if ((pos = getArcField(index->line, 2)) != NULL) { + r->msg[0] = '\0'; + strncat(r->msg, pos, sizeof(pos) - 1); + } + while(linput(index->file, index->line, sizeof(index->line) - 1) + && index->line[0] != '\0') { + char *const line = index->line; + char *value = strchr(line, ':'); + + if (value != NULL) { + *value = '\0'; + for(value++; *value == ' ' || *value == '\t'; value++) ; + HTTP_READFIELD_INT(line, value, "Content-Length", r->size); // size + HTTP_READFIELD_STRING(line, value, "Content-Type", r->contenttype); // contenttype + HTTP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified + HTTP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag + HTTP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved + HTTP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo); // Content-disposition + if (line[0] != '\0') { + int len = r->headers ? ((int) strlen(r->headers)) : 0; + int nlen = + (int) (strlen(line) + 2 + strlen(value) + sizeof("\r\n") + 1); + r->headers = realloc(r->headers, len + nlen); + r->headers[len] = '\0'; + strcat(r->headers, line); + strcat(r->headers, ": "); + strcat(r->headers, value); + strcat(r->headers, "\r\n"); + } + } + } + + /* FIXME charset */ + if (r->contenttype[0] != '\0') { + char *pos = strchr(r->contenttype, ';'); + + if (pos != NULL) { + /*char *chs = strchr(pos, "charset="); */ + /*HTTP_READFIELD_STRING(line, value, "X-Charset", r->charset); */ + *pos = 0; + if ((pos = strchr(r->contenttype, ' ')) != NULL) { + *pos = 0; + } + } + } + + /* Read data */ + if (r->statuscode != STATUSCODE_INVALID) { /* Can continue */ + if (flags & FETCH_BODY) { + long int fposCurrent = ftell(index->file); + long int metaSize = fposCurrent - fposMeta; + long int fetchSize = (long int) r->size; + + if (fetchSize <= 0) { + fetchSize = dataLength - metaSize; + } else if (fetchSize > dataLength - metaSize) { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Truncated Data"); + } + r->size = 0; + if (r->statuscode != STATUSCODE_INVALID) { + r->adr = (char *) malloc(fetchSize); + if (r->adr != NULL) { + if (fetchSize > 0 + && (r->size = + (int) fread(r->adr, 1, fetchSize, + index->file)) != fetchSize) { int last_errno = errno; - r->statuscode=STATUSCODE_INVALID; - sprintf(r->msg,"Read error in cache disk data: %s", strerror(last_errno)); - } - } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"Read error (memory exhausted) from cache"); - } - } - } - } - - } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg, "Cache Read Error : Read Header Error"); - } - - } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg, "Cache Read Error : Read Header Error"); - } - } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg, "Cache Read Error : Seek Error"); - } - - } else { - r->statuscode=STATUSCODE_INVALID; - strcpy(r->msg,"File Cache Entry Not Found"); - } - if (r->location[0] != '\0') { - r->location = strdup(r->location); - } else { - r->location = NULL; - } + + r->statuscode = STATUSCODE_INVALID; + sprintf(r->msg, "Read error in cache disk data: %s", + strerror(last_errno)); + } + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Read error (memory exhausted) from cache"); + } + } + } + } + + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Read Header Error"); + } + + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Read Header Error"); + } + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Cache Read Error : Seek Error"); + } + + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "File Cache Entry Not Found"); + } + if (r->location[0] != '\0') { + r->location = strdup(r->location); + } else { + r->location = NULL; + } return r; } -static int PT_LookupCache__Arc(PT_Index index, const char* url) { - int retCode; - MutexLock(&index->slots.formatArc.fileLock); - { - retCode = PT_LookupCache__Arc_u(index, url); - } - MutexUnlock(&index->slots.formatArc.fileLock); - return retCode; +static int PT_LookupCache__Arc(PT_Index index, const char *url) { + int retCode; + + MutexLock(&index->slots.formatArc.fileLock); + { + retCode = PT_LookupCache__Arc_u(index, url); + } + MutexUnlock(&index->slots.formatArc.fileLock); + return retCode; } -static int PT_LookupCache__Arc_u(PT_Index index_, const char* url) { - if (index_ != NULL) { - PT_Index__New cache = (PT_Index__New) &index_->slots.formatNew; - if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0) - return 0; - if (strncmp(url, "http://", 7) == 0) - url += 7; - if (inthash_read(cache->hash, url, NULL)) - return 1; - } - return 0; +static int PT_LookupCache__Arc_u(PT_Index index_, const char *url) { + if (index_ != NULL) { + PT_Index__New cache = (PT_Index__New) & index_->slots.formatNew; + + if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0) + return 0; + if (strncmp(url, "http://", 7) == 0) + url += 7; + if (inthash_read(cache->hash, url, NULL)) + return 1; + } + return 0; } typedef struct PT_SaveCache__Arc_t { - PT_Indexes indexes; - FILE *fp; - time_t t; - char filename[64]; - struct tm buff; - char headers[8192]; + PT_Indexes indexes; + FILE *fp; + time_t t; + char filename[64]; + struct tm buff; + char headers[8192]; char md5[32 + 2]; } PT_SaveCache__Arc_t; static int PT_SaveCache__Arc_Fun(void *arg, const char *url, PT_Element element) { - PT_SaveCache__Arc_t *st = (PT_SaveCache__Arc_t*) arg; - FILE * const fp = st->fp; - struct tm* tm = convert_time_rfc822(&st->buff, element->lastmodified); - int size_headers; - - sprintf(st->headers, - "HTTP/1.0 %d %s" "\r\n" - "X-Server: ProxyTrack " PROXYTRACK_VERSION "\r\n" - "Content-type: %s%s%s%s" "\r\n" - "Last-modified: %s" "\r\n" - "Content-length: %d" "\r\n" - , - element->statuscode, element->msg, - /**/ - element->contenttype, - (element->charset[0] ? "; charset=\"" : ""), - (element->charset[0] ? element->charset : ""), - (element->charset[0] ? "\"" : ""), - /**/ - element->lastmodified, - (int) element->size - ); - if (element->location != NULL && element->location[0] != '\0') { - sprintf(st->headers + strlen(st->headers), "Location: %s" "\r\n", element->location); - } - if (element->headers != NULL) { - if ( strlen(element->headers) < sizeof(st->headers) - strlen(element->headers) - 1 ) { - strcat(st->headers, element->headers); - } - } - strcat(st->headers, "\r\n"); - size_headers = (int) strlen(st->headers); - - /* doc == <nl><URL-record><nl><network_doc> */ + PT_SaveCache__Arc_t *st = (PT_SaveCache__Arc_t *) arg; + FILE *const fp = st->fp; + struct tm *tm = convert_time_rfc822(&st->buff, element->lastmodified); + int size_headers; + + sprintf(st->headers, + "HTTP/1.0 %d %s" "\r\n" "X-Server: ProxyTrack " PROXYTRACK_VERSION + "\r\n" "Content-type: %s%s%s%s" "\r\n" "Last-modified: %s" "\r\n" + "Content-length: %d" "\r\n", element->statuscode, element->msg, + /**/ element->contenttype, + (element->charset[0] ? "; charset=\"" : ""), + (element->charset[0] ? element->charset : ""), + (element->charset[0] ? "\"" : ""), /**/ element->lastmodified, + (int) element->size); + if (element->location != NULL && element->location[0] != '\0') { + sprintf(st->headers + strlen(st->headers), "Location: %s" "\r\n", + element->location); + } + if (element->headers != NULL) { + if (strlen(element->headers) < + sizeof(st->headers) - strlen(element->headers) - 1) { + strcat(st->headers, element->headers); + } + } + strcat(st->headers, "\r\n"); + size_headers = (int) strlen(st->headers); + + /* doc == <nl><URL-record><nl><network_doc> */ /* Format: URL IP date mime result checksum location offset filename length */ - if (element->adr != NULL) { - domd5mem(element->adr, element->size, st->md5, 1); - } else { - strcpy(st->md5, "-"); - } - fprintf(fp, - /* nl */ - "\n" - /* URL-record */ - "%s%s %s %04d%02d%02d%02d%02d%02d %s %d %s %s %ld %s %ld" - /* nl */ - "\n", - /* args */ - ( link_has_authority(url) ? "" : "http://" ), url, - "0.0.0.0", - tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec, - element->contenttype, - element->statuscode, - st->md5, ( element->location ? element->location : "-" ), - (long int)ftell(fp), st->filename, - (long int)( size_headers + element->size )); - /* network_doc */ - if (fwrite(st->headers, 1, size_headers, fp) != size_headers - || ( element->size > 0 && fwrite(element->adr, 1, element->size, fp) != element->size ) - ) { - return 1; /* Error */ - } - - return 0; + if (element->adr != NULL) { + domd5mem(element->adr, element->size, st->md5, 1); + } else { + strcpy(st->md5, "-"); + } + fprintf(fp, + /* nl */ + "\n" + /* URL-record */ + "%s%s %s %04d%02d%02d%02d%02d%02d %s %d %s %s %ld %s %ld" + /* nl */ + "\n", + /* args */ + (link_has_authority(url) ? "" : "http://"), url, "0.0.0.0", + tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, + tm->tm_min, tm->tm_sec, element->contenttype, element->statuscode, + st->md5, (element->location ? element->location : "-"), + (long int) ftell(fp), st->filename, + (long int) (size_headers + element->size)); + /* network_doc */ + if (fwrite(st->headers, 1, size_headers, fp) != size_headers + || (element->size > 0 + && fwrite(element->adr, 1, element->size, fp) != element->size) + ) { + return 1; /* Error */ + } + + return 0; } static int PT_SaveCache__Arc(PT_Indexes indexes, const char *filename) { - FILE *fp = fopen(filename, "wb"); - if (fp != NULL) { - PT_SaveCache__Arc_t st; - int ret; - time_t t = PT_GetTimeIndex(indexes); - struct tm tm = PT_GetTime(t); - - /* version-2-block == - filedesc://<path><sp><ip_address><sp><date><sp>text/plain<sp>200<sp>-<sp>-<sp>0<sp><filename><sp><length><nl> - 2<sp><reserved><sp><origin-code><nl> - URL<sp>IP-address<sp>Archive-date<sp>Content-type<sp>Result-code<sp>Checksum<sp>Location<sp> Offset<sp>Filename<sp>Archive-length<nl> - <nl> */ - const char* prefix = - "2 0 HTTrack Website Copier" "\n" - "URL IP-address Archive-Date Content-Type Result-code Checksum Location Offset Filename Archive-length" "\n" "\n"; - sprintf(st.filename, "httrack_%d.arc", (int) t); - fprintf(fp, "filedesc://%s 0.0.0.0 %04d%02d%02d%02d%02d%02d text/plain 200 - - 0 %s %d" "\n" - "%s", - st.filename, - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, - st.filename, (int)strlen(prefix), prefix); - st.fp = fp; - st.indexes = indexes; - st.t = t; - ret = PT_EnumCache(indexes, PT_SaveCache__Arc_Fun, (void *)&st); - fclose(fp); - if (ret != 0) - (void) unlink(filename); - return ret; - } - return -1; + FILE *fp = fopen(filename, "wb"); + + if (fp != NULL) { + PT_SaveCache__Arc_t st; + int ret; + time_t t = PT_GetTimeIndex(indexes); + struct tm tm = PT_GetTime(t); + + /* version-2-block == + filedesc://<path><sp><ip_address><sp><date><sp>text/plain<sp>200<sp>-<sp>-<sp>0<sp><filename><sp><length><nl> + 2<sp><reserved><sp><origin-code><nl> + URL<sp>IP-address<sp>Archive-date<sp>Content-type<sp>Result-code<sp>Checksum<sp>Location<sp> Offset<sp>Filename<sp>Archive-length<nl> + <nl> */ + const char *prefix = + "2 0 HTTrack Website Copier" "\n" + "URL IP-address Archive-Date Content-Type Result-code Checksum Location Offset Filename Archive-length" + "\n" "\n"; + sprintf(st.filename, "httrack_%d.arc", (int) t); + fprintf(fp, + "filedesc://%s 0.0.0.0 %04d%02d%02d%02d%02d%02d text/plain 200 - - 0 %s %d" + "\n" "%s", st.filename, tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, st.filename, + (int) strlen(prefix), prefix); + st.fp = fp; + st.indexes = indexes; + st.t = t; + ret = PT_EnumCache(indexes, PT_SaveCache__Arc_Fun, (void *) &st); + fclose(fp); + if (ret != 0) + (void) unlink(filename); + return ret; + } + return -1; } diff --git a/src/proxy/store.h b/src/proxy/store.h index b8fd4db..97d0e0c 100644 --- a/src/proxy/store.h +++ b/src/proxy/store.h @@ -50,66 +50,68 @@ typedef struct _PT_CacheItem _PT_CacheItem; typedef struct _PT_CacheItem *PT_CacheItem; typedef struct _PT_Element { - int indexId; // index identifier, if suitable (!= -1) - // - int statuscode; // status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945) - char* adr; // adresse du bloc de mémoire, NULL=vide - char* headers; // adresse des en têtes si présents (RFC822 format) - size_t size; // taille fichier - char msg[1024]; // error message ("\0"=undefined) - char contenttype[64]; // content-type ("text/html" par exemple) - char charset[64]; // charset ("iso-8859-1" par exemple) - char* location; // on copie dedans éventuellement la véritable 'location' - char lastmodified[64]; // Last-Modified - char etag[64]; // Etag - char cdispo[256]; // Content-Disposition coupé + int indexId; // index identifier, if suitable (!= -1) + // + int statuscode; // status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945) + char *adr; // adresse du bloc de mémoire, NULL=vide + char *headers; // adresse des en têtes si présents (RFC822 format) + size_t size; // taille fichier + char msg[1024]; // error message ("\0"=undefined) + char contenttype[64]; // content-type ("text/html" par exemple) + char charset[64]; // charset ("iso-8859-1" par exemple) + char *location; // on copie dedans éventuellement la véritable 'location' + char lastmodified[64]; // Last-Modified + char etag[64]; // Etag + char cdispo[256]; // Content-Disposition coupé } _PT_Element; typedef struct _PT_Element *PT_Element; typedef enum PT_Fetch_Flags { - FETCH_HEADERS, // fetch headers - FETCH_BODY // fetch body + FETCH_HEADERS, // fetch headers + FETCH_BODY // fetch body } PT_Fetch_Flags; /* Locking */ #ifdef _WIN32 -typedef void* PT_Mutex; +typedef void *PT_Mutex; #else typedef pthread_mutex_t PT_Mutex; #endif -void MutexInit(PT_Mutex *pMutex); -void MutexLock(PT_Mutex *pMutex); -void MutexUnlock(PT_Mutex *pMutex); -void MutexFree(PT_Mutex *pMutex); +void MutexInit(PT_Mutex * pMutex); +void MutexLock(PT_Mutex * pMutex); +void MutexUnlock(PT_Mutex * pMutex); +void MutexFree(PT_Mutex * pMutex); /* Indexes */ PT_Indexes PT_New(void); void PT_Delete(PT_Indexes index); -PT_Element PT_ReadIndex(PT_Indexes indexes, const char* url, int flags); -int PT_LookupIndex(PT_Indexes indexes, const char* url); +PT_Element PT_ReadIndex(PT_Indexes indexes, const char *url, int flags); +int PT_LookupIndex(PT_Indexes indexes, const char *url); int PT_AddIndex(PT_Indexes index, const char *path); int PT_RemoveIndex(PT_Indexes index, int indexId); -int PT_IndexMerge(PT_Indexes indexes, PT_Index *pindex); +int PT_IndexMerge(PT_Indexes indexes, PT_Index * pindex); PT_Index PT_GetIndex(PT_Indexes indexes, int indexId); time_t PT_GetTimeIndex(PT_Indexes indexes); /* Indexes list */ PT_Element PT_Index_HTML_BuildRootInfo(PT_Indexes indexes); -char ** PT_Enumerate(PT_Indexes indexes, const char *url, int subtree); +char **PT_Enumerate(PT_Indexes indexes, const char *url, int subtree); void PT_Enumerate_Delete(char ***plist); -int PT_EnumCache(PT_Indexes indexes, int (*callback)(void *, const char *url, PT_Element), void *arg); +int PT_EnumCache(PT_Indexes indexes, + int (*callback) (void *, const char *url, PT_Element), + void *arg); int PT_SaveCache(PT_Indexes indexes, const char *filename); /* Index */ PT_Index PT_LoadCache(const char *filename); -void PT_Index_Delete(PT_Index *pindex); -PT_Element PT_ReadCache(PT_Index index, const char* url, int flags); -int PT_LookupCache(PT_Index index, const char* url); +void PT_Index_Delete(PT_Index * pindex); +PT_Element PT_ReadCache(PT_Index index, const char *url, int flags); +int PT_LookupCache(PT_Index index, const char *url); time_t PT_Index_Timestamp(PT_Index index); /* Elements*/ PT_Element PT_ElementNew(void); -void PT_Element_Delete(PT_Element *pentry); +void PT_Element_Delete(PT_Element * pentry); #endif |