From 1ecdfbe3b855b1f13bcddccc20135f7e35f171a6 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Mon, 19 Mar 2012 13:00:51 +0000 Subject: httrack 3.43.12 --- src/Makefile.am | 2 +- src/Makefile.in | 5 +- src/hts-indextmpl.h | 12 +- src/htsback.c | 356 +++++++++++++++++++--------- src/htsback.h | 2 + src/htsbase.h | 9 +- src/htsbasenet.h | 10 +- src/htscache.c | 29 ++- src/htscache.h | 1 + src/htscatchurl.c | 4 +- src/htscore.c | 79 ++++--- src/htscore.h | 6 +- src/htscoremain.c | 81 +++++-- src/htsftp.c | 4 +- src/htsglobal.h | 14 +- src/htsindex.c | 2 +- src/htslib.c | 143 +++++++++++- src/htslib.h | 38 ++- src/htsmd5.c | 14 -- src/htsmodules.c | 47 ++-- src/htsname.c | 43 +++- src/htsname.h | 4 + src/htsnet.h | 4 +- src/htsopt.h | 2 +- src/htsparse.c | 351 +++++++++++++++------------- src/htsparse.h | 26 ++- src/htsserver.c | 7 +- src/htswizard.c | 30 +-- src/httrack.c | 28 ++- src/httrack.vcproj | 612 +++++++++++++++++++++++++++++++++++++++++-------- src/minizip/mztools.c | 572 ++++++++++++++++++++++----------------------- src/proxy/proxytrack.c | 5 +- src/webhttrack | 21 +- 33 files changed, 1723 insertions(+), 840 deletions(-) (limited to 'src') diff --git a/src/Makefile.am b/src/Makefile.am index c609b07..9dfec26 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -69,7 +69,7 @@ libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(DL_LIBS) $(SOCKET_LIBS) libhttrack_la_LDFLAGS = -version-info $(VERSION_INFO) libhtsjava_la_SOURCES = htsjava.c htsjava.h -libhtsjava_la_LIBADD = $(THREADS_LIBS) $(DL_LIBS) +libhtsjava_la_LIBADD = $(THREADS_LIBS) $(DL_LIBS) -lhttrack libhtsjava_la_LDFLAGS = -version-info $(VERSION_INFO) EXTRA_DIST = httrack.h webhttrack \ diff --git a/src/Makefile.in b/src/Makefile.in index 1351f2f..2380102 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -61,6 +61,7 @@ DEFAULT_CFLAGS = @DEFAULT_CFLAGS@ DEFS = @DEFS@ DEPDIR = @DEPDIR@ DL_LIBS = @DL_LIBS@ +DSYMUTIL = @DSYMUTIL@ ECHO = @ECHO@ ECHO_C = @ECHO_C@ ECHO_N = @ECHO_N@ @@ -85,6 +86,7 @@ MAINT = @MAINT@ MAINTAINER_MODE_FALSE = @MAINTAINER_MODE_FALSE@ MAINTAINER_MODE_TRUE = @MAINTAINER_MODE_TRUE@ MAKEINFO = @MAKEINFO@ +NMEDIT = @NMEDIT@ OBJEXT = @OBJEXT@ PACKAGE = @PACKAGE@ PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ @@ -94,6 +96,7 @@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ RANLIB = @RANLIB@ +SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ SOCKET_LIBS = @SOCKET_LIBS@ @@ -221,7 +224,7 @@ libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(DL_LIBS) $(SOCKET_LIBS) libhttrack_la_LDFLAGS = -version-info $(VERSION_INFO) libhtsjava_la_SOURCES = htsjava.c htsjava.h -libhtsjava_la_LIBADD = $(THREADS_LIBS) $(DL_LIBS) +libhtsjava_la_LIBADD = $(THREADS_LIBS) $(DL_LIBS) -lhttrack libhtsjava_la_LDFLAGS = -version-info $(VERSION_INFO) EXTRA_DIST = httrack.h webhttrack \ diff --git a/src/hts-indextmpl.h b/src/hts-indextmpl.h index 24706d5..c5ad960 100644 --- a/src/hts-indextmpl.h +++ b/src/hts-indextmpl.h @@ -174,7 +174,7 @@ regen: "
"LF\ "
"LF\ "
"LF\ - " Mirror and index made by HTTrack Website Copier [XR&CO'2007]"LF\ + " Mirror and index made by HTTrack Website Copier [XR&CO'2008]"LF\ "
"LF\ " %s"LF\ " "LF\ @@ -193,7 +193,7 @@ regen: ""LF\ ""LF\ " "LF\ - " "LF\ + " "LF\ " "LF\ "
© 2007 Xavier Roche & other contributors - Web Design: Kauler Leto.© 2008 Xavier Roche & other contributors - Web Design: Kauler Leto.
"LF\ ""LF\ @@ -324,7 +324,7 @@ regen: " "LF\ "
"LF\ "
"LF\ - " Mirror and index made by HTTrack Website Copier [XR&CO'2007]"LF\ + " Mirror and index made by HTTrack Website Copier [XR&CO'2008]"LF\ "
"LF\ " %s"LF\ " "LF\ @@ -342,7 +342,7 @@ regen: ""LF\ ""LF\ " "LF\ - " "LF\ + " "LF\ " "LF\ "
© 2007 Xavier Roche & other contributors - Web Design: Kauler Leto.© 2008 Xavier Roche & other contributors - Web Design: Kauler Leto.
"LF\ ""LF\ @@ -483,7 +483,7 @@ regen: ""LF\ ""LF\ " "LF\ - " "LF\ + " "LF\ " "LF\ "
© 2007 Xavier Roche & other contributors - Web Design: Kauler Leto.© 2008 Xavier Roche & other contributors - Web Design: Kauler Leto.
"LF\ ""LF\ @@ -620,7 +620,7 @@ regen: ""LF\ ""LF\ " "LF\ - " "LF\ + " "LF\ " "LF\ "
© 2007 Xavier Roche & other contributors - Web Design: Kauler Leto.© 2008 Xavier Roche & other contributors - Web Design: Kauler Leto.
"LF\ ""LF\ diff --git a/src/htsback.c b/src/htsback.c index 91245a7..723ad60 100644 --- a/src/htsback.c +++ b/src/htsback.c @@ -479,8 +479,23 @@ int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p) { back[p].compressed_size=back[p].r.size; // en mémoire -> passage sur disque if (!back[p].r.is_write) { +#if 1 +#ifdef _WIN32 +#undef tempnam +#define tempnam _tempnam +#endif + char *const tmp = tempnam(NULL, "httrack_temporaryGzipFile_"); + if (tmp != NULL) { + strcpybuff(back[p].tmpfile_buffer, tmp); + free(tmp); + back[p].tmpfile = back[p].tmpfile_buffer; + } else { + back[p].tmpfile = NULL; + } +#else back[p].tmpfile_buffer[0]='\0'; - back[p].tmpfile=tmpnam(back[p].tmpfile_buffer); + back[p].tmpfile=tmpnam(back[p].tmpfile_buffer); +#endif if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') { back[p].r.out=fopen(back[p].tmpfile,"wb"); if (back[p].r.out) { @@ -497,7 +512,7 @@ int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p) { } else { back[p].tmpfile[0]='\0'; back[p].r.statuscode=STATUSCODE_INVALID; - strcpybuff(back[p].r.msg,"Open error when decompressing"); + strcpybuff(back[p].r.msg,"Open error when decompressing (can not create a temporary file)"); } } } @@ -546,6 +561,11 @@ int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p) { back[p].r.adr = NULL; } + /* remove reference file, if any */ + if (back[p].r.is_write) { + url_savename_refname_remove(opt, back[p].url_adr, back[p].url_fil); + } + /* ************************************************************************ REAL MEDIA HACK Check if we have to load locally the file @@ -884,6 +904,46 @@ int back_unserialize(FILE *fp, lien_back** dst) { return 1; /* error */ } +/* serialize a reference ; used to store references of files being downloaded in case of broken download */ +int back_serialize_ref(httrackp* opt, const lien_back* src) { + char *filename = url_savename_refname_fullpath(opt, src->url_adr, src->url_fil); + FILE *fp = fopen(filename, "wb"); + if (fp == NULL) { +#ifdef _WIN32 + if (mkdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME)) == 0) +#else + if (mkdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME), S_IRWXU | S_IRWXG | S_IRWXO) == 0) +#endif + { + filename = url_savename_refname_fullpath(opt, src->url_adr, src->url_fil); + fp = fopen(filename, "wb"); + } + } + if (fp != NULL) { + int ser = back_serialize(fp, src); + fclose(fp); + return ser; + } + return 1; +} + +/* unserialize a reference ; used to store references of files being downloaded in case of broken download */ +int back_unserialize_ref(httrackp* opt, const char *adr, const char *fil, lien_back** dst) { + char *filename = url_savename_refname_fullpath(opt, adr, fil); + FILE *fp = fopen(filename, "rb"); + if (fp != NULL) { + int ser = back_unserialize(fp, dst); + fclose(fp); + if (ser != 0) { /* back_unserialize_ref() != 0 does not need cleaning up */ + back_clear_entry(*dst); /* delete entry content */ + freet(*dst); /* delete item */ + *dst = NULL; + } + return ser; + } + return 1; +} + // clear, or leave for keep-alive int back_maydelete(httrackp* opt,cache_back* cache,struct_back* sback, int p) { lien_back* const back = sback->lnk; @@ -1253,6 +1313,7 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* int p=0; char catbuff[CATBUFF_SIZE]; char catbuff2[CATBUFF_SIZE]; + lien_back* itemback = NULL; #if (defined(_DEBUG) || defined(DEBUG)) if (!test && back_exist(sback,opt,adr,fil,save)) { @@ -1399,31 +1460,41 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* a+=strlen(buff); sscanf(a,"%d",&pos); // lire position #endif + +#if HTS_FAST_CACHE==0 if (pos<0) { // pas de mise en cache data, vérifier existence +#endif /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */ - if (!IS_DELAYED_EXT(save) && fsize(fconv(catbuff,save)) <= 0) { // fichier existe pas ou est vide! + if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize(fconv(catbuff,save)) <= 0) { // fichier final n'existe pas ou est vide! int found=0; /* It is possible that the file has been moved due to changes in build structure */ { char BIGSTK previous_save[HTS_URLMAXSIZE*2]; + htsblk r; previous_save[0] = '\0'; - back[p].r = cache_readex(opt, cache, adr, fil, NULL, back[p].location_buffer, previous_save, 0); - if (previous_save[0] != '\0' && fexist(fconv(catbuff,previous_save))) { - rename(fconv(catbuff,previous_save), fconv(catbuff2,save)); - if (fexist(fconv(catbuff,save))) { - found = 1; - if ((opt->debug>1) && (opt->log!=NULL)) { - HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File '%s' has been renamed since last mirror to '%s' ; applying changes"LF, previous_save, save); test_flush; - } - } else { - if ((opt->debug>0) && (opt->log!=NULL)) { - HTS_LOG(opt,LOG_ERROR); fprintf(opt->log,"Could not rename '%s' to '%s' ; will have to retransfer it"LF, previous_save, save); test_flush; + r = cache_readex(opt, cache, adr, fil, /*head*/NULL, /*bound to back[p] (temporary)*/back[p].location_buffer, previous_save, /*ro*/1); + /* Is supposed to be on disk only */ + if (r.is_write && previous_save[0] != '\0') { + /* Exists, but with another (old) filename: rename (almost) silently */ + if (strcmp(previous_save, save) != 0 && fexist(fconv(catbuff, previous_save))) { + rename(fconv(catbuff, previous_save), fconv(catbuff2,save)); + if (fexist(fconv(catbuff,save))) { + found = 1; + if ((opt->debug>1) && (opt->log!=NULL)) { + HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File '%s' has been renamed since last mirror to '%s' ; applying changes"LF, previous_save, save); test_flush; + } + } else { + if ((opt->debug>0) && (opt->log!=NULL)) { + HTS_LOG(opt,LOG_ERROR); fprintf(opt->log,"Could not rename '%s' to '%s' ; will have to retransfer it"LF, previous_save, save); test_flush; + } } } } + back[p].location_buffer[0] = '\0'; } + /* Not found ? */ if (!found) { #if HTS_FAST_CACHE hash_pos_return=0; @@ -1438,22 +1509,29 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* FILE* fp=fopen(fconv(catbuff,save),"wb"); if (fp) fclose(fp); if (opt->log!=NULL) { - HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"File must have been erased by user, ignoring: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush; + HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"Previous file not found (erased by user ?), ignoring: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush; } } + } else { + if (opt->log!=NULL) { + HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"Previous file not found (erased by user ?), recatching: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush; + } } } } // fsize() <= 0 +#if HTS_FAST_CACHE==0 } +#endif } } // - } else + } else { #if HTS_FAST_CACHE hash_pos_return=0; #else a=NULL; #endif + } // Existe pas en cache, ou bien pas de cache présent #if HTS_FAST_CACHE @@ -1567,83 +1645,113 @@ int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* printf("..is modified test %s\n",back[p].send_too); #endif } - // Okay, pas trouvé dans le cache - // Et si le fichier existe sur disque? - // Pas dans le cache: fichier n'a pas été transféré du tout, donc pas sur disque? - } else { - if (fexist(save)) { // fichier existe? aghl! - off_t sz=fsize(save); - // Bon, là il est possible que le fichier ait été partiellement transféré - // (s'il l'avait été en totalité il aurait été inscrit dans le cache ET existerait sur disque) - // PAS de If-Modified-Since, on a pas connaissance des données à la date du cache - // On demande juste les données restantes si le date est valide (206), tout sinon (200) - if ((ishtml(opt,save) != 1) && (ishtml(opt,back[p].url_fil)!=1)) { // NON HTML (liens changés!!) - if (sz>0) { // Fichier non vide? (question bête, sinon on transfert tout!) - char lastmodified[256]; - get_filetime_rfc822(save, lastmodified); - if (strnotempty(lastmodified)) { /* pas de If-.. possible */ + } + /* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */ + else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) { + const long file_size = fsize(itemback->url_sav); + /* Found file on disk */ + if (file_size > 0) { + char *send_too = back[p].send_too; + sprintf(send_too, "Range: bytes="LLintP"-\r\n", (LLint) file_size); + send_too += strlen(send_too); + /* add etag information */ + if (strnotempty(itemback->r.etag)) { + sprintf(send_too,"If-Match: %s\r\n", itemback->r.etag); + send_too += strlen(send_too); + } + /* add date information */ + if (strnotempty(itemback->r.lastmodified)) { + sprintf(send_too,"If-Unmodified-Since: %s\r\n", itemback->r.lastmodified); + send_too += strlen(send_too); + } + back[p].http11=1; /* 1.1 */ + back[p].range_req_size=(LLint) file_size; + back[p].r.req.range_used=1; + back[p].is_update=1; /* this is an update of a file */ + back[p].r.req.nocompression=1; /* Do not compress when updating! */ + } else { + /* broken ; remove */ + url_savename_refname_remove(opt, adr, fil); + } + /* cleanup */ + back_clear_entry(itemback); /* delete entry content */ + freet(itemback); /* delete item */ + itemback = NULL; + } + /* Not in cache or temporary cache ; found on disk ? (hack) */ + else if (fexist(save)) { + off_t sz=fsize(save); + // Bon, là il est possible que le fichier ait été partiellement transféré + // (s'il l'avait été en totalité il aurait été inscrit dans le cache ET existerait sur disque) + // PAS de If-Modified-Since, on a pas connaissance des données à la date du cache + // On demande juste les données restantes si le date est valide (206), tout sinon (200) + if ((ishtml(opt,save) != 1) && (ishtml(opt,back[p].url_fil)!=1)) { // NON HTML (liens changés!!) + if (sz>0) { // Fichier non vide? (question bête, sinon on transfert tout!) + char lastmodified[256]; + get_filetime_rfc822(save, lastmodified); + if (strnotempty(lastmodified)) { /* pas de If-.. possible */ #if DEBUGCA - printf("..if unmodified since %s size "LLintP"\n", lastmodified, (LLint)sz); + printf("..if unmodified since %s size "LLintP"\n", lastmodified, (LLint)sz); #endif - if ((opt->debug>1) && (opt->log!=NULL)) { - HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File partially present ("LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush; - } - - /* impossible - don't have etag or date - if (strnotempty(back[p].r.etag)) { // ETag (RFC2616) - sprintf(back[p].send_too,"If-None-Match: %s\r\n",back[p].r.etag); - back[p].http11=1; // En tête 1.1 - } else if (strnotempty(back[p].r.lastmodified)) { - sprintf(back[p].send_too,"If-Unmodified-Since: %s\r\n",back[p].r.lastmodified); + if ((opt->debug>1) && (opt->log!=NULL)) { + HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File partially present ("LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush; + } + + /* impossible - don't have etag or date + if (strnotempty(back[p].r.etag)) { // ETag (RFC2616) + sprintf(back[p].send_too,"If-None-Match: %s\r\n",back[p].r.etag); + back[p].http11=1; // En tête 1.1 + } else if (strnotempty(back[p].r.lastmodified)) { + sprintf(back[p].send_too,"If-Unmodified-Since: %s\r\n",back[p].r.lastmodified); + back[p].http11=1; // En tête 1.1 + } else + */ + if (strlen(lastmodified)) { + sprintf(back[p].send_too, + "If-Unmodified-Since: %s\r\nRange: bytes="LLintP"-\r\n" + , lastmodified, (LLint)sz); back[p].http11=1; // En tête 1.1 - } else - */ - if (strlen(lastmodified)) { - sprintf(back[p].send_too, - "If-Unmodified-Since: %s\r\nRange: bytes="LLintP"-\r\n" - , lastmodified, (LLint)sz); - back[p].http11=1; // En tête 1.1 - back[p].range_req_size=sz; - back[p].r.req.range_used=1; - back[p].r.req.nocompression=1; - } else { - if ((opt->debug>0) && (opt->log!=NULL)) { - HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"Could not find timestamp for partially present file, restarting (lost "LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush; - } - } - - } else { + back[p].is_update=1; /* this is an update of a file */ + back[p].range_req_size=sz; + back[p].r.req.range_used=1; + back[p].r.req.nocompression=1; + } else { if ((opt->debug>0) && (opt->log!=NULL)) { - HTS_LOG(opt,LOG_WARNING); - /* - if (opt->http10) - fprintf(opt->log,"File partially present (%d bytes) retransfered due to HTTP/1.0 settings: %s%s"LF,sz,back[p].url_adr,back[p].url_fil); - else - */ - fprintf(opt->log,"File partially present ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); - test_flush; + HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"Could not find timestamp for partially present file, restarting (lost "LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush; } - /* Sinon requête normale... */ - back[p].http11=0; } - } else if (opt->norecatch) { // tester norecatch - filenote(&opt->state.strc,save,NULL); // ne pas purger tout de même - file_notify(opt,back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified); - back[p].status=STATUS_READY; // OK prêt - back_set_finished(sback, p); - back[p].r.statuscode=STATUSCODE_INVALID; // erreur - strcpybuff(back[p].r.msg,"Null-size file not recaught"); - return 0; - } - } else { - if ((opt->debug>0) && (opt->log!=NULL)) { - HTS_LOG(opt,LOG_WARNING); - fprintf(opt->log,"HTML file ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); - test_flush; + + } else { + if ((opt->debug>0) && (opt->log!=NULL)) { + HTS_LOG(opt,LOG_WARNING); + /* + if (opt->http10) + fprintf(opt->log,"File partially present (%d bytes) retransfered due to HTTP/1.0 settings: %s%s"LF,sz,back[p].url_adr,back[p].url_fil); + else + */ + fprintf(opt->log,"File partially present ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); + test_flush; + } + /* Sinon requête normale... */ + back[p].http11=0; } - /* Sinon requête normale... */ - back[p].http11=0; + } else if (opt->norecatch) { // tester norecatch + filenote(&opt->state.strc,save,NULL); // ne pas purger tout de même + file_notify(opt,back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified); + back[p].status=STATUS_READY; // OK prêt + back_set_finished(sback, p); + back[p].r.statuscode=STATUSCODE_INVALID; // erreur + strcpybuff(back[p].r.msg,"Null-size file not recaught"); + return 0; } + } else { + if ((opt->debug>0) && (opt->log!=NULL)) { + HTS_LOG(opt,LOG_WARNING); + fprintf(opt->log,"HTML file ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); + test_flush; + } + /* Sinon requête normale... */ + back[p].http11=0; } } } @@ -2264,24 +2372,27 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti // vérification de sécurité if (back[i].r.soc!=INVALID_SOCKET) { // hey, you never know.. - do_wait=1; - - // noter socket read - FD_SET(back[i].r.soc,&fds); - - // noter socket error - FD_SET(back[i].r.soc,&fds_e); - - // incrémenter nombre de sockets - nsockets++; - - // calculer max - if (max_c) { - max_c=0; - nfds=back[i].r.soc; - } else if (back[i].r.soc>nfds) { - // ID socket la plus élevée - nfds=back[i].r.soc; + // Do not endlessly wait when receiving SSL http data (Patrick Pfeifer) + if (!(back[i].r.ssl && back[i].status > 0 && back[i].status < 1000)) { + do_wait=1; + + // noter socket read + FD_SET(back[i].r.soc,&fds); + + // noter socket error + FD_SET(back[i].r.soc,&fds_e); + + // incrémenter nombre de sockets + nsockets++; + + // calculer max + if (max_c) { + max_c=0; + nfds=back[i].r.soc; + } else if (back[i].r.soc>nfds) { + // ID socket la plus élevée + nfds=back[i].r.soc; + } } } else { back[i].r.statuscode=STATUSCODE_CONNERROR; @@ -2623,8 +2734,23 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti /* .gz are *NOT* depacked!! */ (strfield(get_ext(catbuff,back[i].url_sav),"gz") == 0) ) { +#if 1 +#ifdef _WIN32 +#undef tempnam +#define tempnam _tempnam +#endif + char *const tmp = tempnam(NULL, "httrack_temporaryGzipFile_"); + if (tmp != NULL) { + strcpybuff(back[i].tmpfile_buffer, tmp); + free(tmp); + back[i].tmpfile = back[i].tmpfile_buffer; + } else { + back[i].tmpfile = NULL; + } +#else back[i].tmpfile_buffer[0]='\0'; - back[i].tmpfile=tmpnam(back[i].tmpfile_buffer); + back[i].tmpfile=tmpnam(back[p].tmpfile_buffer); +#endif if (back[i].tmpfile != NULL && back[i].tmpfile[0]) { if ((back[i].r.out=fopen(back[i].tmpfile,"wb")) == NULL) { last_errno = errno; @@ -2662,13 +2788,19 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti test_flush; } back[i].r.is_write=0; // erreur, abandonner -#if HDEBUG - printf("..error!\n"); -#endif - } + } else { #ifndef _WIN32 - else chmod(back[i].url_sav,HTS_ACCESS_FILE); + chmod(back[i].url_sav, HTS_ACCESS_FILE); #endif + /* create a temporary reference file in case of broken mirror */ + if (back[i].r.out != NULL) { + if (back_serialize_ref(opt, &back[i]) != 0) { + if (opt->log != NULL) { + HTS_LOG(opt,LOG_WARNING); fprintf(opt->log, "Could not create temporary reference file for %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush; + } + } + } + } } else { // on coupe tout! if ((opt->debug>1) && (opt->log!=NULL)) { HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File cancelled (non HTML): %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush; @@ -3409,6 +3541,12 @@ void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_ti if (back[i].r.totalsize>0) back[i].r.totalsize+=sz; // plus en fait fseek(back[i].r.out,0,SEEK_END); // à la fin + /* create a temporary reference file in case of broken mirror */ + if (back_serialize_ref(opt, &back[i]) != 0) { + if (opt->log != NULL) { + HTS_LOG(opt,LOG_WARNING); fprintf(opt->log, "Could not create temporary reference file for %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush; + } + } #if HDEBUG printf("continue interrupted file\n"); #endif diff --git a/src/htsback.h b/src/htsback.h index 57751d1..eb3f8ef 100644 --- a/src/htsback.h +++ b/src/htsback.h @@ -100,6 +100,8 @@ void back_move(lien_back* src, lien_back* dst); void back_copy_static(const lien_back* src, lien_back* dst); int back_serialize(FILE *fp, const lien_back* src); int back_unserialize(FILE *fp, lien_back** dst); +int back_serialize_ref(httrackp* opt, const lien_back* src); +int back_unserialize_ref(httrackp* opt, const char *adr, const char *fil, lien_back** dst); void back_set_finished(struct_back* sback, int p); void back_set_locked(struct_back* sback, int p); void back_set_unlocked(struct_back* sback, int p); diff --git a/src/htsbase.h b/src/htsbase.h index e3aec57..7340598 100644 --- a/src/htsbase.h +++ b/src/htsbase.h @@ -48,16 +48,13 @@ extern "C" { #include #include -#ifdef HAVE_UNISTD_H +#ifndef _WIN32 #include #endif -#if ( defined(_WIN32) ||defined(HAVE_SYS_TYPES_H) ) #include -#endif -#ifdef HAVE_SYS_STAT_H #include -#endif -#ifdef HAVE_DLFCN_H + +#ifndef _WIN32 #include #endif diff --git a/src/htsbasenet.h b/src/htsbasenet.h index 26f1392..d717654 100644 --- a/src/htsbasenet.h +++ b/src/htsbasenet.h @@ -47,9 +47,15 @@ Please visit our Website: http://www.httrack.com #ifndef _WIN32_WCE #undef HTS_USESCOPEID #define WIN32_LEAN_AND_MEAN -#include +// KB955045 (http://support.microsoft.com/kb/955045) +// To execute an application using this function on earlier versions of Windows +// (Windows 2000, Windows NT, and Windows Me/98/95), then it is mandatary to #include Ws2tcpip.h +// and also Wspiapi.h. When the Wspiapi.h header file is included, the 'getaddrinfo' function is +// #defined to the 'WspiapiGetAddrInfo' inline function in Wspiapi.h. #include -#include +#include +//#include +//#include #else #include #include diff --git a/src/htscache.c b/src/htscache.c index 068da05..d3698cf 100644 --- a/src/htscache.c +++ b/src/htscache.c @@ -541,6 +541,22 @@ htsblk cache_read_ro(httrackp* opt,cache_back* cache,const char* adr,const char* return cache_readex(opt,cache,adr,fil,save,location,NULL,1); } +htsblk cache_read_including_broken(httrackp* opt,cache_back* cache,const char* adr,const char* fil) { + htsblk r = cache_read(opt,cache,adr,fil,NULL,NULL); + if (r.statuscode = -1) { + lien_back *itemback = NULL; + if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) { + r = itemback->r; + /* cleanup */ + back_clear_entry(itemback); /* delete entry content */ + freet(itemback); /* delete item */ + itemback = NULL; + return r; + } + } + return r; +} + static htsblk cache_readex_old(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location, char* return_save, int readonly); @@ -681,9 +697,9 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,c } } else if (!readonly && r.statuscode==HTTP_OK && !is_hypertext_mime(opt,r.contenttype, fil) && strnotempty(save)) { // pas HTML, écrire sur disk directement - r.is_write=1; // écrire - if (!dataincache) { + + if (!dataincache) { if (fexist(fconv(catbuff, save))) { // un fichier existe déja //if (fsize(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree) ok=1; // plus rien à faire @@ -701,7 +717,7 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,c strcpybuff(r.msg,"File deleted by user not recaught"); ok=1; // ne pas récupérer (et pas d'erreur) } else { - file_notify(opt,adr, fil, save, 1, 1, 0); + // ????? file_notify(opt,adr, fil, save, 1, 1, 0); r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Previous cache file not found"); ok=1; // ne pas récupérer @@ -792,8 +808,11 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,const char* adr,c } } } - } // si save==null, ne rien charger (juste en tête) - + } + // si save==null, ne rien charger (juste en tête) + else if (r.statuscode==HTTP_OK && !is_hypertext_mime(opt,r.contenttype, fil)) { // pas HTML, écrire sur disk directement + r.is_write = 1; /* supposed to be on disk (informational) */ + } } else { r.statuscode=STATUSCODE_INVALID; diff --git a/src/htscache.h b/src/htscache.h index 7a4bb5c..b8dd037 100644 --- a/src/htscache.h +++ b/src/htscache.h @@ -66,6 +66,7 @@ void cache_mayadd(httrackp* opt,cache_back* cache,htsblk* r,const char* url_adr, void cache_add(httrackp* opt,cache_back* cache,const htsblk *r,const char* url_adr,const char* url_fil,const char* url_save,int all_in_cache,const char* path_prefix); htsblk cache_read(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location); htsblk cache_read_ro(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location); +htsblk cache_read_including_broken(httrackp* opt,cache_back* cache,const char* adr,const char* fil); htsblk cache_readex(httrackp* opt,cache_back* cache,const char* adr,const char* fil,const char* save,char* location,char* return_save,int readonly); htsblk* cache_header(httrackp* opt,cache_back* cache,const char* adr,const char* fil,htsblk* r); void cache_init(cache_back* cache,httrackp* opt); diff --git a/src/htscatchurl.c b/src/htscatchurl.c index 74a2439..2cb10f0 100644 --- a/src/htscatchurl.c +++ b/src/htscatchurl.c @@ -160,9 +160,7 @@ HTSEXT_API int catch_url(T_SOC soc,char* url,char* method,char* data) { // connexion (accept) if (soc != INVALID_SOCKET) { T_SOC soc2; - struct sockaddr dummyaddr; - int dummylen = sizeof(struct sockaddr); - while ( (soc2 = (T_SOC) accept(soc,&dummyaddr,&dummylen)) == INVALID_SOCKET); + while ( (soc2 = (T_SOC) accept(soc, NULL, NULL)) == INVALID_SOCKET); /* #ifdef _WIN32 closesocket(soc); diff --git a/src/htscore.c b/src/htscore.c index 41c27a0..bfc5643 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -779,6 +779,8 @@ int httpmirror(char* url1, httrackp* opt) { do { int error=0; // si error alors sauter int store_errpage=0; // c'est une erreur mais on enregistre le html + int is_binary=0; // is a binary file + int is_loaded_from_file=0; // has been loaded from a file (implies is_write=1) char BIGSTK loc[HTS_URLMAXSIZE*2]; // adresse de relocation // Ici on charge le fichier (html, gif..) en mémoire @@ -1010,7 +1012,35 @@ int httpmirror(char* url1, httrackp* opt) { } } } - + + /* Load file if necessary and decode. */ \ +#define LOAD_IN_MEMORY_IF_NECESSARY() do { \ + if ( \ + may_be_hypertext_mime(opt,r.contenttype, urlfil) /* Is HTML or Js, .. */ \ + && (liens[ptr]->depth>0) /* Depth > 0 (recurse depth) */ \ + && (r.adr==NULL) /* HTML Data exists */ \ + && (!store_errpage) /* Not an html error page */ \ + && (savename[0]!='\0') /* Output filename exists */ \ + ) \ + { \ + is_loaded_from_file = 1; \ + r.adr = readfile2(savename, &r.size); \ + if (r.adr != NULL) { \ + if ( (opt->debug>0) && (opt->log!=NULL) ) { \ + HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"File successfully loaded for parsing: %s%s (%d bytes)"LF,urladr,urlfil,(int)r.size); \ + test_flush; \ + } \ + } else { \ + if ( opt->log != NULL ) { \ + HTS_LOG(opt,LOG_ERROR); fprintf(opt->log,"File could not be loaded for parsing: %s%s"LF,urladr,urlfil); \ + test_flush; \ + } \ + } \ + } \ +} while(0) + /* Load file and decode if necessary, before content-binary check. (3.43) */ + LOAD_IN_MEMORY_IF_NECESSARY(); + // ------------------------------------ // BOGUS MIME TYPE HACK II (the revenge) // Check if we have a bogus MIME type @@ -1021,7 +1051,7 @@ int httpmirror(char* url1, httrackp* opt) { unsigned int map[256]; int i; unsigned int nspec = 0; - map_characters((unsigned char*)r.adr, (unsigned int)r.size, (unsigned int*)map); + map_characters((unsigned char*)r.adr, (unsigned int)r.size, &map[0]); for(i = 1 ; i < 32 ; i++) { // null chars ignored.. if (!is_realspace(i) && i != 27 /* Damn you ISO2022-xx! */ @@ -1136,6 +1166,7 @@ int httpmirror(char* url1, httrackp* opt) { #undef CH_ADD_RNG1 #undef CH_ADD_RNG2 } else if ((nspec > r.size / 100) && (nspec > 10)) { // too many special characters + is_binary = 1; strcpybuff(r.contenttype,"application/octet-stream"); if (opt->log) { HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"File not parsed, looks like binary: %s%s"LF,urladr,urlfil); @@ -1144,9 +1175,11 @@ int httpmirror(char* url1, httrackp* opt) { } /* This hack allows to avoid problems with parsing '\0' characters */ - for(i = 0 ; i < r.size ; i++) { - if (r.adr[i] == '\0') r.adr[i] = ' '; - } + if (!is_binary) { + for(i = 0 ; i < r.size ; i++) { + if (r.adr[i] == '\0') r.adr[i] = ' '; + } + } } @@ -1302,31 +1335,10 @@ int httpmirror(char* url1, httrackp* opt) { io_flush; } #endif - - /* Load file if necessary */ - if ( - may_be_hypertext_mime(opt,r.contenttype, urlfil) /* Is HTML or Js, .. */ - && (liens[ptr]->depth>0) /* Depth > 0 (recurse depth) */ - && (r.adr==NULL) /* HTML Data exists */ - && (!store_errpage) /* Not an html error page */ - && (savename[0]!='\0') /* Output filename exists */ - ) - { - r.adr = readfile2(savename, &r.size); - (void) unlink(fconv(OPT_GET_BUFF(opt),savename)); - if (r.adr != NULL) { - if ( (opt->debug>0) && (opt->log!=NULL) ) { - HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"File successfully loaded for parsing: %s%s (%d bytes)"LF,urladr,urlfil,(int)r.size); - test_flush; - } - } else { - if ( opt->log != NULL ) { - HTS_LOG(opt,LOG_ERROR); fprintf(opt->log,"File could not be loaded for parsing: %s%s"LF,urladr,urlfil); - test_flush; - } - } - } + /* Load file and decode if necessary, after redirect check. */ + LOAD_IN_MEMORY_IF_NECESSARY(); + // ------------------------------------------------------ // ok, fichier chargé localement // ------------------------------------------------------ @@ -1374,6 +1386,8 @@ int httpmirror(char* url1, httrackp* opt) { // traiter if ( + ! is_binary + && ( (is_hypertext_mime(opt,r.contenttype, urlfil)) /* Is HTML or Js, .. */ || (may_be_hypertext_mime(opt,r.contenttype, urlfil) && r.adr != NULL ) /* Is real media, .. */ ) @@ -1386,6 +1400,13 @@ int httpmirror(char* url1, httrackp* opt) { // -- -- -- -- // Parsing HTML if (!error) { + + /* Remove file if being processed */ + if (is_loaded_from_file) { + (void) unlink(fconv(OPT_GET_BUFF(opt),savename)); + is_loaded_from_file = 0; + } + /* Info for wrappers */ if ( (opt->debug>0) && (opt->log!=NULL) ) { HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: check-html: %s%s"LF,urladr,urlfil); diff --git a/src/htscore.h b/src/htscore.h index 5e88313..c115bc1 100644 --- a/src/htscore.h +++ b/src/htscore.h @@ -43,12 +43,8 @@ Please visit our Website: http://www.httrack.com /* specific definitions */ #include "htsbase.h" // Includes & définitions -#if ( defined(_WIN32) ||defined(HAVE_SYS_TYPES_H) ) #include -#endif -#ifdef HAVE_SYS_STAT_H #include -#endif #ifdef _WIN32 #ifndef _WIN32_WCE #include @@ -57,7 +53,7 @@ Please visit our Website: http://www.httrack.com #include #endif #else -#ifdef HAVE_UNISTD_H +#ifndef _WIN32 #include #endif #endif diff --git a/src/htscoremain.c b/src/htscoremain.c index a118c5f..bfaebaf 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -140,7 +140,19 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { // Create options _DEBUG_HEAD=0; // pas de debuggage en têtes - + + /* command max-size check (3.43 ; 3.42-4) */ + { + int i; + for(i = 0 ; i < argc ; i++) { + if (strlen(argv[i]) >= HTS_CDLMAXSIZE) { + HTS_PANIC_PRINTF("argument too long"); + htsmain_free(); + return -1; + } + } + } + /* Init root dir */ hts_rootdir(argv[0]); @@ -231,7 +243,6 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { /* Vérifier argv[] non vide */ if (strnotempty(argv[na])) { - assertf(strlen(argv[na]) < HTS_CDLMAXSIZE); /* Vérifier Commande (alias) */ result=optalias_check(argc,(const char * const *)argv,na, @@ -262,6 +273,9 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { help(argv[0],!opt->quiet); htsmain_free(); return 0; + } else if (strcmp(tmp_argv[0],"-#h")==0) { + printf("HTTrack version "HTTRACK_VERSION"%s\n", hts_get_version_info(opt)); + return 0; } else { if (strncmp(tmp_argv[0],"--",2)) { /* pas */ if ((strchr(tmp_argv[0],'q')!=NULL)) @@ -535,7 +549,7 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { // } else if (strfield2(argv[i]+2,"updatehttrack")) { #ifdef _WIN32 - char s[HTS_CDLMAXSIZE]; + char s[HTS_CDLMAXSIZE + 256]; sprintf(s,"%s not available in this version",argv[i]); HTS_PANIC_PRINTF(s); htsmain_free(); @@ -562,7 +576,7 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { } // else { - char s[HTS_CDLMAXSIZE]; + char s[HTS_CDLMAXSIZE + 256]; sprintf(s,"%s not recognized",argv[i]); HTS_PANIC_PRINTF(s); htsmain_free(); @@ -728,10 +742,10 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { } } else { // aucune URL définie et pas de cache - if (argc > 1 && strcmp(argv[0], "-#h") == 0) { - printf("HTTrack version "HTTRACK_VERSION"%s\n", hts_get_version_info(opt)); - exit(0); - } + //if (argc > 1 && strcmp(argv[1], "-#h") == 0) { + // printf("HTTrack version "HTTRACK_VERSION"%s\n", hts_get_version_info(opt)); + // exit(0); + //} if (opt->quiet) { help(argv[0],!opt->quiet); htsmain_free(); @@ -819,10 +833,10 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { for(na=1;na 0) { /* don't stick! (3.43) */ + url[cl] = ' '; + cl++; + } if (fread(url + cl, 1, fz, fp) != fz) { HTS_PANIC_PRINTF("File url list could not be read"); htsmain_free(); @@ -1325,7 +1343,7 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { break; default: { - char s[HTS_CDLMAXSIZE]; + char s[HTS_CDLMAXSIZE + 256]; sprintf(s,"invalid option %%%c\n",*com); HTS_PANIC_PRINTF(s); htsmain_free(); @@ -1372,7 +1390,7 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { break; default: { - char s[HTS_CDLMAXSIZE]; + char s[HTS_CDLMAXSIZE + 256]; sprintf(s,"invalid option %%%c\n",*com); HTS_PANIC_PRINTF(s); htsmain_free(); @@ -1587,11 +1605,13 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { return 0; break; case '~': /* internal lib test */ - { - char thisIsATestYouShouldSeeAnError[12]; - strcpybuff(thisIsATestYouShouldSeeAnError, "0123456789012345678901234567890123456789"); - return 0; - } + //Disabled because choke on GCC 4.3 (toni from links2linux.de) + //{ + // char thisIsATestYouShouldSeeAnError[12]; + // const char *const bufferOverflowTest = "0123456789012345678901234567890123456789"; + // strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest); + // return 0; + //} break; case 'f': opt->flush=1; break; case 'h': @@ -1773,7 +1793,7 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { break; // default: { - char s[HTS_CDLMAXSIZE]; + char s[HTS_CDLMAXSIZE + 256]; sprintf(s,"invalid option %c\n",*com); HTS_PANIC_PRINTF(s); htsmain_free(); @@ -1958,13 +1978,13 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { else opt->errlog=opt->log; if (opt->log==NULL) { - char s[HTS_CDLMAXSIZE]; + char s[HTS_CDLMAXSIZE + 256]; sprintf(s,"Unable to create log file %s",fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-log.txt")); HTS_PANIC_PRINTF(s); htsmain_free(); return -1; } else if (opt->errlog==NULL) { - char s[HTS_CDLMAXSIZE]; + char s[HTS_CDLMAXSIZE + 256]; sprintf(s,"Unable to create log file %s",fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-err.txt")); HTS_PANIC_PRINTF(s); htsmain_free(); @@ -2200,6 +2220,27 @@ HTSEXT_API int hts_main2(int argc, char **argv, httrackp *opt) { } } + /* Not or cleanly interrupted; erase hts-cache/ref temporary directory */ + if (opt->state.exit_xh == 0) { + // erase ref files if not interrupted + DIR *dir; + struct dirent *entry; + for(dir = opendir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME)) + ; dir != NULL && ( entry = readdir(dir) ) != NULL + ; ) + { + if (entry->d_name[0] != '\0' && entry->d_name[0] != '.') { + char *f = OPT_GET_BUFF(opt); + sprintf(f, "%s/%s", CACHE_REFNAME, entry->d_name); + (void)unlink(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), f)); + } + } + if (dir != NULL) { + (void) closedir(dir); + } + (void)rmdir(fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), CACHE_REFNAME)); + } + /* Info for wrappers */ if ( (opt->debug>0) && (opt->log!=NULL) ) { HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: free"LF); diff --git a/src/htsftp.c b/src/htsftp.c index e8797ca..f6df65e 100644 --- a/src/htsftp.c +++ b/src/htsftp.c @@ -653,9 +653,7 @@ int run_launch_ftp(FTPDownloadStruct *pStruct) { _CHECK_HALT_FTP; if (line[0]=='1') { //T_SOC soc_dat; - struct sockaddr dummyaddr; - int dummylen = sizeof(struct sockaddr); - if ( (soc_dat=accept(soc_servdat,&dummyaddr,&dummylen)) == INVALID_SOCKET) { + if ( (soc_dat=accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) { strcpybuff(back->r.msg,"Unable to accept connection"); // back->status=STATUS_FTP_READY; // fini back->r.statuscode=STATUSCODE_INVALID; diff --git a/src/htsglobal.h b/src/htsglobal.h index 5337659..be06e5d 100644 --- a/src/htsglobal.h +++ b/src/htsglobal.h @@ -39,9 +39,9 @@ Please visit our Website: http://www.httrack.com #ifndef HTTRACK_GLOBAL_DEFH #define HTTRACK_GLOBAL_DEFH -// Version -#define HTTRACK_VERSION "3.42-3" -#define HTTRACK_VERSIONID "3.42.3" +// Version (also check external version information) +#define HTTRACK_VERSION "3.43-12" +#define HTTRACK_VERSIONID "3.43.12" #define HTTRACK_AFF_VERSION "3.x" #define HTTRACK_LIB_VERSION "2.0" @@ -78,9 +78,11 @@ Please visit our Website: http://www.httrack.com // WIN32 #ifndef _WIN32_WCE +/* #define HAVE_SYS_STAT_H 1 #define HAVE_SYS_TYPES_H 1 #define HAVE_SYS_STAT_H 1 +*/ #ifndef DLLIB #define DLLIB 1 #endif @@ -104,12 +106,16 @@ Please visit our Website: http://www.httrack.com #undef HTS_MAKE_KEYWORD_INDEX #ifdef HTS_CECOMPAT #define HTS_DO_NOT_USE_FTIME 1 +/* #undef HAVE_SYS_STAT_H #undef HAVE_SYS_TYPES_H +*/ #else #undef HTS_DO_NOT_USE_FTIME +/* #define HAVE_SYS_STAT_H 1 #define HAVE_SYS_TYPES_H 1 +*/ #endif #define HTS_DLOPEN 0 @@ -260,7 +266,7 @@ Please visit our Website: http://www.httrack.com #endif /* Copyright (C) Xavier Roche and other contributors */ -#define HTTRACK_AFF_AUTHORS "[XR&CO'2007]" +#define HTTRACK_AFF_AUTHORS "[XR&CO'2010]" #define HTS_DEFAULT_FOOTER "" #define HTTRACK_WEB "http://www.httrack.com" #define HTS_UPDATE_WEBSITE "http://www.httrack.com/update.php3?Product=HTTrack&Version="HTTRACK_VERSIONID"&VersionStr="HTTRACK_VERSION"&Platform=%d&Language=%s" diff --git a/src/htsindex.c b/src/htsindex.c index 4a7bd67..4ab69d0 100644 --- a/src/htsindex.c +++ b/src/htsindex.c @@ -172,7 +172,7 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char* } // Check MIME type - if (strfield2(mime,"text/html")) { + if (is_html_mime_type(mime)) { inscript=0; } // FIXME - temporary fix for image/svg+xml (svg) diff --git a/src/htslib.c b/src/htslib.c index aba65fb..ff41d4f 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -239,6 +239,14 @@ const char* hts_detectandleave[] = { const char* hts_mime_keep[] = { "application/octet-stream", "text/plain", + "application/xml", + "text/xml", + "" +}; + +/* bogus servers returns these mime types when the extension is seen within the filename */ +const char* hts_mime_bogus_multiple[] = { + "application/x-wais-source", /* src (src.rpm) */ "" }; @@ -3097,15 +3105,19 @@ int ishtml(httrackp *opt,const char* fil) { *a = '\0'; } if (get_userhttptype(opt, mime, fil_noquery)) { - if (strfield2(mime, "text/html")) { + if (is_html_mime_type(mime)) { return 1; } else { return 0; } } + if (!strnotempty(fil_noquery)) { + return -2; + } + /* Search for known ext */ - for (a = fil_noquery + strlen(fil_noquery) - 1 ; (*a!='.') && (*a!='/') && ( a > fil_noquery ) ; a-- ); + for (a = fil_noquery + strlen(fil_noquery) - 1 ; *a != '.' && *a != '/' && a > fil_noquery ; a-- ); if (*a == '.') { // a une extension char BIGSTK fil_noquery[HTS_URLMAXSIZE*2]; char* b; @@ -3690,15 +3702,15 @@ HTSEXT_API char* unescape_http(char *catbuff, const char* s) { } // unescape in URL/URI ONLY what has to be escaped, to form a standard URL/URI -// DOES NOT DECODE %25 +// DOES NOT DECODE %25 (part of CHAR_DELIM) HTSEXT_API char* unescape_http_unharm(char *catbuff, const char* s, int no_high) { int i,j=0; for (i=0;i<(int) strlen(s);i++) { if (s[i]=='%') { int nchar=(char) ehex(s+i+1); - int test = ( CHAR_RESERVED(nchar) - || ( nchar != '%' && CHAR_DELIM(nchar) ) + int test = ( ( CHAR_RESERVED(nchar) && nchar != '+' ) /* %2B => + (not in query!) */ + || CHAR_DELIM(nchar) || CHAR_UNWISE(nchar) || CHAR_LOW(nchar) /* CHAR_SPECIAL */ || CHAR_XXAVOID(nchar) @@ -3810,9 +3822,13 @@ HTSEXT_API void x_escape_http(char* s,int mode) { || CHAR_XXAVOID(*s) ); } else if (mode==30) { // échapper que ce qui est nécessaire - test = ( - CHAR_LOW(*s) - || CHAR_XXAVOID(*s) ); + test = + ( *s != '/' && CHAR_RESERVED(*s) ) + || CHAR_DELIM(*s) + || CHAR_UNWISE(*s) + || CHAR_SPECIAL(*s) + || CHAR_XXAVOID(*s) + ; } if (test) { @@ -4143,7 +4159,7 @@ HTSEXT_API int is_knowntype(httrackp *opt,const char *fil) { ext = get_ext(catbuff, fil); while(strnotempty(hts_mime[j][1])) { if (strfield2(hts_mime[j][1], ext)) { - if (strfield2(hts_mime[j][0], "text/html")) + if (is_html_mime_type(hts_mime[j][0])) return 2; else return 1; @@ -4189,7 +4205,7 @@ HTSEXT_API int is_userknowntype(httrackp *opt,const char *fil) { get_userhttptype(opt, mime, fil); if (!strnotempty(mime)) return 0; - else if (strfield2(mime,"text/html")) + else if (is_html_mime_type(mime)) return 2; else return 1; @@ -4229,6 +4245,43 @@ int may_unknown(httrackp *opt,const char* st) { return 0; } +/* returns 1 if the mime/filename seems to be bogus because of badly recognized multiple extension + ; such as "application/x-wais-source" for "httrack-3.42-1.el5.src.rpm" + reported by Hippy Dave 08/2008 (3.43) */ +int may_bogus_multiple(httrackp *opt, const char* mime, const char *filename) { + int j; + for(j = 0 ; strnotempty(hts_mime_bogus_multiple[j]) ; j++) { + if (strfield2(hts_mime_bogus_multiple[j], mime)) { /* found mime type in suspicious list */ + char ext[64]; + ext[0] = '\0'; + give_mimext(ext, mime); + if (ext[0] != 0) { /* we have an extension for that */ + const size_t ext_size = strlen(ext); + const char *file = strrchr(filename, '/'); /* fetch terminal filename */ + if (file != NULL) { + int i; + for(i = 0 ; file[i] != 0 ; i++) { + if (i > 0 && file[i - 1] == '.' && strncasecmp(&file[i], ext, ext_size) == 0 + && ( file[i + ext_size] == 0 || file[i + ext_size] == '.' || file[i + ext_size] == '?' ) ) { + return 1; /* is ambiguous */ + } + } + } + } + return 0; + } + } + return 0; +} + +/* filename extension should not be changed because potentially bogus ; replaces may_unknown() (3.43) */ +int may_unknown2(httrackp *opt, const char* mime, const char *filename) { + int ret = may_unknown(opt, mime); + if (ret == 0) { + ret = may_bogus_multiple(opt, mime, filename); + } + return ret; +} // -- Utils fichiers @@ -5568,5 +5621,75 @@ HTSEXT_API int hts_resetvar(void) { return 0; } +#ifdef _WIN32 + +typedef struct dirent dirent; +DIR *opendir(const char *name) { + WIN32_FILE_ATTRIBUTE_DATA st; + DIR *dir; + size_t len; + int i; + if (name == NULL || *name == '\0') { + errno = ENOENT; + return NULL; + } + if (!GetFileAttributesEx(name, GetFileExInfoStandard, &st) + || ( st.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) == 0) { + errno = ENOENT; + return NULL; + } + dir = calloc(sizeof(DIR), 1); + if (dir == NULL) { + errno = ENOMEM; + return NULL; + } + len = strlen(name); + dir->h = INVALID_HANDLE_VALUE; + dir->name = malloc(len + 2 + 1); + strcpy(dir->name, name); + for(i = 0 ; dir->name[i] != '\0' ; i++) { + if (dir->name[i] == '/') { + dir->name[i] = '\\'; + } + } + strcat(dir->name, "\\*"); + return dir; +} + +struct dirent *readdir(DIR *dir) { + WIN32_FIND_DATAA find; + if (dir->h == INVALID_HANDLE_VALUE) { + dir->h = FindFirstFileA(dir->name, &find); + } else { + if (!FindNextFile(dir->h, &find)) { + FindClose(dir->h); + dir->h = INVALID_HANDLE_VALUE; + } + } + if (dir->h != INVALID_HANDLE_VALUE) { + dir->entry.d_name[0] = 0; + strncat(dir->entry.d_name, find.cFileName, HTS_DIRENT_SIZE - 1); + return &dir->entry; + } + errno = ENOENT; + return NULL; +} + +int closedir(DIR *dir) { + if (dir != NULL) { + if (dir->h != INVALID_HANDLE_VALUE) { + CloseHandle(dir->h); + } + if (dir->name != NULL) { + free(dir->name); + } + free(dir); + return 0; + } + errno = EBADF; + return -1; +} +#endif + // Fin diff --git a/src/htslib.h b/src/htslib.h index ae5fde7..19d7adc 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -62,6 +62,12 @@ typedef struct t_dnscache t_dnscache; #include "htsnet.h" #include "htsdefines.h" +/* readdir() */ +#ifndef _WIN32 +#include +#include +#endif + /* cookies et auth */ #include "htsbauth.h" @@ -349,6 +355,8 @@ HTSEXT_API int is_dyntype(const char *fil); HTSEXT_API char* get_ext(char *catbuff, const char *fil); #endif int may_unknown(httrackp *opt,const char* st); +int may_bogus_multiple(httrackp *opt, const char* mime, const char *filename); +int may_unknown2(httrackp *opt,const char* mime, const char *filename); #ifndef HTTRACK_DEFLIB HTSEXT_API char* jump_identification(const char*); HTSEXT_API char* jump_normalized(const char*); @@ -546,8 +554,14 @@ HTS_STATIC int strcmpnocase(char* a,char* b) { #else #define OPT_MMS(a) (0) #endif -#define is_hypertext_mime__(a) \ + +#define is_html_mime_type(a) \ ( (strfield2((a),"text/html")!=0)\ + || (strfield2((a),"application/xhtml+xml")!=0) \ + ) +#define is_hypertext_mime__(a) \ + ( \ + is_html_mime_type(a)\ || (strfield2((a),"application/x-javascript")!=0) \ || (strfield2((a),"text/css")!=0) \ /*|| (strfield2((a),"text/vnd.wap.wml")!=0)*/ \ @@ -564,7 +578,6 @@ HTS_STATIC int strcmpnocase(char* a,char* b) { || OPT_MMS(a) \ ) - /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE @@ -634,6 +647,25 @@ HTS_STATIC char *getcwd_ce(char *buffer, int maxlen) { #define getcwd getcwd_ce #endif +/* dirent() compatibility */ +#ifdef _WIN32 +#define HTS_DIRENT_SIZE 256 +struct dirent { + ino_t d_ino; /* ignored */ + off_t d_off; /* ignored */ + unsigned short d_reclen; /* ignored */ + unsigned char d_type; /* ignored */ + char d_name[HTS_DIRENT_SIZE]; /* filename */ +}; +typedef struct DIR DIR; +struct DIR { + HANDLE h; + struct dirent entry; + char *name; +}; +DIR *opendir(const char *name); +struct dirent *readdir(DIR *dir); +int closedir(DIR *dir); #endif - +#endif diff --git a/src/htsmd5.c b/src/htsmd5.c index 114ae7a..c026225 100644 --- a/src/htsmd5.c +++ b/src/htsmd5.c @@ -52,24 +52,11 @@ int domd5mem(const unsigned char * buf, size_t len, unsigned char * digest, int asAscii) { int endian = 1; unsigned char bindigest[16]; -#if 1 -//#ifndef _WIN32_WCE MD5_CTX ctx; MD5Init(&ctx, * ( (char*) &endian)); MD5Update(&ctx, buf, (unsigned int) len); MD5Final(bindigest, &ctx); -#else - /* Broken md5.. temporary hack */ - int i; - memset(bindigest, 0, 16); - if (len > 0) { - for(i = 0 ; i < len + 16 ; i++) { - bindigest[i % 16] ^= ( buf[i % len] + i + len ); - bindigest[(i - 1) % 16] ^= bindigest[ ( i + buf[i % len]*buf[(i-1) % len] ) % 16]; - } - } -#endif if (!asAscii) { memcpy(digest, bindigest, 16); @@ -80,7 +67,6 @@ int domd5mem(const unsigned char * buf, size_t len, bindigest[4], bindigest[5], bindigest[6], bindigest[7], bindigest[8], bindigest[9], bindigest[10], bindigest[11], bindigest[12], bindigest[13], bindigest[14], bindigest[15]); - } return 0; diff --git a/src/htsmodules.c b/src/htsmodules.c index 8f7facc..07291bd 100644 --- a/src/htsmodules.c +++ b/src/htsmodules.c @@ -263,30 +263,35 @@ void htspe_init(void) { handle = LoadLibraryA((char*)"ssleay32"); #else /* We are compatible with 0.9.6/7/8/8b and potentially above */ - handle = dlopen("libssl.so.0.9.8g", RTLD_LAZY); /* added 8g release too (Debarshi Ray) */ - if (handle == NULL) { - handle = dlopen("libssl.so.0.9.8b", RTLD_LAZY); - } - if (handle == NULL) { - handle = dlopen("libssl.so.0.9.8", RTLD_LAZY); - } - if (handle == NULL) { - handle = dlopen("libssl.so.0.9.7", RTLD_LAZY); - } - if (handle == NULL) { - handle = dlopen("libssl.so.0.9.6", RTLD_LAZY); - } - if (handle == NULL) { - /* Try harder with .0 if any */ - handle = dlopen("libssl.so.0", RTLD_LAZY); - } - if (handle == NULL) { - /* Try harder with devel link */ - handle = dlopen("libssl.so", RTLD_LAZY); + static const char *const libs[] = { + "libssl.so.1.0", + "libssl.so.1", + "libssl.so.1.0.0", + /* */ + "libssl.so.0", + "libssl.so.0.9", + "libssl.so.0.9.8p", + "libssl.so.0.9.8o", + "libssl.so.0.9.8n", + "libssl.so.0.9.8m", + "libssl.so.0.9.8l", + "libssl.so.0.9.8k", /* (Debarshi Ray) */ + "libssl.so.0.9.8j", /* (Debarshi Ray) */ + "libssl.so.0.9.8g", /* Added 8g release too (Debarshi Ray) */ + "libssl.so.0.9.8b", + "libssl.so.0.9.8", + "libssl.so.0.9.7", + "libssl.so.0.9.6", + "libssl.so", /* Try harder with devel link */ + NULL + }; + int i; + for(i = 0, handle = NULL ; handle == NULL && libs[i] != NULL ; i++) { + handle = dlopen(libs[i], RTLD_LAZY); } #endif ssl_handle = handle; - if (handle) { + if (handle != NULL) { SSL_shutdown = (t_SSL_shutdown) DynamicGet(handle, (char*)"SSL_shutdown"); SSL_free = (t_SSL_free) DynamicGet(handle, (char*)"SSL_free"); SSL_new = (t_SSL_new) DynamicGet(handle, (char*)"SSL_new"); diff --git a/src/htsname.c b/src/htsname.c index 1a48f9f..0a24a5a 100644 --- a/src/htsname.c +++ b/src/htsname.c @@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com #include "htscore.h" #include "htsname.h" +#include "md5.h" #include "htsmd5.h" #include "htstools.h" #include @@ -112,7 +113,8 @@ int url_savename(char* adr_complete, char* fil_complete, char* save, struct_back* sback, cache_back* cache, hash_struct* hash, int ptr, int numero_passe, const lien_back* headers) { char catbuff[CATBUFF_SIZE]; - const char* mime_type = ( headers && HTTP_IS_OK(headers->r.statuscode) ) ? headers->r.contenttype : NULL; + const char* mime_type = ( headers && !HTTP_IS_REDIRECT(headers->r.statuscode) ) ? headers->r.contenttype : NULL; + /*const char* mime_type = ( headers && HTTP_IS_OK(headers->r.statuscode) ) ? headers->r.contenttype : NULL;*/ lien_back* const back = sback->lnk; /* */ char BIGSTK newfil[HTS_URLMAXSIZE*2]; /* ="" */ @@ -349,7 +351,7 @@ int url_savename(char* adr_complete, char* fil_complete, char* save, if (!( (opt->check_type==1) && (fil[strlen(fil)-1]=='/') )) // slash doit être html? if ( opt->savename_delayed == 2 || (ishtest=ishtml(opt,fil)) < 0) { // on ne sait pas si c'est un html ou un fichier.. // lire dans le cache - htsblk r = cache_read(opt,cache,adr,fil,NULL,NULL); // test uniquement + htsblk r = cache_read_including_broken(opt,cache,adr,fil); // test uniquement if (r.statuscode != -1) { // pas d'erreur de lecture cache char s[16]; s[0]='\0'; if ( (opt->debug>1) && (opt->log!=NULL) ) { @@ -360,7 +362,7 @@ int url_savename(char* adr_complete, char* fil_complete, char* save, ext_chg=2; /* change filename */ strcpybuff(ext,r.cdispo); } - else if (!may_unknown(opt,r.contenttype) || ishtest == -2) { // on peut patcher à priori? + else if (!may_unknown2(opt,r.contenttype,fil) || ishtest == -2) { // on peut patcher à priori? give_mimext(s,r.contenttype); // obtenir extension if (strnotempty(s)>0) { // on a reconnu l'extension ext_chg=1; @@ -395,7 +397,7 @@ int url_savename(char* adr_complete, char* fil_complete, char* save, mime_from_file[0] = 0; get_httptype(opt, mime_from_file, fil, 1); if (!strnotempty(mime_from_file) || strcasecmp(mime_type, mime_from_file) != 0) { /* different mime for this type */ - if (!may_unknown(opt, mime_type)) { + if (!may_unknown2(opt, mime_type, fil)) { ext_chg = 1; } } else { @@ -599,7 +601,7 @@ int url_savename(char* adr_complete, char* fil_complete, char* save, ext_chg=2; /* change filename */ strcpybuff(ext,back[b].r.cdispo); } - else if (!may_unknown(opt,back[b].r.contenttype) || ishtest == -2 ) { // on peut patcher à priori? (pas interdit ou pas de type) + else if (!may_unknown2(opt, back[b].r.contenttype, back[b].url_fil) || ishtest == -2 ) { // on peut patcher à priori? (pas interdit ou pas de type) give_mimext(s,back[b].r.contenttype); // obtenir extension if (strnotempty(s)>0) { // on a reconnu l'extension ext_chg=1; @@ -1483,4 +1485,35 @@ void url_savename_addstr(char* d,char* s) { d[i]='\0'; } +/* "filename" should be at least 64 bytes. */ +void url_savename_refname(const char *adr, const char *fil, char *filename) { + unsigned char bindigest[16]; + MD5_CTX ctx; + MD5Init(&ctx, 0); + MD5Update(&ctx, adr, (unsigned int) strlen(adr)); + MD5Update(&ctx, ",", 1); + MD5Update(&ctx, fil, (unsigned int) strlen(fil)); + MD5Final(bindigest, &ctx); + sprintf(filename, CACHE_REFNAME "/" + "%02x%02x%02x%02x%02x%02x%02x%02x" + "%02x%02x%02x%02x%02x%02x%02x%02x" + ".ref", + bindigest[0], bindigest[1], bindigest[2], bindigest[3], + bindigest[4], bindigest[5], bindigest[6], bindigest[7], + bindigest[8], bindigest[9], bindigest[10], bindigest[11], + bindigest[12], bindigest[13], bindigest[14], bindigest[15]); +} + +char *url_savename_refname_fullpath(httrackp* opt, const char *adr, const char *fil) { + char digest_filename[64]; + url_savename_refname(adr, fil, digest_filename); + return fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log), digest_filename); +} + +/* remove refname if any */ +void url_savename_refname_remove(httrackp* opt, const char *adr, const char *fil) { + char *filename = url_savename_refname_fullpath(opt, adr, fil); + (void) unlink(filename); +} + #undef test_flush diff --git a/src/htsname.h b/src/htsname.h index 9d98622..0bb4c2c 100644 --- a/src/htsname.h +++ b/src/htsname.h @@ -54,6 +54,7 @@ HTS_STATIC int strendwith_(const char* a, const char* b) { } return (j == -1); } +#define CACHE_REFNAME "hts-cache/ref" /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE @@ -98,6 +99,9 @@ int url_savename(char* adr_complete, char* fil_complete, char* save, void standard_name(char* b,char* dot_pos,char* nom_pos,char* fil_complete,int short_ver); void url_savename_addstr(char* d,char* s); char* url_md5(char* digest_buffer, char* fil_complete); +void url_savename_refname(const char *adr, const char *fil, char *filename); +char *url_savename_refname_fullpath(httrackp* opt, const char *adr, const char *fil); +void url_savename_refname_remove(httrackp* opt, const char *adr, const char *fil); #endif #endif diff --git a/src/htsnet.h b/src/htsnet.h index 7c7fbe3..14b00f0 100644 --- a/src/htsnet.h +++ b/src/htsnet.h @@ -65,8 +65,8 @@ Please visit our Website: http://www.httrack.com #endif #include /* gethostname & co */ -#ifdef HAVE_UNISTD_H - #include +#ifndef _WIN32 +#include #endif /* inet_addr */ #include diff --git a/src/htsopt.h b/src/htsopt.h index 0b304f8..db50dd2 100644 --- a/src/htsopt.h +++ b/src/htsopt.h @@ -217,7 +217,7 @@ struct htsoptstate { int verif_external_status; t_dnscache *dns_cache; /* HTML parsing state */ - char _hts_errmsg[1024]; + char _hts_errmsg[HTS_CDLMAXSIZE + 256]; int _hts_in_html_parsing; int _hts_in_html_done; int _hts_in_html_poll; diff --git a/src/htsparse.c b/src/htsparse.c index b39b41f..34ef831 100644 --- a/src/htsparse.c +++ b/src/htsparse.c @@ -313,30 +313,50 @@ Please visit our Website: http://www.httrack.com } \ } -#define ENGINE_LOAD_CONTEXT() \ - ENGINE_LOAD_CONTEXT_BASE(); \ +#define ENGINE_DEFINE_CONTEXT() \ + ENGINE_DEFINE_CONTEXT_BASE(); \ + /* */ \ + htsblk* const r HTS_UNUSED = stre->r_; \ + hash_struct* const hash HTS_UNUSED = stre->hash_; \ + char* const codebase HTS_UNUSED = stre->codebase; \ + char* const base HTS_UNUSED = stre->base; \ + /* */ \ + const char * const template_header HTS_UNUSED = stre->template_header_; \ + const char * const template_body HTS_UNUSED = stre->template_body_; \ + const char * const template_footer HTS_UNUSED = stre->template_footer_; \ + /* */ \ + char* const makeindex_firstlink = stre->makeindex_firstlink_; \ + /* */ \ + /* */ \ + int error = * stre->error_; \ + int store_errpage = * stre->store_errpage_; \ + int lien_max = *stre->lien_max_; \ /* */ \ - htsblk* r HTS_UNUSED = stre->r_; \ - hash_struct* hash HTS_UNUSED = stre->hash_; \ - int lien_max HTS_UNUSED = *stre->lien_max_; \ + int makeindex_done = *stre->makeindex_done_; \ + FILE* makeindex_fp = *stre->makeindex_fp_; \ + int makeindex_links = *stre->makeindex_links_; \ /* */ \ - int error HTS_UNUSED = * stre->error_; \ - int store_errpage HTS_UNUSED = * stre->store_errpage_; \ - char* codebase HTS_UNUSED = stre->codebase; \ - char* base HTS_UNUSED = stre->base; \ + LLint stat_fragment = *stre->stat_fragment_; \ + TStamp makestat_time = stre->makestat_time; \ + FILE* makestat_fp = stre->makestat_fp + +#define ENGINE_SET_CONTEXT() \ + ENGINE_SET_CONTEXT_BASE(); \ /* */ \ - int makeindex_done HTS_UNUSED = *stre->makeindex_done_; \ - FILE* makeindex_fp HTS_UNUSED = *stre->makeindex_fp_; \ - int makeindex_links HTS_UNUSED = *stre->makeindex_links_; \ - char* makeindex_firstlink HTS_UNUSED = stre->makeindex_firstlink_; \ + error = * stre->error_; \ + store_errpage = * stre->store_errpage_; \ + lien_max = *stre->lien_max_; \ /* */ \ - char *template_header HTS_UNUSED = stre->template_header_; \ - char *template_body HTS_UNUSED = stre->template_body_; \ - char *template_footer HTS_UNUSED = stre->template_footer_; \ + makeindex_done = *stre->makeindex_done_; \ + makeindex_fp = *stre->makeindex_fp_; \ + makeindex_links = *stre->makeindex_links_; \ /* */ \ - LLint stat_fragment HTS_UNUSED = *stre->stat_fragment_; \ - TStamp makestat_time HTS_UNUSED = stre->makestat_time; \ - FILE* makestat_fp HTS_UNUSED = stre->makestat_fp + stat_fragment = *stre->stat_fragment_; \ + makestat_time = stre->makestat_time; \ + makestat_fp = stre->makestat_fp + +#define ENGINE_LOAD_CONTEXT() \ + ENGINE_DEFINE_CONTEXT() #define ENGINE_SAVE_CONTEXT() \ ENGINE_SAVE_CONTEXT_BASE(); \ @@ -382,9 +402,9 @@ Please visit our Website: http://www.httrack.com /* Main parser */ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { + char catbuff[CATBUFF_SIZE]; /* Load engine variables */ ENGINE_LOAD_CONTEXT(); - char catbuff[CATBUFF_SIZE]; { char* cAddr = r->adr; @@ -428,6 +448,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { } if (!error) { + time_t user_interact_timestamp = 0; int detect_title=0; // détection du title int back_add_stats = opt->state.back_add_stats; // @@ -1331,76 +1352,76 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { expected='('; // parenthèse expected_end=")"; // fin: parenthèse } - if (!nc) if ( (nc = strfield(adr,"url")) && (!isalnum(*(adr - 1))) - && *(adr - 1) != '_' - ) { // url(url) - expected='('; // parenthèse - expected_end=")"; // fin: parenthèse - can_avoid_quotes=1; - quotes_replacement=')'; - } - if (!nc) if ( (nc = strfield(adr,"import")) ) { // import "url" - if (is_space(*(adr+nc))) { - expected=0; // no char expected - } else - nc=0; - } - if (nc) { - char *a; - a=adr+nc; + if (!nc && (nc = strfield(adr,"url")) && (!isalnum(*(adr - 1))) && *(adr - 1) != '_') { // url(url) + expected='('; // parenthèse + expected_end=")"; // fin: parenthèse + can_avoid_quotes=1; + quotes_replacement=')'; + } else { + nc = 0; + } + if (!nc) if ( (nc = strfield(adr,"import")) ) { // import "url" + if (is_space(*(adr+nc))) { + expected=0; // no char expected + } else + nc=0; + } + if (nc) { + char *a; + a=adr+nc; + while(is_realspace(*a)) a++; + if ((*a == expected) || (!expected)) { + if (expected) + a++; while(is_realspace(*a)) a++; - if ((*a == expected) || (!expected)) { - if (expected) + if ((*a==34) || (*a=='\'') || (can_avoid_quotes)) { + char *b,*c; + int ndelim=1; + if ((*a==34) || (*a=='\'')) a++; - while(is_realspace(*a)) a++; - if ((*a==34) || (*a=='\'') || (can_avoid_quotes)) { - char *b,*c; - int ndelim=1; - if ((*a==34) || (*a=='\'')) - a++; - else - ndelim=0; - b=a; - if (ndelim) { - while((*b!=34) && (*b!='\'') && (*b!='\0')) b++; - } - else { - while((*b != quotes_replacement) && (*b!='\0')) b++; - } - c=b--; c+=ndelim; - while(*c==' ') c++; - if ((strchr(expected_end,*c)) || (*c=='\n') || (*c=='\r')) { - c-=(ndelim+1); - if ((int) (c - a + 1)) { - if (ensure_not_mime) { - int i = 0; - while(a != NULL && hts_main_mime[i] != NULL && hts_main_mime[i][0] != '\0') { - int p; - if ((p=strfield(a, hts_main_mime[i])) && a[p] == '/') { - a=NULL; - } - i++; + else + ndelim=0; + b=a; + if (ndelim) { + while((*b!=34) && (*b!='\'') && (*b!='\0')) b++; + } + else { + while((*b != quotes_replacement) && (*b!='\0')) b++; + } + c=b--; c+=ndelim; + while(*c==' ') c++; + if ((strchr(expected_end,*c)) || (*c=='\n') || (*c=='\r')) { + c-=(ndelim+1); + if ((int) (c - a + 1)) { + if (ensure_not_mime) { + int i = 0; + while(a != NULL && hts_main_mime[i] != NULL && hts_main_mime[i][0] != '\0') { + int p; + if ((p=strfield(a, hts_main_mime[i])) && a[p] == '/') { + a=NULL; } + i++; } - if (a != NULL) { - if ((opt->debug>1) && (opt->log!=NULL)) { - char str[512]; - str[0]='\0'; - strncatbuff(str,a,minimum((int) (c - a + 1),32)); - HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"link detected in javascript: %s"LF,str); test_flush; - } - p=(int) (a - adr); // p non nul: TRAITER CHAINE COMME FICHIER - if (can_avoid_quotes) { - ending_p=quotes_replacement; - } + } + if (a != NULL) { + if ((opt->debug>1) && (opt->log!=NULL)) { + char str[512]; + str[0]='\0'; + strncatbuff(str,a,minimum((int) (c - a + 1),32)); + HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"link detected in javascript: %s"LF,str); test_flush; + } + p=(int) (a - adr); // p non nul: TRAITER CHAINE COMME FICHIER + if (can_avoid_quotes) { + ending_p=quotes_replacement; } } } + } - } } } + } } /* HTSPARSE_NO_JAVASCRIPT */ @@ -1932,12 +1953,12 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { unescape_amp(lien); unescape_amp(query); // décoder l'inutile (%2E par exemple) et coder espaces - // XXXXXXXXXXXXXXXXX strcpybuff(lien,unescape_http(lien)); - //strcpybuff(lien,unescape_http_unharm(lien, (no_esc_utf)?0:1)); + // Bad: strcpybuff(lien,unescape_http(lien)); + // Bad: strcpybuff(lien,unescape_http_unharm(lien, (no_esc_utf)?0:1)); /* Never unescape high-chars (we don't know the encoding!!) */ strcpybuff(lien,unescape_http_unharm(catbuff,lien, 1)); /* note: '%' is still escaped */ escape_remove_control(lien); - escape_spc_url(lien); + // ???? No! escape_spc_url(lien); strcatbuff(lien,query); /* restore */ } @@ -2114,7 +2135,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { strcatbuff(lien, "/"); while( (a > lien) && (*a) && (*a!='/')) a--; if (*a=='/') { // ok on a repéré le dernier / - if (start_of_filename != NULL && a >= start_of_filename) { + if (start_of_filename != NULL && a + 1 >= start_of_filename) { *(a+1)='\0'; // couper } } else { @@ -2352,6 +2373,8 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { if ((p_type!=2) && (p_type!=-2)) { // pas base href ou codebase if (forbidden_url!=1) { char BIGSTK last_adr[HTS_URLMAXSIZE*2]; + + /* Calc */ last_adr[0]='\0'; //char last_fil[HTS_URLMAXSIZE*2]=""; strcpybuff(last_adr,adr); // ancienne adresse @@ -2398,8 +2421,22 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { && forbidden_url == 0 && IS_DELAYED_EXT(save) ) - { // pas d'erreur, on continue + { + time_t t; + + // pas d'erreur, on continue r_sv = hts_wait_delayed(str, adr, fil, save, parenturladr, parenturlfil, former_adr, former_fil, &forbidden_url); + + /* User interaction, because hts_wait_delayed can be slow.. (3.43) */ + t = time(NULL); + if (user_interact_timestamp == 0 || t - user_interact_timestamp > 0) { + user_interact_timestamp = t; + ENGINE_SAVE_CONTEXT(); + { + hts_mirror_process_user_interaction(str, stre); + } + ENGINE_SET_CONTEXT(); + } } // record! @@ -2715,6 +2752,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { if (!in_media) { // In media (such as real audio): don't patch /* Never escape high-chars (we don't know the encoding!!) */ escape_uri_utf(tempo); + //if (!no_esc_utf) // escape_uri(tempo); // escape with %xx //else { @@ -3596,18 +3634,13 @@ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre) } - - /* -Wait for next file and -check 301, 302, .. statuscodes (moved) + Process pause, link adding.. */ -int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* stre) { +void hts_mirror_process_user_interaction(htsmoduleStruct* str, htsmoduleStructExtended* stre) { + int b; /* Load engine variables */ ENGINE_LOAD_CONTEXT(); - /* */ - int b; - int n; #if BDEBUG==1 printf("\nBack test..\n"); @@ -3664,7 +3697,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* } *stre->exit_xh_=1; // exit requested XH_uninit; - return 0; + return ; } } } @@ -3731,7 +3764,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* } //if (opt->getmode & 1) { if (fp) { fclose(fp); fp=NULL; } } XH_uninit; // désallocation mémoire & buffers - return 0; + return ; } } else { if ( (opt->debug>0) && (opt->log!=NULL) ) { @@ -3781,15 +3814,35 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* } *stre->exit_xh_=1; // exit requested XH_uninit; - return 0; + return ; } Sleep(100); // pause } opt->state._hts_in_html_parsing = prev; } + ENGINE_SAVE_CONTEXT(); + return ; +} + +/* +Wait for next file and +check 301, 302, .. statuscodes (moved) +*/ +int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* stre) { + /* Load engine variables */ + ENGINE_DEFINE_CONTEXT(); + int b; + int n; + + /* User interaction */ + ENGINE_SAVE_CONTEXT(); + { + hts_mirror_process_user_interaction(str, stre); + } + ENGINE_SET_CONTEXT(); // si le fichier n'est pas en backing, le mettre.. - if (!back_exist(sback,opt,urladr,urlfil,savename)) { + if (!back_exist(str->sback,str->opt,urladr,urlfil,savename)) { #if BDEBUG==1 printf("crash backing: %s%s\n",liens[ptr]->adr,liens[ptr]->fil); #endif @@ -3826,6 +3879,12 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* // ------------------------------------------------------------ // attendre que le fichier actuel soit prêt - BOUCLE D'ATTENTE do { + /* User interaction */ + ENGINE_SAVE_CONTEXT(); + { + hts_mirror_process_user_interaction(str, stre); + } + ENGINE_SET_CONTEXT(); // index du lien actuel b=back_index(opt,sback,urladr,urlfil,savename); @@ -4093,49 +4152,6 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* back_maydelete(opt,cache,sback,b); } - // progression -#if 0 - if (opt->aff_progress) { - TStamp tl=time_local(); - if ((tl-HTS_STAT.stat_timestart)>0) { - char s[32]; - int i=0; - lastime=tl; - _CLRSCR; _GOTOXY("1","1"); - printf("Rate=%d B/sec\n",(int) (HTS_STAT.HTS_TOTAL_RECV/(tl-HTS_STAT.stat_timestart))); - while(i=0) { // loading.. - s[0]='\0'; - if (strlen(back[i].url_fil)>16) - strcatbuff(s,back[i].url_fil+strlen(back[i].url_fil)-16); - else - strncatbuff(s,back[i].url_fil,16); - printf("%s : ",s); - - printf("["); - if (back[i].r.totalsize>0) { - int p; - int j; - p=(int)((back[i].r.size*10)/back[i].r.totalsize); - p=minimum(10,p); - for(j=0;jsavename_delayed != 0 @@ -4385,20 +4398,12 @@ int hts_wait_delayed(htsmoduleStruct* str, /* Error */ if (HTTP_IS_ERROR(back[b].r.statuscode)) { - /* 'no error page' selected or file discarded by size rules! */ - if (!opt->errpage || ( back[b].r.statuscode == STATUSCODE_TOO_BIG ) ) { - /* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */ - *forbidden_url = 1; /* Forbidden! */ - if (opt->log != NULL) { - if (back[b].r.statuscode == STATUSCODE_TOO_BIG) { - HTS_LOG(opt,LOG_ERROR); fprintf(opt->log,"link not taken because of its size (%d bytes) at %s%s"LF,(int)back[b].r.totalsize,adr,fil); - } else { - HTS_LOG(opt,LOG_ERROR); fprintf(opt->log,"link not taken because of error (%d '%s') at %s%s"LF,back[b].r.statuscode,back[b].r.msg,adr,fil); - } - test_flush; - } - break; - } + /* seen as in error */ + in_error = back[b].r.statuscode; + in_error_msg[0] = 0; + strncat(in_error_msg, back[b].r.msg, sizeof(in_error_msg) - 1); + in_error_size = back[b].r.totalsize; + /* don't break, even with "don't take error pages" switch, because we need to process the slot anyway (and cache the error) */ } /* Moved! */ else if (HTTP_IS_REDIRECT(back[b].r.statuscode)) @@ -4541,12 +4546,38 @@ int hts_wait_delayed(htsmoduleStruct* str, } // while(IS_DELAYED_EXT(save)) + if (in_error != 0) { + /* 'no error page' selected or file discarded by size rules! */ + if (!opt->errpage || ( in_error == STATUSCODE_TOO_BIG ) ) { + /* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */ +#if 0 + /* No (3.43) - don't do that. We must not post-exclude an authorized link, because this will prevent the cache + system from processing it, leading to refetch it endlessly. Just accept it, and handle the error as + usual during parsing. + */ + *forbidden_url = 1; /* Forbidden! */ +#endif + if (opt->log != NULL && opt->debug > 0) { + if (in_error == STATUSCODE_TOO_BIG) { + HTS_LOG(opt, LOG_INFO); fprintf(opt->log,"link not taken because of its size (%d bytes) at %s%s"LF,(int)in_error_size,adr,fil); + } else { + HTS_LOG(opt, LOG_INFO); fprintf(opt->log,"link not taken because of error (%d '%s') at %s%s"LF,in_error,in_error_msg,adr,fil); + } + test_flush; + } + } + } + // error if (*forbidden_url != 1 && IS_DELAYED_EXT(save)) { *forbidden_url = 1; if (opt->log!=NULL) { - HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"link is probably looping, type unknown, aborting: %s%s"LF, adr, fil); + if (in_error) { + HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"link in error (%d '%s'), type unknown, aborting: %s%s"LF, in_error, in_error_msg, adr, fil); + } else { + HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"link is probably looping, type unknown, aborting: %s%s"LF, adr, fil); + } test_flush; } } diff --git a/src/htsparse.h b/src/htsparse.h index afd9e88..638c409 100644 --- a/src/htsparse.h +++ b/src/htsparse.h @@ -120,6 +120,11 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre); */ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre); +/* + Process user intercations: pause, add link, delete link.. +*/ +void hts_mirror_process_user_interaction(htsmoduleStruct* str, htsmoduleStructExtended* stre); + /* Get the next file on the queue, waiting for it, handling other files in background.. Parameters: The public module structure, and the private module variables @@ -141,7 +146,7 @@ int hts_wait_delayed(htsmoduleStruct* str, /* Context state */ -#define ENGINE_LOAD_CONTEXT_BASE() \ +#define ENGINE_DEFINE_CONTEXT_BASE() \ lien_url** const liens HTS_UNUSED = (lien_url**) str->liens; \ httrackp* const opt HTS_UNUSED = (httrackp*) str->opt; \ struct_back* const sback HTS_UNUSED = (struct_back*) str->sback; \ @@ -151,11 +156,20 @@ int hts_wait_delayed(htsmoduleStruct* str, hash_struct* const hashptr HTS_UNUSED = (hash_struct*) str->hashptr; \ const int numero_passe HTS_UNUSED = str->numero_passe; \ const int add_tab_alloc HTS_UNUSED = str->add_tab_alloc; \ - /* */ \ - int lien_tot HTS_UNUSED = *str->lien_tot_; \ - int ptr HTS_UNUSED = *str->ptr_; \ - size_t lien_size HTS_UNUSED = *str->lien_size_; \ - char* lien_buffer HTS_UNUSED = *str->lien_buffer_ + /* variable */ \ + int lien_tot = *str->lien_tot_; \ + int ptr = *str->ptr_; \ + size_t lien_size = *str->lien_size_; \ + char* lien_buffer = *str->lien_buffer_ + +#define ENGINE_SET_CONTEXT_BASE() \ + lien_tot = *str->lien_tot_; \ + ptr = *str->ptr_; \ + lien_size = *str->lien_size_; \ + lien_buffer = *str->lien_buffer_ + +#define ENGINE_LOAD_CONTEXT_BASE() \ + ENGINE_DEFINE_CONTEXT_BASE() #define ENGINE_SAVE_CONTEXT_BASE() \ /* Apply changes */ \ diff --git a/src/htsserver.c b/src/htsserver.c index abcd1be..855db75 100644 --- a/src/htsserver.c +++ b/src/htsserver.c @@ -302,7 +302,7 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { }; initStrElt initStr[] = { { "user", "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" }, - { "footer", "" }, + { "footer", "" }, { "url2", "+*.png +*.gif +*.jpg +*.css +*.js -ad.doubleclick.net/*" }, { NULL, NULL } }; @@ -332,8 +332,6 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { char line[8192]; char line2[1024]; T_SOC soc_c; - struct sockaddr dummyaddr; - int dummylen = sizeof(struct sockaddr); LLint length = 0; char* error_redirect = NULL; @@ -349,7 +347,6 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { StringCat(tmpbuff, ""); StringCat(tmpbuff2, ""); StringCat(fspath, ""); - memset(&dummyaddr, 0, sizeof(dummyaddr)); /* UnLock */ webhttrack_release(); @@ -360,7 +357,7 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { #endif /* Accept */ - while ( (soc_c = (T_SOC) accept(soc, &dummyaddr, &dummylen)) == INVALID_SOCKET); + while ( (soc_c = (T_SOC) accept(soc, NULL, NULL)) == INVALID_SOCKET); /* Lock */ webhttrack_lock(); diff --git a/src/htswizard.c b/src/htswizard.c index 80cbbda..403ce34 100644 --- a/src/htswizard.c +++ b/src/htswizard.c @@ -684,30 +684,12 @@ static int hts_acceptlink_(httrackp* opt, // si primaire (plus bas) alors ... if ((ptr!=0) && (force_mirror==0)) { - HTS_REQUEST_START; - HT_PRINT("\n"); - HT_PRINT("At "); HT_PRINT(urladr); HT_PRINT(", there is a link ("); HT_PRINT(adr); HT_PRINT("/"); HT_PRINT(fil); HT_PRINT(") which goes outside the address."LF); - HT_PRINT("What should I do? (press a key + enter)"LF LF); - HT_PRINT("* Ignore all further links" LF); - HT_PRINT("0 Ignore this link (default if empty entry)"LF); - HT_PRINT("1 Ignore directory and lower structures"LF); - HT_PRINT("2 Ignore all domain"LF); - //HT_PRINT("3 (Ignore location, not implemented)\n"); - HT_PRINT(LF); - HT_PRINT("4 Get only this page/link"LF); - HT_PRINT("5 Mirror this link (useful)"LF); - HT_PRINT("6 Mirror links located in the same domain"LF); - HT_PRINT(LF); - HTS_REQUEST_END; - { - char BIGSTK tempo[HTS_URLMAXSIZE*2]; - tempo[0]='\0'; - strcatbuff(tempo,adr); - strcatbuff(tempo,"/"); - strcatbuff(tempo,fil); - s = RUN_CALLBACK1(opt, query3, tempo); - } - if (strnotempty(s)==0) // entrée + char BIGSTK tempo[HTS_URLMAXSIZE*2]; + tempo[0]='\0'; + strcatbuff(tempo,adr); + strcatbuff(tempo,fil); + s = RUN_CALLBACK1(opt, query3, tempo); + if (strnotempty(s)==0) // entrée n=0; else if (isdigit((unsigned char)*s)) sscanf(s,"%d",&n); diff --git a/src/httrack.c b/src/httrack.c index 778ca41..d1f2f77 100644 --- a/src/httrack.c +++ b/src/httrack.c @@ -568,20 +568,38 @@ static int __cdecl htsshow_loop(t_hts_callbackarg *carg, httrackp *opt, lien_bac } static const char* __cdecl htsshow_query(t_hts_callbackarg *carg, httrackp *opt, const char* question) { static char s[12]=""; /* ok */ - printf("%s\nPress to confirm, to abort\n",question); - io_flush; linput(stdin,s,4); + printf("%s\nPress to confirm, to abort\n", question); + io_flush; + linput(stdin,s,4); return s; } static const char* __cdecl htsshow_query2(t_hts_callbackarg *carg, httrackp *opt, const char* question) { static char s[12]=""; /* ok */ - printf("%s\nPress to confirm, to abort\n",question); - io_flush; linput(stdin,s,4); + printf("%s\nPress to confirm, to abort\n", question); + io_flush; + linput(stdin,s,4); return s; } static const char* __cdecl htsshow_query3(t_hts_callbackarg *carg, httrackp *opt, const char* question) { static char line[256]; /* ok */ + printf( + "\n" + "A link, %s, is located beyond this mirror scope.\n" + "What should I do? (type in the choice + enter)\n\n" + "* Ignore all further links and do not ask any more questions\n" + "0 Ignore this link (default if empty entry)\n" + "1 Ignore directory and lower structures\n" + "2 Ignore all domain\n" + "\n" + "4 Get only this page/link, but not links inside this page\n" + "5 Mirror this link (useful)\n" + "6 Mirror all links located on the same domain as this link\n" + "\n", + question); do { - io_flush; linput(stdin,line,206); + printf(">> "); + io_flush; + linput(stdin, line, 200); } while(!strnotempty(line)); printf("ok..\n"); return line; diff --git a/src/httrack.vcproj b/src/httrack.vcproj index 2e8bd28..95b63ae 100755 --- a/src/httrack.vcproj +++ b/src/httrack.vcproj @@ -1,36 +1,62 @@ + RootNamespace="httrack" + TargetFrameworkVersion="131072" + > + Name="Win32" + /> + + + + ATLMinimizesCRunTimeLibraryUsage="false" + CharacterSet="2" + > + + + + + + CompileAs="0" + /> + Name="VCManagedResourceCompilerTool" + /> + + + RandomizedBaseAddress="1" + DataExecutionPrevention="0" + TargetMachine="1" + /> + + + + + + + + + + + + + - + HeaderFileName="" + /> + Name="VCCLCompilerTool" + AdditionalOptions="/Zm200 " + Optimization="4" + InlineFunctionExpansion="2" + EnableIntrinsicFunctions="true" + FavorSizeOrSpeed="1" + OmitFramePointers="true" + AdditionalIncludeDirectories=""C:\Dev\zlib-1.2.3\";C:\Dev\openssl\include;C:\Dev\Winhttrack" + PreprocessorDefinitions="WIN32;_CONSOLE;HTS_ANALYSTE_CONSOLE" + RuntimeLibrary="2" + UsePrecompiledHeader="0" + PrecompiledHeaderFile="c:\temp\vcpp/httrack.pch" + AssemblerOutput="2" + AssemblerListingLocation="c:\temp\vcpp/" + ObjectFile="c:\temp\vcpp/" + ProgramDataBaseFileName="c:\temp\vcpp/" + BrowseInformation="1" + WarningLevel="3" + SuppressStartupBanner="true" + Detect64BitPortabilityProblems="true" + DebugInformationFormat="3" + CompileAs="0" + /> + Name="VCManagedResourceCompilerTool" + /> + Culture="1033" + /> + + + + + Name="VCXDCMakeTool" + /> + Name="VCBscMakeTool" + /> + Name="VCFxCopTool" + /> + Name="VCAppVerifierTool" + /> + Name="VCPostBuildEventTool" + /> + ATLMinimizesCRunTimeLibraryUsage="false" + CharacterSet="2" + > + + + + + + CompileAs="0" + /> + + + Name="VCPreLinkEventTool" + /> + RandomizedBaseAddress="1" + DataExecutionPrevention="0" + TargetMachine="1" + /> + + + + + + + + + + + + + - + HeaderFileName="" + /> + Name="VCCLCompilerTool" + AdditionalOptions="/Zm200 " + Optimization="0" + AdditionalIncludeDirectories=""C:\Dev\zlib-1.2.3\";C:\Dev\openssl\include;C:\Dev\Winhttrack" + PreprocessorDefinitions="_DEBUG;WIN32;_CONSOLE;HTS_ANALYSTE_CONSOLE;_CRT_SECURE_NO_WARNINGS" + RuntimeLibrary="3" + BufferSecurityCheck="true" + RuntimeTypeInfo="true" + PrecompiledHeaderFile="c:\temp\vcpp/httrack.pch" + AssemblerOutput="2" + AssemblerListingLocation="c:\temp\vcpp/" + ObjectFile="c:\temp\vcpp/" + ProgramDataBaseFileName="c:\temp\vcpp/" + BrowseInformation="1" + BrowseInformationFile="c:\temp\vcpp/" + WarningLevel="3" + SuppressStartupBanner="true" + Detect64BitPortabilityProblems="true" + DebugInformationFormat="3" + CompileAs="0" + /> + Name="VCManagedResourceCompilerTool" + /> + Culture="1033" + /> + Name="VCPreLinkEventTool" + /> + Name="VCLinkerTool" + AdditionalDependencies="Ws2_32.lib libhttrack.lib" + OutputFile="C:\temp\httrack64\httrack.exe" + LinkIncremental="1" + SuppressStartupBanner="true" + AdditionalLibraryDirectories="C:\Dev\openssl\lib,C:\temp\zlib,C:\Dev\openssl\lib\out32dll,C:\temp\Debuglib" + GenerateDebugInformation="true" + ProgramDatabaseFile=".\Debug/httrack.pdb" + SubSystem="1" + RandomizedBaseAddress="1" + DataExecutionPrevention="0" + TargetMachine="17" + /> + + Name="VCManifestTool" + /> + Name="VCXDCMakeTool" + /> + Name="VCBscMakeTool" + /> + + + + ATLMinimizesCRunTimeLibraryUsage="false" + CharacterSet="2" + > + + + + + + SuppressStartupBanner="true" + Detect64BitPortabilityProblems="true" + CompileAs="0" + /> + Name="VCManagedResourceCompilerTool" + /> + + + RandomizedBaseAddress="1" + DataExecutionPrevention="0" + TargetMachine="1" + /> + + + + + + + + + + + + + + HeaderFileName="" + /> - + Name="VCCLCompilerTool" + AdditionalOptions="/Zm200 " + Optimization="3" + InlineFunctionExpansion="2" + AdditionalIncludeDirectories=""C:\Dev\zlib-1.2.3\";C:\Dev\openssl\include;C:\Dev\Winhttrack" + PreprocessorDefinitions="NDEBUG;WIN32;_CONSOLE;HTS_ANALYSTE_CONSOLE;_CRT_SECURE_NO_WARNINGS" + StringPooling="true" + RuntimeLibrary="2" + EnableFunctionLevelLinking="true" + UsePrecompiledHeader="0" + PrecompiledHeaderFile="c:\temp\vcpp/httrack.pch" + AssemblerListingLocation="c:\temp\vcpp/" + ObjectFile="c:\temp\vcpp/" + ProgramDataBaseFileName="c:\temp\vcpp/" + WarningLevel="3" + SuppressStartupBanner="true" + Detect64BitPortabilityProblems="true" + CompileAs="0" + /> + Name="VCManagedResourceCompilerTool" + /> + Culture="1033" + /> + + + + + Name="VCXDCMakeTool" + /> + Name="VCBscMakeTool" + /> + Name="VCFxCopTool" + /> + Name="VCAppVerifierTool" + /> + Name="VCPostBuildEventTool" + /> + RelativePath="httrack.c" + > + Name="Release avec debug|Win32" + > + BrowseInformation="1" + /> + Name="Release avec debug|x64" + > + + + + BrowseInformation="1" + /> + Name="Debug|x64" + > + + + + + + + PreprocessorDefinitions="" + /> + RelativePath=".\httrack.h" + > diff --git a/src/minizip/mztools.c b/src/minizip/mztools.c index 6021c49..a09bfb7 100644 --- a/src/minizip/mztools.c +++ b/src/minizip/mztools.c @@ -1,287 +1,287 @@ -/* - Additional tools for Minizip - Code: Xavier Roche '2004 - License: Same as ZLIB (www.gzip.org) -*/ - -/* Code */ -#include -#ifndef _WIN32_WCE +/* + Additional tools for Minizip + Code: Xavier Roche '2004 + License: Same as ZLIB (www.gzip.org) +*/ + +/* Code */ +#include +#ifndef _WIN32_WCE +#include +#include +#else #include -#include -#else -#include -#include -#include "celib.h" -#endif -#include "zlib.h" -#include "unzip.h" - -#define READ_8(adr) ((unsigned char)*(adr)) -#define READ_16(adr) ( READ_8(adr) | (READ_8(adr+1) << 8) ) -#define READ_32(adr) ( READ_16(adr) | (READ_16((adr)+2) << 16) ) - -#define WRITE_8(buff, n) do { \ - *((unsigned char*)(buff)) = (unsigned char) ((n) & 0xff); \ -} while(0) -#define WRITE_16(buff, n) do { \ - WRITE_8((unsigned char*)(buff), n); \ - WRITE_8(((unsigned char*)(buff)) + 1, (n) >> 8); \ -} while(0) -#define WRITE_32(buff, n) do { \ - WRITE_16((unsigned char*)(buff), (n) & 0xffff); \ - WRITE_16((unsigned char*)(buff) + 2, (n) >> 16); \ -} while(0) - -extern int ZEXPORT unzRepair(file, fileOut, fileOutTmp, nRecovered, bytesRecovered) -const char* file; -const char* fileOut; -const char* fileOutTmp; -uLong* nRecovered; -uLong* bytesRecovered; -{ - int err = Z_OK; - FILE* fpZip = fopen(file, "rb"); - FILE* fpOut = fopen(fileOut, "wb"); - FILE* fpOutCD = fopen(fileOutTmp, "wb"); - if (fpZip != NULL && fpOut != NULL) { - int entries = 0; - uLong totalBytes = 0; - char header[30]; - char filename[256]; - char extra[1024]; - int offset = 0; - int offsetCD = 0; - while ( fread(header, 1, 30, fpZip) == 30 ) { - int currentOffset = offset; - - /* File entry */ - if (READ_32(header) == 0x04034b50) { - unsigned int version = READ_16(header + 4); - unsigned int gpflag = READ_16(header + 6); - unsigned int method = READ_16(header + 8); - unsigned int filetime = READ_16(header + 10); - unsigned int filedate = READ_16(header + 12); - unsigned int crc = READ_32(header + 14); /* crc */ - unsigned int cpsize = READ_32(header + 18); /* compressed size */ - unsigned int uncpsize = READ_32(header + 22); /* uncompressed sz */ - unsigned int fnsize = READ_16(header + 26); /* file name length */ - unsigned int extsize = READ_16(header + 28); /* extra field length */ - filename[0] = extra[0] = '\0'; - - /* Header */ - if (fwrite(header, 1, 30, fpOut) == 30) { - offset += 30; - } else { - err = Z_ERRNO; - break; - } - - /* Filename */ - if (fnsize > 0) { - if (fread(filename, 1, fnsize, fpZip) == fnsize) { - if (fwrite(filename, 1, fnsize, fpOut) == fnsize) { - offset += fnsize; - } else { - err = Z_ERRNO; - break; - } - } else { - err = Z_ERRNO; - break; - } - } else { - err = Z_STREAM_ERROR; - break; - } - - /* Extra field */ - if (extsize > 0) { - if (fread(extra, 1, extsize, fpZip) == extsize) { - if (fwrite(extra, 1, extsize, fpOut) == extsize) { - offset += extsize; - } else { - err = Z_ERRNO; - break; - } - } else { - err = Z_ERRNO; - break; - } - } - - /* Data */ - { - int dataSize = cpsize; - if (dataSize == 0) { - dataSize = uncpsize; - } - if (dataSize > 0) { - char* data = malloc(dataSize); - if (data != NULL) { - if ((int)fread(data, 1, dataSize, fpZip) == dataSize) { - if ((int)fwrite(data, 1, dataSize, fpOut) == dataSize) { - offset += dataSize; - totalBytes += dataSize; - } else { - err = Z_ERRNO; - } - } else { - err = Z_ERRNO; - } - free(data); - if (err != Z_OK) { - break; - } - } else { - err = Z_MEM_ERROR; - break; - } - } - } - - /* Central directory entry */ - { - char header[46]; - char* comment = ""; - int comsize = (int) strlen(comment); - WRITE_32(header, 0x02014b50); - WRITE_16(header + 4, version); - WRITE_16(header + 6, version); - WRITE_16(header + 8, gpflag); - WRITE_16(header + 10, method); - WRITE_16(header + 12, filetime); - WRITE_16(header + 14, filedate); - WRITE_32(header + 16, crc); - WRITE_32(header + 20, cpsize); - WRITE_32(header + 24, uncpsize); - WRITE_16(header + 28, fnsize); - WRITE_16(header + 30, extsize); - WRITE_16(header + 32, comsize); - WRITE_16(header + 34, 0); /* disk # */ - WRITE_16(header + 36, 0); /* int attrb */ - WRITE_32(header + 38, 0); /* ext attrb */ - WRITE_32(header + 42, currentOffset); - /* Header */ - if (fwrite(header, 1, 46, fpOutCD) == 46) { - offsetCD += 46; - - /* Filename */ - if (fnsize > 0) { - if (fwrite(filename, 1, fnsize, fpOutCD) == fnsize) { - offsetCD += fnsize; - } else { - err = Z_ERRNO; - break; - } - } else { - err = Z_STREAM_ERROR; - break; - } - - /* Extra field */ - if (extsize > 0) { - if (fwrite(extra, 1, extsize, fpOutCD) == extsize) { - offsetCD += extsize; - } else { - err = Z_ERRNO; - break; - } - } - - /* Comment field */ - if (comsize > 0) { - if ((int)fwrite(comment, 1, comsize, fpOutCD) == comsize) { - offsetCD += comsize; - } else { - err = Z_ERRNO; - break; - } - } - - - } else { - err = Z_ERRNO; - break; - } - } - - /* Success */ - entries++; - - } else { - break; - } - } - - /* Final central directory */ - { - int entriesZip = entries; - char header[22]; - char* comment = ""; // "ZIP File recovered by zlib/minizip/mztools"; - int comsize = (int) strlen(comment); - if (entriesZip > 0xffff) { - entriesZip = 0xffff; - } - WRITE_32(header, 0x06054b50); - WRITE_16(header + 4, 0); /* disk # */ - WRITE_16(header + 6, 0); /* disk # */ - WRITE_16(header + 8, entriesZip); /* hack */ - WRITE_16(header + 10, entriesZip); /* hack */ - WRITE_32(header + 12, offsetCD); /* size of CD */ - WRITE_32(header + 16, offset); /* offset to CD */ - WRITE_16(header + 20, comsize); /* comment */ - - /* Header */ - if (fwrite(header, 1, 22, fpOutCD) == 22) { - - /* Comment field */ - if (comsize > 0) { - if ((int)fwrite(comment, 1, comsize, fpOutCD) != comsize) { - err = Z_ERRNO; - } - } - - } else { - err = Z_ERRNO; - } - } - - /* Final merge (file + central directory) */ - fclose(fpOutCD); - if (err == Z_OK) { - fpOutCD = fopen(fileOutTmp, "rb"); - if (fpOutCD != NULL) { - int nRead; - char buffer[8192]; - while ( (nRead = fread(buffer, 1, sizeof(buffer), fpOutCD)) > 0) { - if ((int)fwrite(buffer, 1, nRead, fpOut) != nRead) { - err = Z_ERRNO; - break; - } - } - fclose(fpOutCD); - } - } - - /* Close */ - fclose(fpZip); - fclose(fpOut); - - /* Wipe temporary file */ - (void)remove(fileOutTmp); - - /* Number of recovered entries */ - if (err == Z_OK) { - if (nRecovered != NULL) { - *nRecovered = entries; - } - if (bytesRecovered != NULL) { - *bytesRecovered = totalBytes; - } - } - } else { - err = Z_STREAM_ERROR; - } - return err; -} +#include +#include "celib.h" +#endif +#include "zlib.h" +#include "unzip.h" + +#define READ_8(adr) ((unsigned char)*(adr)) +#define READ_16(adr) ( READ_8(adr) | (READ_8(adr+1) << 8) ) +#define READ_32(adr) ( READ_16(adr) | (READ_16((adr)+2) << 16) ) + +#define WRITE_8(buff, n) do { \ + *((unsigned char*)(buff)) = (unsigned char) ((n) & 0xff); \ +} while(0) +#define WRITE_16(buff, n) do { \ + WRITE_8((unsigned char*)(buff), n); \ + WRITE_8(((unsigned char*)(buff)) + 1, (n) >> 8); \ +} while(0) +#define WRITE_32(buff, n) do { \ + WRITE_16((unsigned char*)(buff), (n) & 0xffff); \ + WRITE_16((unsigned char*)(buff) + 2, (n) >> 16); \ +} while(0) + +extern int ZEXPORT unzRepair(file, fileOut, fileOutTmp, nRecovered, bytesRecovered) +const char* file; +const char* fileOut; +const char* fileOutTmp; +uLong* nRecovered; +uLong* bytesRecovered; +{ + int err = Z_OK; + FILE* fpZip = fopen(file, "rb"); + FILE* fpOut = fopen(fileOut, "wb"); + FILE* fpOutCD = fopen(fileOutTmp, "wb"); + if (fpZip != NULL && fpOut != NULL) { + int entries = 0; + uLong totalBytes = 0; + char header[30]; + char filename[1024]; + char extra[1024]; + int offset = 0; + int offsetCD = 0; + while ( fread(header, 1, 30, fpZip) == 30 ) { + int currentOffset = offset; + + /* File entry */ + if (READ_32(header) == 0x04034b50) { + unsigned int version = READ_16(header + 4); + unsigned int gpflag = READ_16(header + 6); + unsigned int method = READ_16(header + 8); + unsigned int filetime = READ_16(header + 10); + unsigned int filedate = READ_16(header + 12); + unsigned int crc = READ_32(header + 14); /* crc */ + unsigned int cpsize = READ_32(header + 18); /* compressed size */ + unsigned int uncpsize = READ_32(header + 22); /* uncompressed sz */ + unsigned int fnsize = READ_16(header + 26); /* file name length */ + unsigned int extsize = READ_16(header + 28); /* extra field length */ + filename[0] = extra[0] = '\0'; + + /* Header */ + if (fwrite(header, 1, 30, fpOut) == 30) { + offset += 30; + } else { + err = Z_ERRNO; + break; + } + + /* Filename */ + if (fnsize > 0) { + if (fnsize < sizeof(filename) && fread(filename, 1, fnsize, fpZip) == fnsize) { + if (fwrite(filename, 1, fnsize, fpOut) == fnsize) { + offset += fnsize; + } else { + err = Z_ERRNO; + break; + } + } else { + err = Z_ERRNO; + break; + } + } else { + err = Z_STREAM_ERROR; + break; + } + + /* Extra field */ + if (extsize > 0) { + if (extsize < sizeof(extra) && fread(extra, 1, extsize, fpZip) == extsize) { + if (fwrite(extra, 1, extsize, fpOut) == extsize) { + offset += extsize; + } else { + err = Z_ERRNO; + break; + } + } else { + err = Z_ERRNO; + break; + } + } + + /* Data */ + { + int dataSize = cpsize; + if (dataSize == 0) { + dataSize = uncpsize; + } + if (dataSize > 0) { + char* data = malloc(dataSize); + if (data != NULL) { + if ((int)fread(data, 1, dataSize, fpZip) == dataSize) { + if ((int)fwrite(data, 1, dataSize, fpOut) == dataSize) { + offset += dataSize; + totalBytes += dataSize; + } else { + err = Z_ERRNO; + } + } else { + err = Z_ERRNO; + } + free(data); + if (err != Z_OK) { + break; + } + } else { + err = Z_MEM_ERROR; + break; + } + } + } + + /* Central directory entry */ + { + char header[46]; + char* comment = ""; + int comsize = (int) strlen(comment); + WRITE_32(header, 0x02014b50); + WRITE_16(header + 4, version); + WRITE_16(header + 6, version); + WRITE_16(header + 8, gpflag); + WRITE_16(header + 10, method); + WRITE_16(header + 12, filetime); + WRITE_16(header + 14, filedate); + WRITE_32(header + 16, crc); + WRITE_32(header + 20, cpsize); + WRITE_32(header + 24, uncpsize); + WRITE_16(header + 28, fnsize); + WRITE_16(header + 30, extsize); + WRITE_16(header + 32, comsize); + WRITE_16(header + 34, 0); /* disk # */ + WRITE_16(header + 36, 0); /* int attrb */ + WRITE_32(header + 38, 0); /* ext attrb */ + WRITE_32(header + 42, currentOffset); + /* Header */ + if (fwrite(header, 1, 46, fpOutCD) == 46) { + offsetCD += 46; + + /* Filename */ + if (fnsize > 0) { + if (fwrite(filename, 1, fnsize, fpOutCD) == fnsize) { + offsetCD += fnsize; + } else { + err = Z_ERRNO; + break; + } + } else { + err = Z_STREAM_ERROR; + break; + } + + /* Extra field */ + if (extsize > 0) { + if (fwrite(extra, 1, extsize, fpOutCD) == extsize) { + offsetCD += extsize; + } else { + err = Z_ERRNO; + break; + } + } + + /* Comment field */ + if (comsize > 0) { + if ((int)fwrite(comment, 1, comsize, fpOutCD) == comsize) { + offsetCD += comsize; + } else { + err = Z_ERRNO; + break; + } + } + + + } else { + err = Z_ERRNO; + break; + } + } + + /* Success */ + entries++; + + } else { + break; + } + } + + /* Final central directory */ + { + int entriesZip = entries; + char header[22]; + char* comment = ""; // "ZIP File recovered by zlib/minizip/mztools"; + int comsize = (int) strlen(comment); + if (entriesZip > 0xffff) { + entriesZip = 0xffff; + } + WRITE_32(header, 0x06054b50); + WRITE_16(header + 4, 0); /* disk # */ + WRITE_16(header + 6, 0); /* disk # */ + WRITE_16(header + 8, entriesZip); /* hack */ + WRITE_16(header + 10, entriesZip); /* hack */ + WRITE_32(header + 12, offsetCD); /* size of CD */ + WRITE_32(header + 16, offset); /* offset to CD */ + WRITE_16(header + 20, comsize); /* comment */ + + /* Header */ + if (fwrite(header, 1, 22, fpOutCD) == 22) { + + /* Comment field */ + if (comsize > 0) { + if ((int)fwrite(comment, 1, comsize, fpOutCD) != comsize) { + err = Z_ERRNO; + } + } + + } else { + err = Z_ERRNO; + } + } + + /* Final merge (file + central directory) */ + fclose(fpOutCD); + if (err == Z_OK) { + fpOutCD = fopen(fileOutTmp, "rb"); + if (fpOutCD != NULL) { + int nRead; + char buffer[8192]; + while ( (nRead = fread(buffer, 1, sizeof(buffer), fpOutCD)) > 0) { + if ((int)fwrite(buffer, 1, nRead, fpOut) != nRead) { + err = Z_ERRNO; + break; + } + } + fclose(fpOutCD); + } + } + + /* Close */ + fclose(fpZip); + fclose(fpOut); + + /* Wipe temporary file */ + (void)remove(fileOutTmp); + + /* Number of recovered entries */ + if (err == Z_OK) { + if (nRecovered != NULL) { + *nRecovered = entries; + } + if (bytesRecovered != NULL) { + *bytesRecovered = totalBytes; + } + } + } else { + err = Z_STREAM_ERROR; + } + return err; +} diff --git a/src/proxy/proxytrack.c b/src/proxy/proxytrack.c index 195c1a5..78775ab 100644 --- a/src/proxy/proxytrack.c +++ b/src/proxy/proxytrack.c @@ -1365,10 +1365,7 @@ static int proxytrack_process_HTTP_threaded(PT_Indexes indexes, T_SOC soc) { static int proxytrack_start_HTTP(PT_Indexes indexes, T_SOC soc) { while(soc != INVALID_SOCKET) { T_SOC soc_c; - struct sockaddr clientAddr; - int clientAddrLen = sizeof(struct sockaddr); - memset(&clientAddr, 0, sizeof(clientAddr)); - if ( (soc_c = (T_SOC) accept(soc, &clientAddr, &clientAddrLen)) != INVALID_SOCKET) { + if ( (soc_c = (T_SOC) accept(soc, NULL, NULL)) != INVALID_SOCKET) { if (!proxytrack_process_HTTP_threaded(indexes, soc_c)) { CRITICAL("proxytrack_start_HTTP::Can not fork a thread"); } diff --git a/src/webhttrack b/src/webhttrack index 430b026..314e070 100755 --- a/src/webhttrack +++ b/src/webhttrack @@ -4,14 +4,21 @@ # Initializes the htsserver GUI frontend and launch the default browser BROWSEREXE= -SRCHBROWSEREXE="x-www-browser www-browser iceape mozilla firefox firebird galeon konqueror opera netscape" +SRCHBROWSEREXE="x-www-browser www-browser iceape mozilla firefox icecat iceweasel abrowser firebird galeon konqueror opera netscape" if test -n "${BROWSER}"; then # sensible-browser will f up if BROWSER is not set SRCHBROWSEREXE="xdg-open sensible-browser ${SRCHBROWSEREXE}" fi -SRCHPATH="/usr/local/bin /usr/share/bin /usr/bin /usr/lib/httrack /usr/local/lib/httrack /usr/local/share/httrack /sw/bin ${HOME}/usr/bin ${HOME}/bin" +# Patch for Darwin/Mac by Ross Williams +if test "`uname -s`" == "Darwin"; then +# Darwin/Mac OS X uses a system 'open' command to find +# the default browser. The -W flag causes it to wait for +# the browser to exit +BROWSEREXE="/usr/bin/open -W" +fi +SRCHPATH="/usr/local/bin /usr/share/bin /usr/bin /usr/lib/httrack /usr/local/lib/httrack /usr/local/share/httrack /opt/local/bin /sw/bin ${HOME}/usr/bin ${HOME}/bin" SRCHPATH="$SRCHPATH "`echo $PATH | tr ":" " "` -SRCHDISTPATH="/usr/share /usr/local /usr /local /usr/local/share ${HOME}/usr ${HOME}/usr/share /sw ${HOME}/usr/local ${HOME}/usr/share" +SRCHDISTPATH="/usr/share /usr/local /usr /local /usr/local/share ${HOME}/usr ${HOME}/usr/share /opt/local/share /sw ${HOME}/usr/local ${HOME}/usr/share" ### # And now some famous cuisine @@ -33,6 +40,12 @@ echo "$1" | grep -q "firebird" [ $? -eq 0 ] && return 0 echo "$1" | grep -q "firefox" [ $? -eq 0 ] && return 0 +echo "$1" | grep -q "iceweasel" +[ $? -eq 0 ] && return 0 +echo "$1" | grep -q "abrowser" +[ $? -eq 0 ] && return 0 +echo "$1" | grep -q "icecat" +[ $? -eq 0 ] && return 0 return 1; } function mozillaloaded { @@ -41,7 +54,7 @@ if ! test -n "${user_name}"; then user_name=`id -un` fi if test -n "${user_name}"; then -ps -e --user "$user_name" | grep -qE "(iceape|mozilla|netscape|firebird|firefox)" +ps -e -U "$user_name" | grep -E "(iceape|mozilla|netscape|firebird|firefox)" | grep -qv "grep -E" else false fi -- cgit v1.2.3