diff options
Diffstat (limited to 'src')
71 files changed, 11539 insertions, 1742 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index bc74182..587535a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -22,14 +22,20 @@ INCLUDES = \ -DDATADIR=\""$(datadir)"\" \ -DLIBDIR=\""$(libdir)"\" -bin_PROGRAMS = httrack htsserver +bin_PROGRAMS = proxytrack httrack htsserver httrack_LDADD = $(THREADS_LIBS) -lhttrack htsserver_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS) -lhttrack +proxytrack_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS) +proxytrack_CFLAGS = $(AM_CFLAGS) -DNO_MALLOCT lib_LTLIBRARIES = libhttrack.la htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h +proxytrack_SOURCES = proxy/main.c \ + proxy/proxytrack.c proxy/store.c \ + htsinthash.c htsmd5.c md5.c \ + minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c whttrackrundir = $(bindir) whttrackrun_SCRIPTS = webhttrack @@ -42,7 +48,9 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \ htsmd5.c htszlib.c htsnostatic.c htswrap.c \ htsmodules.c \ md5.c \ + htsmms.c \ minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \ + mmsrip/error.c mmsrip/mms.c \ hts-indextmpl.h htsalias.h htsback.h htsbase.h \ htsbasenet.h htsbauth.h htscache.h htscatchurl.h \ htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \ @@ -53,8 +61,9 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \ htstools.h htswizard.h htswrap.h htszlib.h \ htsstrings.h httrack-library.h \ md5.h \ - minizip/crypt.h minizip/ioapi.h minizip/mztools.h minizip/unzip.h minizip/zip.h - + htsmms.h \ + minizip/crypt.h minizip/ioapi.h minizip/mztools.h minizip/unzip.h minizip/zip.h \ + mmsrip/error.h mmsrip/mms.h libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(DL_LIBS) $(SOCKET_LIBS) libhttrack_la_LDFLAGS = -version-info $(VERSION_INFO) @@ -62,5 +71,19 @@ libhttrack_la_LDFLAGS = -version-info $(VERSION_INFO) EXTRA_DIST = httrack.h webhttrack \ httrack.dsp httrack.dsw \ webhttrack.dsp webhttrack.dsw \ - minizip/ChangeLogUnzip minizip/iowin32.c minizip/iowin32.h - + minizip/ChangeLogUnzip \ + minizip/iowin32.c \ + minizip/iowin32.h \ + mmsrip/common.h \ + mmsrip/main.c \ + mmsrip/COPYING \ + mmsrip/AUTHORS \ + mmsrip/NEWS \ + mmsrip/README \ + mmsrip/ChangeLog \ + proxy/AUTHORS \ + proxy/COPYING \ + proxy/changelog.txt \ + proxy/proxystrings.h \ + proxy/proxytrack.h \ + proxy/store.h diff --git a/src/Makefile.in b/src/Makefile.in index aa5da2a..888071a 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -1,7 +1,7 @@ -# Makefile.in generated by automake 1.7 from Makefile.am. +# Makefile.in generated by automake 1.7.9 from Makefile.am. # @configure_input@ -# Copyright 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002 +# Copyright 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003 # Free Software Foundation, Inc. # This Makefile.in is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -42,6 +42,7 @@ ACLOCAL = @ACLOCAL@ AMDEP_FALSE = @AMDEP_FALSE@ AMDEP_TRUE = @AMDEP_TRUE@ AMTAR = @AMTAR@ +AR = @AR@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ @@ -52,6 +53,7 @@ CFLAGS = @CFLAGS@ CPP = @CPP@ CPPFLAGS = @CPPFLAGS@ CXX = @CXX@ +CXXCPP = @CXXCPP@ CXXDEPMODE = @CXXDEPMODE@ CXXFLAGS = @CXXFLAGS@ CYGPATH_W = @CYGPATH_W@ @@ -65,6 +67,8 @@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ EGREP = @EGREP@ EXEEXT = @EXEEXT@ +F77 = @F77@ +FFLAGS = @FFLAGS@ INSTALL_DATA = @INSTALL_DATA@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ INSTALL_SCRIPT = @INSTALL_SCRIPT@ @@ -98,8 +102,10 @@ THREADS_LIBS = @THREADS_LIBS@ V6_FLAG = @V6_FLAG@ VERSION = @VERSION@ VERSION_INFO = @VERSION_INFO@ +ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ ac_ct_CXX = @ac_ct_CXX@ +ac_ct_F77 = @ac_ct_F77@ ac_ct_RANLIB = @ac_ct_RANLIB@ ac_ct_STRIP = @ac_ct_STRIP@ am__fastdepCC_FALSE = @am__fastdepCC_FALSE@ @@ -107,6 +113,7 @@ am__fastdepCC_TRUE = @am__fastdepCC_TRUE@ am__fastdepCXX_FALSE = @am__fastdepCXX_FALSE@ am__fastdepCXX_TRUE = @am__fastdepCXX_TRUE@ am__include = @am__include@ +am__leading_dot = @am__leading_dot@ am__quote = @am__quote@ bindir = @bindir@ build = @build@ @@ -160,14 +167,21 @@ INCLUDES = \ -DLIBDIR=\""$(libdir)"\" -bin_PROGRAMS = httrack htsserver +bin_PROGRAMS = proxytrack httrack htsserver httrack_LDADD = $(THREADS_LIBS) -lhttrack htsserver_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS) -lhttrack +proxytrack_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS) +proxytrack_CFLAGS = $(AM_CFLAGS) -DNO_MALLOCT lib_LTLIBRARIES = libhttrack.la htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h +proxytrack_SOURCES = proxy/main.c \ + proxy/proxytrack.c proxy/store.c \ + htsinthash.c htsmd5.c md5.c \ + minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c + whttrackrundir = $(bindir) whttrackrun_SCRIPTS = webhttrack @@ -180,7 +194,9 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \ htsmd5.c htszlib.c htsnostatic.c htswrap.c \ htsmodules.c \ md5.c \ + htsmms.c \ minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \ + mmsrip/error.c mmsrip/mms.c \ hts-indextmpl.h htsalias.h htsback.h htsbase.h \ htsbasenet.h htsbauth.h htscache.h htscatchurl.h \ htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \ @@ -191,7 +207,9 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \ htstools.h htswizard.h htswrap.h htszlib.h \ htsstrings.h httrack-library.h \ md5.h \ - minizip/crypt.h minizip/ioapi.h minizip/mztools.h minizip/unzip.h minizip/zip.h + htsmms.h \ + minizip/crypt.h minizip/ioapi.h minizip/mztools.h minizip/unzip.h minizip/zip.h \ + mmsrip/error.h mmsrip/mms.h libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(DL_LIBS) $(SOCKET_LIBS) @@ -200,9 +218,25 @@ libhttrack_la_LDFLAGS = -version-info $(VERSION_INFO) EXTRA_DIST = httrack.h webhttrack \ httrack.dsp httrack.dsw \ webhttrack.dsp webhttrack.dsw \ - minizip/ChangeLogUnzip minizip/iowin32.c minizip/iowin32.h + minizip/ChangeLogUnzip \ + minizip/iowin32.c \ + minizip/iowin32.h \ + mmsrip/common.h \ + mmsrip/main.c \ + mmsrip/COPYING \ + mmsrip/AUTHORS \ + mmsrip/NEWS \ + mmsrip/README \ + mmsrip/ChangeLog \ + proxy/AUTHORS \ + proxy/COPYING \ + proxy/changelog.txt \ + proxy/proxystrings.h \ + proxy/proxytrack.h \ + proxy/store.h subdir = src +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs CONFIG_HEADER = $(top_builddir)/config.h CONFIG_CLEAN_FILES = @@ -214,10 +248,10 @@ am_libhttrack_la_OBJECTS = htscore.lo htsparse.lo htsback.lo htscache.lo \ htshelp.lo htsjava.lo htslib.lo htscoremain.lo htsname.lo \ htsrobots.lo htstools.lo htswizard.lo htsalias.lo htsthread.lo \ htsindex.lo htsbauth.lo htsmd5.lo htszlib.lo htsnostatic.lo \ - htswrap.lo htsmodules.lo md5.lo ioapi.lo mztools.lo unzip.lo \ - zip.lo + htswrap.lo htsmodules.lo md5.lo htsmms.lo ioapi.lo mztools.lo \ + unzip.lo zip.lo error.lo mms.lo libhttrack_la_OBJECTS = $(am_libhttrack_la_OBJECTS) -bin_PROGRAMS = httrack$(EXEEXT) htsserver$(EXEEXT) +bin_PROGRAMS = proxytrack$(EXEEXT) httrack$(EXEEXT) htsserver$(EXEEXT) PROGRAMS = $(bin_PROGRAMS) am_htsserver_OBJECTS = htsserver.$(OBJEXT) htsweb.$(OBJEXT) @@ -228,29 +262,50 @@ httrack_SOURCES = httrack.c httrack_OBJECTS = httrack.$(OBJEXT) httrack_DEPENDENCIES = httrack_LDFLAGS = +am_proxytrack_OBJECTS = proxytrack-main.$(OBJEXT) \ + proxytrack-proxytrack.$(OBJEXT) proxytrack-store.$(OBJEXT) \ + proxytrack-htsinthash.$(OBJEXT) proxytrack-htsmd5.$(OBJEXT) \ + proxytrack-md5.$(OBJEXT) proxytrack-ioapi.$(OBJEXT) \ + proxytrack-mztools.$(OBJEXT) proxytrack-unzip.$(OBJEXT) \ + proxytrack-zip.$(OBJEXT) +proxytrack_OBJECTS = $(am_proxytrack_OBJECTS) +proxytrack_DEPENDENCIES = +proxytrack_LDFLAGS = SCRIPTS = $(whttrackrun_SCRIPTS) DEFAULT_INCLUDES = -I. -I$(srcdir) -I$(top_builddir) depcomp = $(SHELL) $(top_srcdir)/depcomp am__depfiles_maybe = depfiles -@AMDEP_TRUE@DEP_FILES = ./$(DEPDIR)/htsalias.Plo ./$(DEPDIR)/htsback.Plo \ -@AMDEP_TRUE@ ./$(DEPDIR)/htsbauth.Plo ./$(DEPDIR)/htscache.Plo \ +@AMDEP_TRUE@DEP_FILES = ./$(DEPDIR)/error.Plo ./$(DEPDIR)/htsalias.Plo \ +@AMDEP_TRUE@ ./$(DEPDIR)/htsback.Plo ./$(DEPDIR)/htsbauth.Plo \ +@AMDEP_TRUE@ ./$(DEPDIR)/htscache.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/htscatchurl.Plo ./$(DEPDIR)/htscore.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/htscoremain.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/htsfilters.Plo ./$(DEPDIR)/htsftp.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/htshash.Plo ./$(DEPDIR)/htshelp.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/htsindex.Plo ./$(DEPDIR)/htsinthash.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/htsjava.Plo ./$(DEPDIR)/htslib.Plo \ -@AMDEP_TRUE@ ./$(DEPDIR)/htsmd5.Plo ./$(DEPDIR)/htsmodules.Plo \ -@AMDEP_TRUE@ ./$(DEPDIR)/htsname.Plo ./$(DEPDIR)/htsnostatic.Plo \ +@AMDEP_TRUE@ ./$(DEPDIR)/htsmd5.Plo ./$(DEPDIR)/htsmms.Plo \ +@AMDEP_TRUE@ ./$(DEPDIR)/htsmodules.Plo ./$(DEPDIR)/htsname.Plo \ +@AMDEP_TRUE@ ./$(DEPDIR)/htsnostatic.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/htsparse.Plo ./$(DEPDIR)/htsrobots.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/htsserver.Po ./$(DEPDIR)/htsthread.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/htstools.Plo ./$(DEPDIR)/htsweb.Po \ @AMDEP_TRUE@ ./$(DEPDIR)/htswizard.Plo ./$(DEPDIR)/htswrap.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/htszlib.Plo ./$(DEPDIR)/httrack.Po \ @AMDEP_TRUE@ ./$(DEPDIR)/ioapi.Plo ./$(DEPDIR)/md5.Plo \ -@AMDEP_TRUE@ ./$(DEPDIR)/mztools.Plo ./$(DEPDIR)/unzip.Plo \ +@AMDEP_TRUE@ ./$(DEPDIR)/mms.Plo ./$(DEPDIR)/mztools.Plo \ +@AMDEP_TRUE@ ./$(DEPDIR)/proxytrack-htsinthash.Po \ +@AMDEP_TRUE@ ./$(DEPDIR)/proxytrack-htsmd5.Po \ +@AMDEP_TRUE@ ./$(DEPDIR)/proxytrack-ioapi.Po \ +@AMDEP_TRUE@ ./$(DEPDIR)/proxytrack-main.Po \ +@AMDEP_TRUE@ ./$(DEPDIR)/proxytrack-md5.Po \ +@AMDEP_TRUE@ ./$(DEPDIR)/proxytrack-mztools.Po \ +@AMDEP_TRUE@ ./$(DEPDIR)/proxytrack-proxytrack.Po \ +@AMDEP_TRUE@ ./$(DEPDIR)/proxytrack-store.Po \ +@AMDEP_TRUE@ ./$(DEPDIR)/proxytrack-unzip.Po \ +@AMDEP_TRUE@ ./$(DEPDIR)/proxytrack-zip.Po ./$(DEPDIR)/unzip.Plo \ @AMDEP_TRUE@ ./$(DEPDIR)/zip.Plo COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) @@ -259,11 +314,12 @@ LTCOMPILE = $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) \ CCLD = $(CC) LINK = $(LIBTOOL) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ $(AM_LDFLAGS) $(LDFLAGS) -o $@ -DIST_SOURCES = $(libhttrack_la_SOURCES) $(htsserver_SOURCES) httrack.c +DIST_SOURCES = $(libhttrack_la_SOURCES) $(htsserver_SOURCES) httrack.c \ + $(proxytrack_SOURCES) DATA = $(DevIncludes_DATA) -DIST_COMMON = Makefile.am Makefile.in -SOURCES = $(libhttrack_la_SOURCES) $(htsserver_SOURCES) httrack.c +DIST_COMMON = $(srcdir)/Makefile.in Makefile.am +SOURCES = $(libhttrack_la_SOURCES) $(htsserver_SOURCES) httrack.c $(proxytrack_SOURCES) all: all-am @@ -302,10 +358,6 @@ clean-libLTLIBRARIES: echo "rm -f \"$${dir}/so_locations\""; \ rm -f "$${dir}/so_locations"; \ done -ioapi.lo: minizip/ioapi.c -mztools.lo: minizip/mztools.c -unzip.lo: minizip/unzip.c -zip.lo: minizip/zip.c libhttrack.la: $(libhttrack_la_OBJECTS) $(libhttrack_la_DEPENDENCIES) $(LINK) -rpath $(libdir) $(libhttrack_la_LDFLAGS) $(libhttrack_la_OBJECTS) $(libhttrack_la_LIBADD) $(LIBS) binPROGRAMS_INSTALL = $(INSTALL_PROGRAM) @@ -343,6 +395,9 @@ htsserver$(EXEEXT): $(htsserver_OBJECTS) $(htsserver_DEPENDENCIES) httrack$(EXEEXT): $(httrack_OBJECTS) $(httrack_DEPENDENCIES) @rm -f httrack$(EXEEXT) $(LINK) $(httrack_LDFLAGS) $(httrack_OBJECTS) $(httrack_LDADD) $(LIBS) +proxytrack$(EXEEXT): $(proxytrack_OBJECTS) $(proxytrack_DEPENDENCIES) + @rm -f proxytrack$(EXEEXT) + $(LINK) $(proxytrack_LDFLAGS) $(proxytrack_OBJECTS) $(proxytrack_LDADD) $(LIBS) whttrackrunSCRIPT_INSTALL = $(INSTALL_SCRIPT) install-whttrackrunSCRIPTS: $(whttrackrun_SCRIPTS) @$(NORMAL_INSTALL) @@ -370,6 +425,7 @@ mostlyclean-compile: distclean-compile: -rm -f *.tab.c +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/error.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/htsalias.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/htsback.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/htsbauth.Plo@am__quote@ @@ -386,6 +442,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/htsjava.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/htslib.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/htsmd5.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/htsmms.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/htsmodules.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/htsname.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/htsnostatic.Plo@am__quote@ @@ -401,17 +458,25 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/httrack.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ioapi.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/md5.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mms.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mztools.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proxytrack-htsinthash.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proxytrack-htsmd5.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proxytrack-ioapi.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proxytrack-main.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proxytrack-md5.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proxytrack-mztools.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proxytrack-proxytrack.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proxytrack-store.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proxytrack-unzip.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/proxytrack-zip.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/unzip.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/zip.Plo@am__quote@ -distclean-depend: - -rm -rf ./$(DEPDIR) - .c.o: @am__fastdepCC_TRUE@ if $(COMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" \ @am__fastdepCC_TRUE@ -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ @@ -421,19 +486,19 @@ distclean-depend: .c.obj: @am__fastdepCC_TRUE@ if $(COMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" \ -@am__fastdepCC_TRUE@ -c -o $@ `if test -f '$<'; then $(CYGPATH_W) '$<'; else $(CYGPATH_W) '$(srcdir)/$<'`; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; \ +@am__fastdepCC_TRUE@ -c -o $@ `if test -f '$<'; then $(CYGPATH_W) '$<'; else $(CYGPATH_W) '$(srcdir)/$<'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Po"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/$*.Po' tmpdepfile='$(DEPDIR)/$*.TPo' @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(COMPILE) -c `if test -f '$<'; then $(CYGPATH_W) '$<'; else $(CYGPATH_W) '$(srcdir)/$<'` +@am__fastdepCC_FALSE@ $(COMPILE) -c `if test -f '$<'; then $(CYGPATH_W) '$<'; else $(CYGPATH_W) '$(srcdir)/$<'; fi` .c.lo: @am__fastdepCC_TRUE@ if $(LTCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" \ @am__fastdepCC_TRUE@ -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Plo"; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/$*.Tpo" "$(DEPDIR)/$*.Plo"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/$*.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ @@ -444,7 +509,7 @@ distclean-depend: ioapi.o: minizip/ioapi.c @am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT ioapi.o -MD -MP -MF "$(DEPDIR)/ioapi.Tpo" \ @am__fastdepCC_TRUE@ -c -o ioapi.o `test -f 'minizip/ioapi.c' || echo '$(srcdir)/'`minizip/ioapi.c; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/ioapi.Tpo" "$(DEPDIR)/ioapi.Po"; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/ioapi.Tpo" "$(DEPDIR)/ioapi.Po"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/ioapi.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/ioapi.c' object='ioapi.o' libtool=no @AMDEPBACKSLASH@ @@ -454,19 +519,19 @@ ioapi.o: minizip/ioapi.c ioapi.obj: minizip/ioapi.c @am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT ioapi.obj -MD -MP -MF "$(DEPDIR)/ioapi.Tpo" \ -@am__fastdepCC_TRUE@ -c -o ioapi.obj `if test -f 'minizip/ioapi.c'; then $(CYGPATH_W) 'minizip/ioapi.c'; else $(CYGPATH_W) '$(srcdir)/minizip/ioapi.c'`; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/ioapi.Tpo" "$(DEPDIR)/ioapi.Po"; \ +@am__fastdepCC_TRUE@ -c -o ioapi.obj `if test -f 'minizip/ioapi.c'; then $(CYGPATH_W) 'minizip/ioapi.c'; else $(CYGPATH_W) '$(srcdir)/minizip/ioapi.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/ioapi.Tpo" "$(DEPDIR)/ioapi.Po"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/ioapi.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/ioapi.c' object='ioapi.obj' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/ioapi.Po' tmpdepfile='$(DEPDIR)/ioapi.TPo' @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o ioapi.obj `if test -f 'minizip/ioapi.c'; then $(CYGPATH_W) 'minizip/ioapi.c'; else $(CYGPATH_W) '$(srcdir)/minizip/ioapi.c'` +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o ioapi.obj `if test -f 'minizip/ioapi.c'; then $(CYGPATH_W) 'minizip/ioapi.c'; else $(CYGPATH_W) '$(srcdir)/minizip/ioapi.c'; fi` ioapi.lo: minizip/ioapi.c @am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT ioapi.lo -MD -MP -MF "$(DEPDIR)/ioapi.Tpo" \ @am__fastdepCC_TRUE@ -c -o ioapi.lo `test -f 'minizip/ioapi.c' || echo '$(srcdir)/'`minizip/ioapi.c; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/ioapi.Tpo" "$(DEPDIR)/ioapi.Plo"; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/ioapi.Tpo" "$(DEPDIR)/ioapi.Plo"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/ioapi.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/ioapi.c' object='ioapi.lo' libtool=yes @AMDEPBACKSLASH@ @@ -477,7 +542,7 @@ ioapi.lo: minizip/ioapi.c mztools.o: minizip/mztools.c @am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT mztools.o -MD -MP -MF "$(DEPDIR)/mztools.Tpo" \ @am__fastdepCC_TRUE@ -c -o mztools.o `test -f 'minizip/mztools.c' || echo '$(srcdir)/'`minizip/mztools.c; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/mztools.Tpo" "$(DEPDIR)/mztools.Po"; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/mztools.Tpo" "$(DEPDIR)/mztools.Po"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/mztools.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/mztools.c' object='mztools.o' libtool=no @AMDEPBACKSLASH@ @@ -487,19 +552,19 @@ mztools.o: minizip/mztools.c mztools.obj: minizip/mztools.c @am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT mztools.obj -MD -MP -MF "$(DEPDIR)/mztools.Tpo" \ -@am__fastdepCC_TRUE@ -c -o mztools.obj `if test -f 'minizip/mztools.c'; then $(CYGPATH_W) 'minizip/mztools.c'; else $(CYGPATH_W) '$(srcdir)/minizip/mztools.c'`; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/mztools.Tpo" "$(DEPDIR)/mztools.Po"; \ +@am__fastdepCC_TRUE@ -c -o mztools.obj `if test -f 'minizip/mztools.c'; then $(CYGPATH_W) 'minizip/mztools.c'; else $(CYGPATH_W) '$(srcdir)/minizip/mztools.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/mztools.Tpo" "$(DEPDIR)/mztools.Po"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/mztools.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/mztools.c' object='mztools.obj' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/mztools.Po' tmpdepfile='$(DEPDIR)/mztools.TPo' @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mztools.obj `if test -f 'minizip/mztools.c'; then $(CYGPATH_W) 'minizip/mztools.c'; else $(CYGPATH_W) '$(srcdir)/minizip/mztools.c'` +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mztools.obj `if test -f 'minizip/mztools.c'; then $(CYGPATH_W) 'minizip/mztools.c'; else $(CYGPATH_W) '$(srcdir)/minizip/mztools.c'; fi` mztools.lo: minizip/mztools.c @am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT mztools.lo -MD -MP -MF "$(DEPDIR)/mztools.Tpo" \ @am__fastdepCC_TRUE@ -c -o mztools.lo `test -f 'minizip/mztools.c' || echo '$(srcdir)/'`minizip/mztools.c; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/mztools.Tpo" "$(DEPDIR)/mztools.Plo"; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/mztools.Tpo" "$(DEPDIR)/mztools.Plo"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/mztools.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/mztools.c' object='mztools.lo' libtool=yes @AMDEPBACKSLASH@ @@ -510,7 +575,7 @@ mztools.lo: minizip/mztools.c unzip.o: minizip/unzip.c @am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT unzip.o -MD -MP -MF "$(DEPDIR)/unzip.Tpo" \ @am__fastdepCC_TRUE@ -c -o unzip.o `test -f 'minizip/unzip.c' || echo '$(srcdir)/'`minizip/unzip.c; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/unzip.Tpo" "$(DEPDIR)/unzip.Po"; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/unzip.Tpo" "$(DEPDIR)/unzip.Po"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/unzip.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/unzip.c' object='unzip.o' libtool=no @AMDEPBACKSLASH@ @@ -520,19 +585,19 @@ unzip.o: minizip/unzip.c unzip.obj: minizip/unzip.c @am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT unzip.obj -MD -MP -MF "$(DEPDIR)/unzip.Tpo" \ -@am__fastdepCC_TRUE@ -c -o unzip.obj `if test -f 'minizip/unzip.c'; then $(CYGPATH_W) 'minizip/unzip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/unzip.c'`; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/unzip.Tpo" "$(DEPDIR)/unzip.Po"; \ +@am__fastdepCC_TRUE@ -c -o unzip.obj `if test -f 'minizip/unzip.c'; then $(CYGPATH_W) 'minizip/unzip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/unzip.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/unzip.Tpo" "$(DEPDIR)/unzip.Po"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/unzip.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/unzip.c' object='unzip.obj' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/unzip.Po' tmpdepfile='$(DEPDIR)/unzip.TPo' @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o unzip.obj `if test -f 'minizip/unzip.c'; then $(CYGPATH_W) 'minizip/unzip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/unzip.c'` +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o unzip.obj `if test -f 'minizip/unzip.c'; then $(CYGPATH_W) 'minizip/unzip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/unzip.c'; fi` unzip.lo: minizip/unzip.c @am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT unzip.lo -MD -MP -MF "$(DEPDIR)/unzip.Tpo" \ @am__fastdepCC_TRUE@ -c -o unzip.lo `test -f 'minizip/unzip.c' || echo '$(srcdir)/'`minizip/unzip.c; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/unzip.Tpo" "$(DEPDIR)/unzip.Plo"; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/unzip.Tpo" "$(DEPDIR)/unzip.Plo"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/unzip.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/unzip.c' object='unzip.lo' libtool=yes @AMDEPBACKSLASH@ @@ -543,7 +608,7 @@ unzip.lo: minizip/unzip.c zip.o: minizip/zip.c @am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT zip.o -MD -MP -MF "$(DEPDIR)/zip.Tpo" \ @am__fastdepCC_TRUE@ -c -o zip.o `test -f 'minizip/zip.c' || echo '$(srcdir)/'`minizip/zip.c; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/zip.Tpo" "$(DEPDIR)/zip.Po"; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/zip.Tpo" "$(DEPDIR)/zip.Po"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/zip.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/zip.c' object='zip.o' libtool=no @AMDEPBACKSLASH@ @@ -553,19 +618,19 @@ zip.o: minizip/zip.c zip.obj: minizip/zip.c @am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT zip.obj -MD -MP -MF "$(DEPDIR)/zip.Tpo" \ -@am__fastdepCC_TRUE@ -c -o zip.obj `if test -f 'minizip/zip.c'; then $(CYGPATH_W) 'minizip/zip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/zip.c'`; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/zip.Tpo" "$(DEPDIR)/zip.Po"; \ +@am__fastdepCC_TRUE@ -c -o zip.obj `if test -f 'minizip/zip.c'; then $(CYGPATH_W) 'minizip/zip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/zip.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/zip.Tpo" "$(DEPDIR)/zip.Po"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/zip.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/zip.c' object='zip.obj' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/zip.Po' tmpdepfile='$(DEPDIR)/zip.TPo' @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o zip.obj `if test -f 'minizip/zip.c'; then $(CYGPATH_W) 'minizip/zip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/zip.c'` +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o zip.obj `if test -f 'minizip/zip.c'; then $(CYGPATH_W) 'minizip/zip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/zip.c'; fi` zip.lo: minizip/zip.c @am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT zip.lo -MD -MP -MF "$(DEPDIR)/zip.Tpo" \ @am__fastdepCC_TRUE@ -c -o zip.lo `test -f 'minizip/zip.c' || echo '$(srcdir)/'`minizip/zip.c; \ -@am__fastdepCC_TRUE@ then mv "$(DEPDIR)/zip.Tpo" "$(DEPDIR)/zip.Plo"; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/zip.Tpo" "$(DEPDIR)/zip.Plo"; \ @am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/zip.Tpo"; exit 1; \ @am__fastdepCC_TRUE@ fi @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/zip.c' object='zip.lo' libtool=yes @AMDEPBACKSLASH@ @@ -573,6 +638,402 @@ zip.lo: minizip/zip.c @AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o zip.lo `test -f 'minizip/zip.c' || echo '$(srcdir)/'`minizip/zip.c +error.o: mmsrip/error.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT error.o -MD -MP -MF "$(DEPDIR)/error.Tpo" \ +@am__fastdepCC_TRUE@ -c -o error.o `test -f 'mmsrip/error.c' || echo '$(srcdir)/'`mmsrip/error.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/error.Tpo" "$(DEPDIR)/error.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/error.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='mmsrip/error.c' object='error.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/error.Po' tmpdepfile='$(DEPDIR)/error.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o error.o `test -f 'mmsrip/error.c' || echo '$(srcdir)/'`mmsrip/error.c + +error.obj: mmsrip/error.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT error.obj -MD -MP -MF "$(DEPDIR)/error.Tpo" \ +@am__fastdepCC_TRUE@ -c -o error.obj `if test -f 'mmsrip/error.c'; then $(CYGPATH_W) 'mmsrip/error.c'; else $(CYGPATH_W) '$(srcdir)/mmsrip/error.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/error.Tpo" "$(DEPDIR)/error.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/error.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='mmsrip/error.c' object='error.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/error.Po' tmpdepfile='$(DEPDIR)/error.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o error.obj `if test -f 'mmsrip/error.c'; then $(CYGPATH_W) 'mmsrip/error.c'; else $(CYGPATH_W) '$(srcdir)/mmsrip/error.c'; fi` + +error.lo: mmsrip/error.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT error.lo -MD -MP -MF "$(DEPDIR)/error.Tpo" \ +@am__fastdepCC_TRUE@ -c -o error.lo `test -f 'mmsrip/error.c' || echo '$(srcdir)/'`mmsrip/error.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/error.Tpo" "$(DEPDIR)/error.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/error.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='mmsrip/error.c' object='error.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/error.Plo' tmpdepfile='$(DEPDIR)/error.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o error.lo `test -f 'mmsrip/error.c' || echo '$(srcdir)/'`mmsrip/error.c + +mms.o: mmsrip/mms.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT mms.o -MD -MP -MF "$(DEPDIR)/mms.Tpo" \ +@am__fastdepCC_TRUE@ -c -o mms.o `test -f 'mmsrip/mms.c' || echo '$(srcdir)/'`mmsrip/mms.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/mms.Tpo" "$(DEPDIR)/mms.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/mms.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='mmsrip/mms.c' object='mms.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/mms.Po' tmpdepfile='$(DEPDIR)/mms.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mms.o `test -f 'mmsrip/mms.c' || echo '$(srcdir)/'`mmsrip/mms.c + +mms.obj: mmsrip/mms.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT mms.obj -MD -MP -MF "$(DEPDIR)/mms.Tpo" \ +@am__fastdepCC_TRUE@ -c -o mms.obj `if test -f 'mmsrip/mms.c'; then $(CYGPATH_W) 'mmsrip/mms.c'; else $(CYGPATH_W) '$(srcdir)/mmsrip/mms.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/mms.Tpo" "$(DEPDIR)/mms.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/mms.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='mmsrip/mms.c' object='mms.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/mms.Po' tmpdepfile='$(DEPDIR)/mms.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mms.obj `if test -f 'mmsrip/mms.c'; then $(CYGPATH_W) 'mmsrip/mms.c'; else $(CYGPATH_W) '$(srcdir)/mmsrip/mms.c'; fi` + +mms.lo: mmsrip/mms.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT mms.lo -MD -MP -MF "$(DEPDIR)/mms.Tpo" \ +@am__fastdepCC_TRUE@ -c -o mms.lo `test -f 'mmsrip/mms.c' || echo '$(srcdir)/'`mmsrip/mms.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/mms.Tpo" "$(DEPDIR)/mms.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/mms.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='mmsrip/mms.c' object='mms.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/mms.Plo' tmpdepfile='$(DEPDIR)/mms.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o mms.lo `test -f 'mmsrip/mms.c' || echo '$(srcdir)/'`mmsrip/mms.c + +proxytrack-main.o: proxy/main.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-main.o -MD -MP -MF "$(DEPDIR)/proxytrack-main.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-main.o `test -f 'proxy/main.c' || echo '$(srcdir)/'`proxy/main.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-main.Tpo" "$(DEPDIR)/proxytrack-main.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-main.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='proxy/main.c' object='proxytrack-main.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-main.Po' tmpdepfile='$(DEPDIR)/proxytrack-main.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-main.o `test -f 'proxy/main.c' || echo '$(srcdir)/'`proxy/main.c + +proxytrack-main.obj: proxy/main.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-main.obj -MD -MP -MF "$(DEPDIR)/proxytrack-main.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-main.obj `if test -f 'proxy/main.c'; then $(CYGPATH_W) 'proxy/main.c'; else $(CYGPATH_W) '$(srcdir)/proxy/main.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-main.Tpo" "$(DEPDIR)/proxytrack-main.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-main.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='proxy/main.c' object='proxytrack-main.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-main.Po' tmpdepfile='$(DEPDIR)/proxytrack-main.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-main.obj `if test -f 'proxy/main.c'; then $(CYGPATH_W) 'proxy/main.c'; else $(CYGPATH_W) '$(srcdir)/proxy/main.c'; fi` + +proxytrack-main.lo: proxy/main.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-main.lo -MD -MP -MF "$(DEPDIR)/proxytrack-main.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-main.lo `test -f 'proxy/main.c' || echo '$(srcdir)/'`proxy/main.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-main.Tpo" "$(DEPDIR)/proxytrack-main.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-main.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='proxy/main.c' object='proxytrack-main.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-main.Plo' tmpdepfile='$(DEPDIR)/proxytrack-main.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-main.lo `test -f 'proxy/main.c' || echo '$(srcdir)/'`proxy/main.c + +proxytrack-proxytrack.o: proxy/proxytrack.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-proxytrack.o -MD -MP -MF "$(DEPDIR)/proxytrack-proxytrack.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-proxytrack.o `test -f 'proxy/proxytrack.c' || echo '$(srcdir)/'`proxy/proxytrack.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-proxytrack.Tpo" "$(DEPDIR)/proxytrack-proxytrack.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-proxytrack.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='proxy/proxytrack.c' object='proxytrack-proxytrack.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-proxytrack.Po' tmpdepfile='$(DEPDIR)/proxytrack-proxytrack.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-proxytrack.o `test -f 'proxy/proxytrack.c' || echo '$(srcdir)/'`proxy/proxytrack.c + +proxytrack-proxytrack.obj: proxy/proxytrack.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-proxytrack.obj -MD -MP -MF "$(DEPDIR)/proxytrack-proxytrack.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-proxytrack.obj `if test -f 'proxy/proxytrack.c'; then $(CYGPATH_W) 'proxy/proxytrack.c'; else $(CYGPATH_W) '$(srcdir)/proxy/proxytrack.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-proxytrack.Tpo" "$(DEPDIR)/proxytrack-proxytrack.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-proxytrack.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='proxy/proxytrack.c' object='proxytrack-proxytrack.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-proxytrack.Po' tmpdepfile='$(DEPDIR)/proxytrack-proxytrack.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-proxytrack.obj `if test -f 'proxy/proxytrack.c'; then $(CYGPATH_W) 'proxy/proxytrack.c'; else $(CYGPATH_W) '$(srcdir)/proxy/proxytrack.c'; fi` + +proxytrack-proxytrack.lo: proxy/proxytrack.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-proxytrack.lo -MD -MP -MF "$(DEPDIR)/proxytrack-proxytrack.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-proxytrack.lo `test -f 'proxy/proxytrack.c' || echo '$(srcdir)/'`proxy/proxytrack.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-proxytrack.Tpo" "$(DEPDIR)/proxytrack-proxytrack.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-proxytrack.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='proxy/proxytrack.c' object='proxytrack-proxytrack.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-proxytrack.Plo' tmpdepfile='$(DEPDIR)/proxytrack-proxytrack.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-proxytrack.lo `test -f 'proxy/proxytrack.c' || echo '$(srcdir)/'`proxy/proxytrack.c + +proxytrack-store.o: proxy/store.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-store.o -MD -MP -MF "$(DEPDIR)/proxytrack-store.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-store.o `test -f 'proxy/store.c' || echo '$(srcdir)/'`proxy/store.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-store.Tpo" "$(DEPDIR)/proxytrack-store.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-store.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='proxy/store.c' object='proxytrack-store.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-store.Po' tmpdepfile='$(DEPDIR)/proxytrack-store.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-store.o `test -f 'proxy/store.c' || echo '$(srcdir)/'`proxy/store.c + +proxytrack-store.obj: proxy/store.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-store.obj -MD -MP -MF "$(DEPDIR)/proxytrack-store.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-store.obj `if test -f 'proxy/store.c'; then $(CYGPATH_W) 'proxy/store.c'; else $(CYGPATH_W) '$(srcdir)/proxy/store.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-store.Tpo" "$(DEPDIR)/proxytrack-store.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-store.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='proxy/store.c' object='proxytrack-store.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-store.Po' tmpdepfile='$(DEPDIR)/proxytrack-store.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-store.obj `if test -f 'proxy/store.c'; then $(CYGPATH_W) 'proxy/store.c'; else $(CYGPATH_W) '$(srcdir)/proxy/store.c'; fi` + +proxytrack-store.lo: proxy/store.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-store.lo -MD -MP -MF "$(DEPDIR)/proxytrack-store.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-store.lo `test -f 'proxy/store.c' || echo '$(srcdir)/'`proxy/store.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-store.Tpo" "$(DEPDIR)/proxytrack-store.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-store.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='proxy/store.c' object='proxytrack-store.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-store.Plo' tmpdepfile='$(DEPDIR)/proxytrack-store.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-store.lo `test -f 'proxy/store.c' || echo '$(srcdir)/'`proxy/store.c + +proxytrack-htsinthash.o: htsinthash.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-htsinthash.o -MD -MP -MF "$(DEPDIR)/proxytrack-htsinthash.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-htsinthash.o `test -f 'htsinthash.c' || echo '$(srcdir)/'`htsinthash.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-htsinthash.Tpo" "$(DEPDIR)/proxytrack-htsinthash.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-htsinthash.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='htsinthash.c' object='proxytrack-htsinthash.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-htsinthash.Po' tmpdepfile='$(DEPDIR)/proxytrack-htsinthash.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-htsinthash.o `test -f 'htsinthash.c' || echo '$(srcdir)/'`htsinthash.c + +proxytrack-htsinthash.obj: htsinthash.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-htsinthash.obj -MD -MP -MF "$(DEPDIR)/proxytrack-htsinthash.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-htsinthash.obj `if test -f 'htsinthash.c'; then $(CYGPATH_W) 'htsinthash.c'; else $(CYGPATH_W) '$(srcdir)/htsinthash.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-htsinthash.Tpo" "$(DEPDIR)/proxytrack-htsinthash.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-htsinthash.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='htsinthash.c' object='proxytrack-htsinthash.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-htsinthash.Po' tmpdepfile='$(DEPDIR)/proxytrack-htsinthash.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-htsinthash.obj `if test -f 'htsinthash.c'; then $(CYGPATH_W) 'htsinthash.c'; else $(CYGPATH_W) '$(srcdir)/htsinthash.c'; fi` + +proxytrack-htsinthash.lo: htsinthash.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-htsinthash.lo -MD -MP -MF "$(DEPDIR)/proxytrack-htsinthash.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-htsinthash.lo `test -f 'htsinthash.c' || echo '$(srcdir)/'`htsinthash.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-htsinthash.Tpo" "$(DEPDIR)/proxytrack-htsinthash.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-htsinthash.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='htsinthash.c' object='proxytrack-htsinthash.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-htsinthash.Plo' tmpdepfile='$(DEPDIR)/proxytrack-htsinthash.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-htsinthash.lo `test -f 'htsinthash.c' || echo '$(srcdir)/'`htsinthash.c + +proxytrack-htsmd5.o: htsmd5.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-htsmd5.o -MD -MP -MF "$(DEPDIR)/proxytrack-htsmd5.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-htsmd5.o `test -f 'htsmd5.c' || echo '$(srcdir)/'`htsmd5.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-htsmd5.Tpo" "$(DEPDIR)/proxytrack-htsmd5.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-htsmd5.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='htsmd5.c' object='proxytrack-htsmd5.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-htsmd5.Po' tmpdepfile='$(DEPDIR)/proxytrack-htsmd5.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-htsmd5.o `test -f 'htsmd5.c' || echo '$(srcdir)/'`htsmd5.c + +proxytrack-htsmd5.obj: htsmd5.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-htsmd5.obj -MD -MP -MF "$(DEPDIR)/proxytrack-htsmd5.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-htsmd5.obj `if test -f 'htsmd5.c'; then $(CYGPATH_W) 'htsmd5.c'; else $(CYGPATH_W) '$(srcdir)/htsmd5.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-htsmd5.Tpo" "$(DEPDIR)/proxytrack-htsmd5.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-htsmd5.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='htsmd5.c' object='proxytrack-htsmd5.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-htsmd5.Po' tmpdepfile='$(DEPDIR)/proxytrack-htsmd5.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-htsmd5.obj `if test -f 'htsmd5.c'; then $(CYGPATH_W) 'htsmd5.c'; else $(CYGPATH_W) '$(srcdir)/htsmd5.c'; fi` + +proxytrack-htsmd5.lo: htsmd5.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-htsmd5.lo -MD -MP -MF "$(DEPDIR)/proxytrack-htsmd5.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-htsmd5.lo `test -f 'htsmd5.c' || echo '$(srcdir)/'`htsmd5.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-htsmd5.Tpo" "$(DEPDIR)/proxytrack-htsmd5.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-htsmd5.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='htsmd5.c' object='proxytrack-htsmd5.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-htsmd5.Plo' tmpdepfile='$(DEPDIR)/proxytrack-htsmd5.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-htsmd5.lo `test -f 'htsmd5.c' || echo '$(srcdir)/'`htsmd5.c + +proxytrack-md5.o: md5.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-md5.o -MD -MP -MF "$(DEPDIR)/proxytrack-md5.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-md5.o `test -f 'md5.c' || echo '$(srcdir)/'`md5.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-md5.Tpo" "$(DEPDIR)/proxytrack-md5.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-md5.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='md5.c' object='proxytrack-md5.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-md5.Po' tmpdepfile='$(DEPDIR)/proxytrack-md5.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-md5.o `test -f 'md5.c' || echo '$(srcdir)/'`md5.c + +proxytrack-md5.obj: md5.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-md5.obj -MD -MP -MF "$(DEPDIR)/proxytrack-md5.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-md5.obj `if test -f 'md5.c'; then $(CYGPATH_W) 'md5.c'; else $(CYGPATH_W) '$(srcdir)/md5.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-md5.Tpo" "$(DEPDIR)/proxytrack-md5.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-md5.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='md5.c' object='proxytrack-md5.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-md5.Po' tmpdepfile='$(DEPDIR)/proxytrack-md5.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-md5.obj `if test -f 'md5.c'; then $(CYGPATH_W) 'md5.c'; else $(CYGPATH_W) '$(srcdir)/md5.c'; fi` + +proxytrack-md5.lo: md5.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-md5.lo -MD -MP -MF "$(DEPDIR)/proxytrack-md5.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-md5.lo `test -f 'md5.c' || echo '$(srcdir)/'`md5.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-md5.Tpo" "$(DEPDIR)/proxytrack-md5.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-md5.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='md5.c' object='proxytrack-md5.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-md5.Plo' tmpdepfile='$(DEPDIR)/proxytrack-md5.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-md5.lo `test -f 'md5.c' || echo '$(srcdir)/'`md5.c + +proxytrack-ioapi.o: minizip/ioapi.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-ioapi.o -MD -MP -MF "$(DEPDIR)/proxytrack-ioapi.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-ioapi.o `test -f 'minizip/ioapi.c' || echo '$(srcdir)/'`minizip/ioapi.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-ioapi.Tpo" "$(DEPDIR)/proxytrack-ioapi.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-ioapi.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/ioapi.c' object='proxytrack-ioapi.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-ioapi.Po' tmpdepfile='$(DEPDIR)/proxytrack-ioapi.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-ioapi.o `test -f 'minizip/ioapi.c' || echo '$(srcdir)/'`minizip/ioapi.c + +proxytrack-ioapi.obj: minizip/ioapi.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-ioapi.obj -MD -MP -MF "$(DEPDIR)/proxytrack-ioapi.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-ioapi.obj `if test -f 'minizip/ioapi.c'; then $(CYGPATH_W) 'minizip/ioapi.c'; else $(CYGPATH_W) '$(srcdir)/minizip/ioapi.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-ioapi.Tpo" "$(DEPDIR)/proxytrack-ioapi.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-ioapi.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/ioapi.c' object='proxytrack-ioapi.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-ioapi.Po' tmpdepfile='$(DEPDIR)/proxytrack-ioapi.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-ioapi.obj `if test -f 'minizip/ioapi.c'; then $(CYGPATH_W) 'minizip/ioapi.c'; else $(CYGPATH_W) '$(srcdir)/minizip/ioapi.c'; fi` + +proxytrack-ioapi.lo: minizip/ioapi.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-ioapi.lo -MD -MP -MF "$(DEPDIR)/proxytrack-ioapi.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-ioapi.lo `test -f 'minizip/ioapi.c' || echo '$(srcdir)/'`minizip/ioapi.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-ioapi.Tpo" "$(DEPDIR)/proxytrack-ioapi.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-ioapi.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/ioapi.c' object='proxytrack-ioapi.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-ioapi.Plo' tmpdepfile='$(DEPDIR)/proxytrack-ioapi.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-ioapi.lo `test -f 'minizip/ioapi.c' || echo '$(srcdir)/'`minizip/ioapi.c + +proxytrack-mztools.o: minizip/mztools.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-mztools.o -MD -MP -MF "$(DEPDIR)/proxytrack-mztools.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-mztools.o `test -f 'minizip/mztools.c' || echo '$(srcdir)/'`minizip/mztools.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-mztools.Tpo" "$(DEPDIR)/proxytrack-mztools.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-mztools.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/mztools.c' object='proxytrack-mztools.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-mztools.Po' tmpdepfile='$(DEPDIR)/proxytrack-mztools.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-mztools.o `test -f 'minizip/mztools.c' || echo '$(srcdir)/'`minizip/mztools.c + +proxytrack-mztools.obj: minizip/mztools.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-mztools.obj -MD -MP -MF "$(DEPDIR)/proxytrack-mztools.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-mztools.obj `if test -f 'minizip/mztools.c'; then $(CYGPATH_W) 'minizip/mztools.c'; else $(CYGPATH_W) '$(srcdir)/minizip/mztools.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-mztools.Tpo" "$(DEPDIR)/proxytrack-mztools.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-mztools.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/mztools.c' object='proxytrack-mztools.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-mztools.Po' tmpdepfile='$(DEPDIR)/proxytrack-mztools.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-mztools.obj `if test -f 'minizip/mztools.c'; then $(CYGPATH_W) 'minizip/mztools.c'; else $(CYGPATH_W) '$(srcdir)/minizip/mztools.c'; fi` + +proxytrack-mztools.lo: minizip/mztools.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-mztools.lo -MD -MP -MF "$(DEPDIR)/proxytrack-mztools.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-mztools.lo `test -f 'minizip/mztools.c' || echo '$(srcdir)/'`minizip/mztools.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-mztools.Tpo" "$(DEPDIR)/proxytrack-mztools.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-mztools.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/mztools.c' object='proxytrack-mztools.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-mztools.Plo' tmpdepfile='$(DEPDIR)/proxytrack-mztools.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-mztools.lo `test -f 'minizip/mztools.c' || echo '$(srcdir)/'`minizip/mztools.c + +proxytrack-unzip.o: minizip/unzip.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-unzip.o -MD -MP -MF "$(DEPDIR)/proxytrack-unzip.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-unzip.o `test -f 'minizip/unzip.c' || echo '$(srcdir)/'`minizip/unzip.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-unzip.Tpo" "$(DEPDIR)/proxytrack-unzip.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-unzip.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/unzip.c' object='proxytrack-unzip.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-unzip.Po' tmpdepfile='$(DEPDIR)/proxytrack-unzip.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-unzip.o `test -f 'minizip/unzip.c' || echo '$(srcdir)/'`minizip/unzip.c + +proxytrack-unzip.obj: minizip/unzip.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-unzip.obj -MD -MP -MF "$(DEPDIR)/proxytrack-unzip.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-unzip.obj `if test -f 'minizip/unzip.c'; then $(CYGPATH_W) 'minizip/unzip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/unzip.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-unzip.Tpo" "$(DEPDIR)/proxytrack-unzip.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-unzip.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/unzip.c' object='proxytrack-unzip.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-unzip.Po' tmpdepfile='$(DEPDIR)/proxytrack-unzip.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-unzip.obj `if test -f 'minizip/unzip.c'; then $(CYGPATH_W) 'minizip/unzip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/unzip.c'; fi` + +proxytrack-unzip.lo: minizip/unzip.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-unzip.lo -MD -MP -MF "$(DEPDIR)/proxytrack-unzip.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-unzip.lo `test -f 'minizip/unzip.c' || echo '$(srcdir)/'`minizip/unzip.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-unzip.Tpo" "$(DEPDIR)/proxytrack-unzip.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-unzip.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/unzip.c' object='proxytrack-unzip.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-unzip.Plo' tmpdepfile='$(DEPDIR)/proxytrack-unzip.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-unzip.lo `test -f 'minizip/unzip.c' || echo '$(srcdir)/'`minizip/unzip.c + +proxytrack-zip.o: minizip/zip.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-zip.o -MD -MP -MF "$(DEPDIR)/proxytrack-zip.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-zip.o `test -f 'minizip/zip.c' || echo '$(srcdir)/'`minizip/zip.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-zip.Tpo" "$(DEPDIR)/proxytrack-zip.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-zip.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/zip.c' object='proxytrack-zip.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-zip.Po' tmpdepfile='$(DEPDIR)/proxytrack-zip.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-zip.o `test -f 'minizip/zip.c' || echo '$(srcdir)/'`minizip/zip.c + +proxytrack-zip.obj: minizip/zip.c +@am__fastdepCC_TRUE@ if $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-zip.obj -MD -MP -MF "$(DEPDIR)/proxytrack-zip.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-zip.obj `if test -f 'minizip/zip.c'; then $(CYGPATH_W) 'minizip/zip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/zip.c'; fi`; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-zip.Tpo" "$(DEPDIR)/proxytrack-zip.Po"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-zip.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/zip.c' object='proxytrack-zip.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-zip.Po' tmpdepfile='$(DEPDIR)/proxytrack-zip.TPo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-zip.obj `if test -f 'minizip/zip.c'; then $(CYGPATH_W) 'minizip/zip.c'; else $(CYGPATH_W) '$(srcdir)/minizip/zip.c'; fi` + +proxytrack-zip.lo: minizip/zip.c +@am__fastdepCC_TRUE@ if $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -MT proxytrack-zip.lo -MD -MP -MF "$(DEPDIR)/proxytrack-zip.Tpo" \ +@am__fastdepCC_TRUE@ -c -o proxytrack-zip.lo `test -f 'minizip/zip.c' || echo '$(srcdir)/'`minizip/zip.c; \ +@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/proxytrack-zip.Tpo" "$(DEPDIR)/proxytrack-zip.Plo"; \ +@am__fastdepCC_TRUE@ else rm -f "$(DEPDIR)/proxytrack-zip.Tpo"; exit 1; \ +@am__fastdepCC_TRUE@ fi +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='minizip/zip.c' object='proxytrack-zip.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ depfile='$(DEPDIR)/proxytrack-zip.Plo' tmpdepfile='$(DEPDIR)/proxytrack-zip.TPlo' @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(proxytrack_CFLAGS) $(CFLAGS) -c -o proxytrack-zip.lo `test -f 'minizip/zip.c' || echo '$(srcdir)/'`minizip/zip.c + mostlyclean-libtool: -rm -f *.lo @@ -660,11 +1121,13 @@ top_distdir = .. distdir = $(top_distdir)/$(PACKAGE)-$(VERSION) distdir: $(DISTFILES) - $(mkinstalldirs) $(distdir)/minizip + $(mkinstalldirs) $(distdir)/minizip $(distdir)/mmsrip $(distdir)/proxy @srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's|.|.|g'`; \ list='$(DISTFILES)'; for file in $$list; do \ case $$file in \ $(srcdir)/*) file=`echo "$$file" | sed "s|^$$srcdirstrip/||"`;; \ + $(top_srcdir)/*) file=`echo "$$file" | sed "s|^$$topsrcdirstrip/|$(top_builddir)/|"`;; \ esac; \ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ dir=`echo "$$file" | sed -e 's,/[^/]*$$,,'`; \ @@ -693,7 +1156,6 @@ install-binPROGRAMS: install-libLTLIBRARIES installdirs: $(mkinstalldirs) $(DESTDIR)$(libdir) $(DESTDIR)$(bindir) $(DESTDIR)$(whttrackrundir) $(DESTDIR)$(DevIncludesdir) - install: install-am install-exec: install-exec-am install-data: install-data-am @@ -705,7 +1167,7 @@ install-am: all-am installcheck: installcheck-am install-strip: $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ - INSTALL_STRIP_FLAG=-s \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ `test -z '$(STRIP)' || \ echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install mostlyclean-generic: @@ -713,7 +1175,7 @@ mostlyclean-generic: clean-generic: distclean-generic: - -rm -f Makefile $(CONFIG_CLEAN_FILES) + -rm -f $(CONFIG_CLEAN_FILES) maintainer-clean-generic: @echo "This command is intended for maintainers to use" @@ -724,9 +1186,10 @@ clean-am: clean-binPROGRAMS clean-generic clean-libLTLIBRARIES \ clean-libtool mostlyclean-am distclean: distclean-am - -distclean-am: clean-am distclean-compile distclean-depend \ - distclean-generic distclean-libtool distclean-tags + -rm -rf ./$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-libtool distclean-tags dvi: dvi-am @@ -747,7 +1210,8 @@ install-man: installcheck-am: maintainer-clean: maintainer-clean-am - + -rm -rf ./$(DEPDIR) + -rm -f Makefile maintainer-clean-am: distclean-am maintainer-clean-generic mostlyclean: mostlyclean-am @@ -769,19 +1233,18 @@ uninstall-am: uninstall-DevIncludesDATA uninstall-binPROGRAMS \ .PHONY: CTAGS GTAGS all all-am check check-am clean clean-binPROGRAMS \ clean-generic clean-libLTLIBRARIES clean-libtool ctags \ - distclean distclean-compile distclean-depend distclean-generic \ - distclean-libtool distclean-tags distdir dvi dvi-am info \ - info-am install install-DevIncludesDATA install-am \ - install-binPROGRAMS install-data install-data-am install-exec \ - install-exec-am install-info install-info-am \ - install-libLTLIBRARIES install-man install-strip \ - install-whttrackrunSCRIPTS installcheck installcheck-am \ - installdirs maintainer-clean maintainer-clean-generic \ - mostlyclean mostlyclean-compile mostlyclean-generic \ - mostlyclean-libtool pdf pdf-am ps ps-am tags uninstall \ - uninstall-DevIncludesDATA uninstall-am uninstall-binPROGRAMS \ - uninstall-info-am uninstall-libLTLIBRARIES \ - uninstall-whttrackrunSCRIPTS + distclean distclean-compile distclean-generic distclean-libtool \ + distclean-tags distdir dvi dvi-am info info-am install \ + install-DevIncludesDATA install-am install-binPROGRAMS \ + install-data install-data-am install-exec install-exec-am \ + install-info install-info-am install-libLTLIBRARIES install-man \ + install-strip install-whttrackrunSCRIPTS installcheck \ + installcheck-am installdirs maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags uninstall uninstall-DevIncludesDATA uninstall-am \ + uninstall-binPROGRAMS uninstall-info-am \ + uninstall-libLTLIBRARIES uninstall-whttrackrunSCRIPTS # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. diff --git a/src/hts-indextmpl.h b/src/hts-indextmpl.h index cddbefa..b9aff10 100644 --- a/src/hts-indextmpl.h +++ b/src/hts-indextmpl.h @@ -174,7 +174,7 @@ regen: " <BR>"LF\ " <BR>"LF\ " <H6 ALIGN=\"RIGHT\">"LF\ - " <I>Mirror and index made by HTTrack Website Copier [XR&CO'2005]</I>"LF\ + " <I>Mirror and index made by HTTrack Website Copier [XR&CO'2006]</I>"LF\ " </H6>"LF\ " %s"LF\ " <!-- Thanks for using HTTrack Website Copier! -->"LF\ @@ -193,7 +193,7 @@ regen: ""LF\ "<table width=\"76%%\" border=\"0\" align=\"center\" valign=\"bottom\" cellspacing=\"0\" cellpadding=\"0\">"LF\ " <tr>"LF\ - " <td id=\"footer\"><small>© 2005 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\ + " <td id=\"footer\"><small>© 2006 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\ " </tr>"LF\ "</table>"LF\ ""LF\ @@ -324,7 +324,7 @@ regen: " </TABLE>"LF\ " <BR>"LF\ " <H6 ALIGN=\"RIGHT\">"LF\ - " <I>Mirror and index made by HTTrack Website Copier [XR&CO'2005]</I>"LF\ + " <I>Mirror and index made by HTTrack Website Copier [XR&CO'2006]</I>"LF\ " </H6>"LF\ " %s"LF\ " <!-- Thanks for using HTTrack Website Copier! -->"LF\ @@ -342,7 +342,7 @@ regen: ""LF\ "<table width=\"76%%\" border=\"0\" align=\"center\" valign=\"bottom\" cellspacing=\"0\" cellpadding=\"0\">"LF\ " <tr>"LF\ - " <td id=\"footer\"><small>© 2005 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\ + " <td id=\"footer\"><small>© 2006 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\ " </tr>"LF\ "</table>"LF\ ""LF\ @@ -483,7 +483,7 @@ regen: ""LF\ "<table width=\"76%%\" height=\"100%%\" border=\"0\" align=\"center\" valign=\"bottom\" cellspacing=\"0\" cellpadding=\"0\">"LF\ " <tr>"LF\ - " <td id=\"footer\"><small>© 2005 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\ + " <td id=\"footer\"><small>© 2006 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\ " </tr>"LF\ "</table>"LF\ ""LF\ @@ -620,7 +620,7 @@ regen: ""LF\ "<table width=\"76%%\" height=\"100%%\" border=\"0\" align=\"center\" valign=\"bottom\" cellspacing=\"0\" cellpadding=\"0\">"LF\ " <tr>"LF\ - " <td id=\"footer\"><small>© 2005 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\ + " <td id=\"footer\"><small>© 2006 Xavier Roche & other contributors - Web Design: Kauler Leto.</small></td>"LF\ " </tr>"LF\ "</table>"LF\ ""LF\ diff --git a/src/htsalias.c b/src/htsalias.c index d2e09e1..c6bfef4 100644 --- a/src/htsalias.c +++ b/src/htsalias.c @@ -46,7 +46,15 @@ void linput(FILE* fp,char* s,int max); void hts_lowcase(char* s); #define _NOT_NULL(a) ( (a!=NULL) ? (a) : "" ) -#define is_realspace(c) (strchr(" \x0d\x0a\x09\x0b\x0c",(c))!=NULL) +// #define is_realspace(c) (strchr(" \x0d\x0a\x09\x0b\x0c",(c))!=NULL) +#define is_realspace(c) ( \ + (c) == ' ' \ + || (c) == '\x09' \ + || (c) == '\x0a' \ + || (c) == '\x0b' \ + || (c) == '\x0c' \ + || (c) == '\x0d' \ +) // COPY OF cmdl_ins in htsmain.c // Insert a command in the argc/argv @@ -99,6 +107,7 @@ const char* hts_optalias[][4] = { {"max-files","-m","param",""}, {"max-size","-M","param",""}, {"max-time","-E","param",""}, + {"max-mms-time","-%m","param",""}, {"max-rate","-A","param",""}, {"max-pause","-G","param",""}, {"sockets","-c","param","number of simultaneous connections allowed"},{"socket","-c","param","number of simultaneous connections allowed"},{"connection","-c","param","number of simultaneous connections allowed"}, @@ -109,6 +118,9 @@ const char* hts_optalias[][4] = { {"host-control","-H","param",""}, {"extended-parsing","-%P","param",""}, {"near","-n","single",""}, + {"delayed-type-check","-%N","single",""}, + {"cached-delayed-type-check","-%D","single",""}, + {"delayed-type-check-always","-%N2","single",""}, {"disable-security-limits","-%!","single",""}, {"test","-t","single",""}, {"list","-%L","param1",""}, diff --git a/src/htsback.c b/src/htsback.c index 317d4e7..8a9aac5 100644 --- a/src/htsback.c +++ b/src/htsback.c @@ -66,25 +66,91 @@ Please visit our Website: http://www.httrack.com #else #endif +#if HTS_USEMMS +#include "htsmms.h" +#endif + #undef test_flush #define test_flush if (opt->flush) { if (opt->log) { fflush(opt->log); } if (opt->errlog) { fflush(opt->errlog); } } #define VT_CLREOL "\33[K" +struct_back* back_new(int back_max) { + int i; + struct_back* sback = calloct(1, sizeof(struct_back)); + sback->count = back_max; + sback->lnk = (lien_back*) calloct((back_max + 1), sizeof(lien_back)); + sback->ready = inthash_new(8191); + inthash_value_is_malloc(sback->ready, 1); + // init + for(i = 0 ; i < sback->count ; i++){ + sback->lnk[i].r.location = sback->lnk[i].location_buffer; + sback->lnk[i].status = -1; + sback->lnk[i].r.soc = INVALID_SOCKET; + } + return sback; +} + +void back_free(struct_back** sback) { + if (sback != NULL && *sback != NULL) { + if ((*sback)->lnk != NULL) { + freet((*sback)->lnk); + (*sback)->lnk = NULL; + } + if ((*sback)->ready != NULL) { + inthash_delete((inthash *)&(*sback)->ready); + } + freet(*sback); + *sback = NULL; + } +} + +void back_delete_all(httrackp* opt, cache_back* cache, struct_back* sback) { + if (sback != NULL) { + int i; + // delete live slots + for(i = 0 ; i < sback->count ; i++) { + back_delete(opt, cache, sback, i); + } + // delete stored slots + if (sback->ready != NULL) { + struct_inthash_enum e = inthash_enum_new((inthash)sback->ready); + inthash_chain* item; + while((item = inthash_enum_next(&e))) { + struct_back back1; + back1.count = 1; + back1.lnk = (lien_back*) item->value.ptr; + back1.ready = NULL; + back_delete(opt, cache, &back1, 0); + } + } + } +} + // --- // routines de backing + +static int back_index_ready(struct_back* sback, char* adr, char* fil, char* sav, int getIndex); +static int back_index_fetch(struct_back* sback, char* adr, char* fil, char* sav, int getIndex); + // retourne l'index d'un lien dans un tableau de backing -int back_index(lien_back* back,int back_max,char* adr,char* fil,char* sav) { +int back_index(struct_back* sback,char* adr,char* fil,char* sav) { + return back_index_fetch(sback, adr, fil, sav, 1); +} + +static int back_index_fetch(struct_back* sback, char* adr, char* fil, char* sav, int getIndex) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int i=0; int index=-1; - while( i<back_max ) { + while( i < back_max ) { if (back[i].status>=0) // réception OU prêt if (strfield2(back[i].url_adr,adr)) { if (strcmp(back[i].url_fil,fil)==0) { if (index==-1) /* first time we meet, store it */ index=i; - else if (strcmp(back[i].url_sav,sav)==0) { /* oops, check sav too */ + else if (sav != NULL && strcmp(back[i].url_sav, sav) == 0) { /* oops, check sav too */ index=i; return index; } @@ -92,11 +158,83 @@ int back_index(lien_back* back,int back_max,char* adr,char* fil,char* sav) { } i++; } + // not found in fast repository - search in the storage hashtable + if (index == -1 && sav != NULL) { + index = back_index_ready(sback, adr, fil, sav, getIndex); + } + return index; +} + +static int back_index_ready(struct_back* sback, char* adr, char* fil, char* sav, int getIndex) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + int index=-1; + void* ptr = NULL; + if (inthash_read_pvoid((inthash)sback->ready, sav, &ptr)) { + lien_back* itemback = (lien_back*) ptr; + if (itemback != NULL) { + if (!getIndex) { + return sback->count; // positive (but invalid) result + } else { + // move from hashtable to fast repository + int q = back_search_quick(sback); + if (q != -1) { + deletehttp(&back[q].r); // security check + back_move(itemback, &back[q]); + inthash_remove((inthash)sback->ready, sav); // delete item + back[q].locked = 1; /* locked */ + index = q; + } + } + } + } return index; } +/* Put all backing entries that are ready in the storage hashtable to spare space and CPU */ +int back_cleanup_background(httrackp* opt,cache_back* cache,struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + int nclean = 0; + int i; + for( i = 0 ; i < back_max ; i++ ) { + // ready, not locked and suitable + if (back[i].status == 0 && back[i].locked == 0 + && back[i].url_sav[0] != '\0' + && strcmp(back[i].url_sav, BACK_ADD_TEST) != 0 + && !IS_DELAYED_EXT(back[i].url_sav) + ) + { + lien_back* itemback = calloct(1, sizeof(lien_back)); + /* Security check */ + int checkIndex = back_index_ready(sback, back[i].url_adr, back[i].url_fil, back[i].url_sav, 1); + if (checkIndex != -1) { + if (opt->log) { + fspc(opt->log,"warning"); + fprintf(opt->log,"engine: unexpected duplicate file entry: %s%s -> %s (%d '%s') / %s%s -> %s (%d '%s')"LF, + back[checkIndex].url_adr, back[checkIndex].url_fil, back[checkIndex].url_sav, back[checkIndex].r.statuscode, back[checkIndex].r.msg, + back[i].url_adr, back[i].url_fil, back[i].url_sav, back[i].r.statuscode, back[i].r.msg + ); + test_flush; + } + back_delete(NULL, NULL, sback, checkIndex); +#ifdef _DEBUG + /* This should NOT happend! */ + { int duplicateEntryInBacklog = 1; assertf(!duplicateEntryInBacklog); } +#endif + } + back_move(&back[i], itemback); + inthash_add_pvoid((inthash)sback->ready, itemback->url_sav, itemback); + nclean++; + } + } + return nclean; +} + // nombre d'entrées libres dans le backing -int back_available(lien_back* back,int back_max) { +int back_available(struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int i; int nb=0; for(i=0;i<back_max;i++) @@ -106,23 +244,61 @@ int back_available(lien_back* back,int back_max) { } // retourne estimation de la taille des html et fichiers stockés en mémoire -LLint back_incache(lien_back* back,int back_max) { +LLint back_incache(struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int i; LLint sum=0; for(i=0;i<back_max;i++) if (back[i].status!=-1) if (back[i].r.adr) // ne comptabilier que les blocs en mémoire sum+=max(back[i].r.size,back[i].r.totalsize); + // stored (ready) slots + if (sback->ready != NULL) { + struct_inthash_enum e = inthash_enum_new((inthash)sback->ready); + inthash_chain* item; + while((item = inthash_enum_next(&e))) { + lien_back* ritem = (lien_back*) item->value.ptr; + if (ritem->status!=-1) + if (ritem->r.adr) // ne comptabilier que les blocs en mémoire + sum+=max(ritem->r.size,ritem->r.totalsize); + } + } return sum; } +// retourne estimation de la taille des html et fichiers stockés en mémoire +int back_done_incache(struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + int i; + int n=0; + for(i=0;i<back_max;i++) + if (back[i].status==0) + n++; + // stored (ready) slots + if (sback->ready != NULL) { + struct_inthash_enum e = inthash_enum_new((inthash)sback->ready); + inthash_chain* item; + while((item = inthash_enum_next(&e))) { + lien_back* ritem = (lien_back*) item->value.ptr; + if (ritem->status==0) + n++; + } + } + return n; +} + + // le lien a-t-il été mis en backing? -HTS_INLINE int back_exist(lien_back* back,int back_max,char* adr,char* fil,char* sav) { - return (back_index(back,back_max,adr,fil,sav)>=0); +HTS_INLINE int back_exist(struct_back* sback,char* adr,char* fil,char* sav) { + return (back_index_fetch(sback, adr, fil, sav, /*don't fetch*/0) >= 0); } // nombre de sockets en tâche de fond -int back_nsoc(lien_back* back,int back_max) { +int back_nsoc(struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int n=0; int i; for(i=0;i<back_max;i++) @@ -131,7 +307,9 @@ int back_nsoc(lien_back* back,int back_max) { return n; } -int back_nsoc_overall(lien_back* back,int back_max) { +int back_nsoc_overall(struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int n=0; int i; for(i=0;i<back_max;i++) @@ -145,250 +323,328 @@ int back_nsoc_overall(lien_back* back,int back_max) { // // fermer les paramètres de transfert, // et notamment vérifier les fichiers compressés (décompresser), callback etc. -int back_finalize(httrackp* opt,cache_back* cache,lien_back* back,int p) { - /* Don't store broken files */ - if (back[p].r.totalsize > 0 && back[p].r.size != back[p].r.totalsize && ! opt->tolerant) { - return -1; - } - - /* Store ? */ - if (!back[p].finalized) { - back[p].finalized = 1; - if ( - (back[p].status == 0) // ready - && - (back[p].r.statuscode>0) // not internal error - ) { - if (!back[p].testmode) { // not test mode - char* state="unknown"; - - /* décompression */ +int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + assertf(p >= 0 && p < back_max); + + /* Store ? */ + if (!back[p].finalized) { + back[p].finalized = 1; + + /* Don't store broken files */ + if (back[p].r.totalsize > 0 && back[p].r.size != back[p].r.totalsize && ! opt->tolerant) { + return -1; + } + + if ( + (back[p].status == 0) // ready + && + (back[p].r.statuscode>0) // not internal error + ) + { + if (!back[p].testmode) { // not test mode + char* state="unknown"; + + /* décompression */ #if HTS_USEZLIB - if (gz_is_available && back[p].r.compressed) { - if (back[p].r.size > 0) { - //if ( (back[p].r.adr) && (back[p].r.size>0) ) { - // stats - back[p].compressed_size=back[p].r.size; - // en mémoire -> passage sur disque - if (!back[p].r.is_write) { - back[p].tmpfile_buffer[0]='\0'; - back[p].tmpfile=tmpnam(back[p].tmpfile_buffer); - if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') { - back[p].r.out=fopen(back[p].tmpfile,"wb"); - if (back[p].r.out) { - if ((back[p].r.adr) && (back[p].r.size>0)) { - if (fwrite(back[p].r.adr,1,(INTsys)back[p].r.size,back[p].r.out) != back[p].r.size) { - back[p].r.statuscode=-1; - strcpybuff(back[p].r.msg,"Write error when decompressing"); - } - } else { - back[p].tmpfile[0]='\0'; - back[p].r.statuscode=-1; - strcpybuff(back[p].r.msg,"Empty compressed file"); - } - } else { - back[p].tmpfile[0]='\0'; - back[p].r.statuscode=-1; - strcpybuff(back[p].r.msg,"Open error when decompressing"); - } - } - } - // fermer fichier sortie - if (back[p].r.out!=NULL) { - fclose(back[p].r.out); - back[p].r.out=NULL; - } - // décompression - if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0' && back[p].url_sav[0]) { - LLint size; - filecreateempty(back[p].url_sav); // filenote & co - if ((size = hts_zunpack(back[p].tmpfile,back[p].url_sav))>=0) { - back[p].r.size=back[p].r.totalsize=size; - // fichier -> mémoire - if (!back[p].r.is_write) { - deleteaddr(&back[p].r); - back[p].r.adr=readfile(back[p].url_sav); - if (!back[p].r.adr) { - back[p].r.statuscode=-1; - strcpybuff(back[p].r.msg,"Read error when decompressing"); - } - remove(back[p].url_sav); - } - } - remove(back[p].tmpfile); - } - // stats - HTS_STAT.total_packed+=back[p].compressed_size; - HTS_STAT.total_unpacked+=back[p].r.size; - HTS_STAT.total_packedfiles++; - // unflag - } - } - back[p].r.compressed=0; -#endif - - /* Stats */ - if (cache->txt) { - char flags[32]; - char s[256]; - time_t tt; - struct tm* A; - tt=time(NULL); - A=localtime(&tt); - if (A == NULL) { - int localtime_returned_null=0; - assert(localtime_returned_null); - } - strftime(s,250,"%H:%M:%S",A); - - flags[0]='\0'; - /* input flags */ - if (back[p].is_update) - strcatbuff(flags, "U"); // update request - else - strcatbuff(flags, "-"); - if (back[p].range_req_size) - strcatbuff(flags, "R"); // range request - else - strcatbuff(flags, "-"); - /* state flags */ - if (back[p].r.is_file) // direct to disk - strcatbuff(flags, "F"); - else - strcatbuff(flags, "-"); - /* output flags */ - if (!back[p].r.notmodified) - strcatbuff(flags, "M"); // modified - else - strcatbuff(flags, "-"); - if (back[p].r.is_chunk) // chunked - strcatbuff(flags, "C"); - else - strcatbuff(flags, "-"); - if (back[p].r.compressed) - strcatbuff(flags, "Z"); // gzip - else - strcatbuff(flags, "-"); - /* Err I had to split these.. */ - fprintf(cache->txt,"%s\t", s); - fprintf(cache->txt,LLintP"/", (LLint)back[p].r.size); - fprintf(cache->txt,LLintP,(LLint)back[p].r.totalsize); - fprintf(cache->txt,"\t%s\t",flags); - } - if (back[p].r.statuscode==200) { - if (back[p].r.size>=0) { - if (strcmp(back[p].url_fil,"/robots.txt") !=0 ) { - HTS_STAT.stat_bytes+=back[p].r.size; - HTS_STAT.stat_files++; - } - if ( (!back[p].r.notmodified) && (opt->is_update) ) { - HTS_STAT.stat_updated_files++; // page modifiée - if (opt->log!=NULL) { - fspc(opt->log,"info"); - if (back[p].is_update) { - fprintf(opt->log,"engine: transfer-status: link updated: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav); - } else { - fprintf(opt->log,"engine: transfer-status: link added: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav); - } - test_flush; - } - if (cache->txt) { - if (back[p].is_update) { - state="updated"; - } else { - state="added"; - } - } - } else { - if ( (opt->debug>0) && (opt->log!=NULL) ) { - fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: link recorded: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav); - test_flush; - } - if (cache->txt) { - if (opt->is_update) - state="untouched"; - else - state="added"; - } - } - } else { - if ( (opt->debug>0) && (opt->log!=NULL) ) { - fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: empty file? (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil); - test_flush; - } - if (cache->txt) { - state="empty"; - } - } - } else { - if ( (opt->debug>0) && (opt->log!=NULL) ) { - fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: link error (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil); - } - if (cache->txt) { - state="error"; - } - } - if (cache->txt) { - fprintf(cache->txt, - "%d\t" - "%s ('%s')\t" - "%s\t" - "%s%s\t" - "%s%s\t%s\t" - "(from %s%s)" - LF, - back[p].r.statuscode, - state, escape_check_url_addr(back[p].r.msg), - escape_check_url_addr(back[p].r.contenttype), - ((back[p].r.etag[0])?"etag:":((back[p].r.lastmodified[0])?"date:":"")), escape_check_url_addr((back[p].r.etag[0])?back[p].r.etag:(back[p].r.lastmodified)), - escape_check_url_addr(back[p].url_adr),escape_check_url_addr(back[p].url_fil),escape_check_url_addr(back[p].url_sav), - escape_check_url_addr(back[p].referer_adr),escape_check_url_addr(back[p].referer_fil) - ); - if (opt->flush) - fflush(cache->txt); - } - - /* Cache */ - cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,back[p].url_sav); - - // status finished callback + if (gz_is_available && back[p].r.compressed) { + if (back[p].r.size > 0) { + //if ( (back[p].r.adr) && (back[p].r.size>0) ) { + // stats + back[p].compressed_size=back[p].r.size; + // en mémoire -> passage sur disque + if (!back[p].r.is_write) { + back[p].tmpfile_buffer[0]='\0'; + back[p].tmpfile=tmpnam(back[p].tmpfile_buffer); + if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') { + back[p].r.out=fopen(back[p].tmpfile,"wb"); + if (back[p].r.out) { + if ((back[p].r.adr) && (back[p].r.size>0)) { + if (fwrite(back[p].r.adr,1,(INTsys)back[p].r.size,back[p].r.out) != back[p].r.size) { + back[p].r.statuscode=STATUSCODE_INVALID; + strcpybuff(back[p].r.msg,"Write error when decompressing"); + } + } else { + back[p].tmpfile[0]='\0'; + back[p].r.statuscode=STATUSCODE_INVALID; + strcpybuff(back[p].r.msg,"Empty compressed file"); + } + } else { + back[p].tmpfile[0]='\0'; + back[p].r.statuscode=STATUSCODE_INVALID; + strcpybuff(back[p].r.msg,"Open error when decompressing"); + } + } + } + // fermer fichier sortie + if (back[p].r.out!=NULL) { + fclose(back[p].r.out); + back[p].r.out=NULL; + } + // décompression + if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') { + if (back[p].url_sav[0]) { + LLint size; + file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 1, 1, back[p].r.notmodified); + filecreateempty(back[p].url_sav); // filenote & co + if ((size = hts_zunpack(back[p].tmpfile,back[p].url_sav))>=0) { + back[p].r.size=back[p].r.totalsize=size; + // fichier -> mémoire + if (!back[p].r.is_write) { + deleteaddr(&back[p].r); + back[p].r.adr=readfile(back[p].url_sav); + if (!back[p].r.adr) { + back[p].r.statuscode=STATUSCODE_INVALID; + strcpybuff(back[p].r.msg,"Read error when decompressing"); + } + unlink(back[p].url_sav); + } + } + } + /* encore that no remaining temporary file exists */ + unlink(back[p].tmpfile); + back[p].tmpfile = NULL; + } + // stats + HTS_STAT.total_packed+=back[p].compressed_size; + HTS_STAT.total_unpacked+=back[p].r.size; + HTS_STAT.total_packedfiles++; + // unflag + } + } + back[p].r.compressed=0; +#endif + + /* Write mode to disk */ + if (back[p].r.is_write && back[p].r.adr != NULL) { + freet(back[p].r.adr); + back[p].r.adr = NULL; + } + + /* ************************************************************************ + REAL MEDIA HACK + Check if we have to load locally the file + ************************************************************************ */ + if (back[p].r.statuscode == 200) { // OK (ou 304 en backing) + if (back[p].r.is_write) { // Written file + if (may_be_hypertext_mime(back[p].r.contenttype, back[p].url_fil)) { // to parse! + LLint sz; + sz=fsize(back[p].url_sav); + if (sz>0) { // ok, exists! + if (sz < 8192) { // ok, small file --> to parse! + FILE* fp=fopen(back[p].url_sav,"rb"); + if (fp) { + back[p].r.adr=malloct((int)sz + 2); + if (back[p].r.adr) { + if (fread(back[p].r.adr,1,(INTsys)sz,fp) == sz) { + back[p].r.size=sz; + back[p].r.adr[sz] = '\0'; + back[p].r.is_write = 0; /* not anymore a direct-to-disk file */ + } else { + freet(back[p].r.adr); + back[p].r.size=0; + back[p].r.adr = NULL; + back[p].r.statuscode=STATUSCODE_INVALID; + strcpybuff(back[p].r.msg, ".RAM read error"); + } + fclose(fp); + fp=NULL; + // remove (temporary) file! + unlink(fconv(back[p].url_sav)); + } + if (fp) + fclose(fp); + } + } + } + } + } + } + /* EN OF REAL MEDIA HACK */ + + + /* Stats */ + if (cache->txt) { + char flags[32]; + char s[256]; + time_t tt; + struct tm* A; + tt=time(NULL); + A=localtime(&tt); + if (A == NULL) { + int localtime_returned_null=0; + assert(localtime_returned_null); + } + strftime(s,250,"%H:%M:%S",A); + + flags[0]='\0'; + /* input flags */ + if (back[p].is_update) + strcatbuff(flags, "U"); // update request + else + strcatbuff(flags, "-"); + if (back[p].range_req_size) + strcatbuff(flags, "R"); // range request + else + strcatbuff(flags, "-"); + /* state flags */ + if (back[p].r.is_file) // direct to disk + strcatbuff(flags, "F"); + else + strcatbuff(flags, "-"); + /* output flags */ + if (!back[p].r.notmodified) + strcatbuff(flags, "M"); // modified + else + strcatbuff(flags, "-"); + if (back[p].r.is_chunk) // chunked + strcatbuff(flags, "C"); + else + strcatbuff(flags, "-"); + if (back[p].r.compressed) + strcatbuff(flags, "Z"); // gzip + else + strcatbuff(flags, "-"); + /* Err I had to split these.. */ + fprintf(cache->txt,"%s\t", s); + fprintf(cache->txt,LLintP"/", (LLint)back[p].r.size); + fprintf(cache->txt,LLintP,(LLint)back[p].r.totalsize); + fprintf(cache->txt,"\t%s\t",flags); + } + if (back[p].r.statuscode == 200) { + if (back[p].r.size>=0) { + if (strcmp(back[p].url_fil,"/robots.txt") !=0 ) { + HTS_STAT.stat_bytes+=back[p].r.size; + HTS_STAT.stat_files++; + } + if ( (!back[p].r.notmodified) && (opt->is_update) ) { + HTS_STAT.stat_updated_files++; // page modifiée + if (opt->log!=NULL) { + fspc(opt->log,"info"); + if (back[p].is_update) { + fprintf(opt->log,"engine: transfer-status: link updated: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav); + } else { + fprintf(opt->log,"engine: transfer-status: link added: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav); + } + test_flush; + } + if (cache->txt) { + if (back[p].is_update) { + state="updated"; + } else { + state="added"; + } + } + } else { + if ( (opt->debug>0) && (opt->log!=NULL) ) { + fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: link recorded: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav); + test_flush; + } + if (cache->txt) { + if (opt->is_update) + state="untouched"; + else + state="added"; + } + } + } else { + if ( (opt->debug>0) && (opt->log!=NULL) ) { + fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: empty file? (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil); + test_flush; + } + if (cache->txt) { + state="empty"; + } + } + } else { + if ( (opt->debug>0) && (opt->log!=NULL) ) { + fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: link error (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil); + } + if (cache->txt) { + state="error"; + } + } + if (cache->txt) { + fprintf(cache->txt, + "%d\t" + "%s ('%s')\t" + "%s\t" + "%s%s\t" + "%s%s%s\t%s\t" + "(from %s%s%s)" + LF, + back[p].r.statuscode, + state, escape_check_url_addr(back[p].r.msg), + escape_check_url_addr(back[p].r.contenttype), + ((back[p].r.etag[0])?"etag:":((back[p].r.lastmodified[0])?"date:":"")), escape_check_url_addr((back[p].r.etag[0])?back[p].r.etag:(back[p].r.lastmodified)), + (link_has_authority(back[p].url_adr) ? "" : "http://"),escape_check_url_addr(back[p].url_adr),escape_check_url_addr(back[p].url_fil),escape_check_url_addr(back[p].url_sav), + (link_has_authority(back[p].referer_adr) || !back[p].referer_adr[0]) ? "" : "http://",escape_check_url_addr(back[p].referer_adr),escape_check_url_addr(back[p].referer_fil) + ); + if (opt->flush) + fflush(cache->txt); + } + + /* Cache */ + if (!IS_DELAYED_EXT(back[p].url_sav)) { + cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,back[p].url_sav); + } else { + if (!HTTP_IS_OK(back[p].r.statuscode)) { + if ( (opt->debug>0) && (opt->log!=NULL) ) { + fspc(opt->log,"debug"); fprintf(opt->log,"redirect to %s%s"LF,back[p].url_adr,back[p].url_fil); + } + /* Store only header reference */ + cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,NULL); + } else { + if (opt->log!=NULL) { + fspc(opt->log,"warning"); fprintf(opt->log,"file not stored in cache due to bogus state (incomplete type): %s%s"LF,back[p].url_adr,back[p].url_fil); + } + } + } + + // status finished callback #if HTS_ANALYSTE - hts_htmlcheck_xfrstatus(&back[p]); -#endif - return 0; - } else { // testmode - if (back[p].r.statuscode / 100 >= 3) { /* Store 3XX, 4XX, 5XX test response codes, but NOT 2XX */ - /* Cache */ - cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,NULL); - } - } - } - } - return -1; + if (hts_htmlcheck_xfrstatus != NULL) { + hts_htmlcheck_xfrstatus(&back[p]); + } +#endif + return 0; + } else { // testmode + if (back[p].r.statuscode / 100 >= 3) { /* Store 3XX, 4XX, 5XX test response codes, but NOT 2XX */ + /* Cache */ + cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,NULL); + } + } + } + } + return -1; } /* try to keep the connection alive */ -int back_letlive(httrackp* opt, cache_back* cache, lien_back* back, int p) { - int checkerror; - htsblk* src = &back[p].r; - if (src && !src->is_file - && src->soc != INVALID_SOCKET - && src->statuscode >= 0 /* no timeout errors & co */ - && src->keep_alive_trailers == 0 /* not yet supported (chunk trailers) */ - && ! ( checkerror = check_sockerror(src->soc) ) - /*&& !check_sockdata(src->soc)*/ /* no unexpected data */ - ) { - htsblk tmp; - memset(&tmp, 0, sizeof(tmp)); - /* clear everything but connection: switch, close, and reswitch */ - back_connxfr(src, &tmp); - back_delete(opt, cache, back, p); - //deletehttp(src); - back_connxfr(&tmp, src); - src->req.flush_garbage=1; /* ignore CRLF garbage */ - return 1; - } - return 0; +int back_letlive(httrackp* opt, cache_back* cache, struct_back* sback, int p) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + int checkerror; + htsblk* src = &back[p].r; + assertf(p >= 0 && p < back_max); + if (src && !src->is_file + && src->soc != INVALID_SOCKET + && src->statuscode >= 0 /* no timeout errors & co */ + && src->keep_alive_trailers == 0 /* not yet supported (chunk trailers) */ + && ! ( checkerror = check_sockerror(src->soc) ) + /*&& !check_sockdata(src->soc)*/ /* no unexpected data */ + ) { + htsblk tmp; + memset(&tmp, 0, sizeof(tmp)); + /* clear everything but connection: switch, close, and reswitch */ + back_connxfr(src, &tmp); + back_delete(opt, cache, sback, p); + //deletehttp(src); + back_connxfr(&tmp, src); + src->req.flush_garbage=1; /* ignore CRLF garbage */ + return 1; + } + return 0; } void back_connxfr(htsblk* src, htsblk* dst) { @@ -410,9 +666,34 @@ void back_connxfr(htsblk* src, htsblk* dst) { src->debugid = 0; } +void back_move(lien_back* src, lien_back* dst) { + memcpy(dst, src, sizeof(lien_back)); + memset(src, 0, sizeof(lien_back)); + src->r.soc=INVALID_SOCKET; + src->status=-1; + src->r.location = src->location_buffer; + dst->r.location = dst->location_buffer; +} + +void back_copy_static(const lien_back* src, lien_back* dst) { + memcpy(dst, src, sizeof(lien_back)); + dst->r.soc=INVALID_SOCKET; + dst->r.adr = NULL; + dst->r.headers = NULL; + dst->r.out = NULL; + dst->r.location = dst->location_buffer; + dst->r.fp = NULL; +#if HTS_USEOPENSSL + dst->r.ssl_con = NULL; +#endif +} + // clear, or leave for keep-alive -int back_maydelete(httrackp* opt,cache_back* cache,lien_back* back, int p) { - if (p>=0) { // on sait jamais.. +int back_maydelete(httrackp* opt,cache_back* cache,struct_back* sback, int p) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + assertf(p >= 0 && p < back_max); + if (p >= 0 && p < back_max) { // on sait jamais.. if ( /* Keep-alive authorized by user */ !opt->nokeepalive @@ -429,7 +710,7 @@ int back_maydelete(httrackp* opt,cache_back* cache,lien_back* back, int p) { ) { lien_back tmp; strcpybuff(tmp.url_adr, back[p].url_adr); - if (back_letlive(opt, cache, back, p)) { + if (back_letlive(opt, cache, sback, p)) { strcpybuff(back[p].url_adr, tmp.url_adr); back[p].status = -103; // alive & waiting if ((opt->debug>1) && (opt->log!=NULL)) { @@ -440,14 +721,17 @@ int back_maydelete(httrackp* opt,cache_back* cache,lien_back* back, int p) { return 1; } } - back_delete(opt,cache,back, p); + back_delete(opt,cache,sback, p); } return 0; } // clear, or leave for keep-alive -void back_maydeletehttp(httrackp* opt, cache_back* cache, lien_back* back, int back_max, int p) { +void back_maydeletehttp(httrackp* opt, cache_back* cache, struct_back* sback, int p) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; TStamp lt = 0; + assertf(p >= 0 && p < back_max); if (back[p].r.soc!=INVALID_SOCKET) { int q; if ( @@ -470,7 +754,7 @@ void back_maydeletehttp(httrackp* opt, cache_back* cache, lien_back* back, int b /* Connection delay must not exceed keep-alive timeout */ && ( opt->maxconn <= 0 || ( back[p].r.keep_alive_t > ( 1.0 / opt->maxconn ) ) ) /* Available slot in backing */ - && ( q = back_search(opt, cache, back, back_max) ) >= 0 + && ( q = back_search(opt, cache, sback) ) >= 0 ) { lien_back tmp; @@ -495,13 +779,16 @@ void back_maydeletehttp(httrackp* opt, cache_back* cache, lien_back* back, int b /* attempt to attach a live connection to this slot */ -int back_trylive(httrackp* opt,cache_back* cache,lien_back* back, int back_max, int p) { +int back_trylive(httrackp* opt,cache_back* cache,struct_back* sback, int p) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + assertf(p >= 0 && p < back_max); if (p>=0 && back[p].status != -103) { // we never know.. - int i = back_searchlive(opt,back, back_max, back[p].url_adr); // search slot + int i = back_searchlive(opt,sback, back[p].url_adr); // search slot if (i >= 0 && i != p) { deletehttp(&back[p].r); // security check back_connxfr(&back[i].r, &back[p].r); // transfer live connection settings from i to p - back_delete(opt,cache,back, i); // delete old slot + back_delete(opt,cache,sback, i); // delete old slot back[p].status=100; // ready to connect return 1; // success: will reuse live connection } @@ -510,7 +797,9 @@ int back_trylive(httrackp* opt,cache_back* cache,lien_back* back, int back_max, } /* search for a live position, or, if not possible, try to return a new one */ -int back_searchlive(httrackp* opt, lien_back* back, int back_max, char* search_addr) { +int back_searchlive(httrackp* opt, struct_back* sback, char* search_addr) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int i; /* search for a live socket */ @@ -526,7 +815,9 @@ int back_searchlive(httrackp* opt, lien_back* back, int back_max, char* search_a return -1; } -int back_search(httrackp* opt,cache_back* cache,lien_back* back, int back_max) { +int back_search_quick(struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int i; /* try to find an empty place */ @@ -536,11 +827,24 @@ int back_search(httrackp* opt,cache_back* cache,lien_back* back, int back_max) { } } + /* oops, can't find a place */ + return -1; +} + +int back_search(httrackp* opt,cache_back* cache,struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + int i; + + /* try to find an empty place */ + if ( ( i = back_search_quick(sback) ) != -1) + return i; + /* couldn't find an empty place, try to requisition a keep-alive place */ for(i = 0 ; i < back_max ; i++ ) { if (back[i].status == -103) { /* close this place */ - back_delete(opt,cache,back, i); + back_delete(opt,cache,sback, i); return i; } } @@ -549,17 +853,84 @@ int back_search(httrackp* opt,cache_back* cache,lien_back* back, int back_max) { return -1; } +void back_set_finished(struct_back* sback, int p) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + assertf(p >= 0 && p < back_max); + if (p >= 0 && p < sback->count) { // we never know.. + /* status: finished (waiting to be validated) */ + back[p].status=0; /* finished */ + /* close open r/w streams, if any */ + if (back[p].r.fp!=NULL) { + fclose(back[p].r.fp); + back[p].r.fp=NULL; + } + if (back[p].r.out!=NULL) { // fermer fichier sortie + fclose(back[p].r.out); + back[p].r.out=NULL; + } + } +} + +int back_flush_output(httrackp* opt, cache_back* cache, struct_back* sback, int p) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + assertf(p >= 0 && p < back_max); + if (p >= 0 && p < sback->count) { // on sait jamais.. + /* close input file */ + if (back[p].r.fp!=NULL) { + fclose(back[p].r.fp); + back[p].r.fp=NULL; + } + /* fichier de sortie */ + if (back[p].r.out!=NULL) { // fermer fichier sortie + fclose(back[p].r.out); + back[p].r.out=NULL; + } + /* set file time */ + if (back[p].r.is_write) { // ecriture directe + /* écrire date "remote" */ + if (strnotempty(back[p].url_sav) + && strnotempty(back[p].r.lastmodified) + && fexist(back[p].url_sav)) // normalement existe si on a un fichier de sortie + { + set_filetime_rfc822(back[p].url_sav,back[p].r.lastmodified); + } + /* executer commande utilisateur après chargement du fichier */ + //xx usercommand(opt,0,NULL,back[p].url_sav, back[p].url_adr, back[p].url_fil); + back[p].r.is_write=0; + } + return 1; + } + return 0; +} + // effacer entrée -int back_delete(httrackp* opt, cache_back* cache, lien_back* back, int p) { - if (p>=0) { // on sait jamais.. +int back_set_passe2_ptr(httrackp* opt, cache_back* cache, struct_back* sback, int p, int* pass2_ptr) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + assertf(p >= 0 && p < back_max); + if (p >= 0 && p < sback->count) { // on sait jamais.. + back[p].pass2_ptr = pass2_ptr; + return 1; + } + return 0; +} + +// effacer entrée +int back_delete(httrackp* opt, cache_back* cache, struct_back* sback, int p) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + assertf(p >= 0 && p < back_max); + if (p >= 0 && p < sback->count) { // on sait jamais.. // Vérificateur d'intégrité - #if DEBUG_CHECKINT +#if DEBUG_CHECKINT _CHECKINT(&back[p],"Appel back_delete") - #endif +#endif #if HTS_DEBUG_CLOSESOCK - DEBUG_W("back_delete: #%d\n" _ (int) p); + DEBUG_W("back_delete: #%d\n" _ (int) p); #endif - + // Finalize if (!back[p].finalized) { if ( @@ -569,13 +940,18 @@ int back_delete(httrackp* opt, cache_back* cache, lien_back* back, int p) { && (back[p].r.statuscode>0) // not internal error ) { - if ((opt->debug>1) && (opt->log!=NULL)) { - fspc(opt->log,"debug"); fprintf(opt->log,"File '%s%s' -> %s not yet saved in cache - saving now"LF, back[p].url_adr, back[p].url_fil, back[p].url_sav); test_flush; + if (opt != NULL && opt->debug>1 && opt->log!=NULL) { + fspc(opt->log,"debug"); fprintf(opt->log,"File '%s%s' -> %s not yet saved in cache - saving now"LF, back[p].url_adr, back[p].url_fil, back[p].url_sav); test_flush; + } + } + if (cache != NULL) { + back_finalize(opt, cache, sback, p); } - } - back_finalize(opt, cache, back, p); } back[p].finalized = 0; + + // flush output buffers + (void) back_flush_output(opt, cache, sback, p); // Libérer tous les sockets, handles, buffers.. if (back[p].r.soc!=INVALID_SOCKET) { @@ -597,12 +973,12 @@ int back_delete(httrackp* opt, cache_back* cache, lien_back* back, int p) { back[p].chunk_blocksize=0; back[p].is_chunk=0; } - // if (back[p].r.is_file) { // fermer fichier entrée - if (back[p].r.fp!=NULL) { - fclose(back[p].r.fp); - back[p].r.fp=NULL; - } - // } + + // only for security + if (back[p].tmpfile && back[p].tmpfile[0] != '\0') { + (void) unlink(back[p].tmpfile); + back[p].tmpfile = NULL; + } // headers if (back[p].r.headers != NULL) { @@ -610,36 +986,21 @@ int back_delete(httrackp* opt, cache_back* cache, lien_back* back, int p) { back[p].r.headers = NULL; } - /* fichier de sortie */ - if (back[p].r.out!=NULL) { // fermer fichier sortie - fclose(back[p].r.out); - back[p].r.out=NULL; - } - - if (back[p].r.is_write) { // ecriture directe - /* écrire date "remote" */ - if (strnotempty(back[p].url_sav)) // normalement existe si on a un fichier de sortie - if (strnotempty(back[p].r.lastmodified)) // last-modified existe - if (fexist(back[p].url_sav)) // ainsi que le fichier - set_filetime_rfc822(back[p].url_sav,back[p].r.lastmodified); - - /* executer commande utilisateur après chargement du fichier */ - //xx usercommand(opt,0,NULL,back[p].url_sav, back[p].url_adr, back[p].url_fil); - back[p].r.is_write=0; - } - // Tout nettoyer memset(&back[p], 0, sizeof(lien_back)); back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer; // Le plus important: libérer le champ back[p].status=-1; + return 1; } return 0; } /* Space left on backing stack */ -int back_stack_available(lien_back* back,int back_max) { +int back_stack_available(struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int p=0,n=0; for( ; p < back_max ; p++ ) if ( back[p].status == -1 ) @@ -648,7 +1009,19 @@ int back_stack_available(lien_back* back,int back_max) { } // ajouter un lien en backing -int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test,int* pass2_ptr) { +int back_add_if_not_exists(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test,int* pass2_ptr) { + int index = back_index(sback, adr, fil, save); + if (index < 0) { + return back_add(sback, opt, cache, adr, fil, save, referer_adr, referer_fil, test, pass2_ptr); + } else { + /* Ensure that the reference to pass2_ptr is set */ + return back_set_passe2_ptr(opt,cache,sback,index,pass2_ptr); + } +} + +int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test,int* pass2_ptr) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int p=0; // vérifier cohérence de adr et fil (non vide!) @@ -670,8 +1043,8 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* opt->state.back_add_stats++; // rechercher emplacement - back_clean(opt, cache, back, back_max); - if ( ( p = back_search(opt, cache, back, back_max) ) >= 0) { + back_clean(opt, cache, sback); + if ( ( p = back_search(opt, cache, sback) ) >= 0) { back[p].send_too[0]='\0'; // éventuels paramètres supplémentaires à transmettre au serveur // clear r @@ -719,9 +1092,10 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* /* Stop requested - abort backing */ if (opt->state.stop) { - back[p].r.statuscode=-1; // fatal + back[p].r.statuscode=STATUSCODE_INVALID; // fatal strcpybuff(back[p].r.msg,"mirror stopped by user"); back[p].status=0; // terminé + back_set_finished(sback, p); if ((opt->debug>0) && (opt->log!=NULL)) { fspc(opt->log,"warning"); fprintf(opt->log,"File not added due to mirror cancel: %s%s"LF,adr,fil); test_flush; } @@ -751,8 +1125,6 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* if ((strcmp(adr,"file://")) /* pas fichier */ && ( (!test) || (cache->type==1) ) /* cache prioritaire, laisser passer en test! */ && ( (strnotempty(save)) || (strcmp(fil,"/robots.txt")==0) ) ) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302.. - //if ((!test) && (strcmp(adr,"file://")) - //if ((!test) && (strncmp(adr,"ftp://",6)) && (strcmp(adr,"file://")) #if HTS_FAST_CACHE long int hash_pos; int hash_pos_return=0; @@ -788,7 +1160,8 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* sscanf(a,"%d",&pos); // lire position #endif if (pos<0) { // pas de mise en cache data, vérifier existence - if (fsize(fconv(save)) <= 0) { // fichier existe pas ou est vide! + /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */ + if (!IS_DELAYED_EXT(save) && fsize(fconv(save)) <= 0) { // fichier existe pas ou est vide! int found=0; /* It is possible that the file has been moved due to changes in build structure */ @@ -830,7 +1203,7 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* } } } - } + } // fsize() <= 0 } } } @@ -863,7 +1236,8 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* if (back[p].r.statuscode != -1) { // pas d'erreur de lecture if (!back_checksize(opt,&back[p],0)) { back[p].status=0; // FINI - back[p].r.statuscode=-1; + back_set_finished(sback, p); + back[p].r.statuscode=STATUSCODE_TOO_BIG; if (!back[p].testmode) strcpybuff(back[p].r.msg,"Cached file skipped (too big)"); else @@ -872,7 +1246,7 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* } } - if (back[p].r.statuscode != -1) { // pas d'erreur de lecture + if (back[p].r.statuscode != -1 || IS_DELAYED_EXT(save)) { // pas d'erreur de lecture ou test retardé if ((opt->debug>0) && (opt->log!=NULL)) { if (!test) { fspc(opt->log,"debug"); fprintf(opt->log,"File immediately loaded from cache: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush; @@ -882,11 +1256,13 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* } back[p].r.notmodified=1; // fichier non modifié back[p].status=0; // OK prêt + //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified); // not modified + back_set_finished(sback, p); // finalize transfer if (!test) { if (back[p].r.statuscode>0) { - back_finalize(opt,cache,back,p); + back_finalize(opt,cache,sback,p); } } @@ -906,9 +1282,11 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* LLint save_totalsize=back[p].r.totalsize; back[p].r.totalsize=r.totalsize; if (!back_checksize(opt,&back[p],1)) { - r.statuscode = -1; + r.statuscode = STATUSCODE_INVALID; // back[p].status=0; // FINI + back_set_finished(sback, p); + back[p].r.statuscode=STATUSCODE_TOO_BIG; deletehttp(&back[p].r); back[p].r.soc=INVALID_SOCKET; if (!back[p].testmode) strcpybuff(back[p].r.msg,"File too big"); @@ -1010,8 +1388,10 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* } } else if (opt->norecatch) { // tester norecatch filenote(save,NULL); // ne pas purger tout de même + file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified); back[p].status=0; // OK prêt - back[p].r.statuscode=-1; // erreur + back_set_finished(sback, p); + back[p].r.statuscode=STATUSCODE_INVALID; // erreur strcpybuff(back[p].r.msg,"Null-size file not recaught"); return 0; } @@ -1072,6 +1452,25 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* return 0; } } +#if HTS_USEMMS + else if (strfield(back[p].url_adr,"mms://")) { + MMSDownloadStruct str; + if (back[p].testmode) { + if ((opt->debug>1) && (opt->errlog!=NULL)) { + fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: forbidden test with mms link for back_add"LF); + } + return -1; // erreur pas de test permis + } + if (back[p].r.req.proxy.active) { + fspc(opt->errlog,"warning"); fprintf(opt->errlog,"warning: direct connection for mms links (proxy settings ignored)"LF); + } + back[p].status=1000; // connexion externe + str.pBack = &back[p]; + str.pOpt = opt; + launch_mms(&str); + return 0; + } +#endif #if HTS_USEOPENSSL else if (SSL_is_available && strfield(back[p].url_adr,"https://")) { // let's rock back[p].r.ssl = 1; @@ -1080,7 +1479,7 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* } #endif - if (!back_trylive(opt, cache, back, back_max, p)) { + if (!back_trylive(opt, cache, sback, p)) { #if HTS_XGETHOST #if HDEBUG printf("back_solve..\n"); @@ -1095,6 +1494,7 @@ int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* soc=http_xfopen(0,0,0,back[p].send_too,adr,fil,&(back[p].r)); if (soc==INVALID_SOCKET) { back[p].status=0; // fini, erreur + back_set_finished(sback, p); } } // @@ -1161,6 +1561,7 @@ printf("Xfopen ok, poll..\n"); #else if (soc==INVALID_SOCKET) { // erreur socket back[p].status=0; // FINI + back_set_finished(sback, p); //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc); back[p].r.soc=INVALID_SOCKET; } else { @@ -1275,7 +1676,12 @@ PTHREAD_TYPE PTHREAD_TYPE_FNC Hostlookup(void* iadr_p) { // si c'est un fichier, la résolution est immédiate // idem pour ftp:// void back_solve(lien_back* back) { - if ((!strfield(back->url_adr,"file://")) && (!strfield(back->url_adr,"ftp://"))) { + if ((!strfield(back->url_adr,"file://")) + && ! strfield(back->url_adr,"ftp://") +#if HTS_USEMMS + && ! strfield(back->url_adr,"mms://") +#endif + ) { //## if (back->url_adr[0]!=lOCAL_CHAR) { // qq chose à préparer char* a; if (!(back->r.req.proxy.active)) @@ -1318,7 +1724,12 @@ void back_solve(lien_back* back) { // détermine si le host a pu être résolu int host_wait(lien_back* back) { - if ((!strfield(back->url_adr,"file://")) && (!strfield(back->url_adr,"ftp://"))) { + if ((!strfield(back->url_adr,"file://")) + && (!strfield(back->url_adr,"ftp://")) +#if HTS_USEMMS + && (!strfield(back->url_adr,"mms://")) +#endif + ) { //## if (back->url_adr[0]!=lOCAL_CHAR) { if (!(back->r.req.proxy.active)) { return (hts_dnstest(back->url_adr)); @@ -1334,7 +1745,9 @@ int host_wait(lien_back* back) { // cleanup non-html files in backing to save backing space // and allow faster "save in cache" operation // also cleanup keep-alive sockets and ensure that not too many sockets are being opened -void back_clean(httrackp* opt,cache_back* cache,lien_back* back,int back_max) { +void back_clean(httrackp* opt,cache_back* cache,struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; #if HTS_ANALYSTE int oneMore = ( (_hts_in_html_parsing == 2 && opt->maxsoc >= 2) || (_hts_in_html_parsing == 1 && opt->maxsoc >= 4) ) ? 1 : 0; // testing links #endif @@ -1344,28 +1757,22 @@ void back_clean(httrackp* opt,cache_back* cache,lien_back* back,int back_max) { /* Check autoclean */ if (!back[i].testmode) { // not test mode if (strnotempty(back[i].url_sav)) { // filename exists - if (back[i].r.statuscode==200) { // HTTP "OK" + if (back[i].r.statuscode==200) { // HTTP "OK" if (back[i].r.size>0) { // size>0 - if (back[i].r.is_write // not in memory (on disk, ready) + if (back[i].r.is_write // not in memory (on disk, ready) && !is_hypertext_mime(back[i].r.contenttype, back[i].url_fil) // not HTML/hypertext && !may_be_hypertext_mime(back[i].r.contenttype, back[i].url_fil) // may NOT be parseable mime type - ) { + ) + { if (back[i].pass2_ptr) { - // finalize - // // back_finalize(opt,cache,back,i); - // stats - //HTS_STAT.stat_bytes+=back[i].r.size; - //HTS_STAT.stat_files++; - //if ( (!back[i].r.notmodified) && (opt->is_update) ) { - // HTS_STAT.stat_updated_files++; // page modifiée - //} - //xxxcache_mayadd(opt,cache,&back[i].r,back[i].url_adr,back[i].url_fil,back[i].url_sav); + (void) back_flush_output(opt, cache, sback, i); // flush output buffers usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr, back[i].url_fil); *back[i].pass2_ptr=-1; // Done! + HTS_STAT.stat_background++; if ((opt->debug>0) && (opt->log!=NULL)) { fspc(opt->log,"info"); fprintf(opt->log,"File successfully written in background: %s"LF,back[i].url_sav); test_flush; } - back_maydelete(opt,cache,back,i); // May delete backing entry + back_maydelete(opt,cache,sback,i); // May delete backing entry } } else { if (!back[i].finalized) { @@ -1375,19 +1782,19 @@ void back_clean(httrackp* opt,cache_back* cache,lien_back* back,int back_max) { if ( (opt->debug>1) && (opt->log!=NULL) ) { fspc(opt->log,"debug"); fprintf(opt->log,"file %s%s validated (cached, left in memory)"LF,back[i].url_adr,back[i].url_fil); test_flush; } - back_maydeletehttp(opt, cache, back, back_max, i); + back_maydeletehttp(opt, cache, sback, i); } else { - /* - NOT YET HANDLED CORRECTLY (READ IN NEW CACHE TO DO) + /* + NOT YET HANDLED CORRECTLY (READ IN NEW CACHE TO DO) */ /* Lock the entry but do not keep the html data in memory (in cache) */ if (opt->cache) { htsblk r; - + /* Ensure deleted or recycled socket */ - back_maydeletehttp(opt, cache, back, back_max, i); + back_maydeletehttp(opt, cache, sback, i); assertf(back[i].r.soc == INVALID_SOCKET); - + /* Check header */ cache_header(opt,cache,back[i].url_adr,back[i].url_fil,&r); if (r.statuscode == 200) { @@ -1426,7 +1833,7 @@ void back_clean(httrackp* opt,cache_back* cache,lien_back* back,int back_max) { back[i].url_adr); test_flush; } - back_delete(opt,cache,back, i); // delete backing entry + back_delete(opt,cache,sback, i); // delete backing entry } } } @@ -1434,15 +1841,14 @@ void back_clean(httrackp* opt,cache_back* cache,lien_back* back,int back_max) { for(i=0;i<back_max;i++) { if (back[i].status == 0) { // ready if (back[i].r.soc != INVALID_SOCKET) { - back_maydeletehttp(opt,cache,back, back_max, i); + back_maydeletehttp(opt,cache,sback, i); } - } } /* delete sockets if too many keep-alive'd sockets in background */ if (opt->maxsoc > 0) { int max = opt->maxsoc + oneMore; - int curr = back_nsoc_overall(back, back_max); + int curr = back_nsoc_overall(sback); if (curr > max) { if ((opt->debug>1) && (opt->log!=NULL)) { fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): deleting #%d sockets"LF, @@ -1451,16 +1857,26 @@ void back_clean(httrackp* opt,cache_back* cache,lien_back* back,int back_max) { } for(i = 0 ; i < back_max && curr > max ; i++) { if (back[i].status == -103) { - back_delete(opt,cache,back, i); // delete backing entry + back_delete(opt,cache,sback, i); // delete backing entry curr--; } } } + /* transfer ready slots to the storage hashtable */ + { + int nxfr = back_cleanup_background(opt,cache,sback); + if (nxfr > 0 && (opt->debug>0) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"(htsback): %d slots ready moved to background"LF, nxfr); + test_flush; + } + } } // attente (gestion des buffers des sockets) -void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TStamp stat_timestart) { +void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_timestart) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; unsigned int i_mod; T_SOC nfds=INVALID_SOCKET; fd_set fds,fds_c,fds_e; // fds pour lecture, connect (write), et erreur @@ -1482,7 +1898,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta #if 1 // Cleanup the stack to save space! - back_clean(opt,cache,back,back_max); + back_clean(opt,cache,sback); #endif // recevoir tant qu'il y a des données (avec un maximum de max_loop boucles) @@ -1567,12 +1983,13 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta nfds=back[i].r.soc; } } else { - back[i].r.statuscode=-4; + back[i].r.statuscode=STATUSCODE_CONNERROR; if (back[i].status==100) strcpybuff(back[i].r.msg,"Connect Error"); else strcpybuff(back[i].r.msg,"Receive Error"); back[i].status=0; // terminé + back_set_finished(sback, i); if ((opt->debug>0) && (opt->log!=NULL)) { fspc(opt->log,"warning"); fprintf(opt->log,"Unexpected socket error during pre-loop"LF); test_flush; } @@ -1640,15 +2057,16 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta deletehttp(&back[i].r); } back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=-4; + back[i].r.statuscode=STATUSCODE_CONNERROR; if (back[i].status==100) strcpybuff(back[i].r.msg,"Connect Error"); else strcpybuff(back[i].r.msg,"Receive Error"); if (back[i].status == -103) { /* Keep-alive socket */ - back_delete(opt,cache,back, i); + back_delete(opt,cache,sback, i); } else { back[i].status=0; // terminé + back_set_finished(sback, i); } } } @@ -1682,17 +2100,18 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta SSL_set_connect_state(back[i].r.ssl_con); back[i].status = 102; /* handshake wait */ } else - back[i].r.statuscode=-6; + back[i].r.statuscode=STATUSCODE_SSL_HANDSHAKE; } else - back[i].r.statuscode=-6; + back[i].r.statuscode=STATUSCODE_SSL_HANDSHAKE; } /* Error */ - if (back[i].r.statuscode == -6) { + if (back[i].r.statuscode == STATUSCODE_SSL_HANDSHAKE) { strcpybuff(back[i].r.msg, "bad SSL/TLS handshake"); deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=-5; + back[i].r.statuscode=STATUSCODE_NON_FATAL; back[i].status=0; + back_set_finished(sback, i); } } @@ -1753,9 +2172,10 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta } deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=-5; + back[i].r.statuscode=STATUSCODE_NON_FATAL; back[i].status=0; - } + back_set_finished(sback, i); + } } else { /* got it! */ back[i].status=100; // back to waitconnect } @@ -1763,8 +2183,9 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta strcpybuff(back[i].r.msg, "unexpected SSL/TLS error"); deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=-5; + back[i].r.statuscode=STATUSCODE_NON_FATAL; back[i].status=0; + back_set_finished(sback, i); } } @@ -1791,6 +2212,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta back[i].r.soc=http_xfopen(0,0,0,back[i].send_too,back[i].url_adr,back[i].url_fil,&(back[i].r)); if (back[i].r.soc==INVALID_SOCKET) { back[i].status=0; // fini, erreur + back_set_finished(sback, i); if (back[i].r.soc!=INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(2): deletehttp\n"); @@ -1798,7 +2220,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta deletehttp(&back[i].r); } back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=-5; + back[i].r.statuscode=STATUSCODE_NON_FATAL; if (strnotempty(back[i].r.msg)==0) strcpybuff(back[i].r.msg,"Unable to resolve host name"); } @@ -1825,25 +2247,27 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta } back[i].r.msg[j++]='\0'; fclose(fp); - remove(fconcat(back[i].location_buffer,".ok")); + unlink(fconcat(back[i].location_buffer,".ok")); strcpybuff(fconcat(back[i].location_buffer,".ok"),""); } else { strcpybuff(back[i].r.msg,"Unknown ftp result, check if file is ok"); - back[i].r.statuscode=-1; + back[i].r.statuscode=STATUSCODE_INVALID; } back[i].status=0; + back_set_finished(sback, i); // finalize transfer if (back[i].r.statuscode>0) { - back_finalize(opt,cache,back,i); + back_finalize(opt,cache,sback,i); } } } #endif else if (back[i].status==1001) { // ftp ready back[i].status=0; + back_set_finished(sback, i); // finalize transfer if (back[i].r.statuscode>0) { - back_finalize(opt,cache,back,i); + back_finalize(opt,cache,sback,i); } } else if ((back[i].status>0) && (back[i].status<1000)) { // en réception http @@ -1888,8 +2312,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta if (strnotempty(back[i].url_sav)) { if (strcmp(back[i].url_fil,"/robots.txt")) { if (back[i].r.statuscode==200) { // 'OK' - if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_fil) - ) { // pas HTML + if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_fil)) { // pas HTML if (opt->getmode&2) { // on peut ecrire des non html int fcheck=0; back[i].r.is_write=1; // écrire @@ -1903,11 +2326,13 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta if (back[i].tmpfile != NULL && back[i].tmpfile[0]) back[i].r.out=fopen(back[i].tmpfile,"wb"); } else { + file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 1, 1, back[i].r.notmodified); back[i].r.compressed=0; back[i].r.out=filecreate(back[i].url_sav); } if (back[i].r.out==NULL) { if ((fcheck=check_fatal_io_errno())) { + fspc(opt->log,"error"); fprintf(opt->log,"Mirror aborted: disk full or filesystem problems"LF); test_flush; opt->state.exit_xh=-1; /* fatal error */ } } @@ -1941,10 +2366,11 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta fspc(opt->log,"debug"); fprintf(opt->log,"File cancelled (non HTML): %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush; } back[i].status=0; // terminé + back_set_finished(sback, i); if (!back[i].testmode) - back[i].r.statuscode=-10; // EUHH CANCEL + back[i].r.statuscode=STATUSCODE_INVALID; // EUHH CANCEL else - back[i].r.statuscode=-10; // "TEST OK" + back[i].r.statuscode=STATUSCODE_TEST_OK; // "TEST OK" if (back[i].r.soc!=INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(3): deletehttp\n"); @@ -2021,6 +2447,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta if (retour_fread < 0) { // fin réception back[i].status=0; // terminé + back_set_finished(sback, i); /*KA back[i].r.soc=INVALID_SOCKET; */ #if CHUNKDEBUG==1 if (back[i].is_chunk) @@ -2031,7 +2458,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta strcatbuff(back[i].r.msg, "Interrupted transfer"); else strcatbuff(back[i].r.msg, "No data (connection closed)"); - back[i].r.statuscode=-4; + back[i].r.statuscode=STATUSCODE_CONNERROR; } else if ((back[i].r.statuscode <= 0) && (strnotempty(back[i].r.msg)==0)) { #if HDEBUG printf("error interruped: %s\n",back[i].r.adr); @@ -2040,7 +2467,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta strcatbuff(back[i].r.msg,"Interrupted transfer"); else strcatbuff(back[i].r.msg,"No data (connection closed)"); - back[i].r.statuscode=-4; + back[i].r.statuscode=STATUSCODE_CONNERROR; } // Close socket @@ -2049,12 +2476,14 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta DEBUG_W("back_wait(4): deletehttp\n"); #endif /*KA deletehttp(&back[i].r);*/ - back_maydeletehttp(opt, cache, back, back_max, i); + back_maydeletehttp(opt, cache, sback, i); } // finalize transfer - if (back[i].r.statuscode>0) { - back_finalize(opt,cache,back,i); + if (back[i].r.statuscode>0 + && !IS_DELAYED_EXT(back[i].url_sav) + ) { + back_finalize(opt,cache,sback,i); } if (back[i].r.totalsize>0) { // tester totalsize @@ -2064,7 +2493,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta //#if HTS_CL_IS_FATAL deleteaddr(&back[i].r); if (back[i].r.size<back[i].r.totalsize) - back[i].r.statuscode=-4; // recatch + back[i].r.statuscode=STATUSCODE_CONNERROR; // recatch sprintf(back[i].r.msg,"Incorrect length ("LLintP" Bytes, "LLintP" expected)",(LLint)back[i].r.size,(LLint)back[i].r.totalsize); } else { //#else @@ -2109,15 +2538,17 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta if (sscanf(chunk_data,"%x",&chunk_size) == 1) { if (chunk_size > 0) back[i].chunk_blocksize = chunk_size; /* the data block chunk size */ - else - back[i].chunk_blocksize = -1; /* ending */ - back[i].r.totalsize+=chunk_size; // noter taille - back[i].r.adr=(char*) realloct(back[i].r.adr,(INTsys) back[i].r.totalsize + 1); - if (!back[i].r.adr) { - if (cache->errlog!=NULL) { - fprintf(cache->errlog,"Error: Not enough memory ("LLintP") for %s%s"LF,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil); - } - } + else + back[i].chunk_blocksize = -1; /* ending */ + back[i].r.totalsize+=chunk_size; // noter taille + if (back[i].r.adr != NULL || !back[i].r.is_write) { // Not to disk + back[i].r.adr=(char*) realloct(back[i].r.adr,(INTsys) back[i].r.totalsize + 1); + if (!back[i].r.adr) { + if (cache->errlog!=NULL) { + fprintf(cache->errlog,"Error: Not enough memory ("LLintP") for %s%s"LF,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil); + } + } + } #if CHUNKDEBUG==1 printf("[%d] chunk length: %d - next total "LLintP":\n",(int)back[i].r.soc,(int)chunk_size,(LLint)back[i].r.totalsize); #endif @@ -2177,10 +2608,19 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta /* End */ //if (back[i].status==97) { back[i].status=0; // fin + back_set_finished(sback, i); //} - // finalize transfer - back_finalize(opt,cache,back,i); + // finalize transfer if not temporary + if (!IS_DELAYED_EXT(back[i].url_sav)) { + back_finalize(opt,cache,sback,i); + } else { + if (back[i].r.statuscode == 200) { + if (cache->errlog!=NULL) { + fspc(cache->errlog,"warning"); fprintf(cache->errlog,"Unexpected incomplete type with 200 code at %s%s"LF, back[i].url_adr, back[i].url_fil); + } + } + } if (back[i].r.soc!=INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(5): deletehttp\n"); @@ -2189,7 +2629,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta if (chunk_size < 0) { deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; deleteaddr(&back[i].r); - back[i].r.statuscode=-1; + back[i].r.statuscode=STATUSCODE_INVALID; strcpybuff(back[i].r.msg,"Invalid chunk"); #if CHUNKDEBUG==1 printf("[%d] chunk error\n", (int)back[i].r.soc); @@ -2204,7 +2644,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta if (back[i].r.totalsize!=back[i].r.size) { // pas la même! if (!opt->tolerant) { deleteaddr(&back[i].r); - back[i].r.statuscode=-1; + back[i].r.statuscode=STATUSCODE_INVALID; strcpybuff(back[i].r.msg,"Incorrect length"); } else { // Un warning suffira.. @@ -2323,9 +2763,10 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta fspc(opt->log,"warning"); fprintf(opt->log,"External wrapper aborted transfer, breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush; } back[i].status=0; // FINI + back_set_finished(sback, i); deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; strcpybuff(back[i].r.msg,"External wrapper aborted transfer"); - back[i].r.statuscode = -1; + back[i].r.statuscode = STATUSCODE_INVALID; } } #endif @@ -2352,7 +2793,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta back[i].r.size=0; back[i].r.totalsize=0; back[i].chunk_size=0; - back[i].r.statuscode=-1; + back[i].r.statuscode=STATUSCODE_INVALID; back[i].r.msg[0]='\0'; if ((opt->debug>1) && (opt->log!=NULL)) { fspc(opt->log,"debug"); fprintf(opt->log,"Status 100 detected for %s%s, continuing headers"LF,back[i].url_adr,back[i].url_fil); test_flush; @@ -2370,9 +2811,10 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta // Content-Range: bytes */2830 if (back[i].range_req_size == back[i].r.crange) { filenote(back[i].url_sav,NULL); - //xxusercommand(opt,0,NULL,back[i].url_sav,back[i].url_adr,back[i].url_fil); + file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; back[i].status=0; // READY + back_set_finished(sback, i); back[i].r.size=back[i].r.totalsize=back[i].range_req_size; back[i].r.statuscode=304; // NOT MODIFIED if ((opt->debug>1) && (opt->log!=NULL)) { @@ -2386,7 +2828,22 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta back[i].r.statuscode=200; } - // Various hacks to limit re-transfers when updating a mirror + // 'do not erase already downloaded file' + // on an updated file + // with an error : consider a 304 error + if (!opt->delete_old) { + if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update && !back[i].testmode) { + if (back[i].url_sav[0] && fexist(back[i].url_sav)) { + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"Error ignored %d (%s) because of 'no purge' option for %s%s"LF,back[i].r.statuscode,back[i].r.msg,back[i].url_adr,back[i].url_fil); test_flush; + } + back[i].r.statuscode = 304; + deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; + } + } + } + + // Various hacks to limit re-transfers when updating a mirror // Force update if same size detected if (opt->sizehack) { // We already have the file @@ -2421,6 +2878,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta } if (r.adr) { freet(r.adr); + r.adr = NULL; } } } @@ -2438,9 +2896,10 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta if (back[i].r.totalsize == size) { // same size! deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; back[i].status=0; // READY + back_set_finished(sback, i); back[i].r.size=back[i].r.totalsize; filenote(back[i].url_sav,NULL); - //xxusercommand(opt,0,NULL,back[i].url_sav,back[i].url_adr,back[i].url_fil); + file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); back[i].r.statuscode=304; // NOT MODIFIED if ((opt->debug>1) && (opt->log!=NULL)) { fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (same size file discovered), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush; @@ -2475,9 +2934,10 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta #endif deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; back[i].status=0; // READY + back_set_finished(sback, i); back[i].r.size=back[i].r.totalsize; filenote(back[i].url_sav,NULL); - //xxusercommand(opt,0,NULL,back[i].url_sav,back[i].url_adr,back[i].url_fil); + file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); back[i].r.statuscode=304; // NOT MODIFIED if ((opt->debug>1) && (opt->log!=NULL)) { fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (reget failed), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush; @@ -2506,6 +2966,8 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta if (back[i].r.soc!=INVALID_SOCKET) { if (!back_checksize(opt,&back[i],1)) { back[i].status=0; // FINI + back_set_finished(sback, i); + back[i].r.statuscode=STATUSCODE_TOO_BIG; deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; if (!back[i].testmode) strcpybuff(back[i].r.msg,"File too big"); @@ -2529,6 +2991,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; } back[i].status=0; // terminé + back_set_finished(sback, i); } // traiter une éventuelle erreur 304 (cache à jour utilisable) else if (back[i].r.statuscode==304) { // document à jour dans le cache @@ -2550,7 +3013,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta // hack: // In case of 'if-unmodified-since' hack, a 304 status can be sent // then, force 'ok' status - if (back[i].r.statuscode == -1) { + if (back[i].r.statuscode == STATUSCODE_INVALID) { if (fexist(back[i].url_sav)) { back[i].r.statuscode=200; // OK strcpybuff(back[i].r.msg, "OK (cached)"); @@ -2566,14 +3029,16 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta // Status is okay? if (back[i].r.statuscode!=-1) { // pas d'erreur de lecture back[i].status=0; // OK prêt + back_set_finished(sback, i); back[i].r.notmodified=1; // NON modifié! if ((opt->debug>0) && (opt->log!=NULL)) { fspc(opt->log,"debug"); fprintf(opt->log,"File loaded after test from cache: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush; } // finalize + //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); // not modified if (back[i].r.statuscode>0) { - back_finalize(opt,cache,back,i); + back_finalize(opt,cache,sback,i); } #if DEBUGCA @@ -2583,16 +3048,14 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode); } else { // erreur back[i].status=0; // terminé + back_set_finished(sback, i); //printf("erreur cache\n"); } /********** NO - must complete the body! ********** */ #if 0 - } else if ((back[i].r.statuscode==301) - || (back[i].r.statuscode==302) - || (back[i].r.statuscode==303) - || (back[i].r.statuscode==307) + } else if (HTTP_IS_REDIRECT(back[i].r.statuscode) || (back[i].r.statuscode==412) || (back[i].r.statuscode==416) ) { // Ne pas prendre le html, erreurs connues et gérées @@ -2601,12 +3064,13 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta #endif // Couper connexion /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;*/ - back_maydeletehttp(opt, cache, back, back_max, i); + back_maydeletehttp(opt, cache, sback, i); back[i].status=0; // terminé + back_set_finished(sback, i); // finalize if (back[i].r.statuscode>0) { - back_finalize(opt,cache,back,i); + back_finalize(opt,cache,sback,i); } #endif /********** **************************** ********** */ @@ -2629,6 +3093,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta if (!is_hypertext_mime(back[i].r.contenttype, back[i].url_sav)) { // pas HTML if (opt->getmode&2) { // on peut ecrire des non html **sinon ben euhh sera intercepté plus loin, donc rap sur ce qui va sortir** filenote(back[i].url_sav,NULL); // noter fichier comme connu + file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 1, back[i].r.notmodified); back[i].r.out=fopen(fconv(back[i].url_sav),"ab"); // append if (back[i].r.out) { back[i].r.is_write=1; // écrire @@ -2642,6 +3107,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta #endif } else { // On est dans la m** back[i].status=0; // terminé (voir plus loin) + back_set_finished(sback, i); strcpybuff(back[i].r.msg,"Can not open partial file"); } } @@ -2657,6 +3123,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta back[i].r.totalsize+=sz; // plus en fait if (( fread(back[i].r.adr,1,(INTsys)sz,fp)) != sz) { back[i].status=0; // terminé (voir plus loin) + back_set_finished(sback, i); strcpybuff(back[i].r.msg,"Can not read partial file"); } else { back[i].r.statuscode=200; // Forcer 'OK' @@ -2666,16 +3133,19 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta } } else { back[i].status=0; // terminé (voir plus loin) + back_set_finished(sback, i); strcpybuff(back[i].r.msg,"No memory for partial file"); } fclose(fp); } else { // Argh.. back[i].status=0; // terminé (voir plus loin) + back_set_finished(sback, i); strcpybuff(back[i].r.msg,"Can not open partial file"); } } } else { // Non trouvé?? back[i].status=0; // terminé (voir plus loin) + back_set_finished(sback, i); strcpybuff(back[i].r.msg,"Can not find partial file"); } // Erreur? @@ -2688,7 +3158,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta } back[i].r.soc=INVALID_SOCKET; //back[i].r.statuscode=206; ???????? - back[i].r.statuscode=-5; + back[i].r.statuscode=STATUSCODE_NON_FATAL; if (strnotempty(back[i].r.msg)) strcpybuff(back[i].r.msg,"Error attempting to solve status 206 (partial file)"); } @@ -2699,13 +3169,14 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta if (back[i].r.empty /* ?? && back[i].r.statuscode==200 */) { // empty response // Couper connexion - back_maydeletehttp(opt, cache, back, back_max, i); + back_maydeletehttp(opt, cache, sback, i); /* KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */ back[i].status=0; // terminé + back_set_finished(sback, i); if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct((INTsys) 2)) ) { back[i].r.adr[0] = 0; } - back_finalize(opt,cache,back,i); + back_finalize(opt,cache,sback,i); } else if (!back[i].r.is_chunk) { // pas de chunk //if (back[i].r.http11!=2) { // pas de chunk @@ -2730,13 +3201,14 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta #endif } else { // mode test, ne pas passer en 1!! back[i].status=0; // READY + back_set_finished(sback, i); #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(test ok): deletehttp\n"); #endif deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; if (back[i].r.statuscode==200) { strcpybuff(back[i].r.msg,"Test: OK"); - back[i].r.statuscode=-10; // test réussi + back[i].r.statuscode=STATUSCODE_TEST_OK; // test réussi } else { // test a échoué, on ne change rien sauf que l'erreur est à titre indicatif char tempo[1000]; @@ -2766,7 +3238,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta #if HTS_REMOVE_BAD_FILES if (back[i].status<0) { if (!back[i].testmode) { // pas en test - remove(back[i].url_sav); // éliminer fichier (endommagé) + unlink(back[i].url_sav); // éliminer fichier (endommagé) //printf("&& %s\n",back[i].url_sav); } } @@ -2808,7 +3280,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta deletehttp(&back[i].r); } back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=-2; + back[i].r.statuscode=STATUSCODE_TIMEOUT; if (back[i].status==100) strcpybuff(back[i].r.msg,"Connect Time Out"); else if (back[i].status==101) @@ -2816,10 +3288,12 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta else strcpybuff(back[i].r.msg,"Receive Time Out"); back[i].status=0; // terminé + back_set_finished(sback, i); } else if ((back[i].rateout>0) && (back[i].status<99)) { if (((int) (act-back[i].rateout_time))>=HTS_WATCHRATE) { // checker au bout de 15s if ( (int) ((back[i].r.size)/(act-back[i].rateout_time)) < back[i].rateout ) { // trop lent back[i].status=0; // terminé + back_set_finished(sback, i); if (back[i].r.soc!=INVALID_SOCKET) { #if HTS_DEBUG_CLOSESOCK DEBUG_W("back_wait(rateout): deletehttp\n"); @@ -2827,7 +3301,7 @@ void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TSta deletehttp(&back[i].r); } back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=-3; + back[i].r.statuscode=STATUSCODE_SLOW; strcpybuff(back[i].r.msg,"Transfer Rate Too Low"); } } @@ -2891,22 +3365,37 @@ int back_checkmirror(httrackp* opt) { // octets transférés + add -LLint back_transfered(LLint nb,lien_back* back,int back_max) { +LLint back_transfered(LLint nb,struct_back* sback) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int i; // ajouter octets en instance for(i=0;i<back_max;i++) if ((back[i].status>0) && (back[i].status<99 || back[i].status>=1000)) nb+=back[i].r.size; + // stored (ready) slots + if (sback->ready != NULL) { + struct_inthash_enum e = inthash_enum_new((inthash)sback->ready); + inthash_chain* item; + while((item = inthash_enum_next(&e))) { + lien_back* ritem = (lien_back*) item->value.ptr; + if ((ritem->status>0) && (ritem->status<99 || ritem->status>=1000)) + nb+=ritem->r.size; + } + } return nb; } // infos backing // j: 1 afficher sockets 2 afficher autres 3 tout afficher -void back_info(lien_back* back,int i,int j,FILE* fp) { +void back_info(struct_back* sback,int i,int j,FILE* fp) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + assertf(i >= 0 && i < back_max); if (back[i].status>=0) { char BIGSTK s[HTS_URLMAXSIZE*2+1024]; s[0]='\0'; - back_infostr(back,i,j,s); + back_infostr(sback,i,j,s); strcatbuff(s,LF); fprintf(fp,"%s",s); } @@ -2914,7 +3403,10 @@ void back_info(lien_back* back,int i,int j,FILE* fp) { // infos backing // j: 1 afficher sockets 2 afficher autres 3 tout afficher -void back_infostr(lien_back* back,int i,int j,char* s) { +void back_infostr(struct_back* sback,int i,int j,char* s) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + assertf(i >= 0 && i < back_max); if (back[i].status>=0) { int aff=0; if (j & 1) { diff --git a/src/htsback.h b/src/htsback.h index 9587d7e..90c36db 100644 --- a/src/htsback.h +++ b/src/htsback.h @@ -42,38 +42,63 @@ Please visit our Website: http://www.httrack.com #include "htsbasenet.h" #include "htscore.h" +typedef enum BackStatusCode { + STATUSCODE_INVALID = -1, + STATUSCODE_TIMEOUT = -2, + STATUSCODE_SLOW = -3, + STATUSCODE_CONNERROR = -4, + STATUSCODE_NON_FATAL = -5, + STATUSCODE_SSL_HANDSHAKE = -6, + STATUSCODE_TOO_BIG = -7, + STATUSCODE_TEST_OK = -10 +} BackStatusCode; + /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE +// create/destroy +struct_back* back_new(int back_max); +void back_free(struct_back** sback); + // backing #define BACK_ADD_TEST "(dummy)" #define BACK_ADD_TEST2 "(dummy2)" -int back_index(lien_back* back,int back_max,char* adr,char* fil,char* sav); -int back_available(lien_back* back,int back_max); -LLint back_incache(lien_back* back,int back_max); -HTS_INLINE int back_exist(lien_back* back,int back_max,char* adr,char* fil,char* sav); -int back_nsoc(lien_back* back,int back_max); -int back_nsoc_overall(lien_back* back,int back_max); -int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test,int* pass2_ptr); -int back_stack_available(lien_back* back,int back_max); -int back_search(httrackp* opt, cache_back* cache, lien_back* back, int back_max); -void back_clean(httrackp* opt,cache_back* cache,lien_back* back,int back_max); -void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TStamp stat_timestart); -int back_letlive(httrackp* opt, cache_back* cache, lien_back* back, int p); -int back_searchlive(httrackp* opt, lien_back* back, int back_max, char* search_addr); +int back_index(struct_back* sback,char* adr,char* fil,char* sav); +int back_available(struct_back* sback); +LLint back_incache(struct_back* sback); +int back_done_incache(struct_back* sback); +HTS_INLINE int back_exist(struct_back* sback,char* adr,char* fil,char* sav); +int back_nsoc(struct_back* sback); +int back_nsoc_overall(struct_back* sback); +int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test,int* pass2_ptr); +int back_add_if_not_exists(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test,int* pass2_ptr); +int back_stack_available(struct_back* sback); +int back_search(httrackp* opt, cache_back* cache, struct_back* sback); +int back_search_quick(struct_back* sback); +void back_clean(httrackp* opt,cache_back* cache,struct_back* sback); +int back_cleanup_background(httrackp* opt,cache_back* cache,struct_back* sback); +void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_timestart); +int back_letlive(httrackp* opt, cache_back* cache, struct_back* sback, int p); +int back_searchlive(httrackp* opt, struct_back* sback, char* search_addr); void back_connxfr(htsblk* src, htsblk* dst); -int back_delete(httrackp* opt,cache_back* cache,lien_back* back,int p); -int back_maydelete(httrackp* opt, cache_back* cache, lien_back* back, int p); -void back_maydeletehttp(httrackp* opt, cache_back* cache, lien_back* back, int back_max, int p); -int back_trylive(httrackp* opt,cache_back* cache,lien_back* back, int back_max, int p); -int back_finalize(httrackp* opt,cache_back* cache,lien_back* back,int p); -void back_info(lien_back* back,int i,int j,FILE* fp); -void back_infostr(lien_back* back,int i,int j,char* s); -LLint back_transfered(LLint add,lien_back* back,int back_max); +void back_move(lien_back* src, lien_back* dst); +void back_copy_static(const lien_back* src, lien_back* dst); +void back_set_finished(struct_back* sback, int p); +int back_delete(httrackp* opt,cache_back* cache,struct_back* sback,int p); +int back_flush_output(httrackp* opt, cache_back* cache, struct_back* sback, int p); +int back_set_passe2_ptr(httrackp* opt, cache_back* cache, struct_back* sback, int p, int* pass2_ptr); +void back_delete_all(httrackp* opt, cache_back* cache, struct_back* sback); +int back_maydelete(httrackp* opt, cache_back* cache, struct_back* sback, int p); +void back_maydeletehttp(httrackp* opt, cache_back* cache, struct_back* sback, int p); +int back_trylive(httrackp* opt,cache_back* cache,struct_back* sback, int p); +int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p); +void back_info(struct_back* sback,int i,int j,FILE* fp); +void back_infostr(struct_back* sback,int i,int j,char* s); +LLint back_transfered(LLint add,struct_back* sback); // hostback #if HTS_XGETHOST -void back_solve(lien_back* back); -int host_wait(lien_back* back); +void back_solve(lien_back* sback); +int host_wait(lien_back* sback); #endif int back_checksize(httrackp* opt,lien_back* eback,int check_only_totalsize); int back_checkmirror(httrackp* opt); diff --git a/src/htsbase.h b/src/htsbase.h index 9911d73..7faec95 100644 --- a/src/htsbase.h +++ b/src/htsbase.h @@ -282,32 +282,6 @@ extern HTSEXT_API int htsMemoryFastXfr; } \ } \ } while(0) -#define strncpybuff(A, B, N) do { \ - assertf( (A) != NULL ); \ - if ( ! (B) ) { assertf( 0 ); } \ - if (htsMemoryFastXfr) { \ - if (sizeof(A) != sizeof(char*)) { \ - (A)[sizeof(A) - 1] = '\0'; \ - } \ - strncpy(A, B, N); \ - if (sizeof(A) != sizeof(char*)) { \ - assertf((A)[sizeof(A) - 1] == '\0'); \ - } \ - } else { \ - unsigned int szf = (unsigned int) strlen(B); \ - if (szf > (unsigned int) (N)) szf = (unsigned int) (N); \ - if (sizeof(A) != sizeof(char*)) { \ - assertf(szf + 1 < sizeof(A)); \ - if (szf > 0) { \ - if (szf + 1 < sizeof(A)) { \ - memcpy((A), (B), szf); \ - } \ - } \ - } else { \ - memcpybuff((A), (B), szf); \ - } \ - } \ -} while(0) #else @@ -347,24 +321,12 @@ extern HTSEXT_API int htsMemoryFastXfr; assertf((A)[sizeof(A) - 1] == '\0'); \ } \ } while(0) -#define strncpybuff(A, B, N) do { \ - assertf( (A) != NULL ); \ - if ( ! (B) ) { assertf( 0 ); } \ - if (sizeof(A) != sizeof(char*)) { \ - (A)[sizeof(A) - 1] = '\0'; \ - } \ - strncpy(A, B, N); \ - if (sizeof(A) != sizeof(char*)) { \ - assertf((A)[sizeof(A) - 1] == '\0'); \ - } \ -} while(0) #else #define strcatbuff strcat #define strncatbuff strncat #define strcpybuff strcpy -#define strncpybuff strncpy #endif diff --git a/src/htscache.c b/src/htscache.c index aa9a6c8..de2273c 100644 --- a/src/htscache.c +++ b/src/htscache.c @@ -121,6 +121,14 @@ void cache_mayadd(httrackp* opt,cache_back* cache,htsblk* r,char* url_adr,char* // stocker dans le cache? if (opt->cache) { if (cache_writable(cache)) { + // ensure not a temporary filename (should not happend ?!) + if (IS_DELAYED_EXT(url_save)) { + if (opt->log!=NULL) { + fspc(opt->log,"warning"); fprintf(opt->log, "aborted cache validation: %s%s still has temporary name %s"LF, url_adr, url_fil, url_save); + } + return ; + } + // c'est le seul endroit ou l'on ajoute des elements dans le cache (fichier entier ou header) // on stocke tout fichier "ok", mais également les réponses 404,301,302... if ( @@ -130,10 +138,7 @@ void cache_mayadd(httrackp* opt,cache_back* cache,htsblk* r,char* url_adr,char* /* We don't store 5XX errors, because it might be a server problem */ (r->statuscode==200) /* stocker réponse standard, plus */ || (r->statuscode==204) /* no content */ - || (r->statuscode==301) /* moved perm */ - || (r->statuscode==302) /* moved temp */ - || (r->statuscode==303) /* moved temp */ - || (r->statuscode==307) /* moved temp */ + || HTTP_IS_REDIRECT(r->statuscode) /* redirect */ || (r->statuscode==401) /* authorization */ || (r->statuscode==403) /* unauthorized */ || (r->statuscode==404) /* not found */ @@ -145,7 +150,7 @@ void cache_mayadd(httrackp* opt,cache_back* cache,htsblk* r,char* url_adr,char* // stocker fichiers (et robots.txt) if ( url_save == NULL || (strnotempty(url_save)) || (strcmp(url_fil,"/robots.txt")==0)) { // ajouter le fichier au cache - cache_add(cache,*r,url_adr,url_fil,url_save,opt->all_in_cache); + cache_add(cache,*r,url_adr,url_fil,url_save,opt->all_in_cache,opt->path_html); // // store a reference NOT to redo the same test zillions of times! // (problem reported by Lars Clausen) @@ -206,7 +211,7 @@ struct cache_back_zip_entry { if (line[0] != '\0' && strfield2(line, refline)) { \ strcpybuff(refvalue, value); \ line[0] = '\0'; \ - } \ + } \ } while(0) #define ZIP_READFIELD_INT(line, value, refline, refvalue) do { \ if (line[0] != '\0' && strfield2(line, refline)) { \ @@ -214,19 +219,21 @@ struct cache_back_zip_entry { sscanf(value, "%d", &intval); \ (refvalue) = intval; \ line[0] = '\0'; \ - } \ + } \ } while(0) /* Ajout d'un fichier en cache */ -void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_save,int all_in_cache) { - char BIGSTK filemame[HTS_URLMAXSIZE*4]; +void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_save,int all_in_cache,char* path_prefix) { + char BIGSTK filename[HTS_URLMAXSIZE*4]; int dataincache=0; // put data in cache ? char BIGSTK headers[8192]; int headersSize = 0; int entryBodySize = 0; int entryFilenameSize = 0; zip_fileinfo fi; + char* url_save_suffix = url_save; + int zErr; // robots.txt hack if (url_save == NULL) { @@ -241,10 +248,14 @@ void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_ } /* Data in cache ? */ - if (is_hypertext_mime(r.contenttype, url_fil)) + if (is_hypertext_mime(r.contenttype, url_fil) + || (may_be_hypertext_mime(r.contenttype, url_fil) && r.adr != NULL) + ) + { dataincache=1; - else if (all_in_cache) + } else if (all_in_cache) { dataincache=1; + } } if (r.size < 0) // error @@ -271,6 +282,14 @@ void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_ sprintf(headers + headersSize, "HTTP/1.%c %d %s\r\n", '1', r.statuscode, r.msg); } headersSize += (int) strlen(headers + headersSize); + + if (path_prefix != NULL && path_prefix[0] != '\0' && url_save != NULL && url_save[0] != '\0') { + int prefixLen = (int) strlen(path_prefix); + if (strncmp(url_save, path_prefix, prefixLen) == 0) { + url_save_suffix += prefixLen; + } + } + /* Second line MUST ALWAYS be X-In-Cache */ ZIP_FIELD_INT_FORCE(headers, headersSize, "X-In-Cache", dataincache); ZIP_FIELD_INT(headers, headersSize, "X-StatusCode", r.statuscode); @@ -284,18 +303,18 @@ void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_ ZIP_FIELD_STRING(headers, headersSize, "Content-Disposition", r.cdispo); // Content-disposition ZIP_FIELD_STRING(headers, headersSize, "X-Addr", url_adr); // Original address ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil); // Original URI filename - ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save); // Original save filename + ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save_suffix); // Original save filename entryFilenameSize = (int) ( strlen(url_adr) + strlen(url_fil)); /* Filename */ if (!link_has_authority(url_adr)) { - strcpybuff(filemame, "http://"); + strcpybuff(filename, "http://"); } else { - strcpybuff(filemame, ""); + strcpybuff(filename, ""); } - strcatbuff(filemame, url_adr); - strcatbuff(filemame, url_fil); + strcatbuff(filename, url_adr); + strcatbuff(filename, url_fil); /* Time */ memset(&fi, 0, sizeof(fi)); @@ -312,8 +331,8 @@ void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_ } /* Open file - NOTE: headers in "comment" */ - if (zipOpenNewFileInZip((zipFile) cache->zipOutput, - filemame, + if ((zErr = zipOpenNewFileInZip((zipFile) cache->zipOutput, + filename, &fi, /* Store headers in realtime in the local file directory as extra field @@ -325,19 +344,19 @@ void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_ 0, NULL, /* comment */ Z_DEFLATED, - Z_DEFAULT_COMPRESSION) != Z_OK) + Z_DEFAULT_COMPRESSION)) != Z_OK) { - int zip_disk_write_failed = 0; - assertf(zip_disk_write_failed); + int zip_zipOpenNewFileInZip_failed = 0; + assertf(zip_zipOpenNewFileInZip_failed); } /* Write data in cache */ if (dataincache) { if (r.is_write == 0) { if (r.size > 0 && r.adr != NULL) { - if (zipWriteInFileInZip((zipFile) cache->zipOutput, r.adr, (int) r.size) != Z_OK) { - int zip_disk_write_failed = 0; - assertf(zip_disk_write_failed); + if ((zErr = zipWriteInFileInZip((zipFile) cache->zipOutput, r.adr, (int) r.size)) != Z_OK) { + int zip_zipWriteInFileInZip_failed = 0; + assertf(zip_zipWriteInFileInZip_failed); } } } else { @@ -352,9 +371,9 @@ void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_ do { nl=fread(buff,1,32768,fp); if (nl>0) { - if (zipWriteInFileInZip((zipFile) cache->zipOutput, buff, (int) nl) != Z_OK) { - int zip_disk_write_failed = 0; - assertf(zip_disk_write_failed); + if ((zErr = zipWriteInFileInZip((zipFile) cache->zipOutput, buff, (int) nl)) != Z_OK) { + int zip_zipWriteInFileInZip_failed = 0; + assertf(zip_zipWriteInFileInZip_failed); } } } while(nl>0); @@ -367,15 +386,15 @@ void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_ } /* Close */ - if (zipCloseFileInZip((zipFile) cache->zipOutput) != Z_OK) { - int zip_disk_write_failed = 0; - assertf(zip_disk_write_failed); + if ((zErr = zipCloseFileInZip((zipFile) cache->zipOutput)) != Z_OK) { + int zip_zipCloseFileInZip_failed = 0; + assertf(zip_zipCloseFileInZip_failed); } /* Flush */ - if (zipFlush((zipFile) cache->zipOutput) != 0) { - int zip_disk_write_failed = 0; - assertf(zip_disk_write_failed); + if ((zErr = zipFlush((zipFile) cache->zipOutput)) != 0) { + int zip_zipFlush_failed = 0; + assertf(zip_zipFlush_failed); } } @@ -551,10 +570,13 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,char* adr,char* f char BIGSTK location_default[HTS_URLMAXSIZE*2]; char BIGSTK buff[HTS_URLMAXSIZE*2]; char BIGSTK previous_save[HTS_URLMAXSIZE*2]; + char BIGSTK previous_save_[HTS_URLMAXSIZE*2]; long int hash_pos; int hash_pos_return; htsblk r; memset(&r, 0, sizeof(htsblk)); r.soc=INVALID_SOCKET; + location_default[0] = '\0'; + previous_save[0] = previous_save_[0] = '\0'; if (location) { r.location = location; @@ -625,13 +647,26 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,char* adr,char* f ZIP_READFIELD_STRING(line, value, "Etag", r.etag); // Etag ZIP_READFIELD_STRING(line, value, "Location", r.location); // 'location' pour moved ZIP_READFIELD_STRING(line, value, "Content-Disposition", r.cdispo); // Content-disposition - ZIP_READFIELD_STRING(line, value, "X-Addr", previous_save); // Original address - ZIP_READFIELD_STRING(line, value, "X-Fil", previous_save); // Original URI filename - ZIP_READFIELD_STRING(line, value, "X-Save", previous_save); // Original save filename + //ZIP_READFIELD_STRING(line, value, "X-Addr", ..); // Original address + //ZIP_READFIELD_STRING(line, value, "X-Fil", ..); // Original URI filename + ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename } } while(offset < readSizeHeader && !lineEof); totalHeader = offset; + /* Previous entry */ + if (previous_save_[0] != '\0') { + int pathLen = (int) strlen(opt->path_html); + if (pathLen != 0 && strncmp(previous_save_, opt->path_html, pathLen) != 0) { // old (<3.40) buggy format + sprintf(previous_save, "%s%s", opt->path_html, previous_save_); + } else { + strcpy(previous_save, previous_save_); + } + } + if (return_save != NULL) { + strcpybuff(return_save, previous_save); + } + /* Complete fields */ r.totalsize=r.size; r.adr=NULL; @@ -652,27 +687,33 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,char* adr,char* f else if (!readonly && r.statuscode==200 && !is_hypertext_mime(r.contenttype, fil) && strnotempty(save)) { // pas HTML, écrire sur disk directement r.is_write=1; // écrire - if (fexist(fconv(save))) { // un fichier existe déja - //if (fsize(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree) - ok=1; // plus rien à faire - filenote(save,NULL); // noter comme connu - } + if (!dataincache) { + if (fexist(fconv(save))) { // un fichier existe déja + //if (fsize(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree) + ok=1; // plus rien à faire + filenote(save,NULL); // noter comme connu + file_notify(adr, fil, save, 0, 0, 1); // data in cache + } + } if (!dataincache && !ok) { // Pas de donnée en cache et fichier introuvable : erreur! if (opt->norecatch) { + file_notify(adr, fil, save, 1, 0, 0); filecreateempty(save); // - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"File deleted by user not recaught"); ok=1; // ne pas récupérer (et pas d'erreur) } else { - r.statuscode=-1; + file_notify(adr, fil, save, 1, 1, 0); + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Previous cache file not found"); ok=1; // ne pas récupérer } } - if (!ok) { + if (!ok) { // load from cache + file_notify(adr, fil, save, 1, 1, 1); // data in cache r.out=filecreate(save); #if HDEBUG printf("direct-disk: %s\n",save); @@ -687,8 +728,8 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,char* adr,char* f if (nl>0) { size-=nl; if ((INTsys)fwrite(buff,1,(INTsys)nl,r.out)!=nl) { // erreur - r.statuscode=-1; - strcpybuff(r.msg,"Cache Read Error : Read To Disk"); + r.statuscode=STATUSCODE_INVALID; + sprintf(r.msg,"Cache Read Error : Read To Disk: %s", strerror(errno)); } } } while((nl>0) && (size>0) && (r.statuscode!=-1)); @@ -699,9 +740,8 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,char* adr,char* f #if HTS_WIN==0 chmod(save,HTS_ACCESS_FILE); #endif - //xxusercommand(opt,0,NULL,fconv(save), adr, fil); } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Write Error : Unable to Create File"); //printf("%s\n",save); } @@ -713,26 +753,26 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,char* adr,char* f if (!dataincache) { if (strnotempty(save)) { // Pas de donnée en cache, bizarre car html!!! - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Previous cache file not found (2)"); } else { /* Read in memory from cache */ - if (strnotempty(return_save) && fexist(return_save)) { - FILE* fp = fopen(fconv(return_save), "rb"); + if (strnotempty(previous_save) && fexist(previous_save)) { + FILE* fp = fopen(fconv(previous_save), "rb"); if (fp != NULL) { r.adr=(char*) malloct((INTsys)r.size + 4); - if (adr != NULL) { + if (r.adr != NULL) { if (r.size > 0 && fread(r.adr, 1, (INTsys) r.size, fp) != r.size) { - r.statuscode=-1; - strcpybuff(r.msg,"Read error in cache disk data"); + r.statuscode=STATUSCODE_INVALID; + sprintf(r.msg,"Read error in cache disk data: %s", strerror(errno)); } } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Read error (memory exhausted) from cache"); } fclose(fp); } } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache file not found on disk"); } } @@ -743,13 +783,13 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,char* adr,char* f if (unzReadCurrentFile((unzFile) cache->zipInput, r.adr, (INTsys)r.size) != r.size) { // erreur freet(r.adr); r.adr=NULL; - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Read Error : Read Data"); } else *(r.adr+r.size)='\0'; //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode); } else { // erreur - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Memory Error"); } } @@ -758,21 +798,21 @@ static htsblk cache_readex_new(httrackp* opt,cache_back* cache,char* adr,char* f } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Read Error : Read Header Data"); } unzCloseCurrentFile((unzFile) cache->zipInput); } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Read Error : Open File"); } } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Read Error : Bad Offset"); } } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"File Cache Entry Not Found"); } if (!location) { /* don't export internal buffer */ @@ -934,19 +974,20 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,char* adr,char* f //if (fsize(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree) ok=1; // plus rien à faire filenote(save,NULL); // noter comme connu - //xxusercommand(opt,0,NULL,save,adr,fil); + file_notify(adr, fil, save, 0, 0, 0); //} } if ((pos<0) && (!ok)) { // Pas de donnée en cache et fichier introuvable : erreur! if (opt->norecatch) { + file_notify(adr, fil, save, 1, 0, 0); filecreateempty(save); // - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"File deleted by user not recaught"); ok=1; // ne pas récupérer (et pas d'erreur) } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Previous cache file not found"); ok=1; // ne pas récupérer } @@ -967,7 +1008,7 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,char* adr,char* f if (nl>0) { size-=nl; if ((INTsys)fwrite(buff,1,(INTsys)nl,r.out)!=nl) { // erreur - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Read Error : Read To Disk"); } } @@ -979,9 +1020,8 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,char* adr,char* f #if HTS_WIN==0 chmod(save,HTS_ACCESS_FILE); #endif - //xxusercommand(opt,0,NULL,fconv(save), adr, fil); } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Write Error : Unable to Create File"); //printf("%s\n",save); } @@ -993,26 +1033,26 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,char* adr,char* f if (pos<0) { if (strnotempty(save)) { // Pas de donnée en cache, bizarre car html!!! - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Previous cache file not found (2)"); } else { /* Read in memory from cache */ if (strnotempty(return_save) && fexist(return_save)) { FILE* fp = fopen(fconv(return_save), "rb"); if (fp != NULL) { r.adr=(char*) malloct((INTsys)r.size + 4); - if (adr != NULL) { + if (r.adr != NULL) { if (r.size > 0 && fread(r.adr, 1, (INTsys) r.size, fp) != r.size) { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Read error in cache disk data"); } } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Read error (memory exhausted) from cache"); } fclose(fp); } } else { - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache file not found on disk"); } } @@ -1023,13 +1063,13 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,char* adr,char* f if (fread(r.adr,1,(INTsys)r.size,cache->olddat)!=r.size) { // erreur freet(r.adr); r.adr=NULL; - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Read Error : Read Data"); } else *(r.adr+r.size)='\0'; //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode); } else { // erreur - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Memory Error"); } } @@ -1039,28 +1079,28 @@ static htsblk cache_readex_old(httrackp* opt,cache_back* cache,char* adr,char* f #if DEBUGCA printf("Cache Read Error : Bad Data"); #endif - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Read Error : Bad Data"); } } else { // erreur #if DEBUGCA printf("Cache Read Error : Read Header"); #endif - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Read Error : Read Header"); } } else { #if DEBUGCA printf("Cache Read Error : Seek Failed"); #endif - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"Cache Read Error : Seek Failed"); } } else { #if DEBUGCA printf("File Cache Not Found"); #endif - r.statuscode=-1; + r.statuscode=STATUSCODE_INVALID; strcpybuff(r.msg,"File Cache Entry Not Found"); } if (!location) { /* don't export internal buffer */ @@ -1551,6 +1591,10 @@ void cache_init(cache_back* cache,httrackp* opt) { // lire un fichier.. (compatible \0) char* readfile(char* fil) { + return readfile2(fil, NULL); +} + +char* readfile2(char* fil, LLint* size) { char* adr=NULL; INTsys len=0; len=fsize(fil); @@ -1559,6 +1603,8 @@ char* readfile(char* fil) { fp=fopen(fconv(fil),"rb"); if (fp!=NULL) { // n'existe pas (!) adr=(char*) malloct(len+1); + if (size != NULL) + *size = len; if (adr!=NULL) { if (len > 0 && (INTsys)fread(adr,1,len,fp) != len) { // fichier endommagé ? freet(adr); diff --git a/src/htscache.h b/src/htscache.h index 51dd439..b80a0ee 100644 --- a/src/htscache.h +++ b/src/htscache.h @@ -47,7 +47,7 @@ Please visit our Website: http://www.httrack.com // cache void cache_mayadd(httrackp* opt,cache_back* cache,htsblk* r,char* url_adr,char* url_fil,char* url_save); -void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_save,int all_in_cache); +void cache_add(cache_back* cache,htsblk r,char* url_adr,char* url_fil,char* url_save,int all_in_cache,char* path_prefix); htsblk cache_read(httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* location); htsblk cache_read_ro(httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* location); htsblk cache_readex(httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* location,char* return_save,int readonly); diff --git a/src/htsconfig.h b/src/htsconfig.h index 665c9df..3b99b1f 100644 --- a/src/htsconfig.h +++ b/src/htsconfig.h @@ -55,6 +55,9 @@ Please visit our Website: http://www.httrack.com // nom par défaut pour / en ftp #define DEFAULT_FTP "index.txt" +// nom par défaut pour / en mms +#define DEFAULT_MMS "default.avi" + // extension par défaut pour fichiers n'en ayant pas #define DEFAULT_EXT ".html" #define DEFAULT_EXT_SHORT ".htm" @@ -82,8 +85,12 @@ Please visit our Website: http://www.httrack.com // écriture directe dur disque possible (0/1) #define HTS_DIRECTDISK 1 +// always direct-to-disk (0/1) +#define HTS_DIRECTDISK_ALWAYS 1 + // gérer une table de hachage? -#define HTS_HASH 1 +// REMOVED +// #define HTS_HASH 1 // fast cache (build hash table) #define HTS_FAST_CACHE 1 @@ -100,7 +107,8 @@ Please visit our Website: http://www.httrack.com // case-sensitive pour les dossiers et fichiers (0/1) // [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est pas très utile.. // ..et pas bcp respecté] -#define HTS_CASSE 0 +// REMOVED +// #define HTS_CASSE 0 // Un fichier ayant une taille différente du content-length doit il être annulé? // SEE opt.tolerant and opt.http10 diff --git a/src/htscore.c b/src/htscore.c index ff761ef..48d776f 100644 --- a/src/htscore.c +++ b/src/htscore.c @@ -84,8 +84,10 @@ t_hts_htmlcheck_query2 hts_htmlcheck_query2 = NULL; t_hts_htmlcheck_query3 hts_htmlcheck_query3 = NULL; t_hts_htmlcheck_loop hts_htmlcheck_loop = NULL; t_hts_htmlcheck_check hts_htmlcheck_check = NULL; +t_hts_htmlcheck_check_mime hts_htmlcheck_check_mime = NULL; t_hts_htmlcheck_pause hts_htmlcheck_pause = NULL; t_hts_htmlcheck_filesave hts_htmlcheck_filesave = NULL; +t_hts_htmlcheck_filesave2 hts_htmlcheck_filesave2 = NULL; t_hts_htmlcheck_linkdetected hts_htmlcheck_linkdetected = NULL; t_hts_htmlcheck_linkdetected2 hts_htmlcheck_linkdetected2 = NULL; t_hts_htmlcheck_xfrstatus hts_htmlcheck_xfrstatus = NULL; @@ -180,7 +182,9 @@ char HTbuff[2048]; if ( (opt.debug>0) && (opt.log!=NULL) ) { \ fspc(opt.log,"info"); fprintf(opt.log,"engine: end"LF); \ } \ -hts_htmlcheck_end(); \ +if (hts_htmlcheck_end != NULL) { \ + hts_htmlcheck_end(); \ +} \ } #else #define HTMLCHECK_UNINIT @@ -207,13 +211,8 @@ hts_htmlcheck_end(); \ if (filters) { \ freet(filters); filters=NULL; \ } \ - if (back) { \ - int i; \ - for(i=0;i<back_max;i++) { \ - back_delete(&opt,&cache,back,i); \ - } \ - freet(back); back=NULL; \ - } \ + back_delete_all(&opt,&cache,sback); \ + back_free(&sback); \ checkrobots_free(&robots);\ if (cache.use) { freet(cache.use); cache.use=NULL; } \ if (cache.dat) { fclose(cache.dat); cache.dat=NULL; } \ @@ -251,11 +250,7 @@ hts_htmlcheck_end(); \ // enfin on écrit à l'adresse courante du buffer, qu'on incrémente. on décrémente la taille dispo d'autant ensuite // codebase: si non nul et si .class stockee on le note pour chemin primaire pour classes // FA,FS: former_adr et former_fil, lien original -#if HTS_HASH #define liens_record_sav_len(A) -#else -#define liens_record_sav_len(A) (A)->sav_len=strlen((A)->sav) -#endif #define liens_record(A,F,S,FA,FF,NORM) { \ int notecode=0; \ @@ -347,8 +342,7 @@ int httpmirror(char* url1,httrackp* ptropt) { int ptr; // pointeur actuel sur les liens // int numero_passe=0; // deux passes pour html puis images - int back_max=0; // fichiers qui peuvent être en local - lien_back* back=NULL; // backing en local + struct_back* sback=NULL; htsblk BIGSTK r; // retour de certaines fonctions TStamp lastime=0; // pour affichage infos de tmp en tmp // pour les stats, nombre de fichiers & octets écrits @@ -469,7 +463,7 @@ int httpmirror(char* url1,httrackp* ptropt) { cache_tests=inthash_new(cache_hash_size); if (cache_hashtable==NULL || cache_tests==NULL) { printf("PANIC! : Not enough memory [%d]\n",__LINE__); - filters[0]=NULL; back_max=0; // uniquement a cause du warning de XH_extuninit + filters[0]=NULL; // uniquement a cause du warning de XH_extuninit XH_extuninit; return 0; } @@ -490,7 +484,6 @@ int httpmirror(char* url1,httrackp* ptropt) { opt.maxfilter = maximum(opt.maxfilter, 128); if (filters_init(&filters, opt.maxfilter, 0) == 0) { printf("PANIC! : Not enough memory [%d]\n",__LINE__); - back_max=0; // uniquement a cause du warning de XH_extuninit XH_extuninit; return 0; } @@ -518,7 +511,6 @@ int httpmirror(char* url1,httrackp* ptropt) { // initialiser ptr et lien_tot ptr=0; lien_tot=0; -#if HTS_HASH // initialiser hachage { int i; @@ -527,8 +519,6 @@ int httpmirror(char* url1,httrackp* ptropt) { hash.liens = liens; hash.max_lien=0; } -#endif - // copier adresse(s) dans liste des adresses { @@ -547,7 +537,6 @@ int httpmirror(char* url1,httrackp* ptropt) { primary[0]='\0'; } else { printf("PANIC! : Not enough memory [%d]\n",__LINE__); - back_max=0; // uniquement a cause du warning de XH_extuninit XH_extuninit; return 0; } @@ -593,13 +582,6 @@ int httpmirror(char* url1,httrackp* ptropt) { strcpybuff(filters[filptr],"+"); else strcpybuff(filters[filptr],"-"); - /* - if (strfield(tempo,"http://")) - strcatbuff(filters[filptr],tempo+7); // ignorer http:// - else if (strfield(tempo,"ftp://")) - strcatbuff(filters[filptr],tempo+6); // ignorer ftp:// - else - */ strcatbuff(filters[filptr],tempo); filptr++; @@ -613,7 +595,6 @@ int httpmirror(char* url1,httrackp* ptropt) { fprintf(opt.errlog,"To avoid that: use #F option for more filters (example: -#F5000)"LF); test_flush; } - back_max=0; // uniquement a cause du warning de XH_extuninit XH_extuninit; return 0; } @@ -703,7 +684,6 @@ int httpmirror(char* url1,httrackp* ptropt) { fprintf(opt.errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url))); test_flush; } - back_max=0; // uniquement a cause du warning de XH_extuninit XH_extuninit; // désallocation mémoire & buffers return 0; } @@ -757,20 +737,11 @@ int httpmirror(char* url1,httrackp* ptropt) { // On prévoit large: les fichiers HTML ne prennent que peu de place en mémoire, et les // fichiers non html sont sauvés en direct sur disque. // --> 1024 entrées + 32 entrées par socket en supplément - back_max=opt.maxsoc*32+1024; - //back_max=opt.maxsoc*8+32; - back=(lien_back*) calloct((back_max+1),sizeof(lien_back)); - if (back==NULL) { + sback = back_new(opt.maxsoc*32+1024); + if (sback == NULL) { if (opt.errlog) fprintf(opt.errlog,"Not enough memory, can not allocate %d bytes"LF,(int)((opt.maxsoc+1)*sizeof(lien_back))); return 0; - } else { // copier buffer-location & effacer - int i; - for(i=0;i<back_max;i++){ - back[i].r.location=back[i].location_buffer; - back[i].status=-1; - back[i].r.soc=INVALID_SOCKET; - } } } @@ -805,7 +776,9 @@ int httpmirror(char* url1,httrackp* ptropt) { /* Send options to callback functions */ #if HTS_ANALYSTE - hts_htmlcheck_chopt(&opt); + if (hts_htmlcheck_chopt != NULL) { + hts_htmlcheck_chopt(&opt); + } #endif // attendre une certaine heure.. @@ -846,12 +819,12 @@ int httpmirror(char* url1,httrackp* ptropt) { } #if HTS_ANALYSTE - { + if (hts_htmlcheck_loop != NULL) { int r; if (rollover) - r=hts_htmlcheck_loop(back,back_max,0,0,lien_tot,(int) (opt.waittime-tl+24*3600),NULL); + r=hts_htmlcheck_loop(sback->lnk, sback->count,0,0,lien_tot,(int) (opt.waittime-tl+24*3600),NULL); else - r=hts_htmlcheck_loop(back,back_max,0,0,lien_tot,(int) (opt.waittime-tl),NULL); + r=hts_htmlcheck_loop(sback->lnk, sback->count,0,0,lien_tot,(int) (opt.waittime-tl),NULL); if (!r) { exit_xh=1; // exit requested ok=1; @@ -883,9 +856,11 @@ int httpmirror(char* url1,httrackp* ptropt) { fspc(opt.log,"info"); fprintf(opt.log,"engine: start"LF); } #if HTS_ANALYSTE - if (!hts_htmlcheck_start(&opt)) { - XH_extuninit; - return 1; + if (hts_htmlcheck_start != NULL) { + if (!hts_htmlcheck_start(&opt)) { + XH_extuninit; + return 1; + } } set_wrappers(); // _start() is allowed to set other wrappers #endif @@ -927,9 +902,17 @@ int httpmirror(char* url1,httrackp* ptropt) { ) ) { // sauter si lien annulé (ou fil vide) if ((opt.debug>1) && (opt.log!=NULL)) { - fspc(opt.log,"debug"); fprintf(opt.log,"link #%d seems ready, skipping: %s%s.."LF,ptr,((urladr != NULL)?(urladr):(" ")),((urlfil != NULL)?(urlfil):(" "))); + if (liens[ptr] != NULL && liens[ptr]->pass2 == -1) { + fspc(opt.log,"debug"); fprintf(opt.log,"link #%d is ready, skipping: %s%s.."LF,ptr,((urladr != NULL)?(urladr):(" ")),((urlfil != NULL)?(urlfil):(" "))); + } else { + fspc(opt.log,"debug"); fprintf(opt.log,"link #%d seems ready, skipping: %s%s.."LF,ptr,((urladr != NULL)?(urladr):(" ")),((urlfil != NULL)?(urlfil):(" "))); + } test_flush; } + // remove from stats + if (liens[ptr]->pass2 == -1) { + HTS_STAT.stat_background--; + } ptr++; } } @@ -983,8 +966,7 @@ int httpmirror(char* url1,httrackp* ptropt) { /* */ str.liens = liens; str.opt = &opt; - str.back = back; - str.back_max = back_max; + str.sback = sback; str.cache = &cache; str.hashptr = hashptr; str.numero_passe = numero_passe; @@ -1105,7 +1087,8 @@ int httpmirror(char* url1,httrackp* ptropt) { if (r.statuscode == 200) { // OK (ou 304 en backing) if (r.adr) { // Written file if ( (is_hypertext_mime(r.contenttype, urlfil)) /* Is HTML or Js, .. */ - || (may_be_hypertext_mime(r.contenttype, urlfil) && (r.adr) ) /* Is real media, .. */ + /* NO - real media is real media, and mms is mms, not HTML */ + /*|| (may_be_hypertext_mime(r.contenttype, urlfil) && (r.adr) )*/ /* Is real media, .. */ ) { if (strnotempty(r.cdispo)) { // Content-disposition set! if (ishtml(savename) == 0) { // Non HTML!! @@ -1135,7 +1118,7 @@ int httpmirror(char* url1,httrackp* ptropt) { nspec += map[i]; } } - /* On-the-fly UCS2 to ISO-8859-1 conversion (note: UCS2 should never be used on the net) */ + /* On-the-fly UCS2 to UTF-8 conversion (note: UCS2 should never be used on the net) */ if ( map[0] > r.size/10 && @@ -1143,45 +1126,116 @@ int httpmirror(char* url1,httrackp* ptropt) { && ( ( ((unsigned char) r.adr[0]) == 0xff && ((unsigned char) r.adr[1]) == 0xfe) - || - ( ((unsigned char) r.adr[0]) == 0xfe && ((unsigned char) r.adr[1]) == 0xff) - ) - ) { - int lost=0; - int i; - int swap = (r.adr[0] == 0xff); - for(i = 0 ; i < r.size / 2 - 1 ; i++) { - unsigned int unic = 0; - if (swap) - unic = (r.adr[i*2 + 2] << 8) + r.adr[i*2 + 2 + 1]; - else - unic = r.adr[i*2 + 2] + (r.adr[i*2 + 2 + 1] << 8); - if (unic <= 255) - r.adr[i] = (char) unic; - else { - r.adr[i] = '?'; - lost++; - } - } - r.size = r.size / 2 - 1; - r.adr[r.size] = '\0'; - - if (opt.errlog) { - fspc(opt.errlog,"warning"); fprintf(opt.errlog,"File %s%s converted from UCS2 to 8-bit, %d characters lost during conversion (better to use UTF-8)"LF, urladr, urlfil, lost); - test_flush; - } - } else if ((nspec > r.size / 100) && (nspec > 10)) { // too many special characters - strcpybuff(r.contenttype,"application/octet-stream"); - if (opt.errlog) { - fspc(opt.errlog,"warning"); fprintf(opt.errlog,"File not parsed, looks like binary: %s%s"LF,urladr,urlfil); - test_flush; - } - } - - /* This hack allows to avoid problems with parsing '\0' characters */ - for(i = 0 ; i < r.size ; i++) { - if (r.adr[i] == '\0') r.adr[i] = ' '; - } + || + ( ((unsigned char) r.adr[0]) == 0xfe && ((unsigned char) r.adr[1]) == 0xff) + ) + ) + { +#define CH_ADD(c) do { \ + if (new_offs + 1 > new_capa) { \ + new_capa *= 2; \ + new_adr = (unsigned char*) realloct(new_adr, \ + new_capa); \ + assertf(new_adr != NULL); \ + } \ + new_adr[new_offs++] = (unsigned char) (c); \ +} while(0) +#define CH_ADD_RNG1(c, r, o) do { \ + CH_ADD( (c) / (r) + (o) ); \ + c = (c) % (r); \ +} while(0) +#define CH_ADD_RNG0(c, o) do { \ + CH_ADD_RNG1(c, 1, o); \ +} while(0) +#define CH_ADD_RNG2(c, r, r2, o) do { \ + CH_ADD_RNG1(c, (r) * (r2), o); \ +} while(0) + int new_capa = r.size / 2 + 1; + int new_offs = 0; + unsigned char* prev_adr = (unsigned char*) r.adr; + unsigned char* new_adr = (unsigned char*) malloct(new_capa); + int i; + int swap = (((unsigned char)r.adr[0]) == 0xff); + assertf(new_adr != NULL); + /* + See http://www.unicode.org/reports/tr28/tr28-3.html#conformance + U+0000..U+007F 00..7F + U+0080..U+07FF C2..DF 80..BF + U+0800..U+0FFF E0 A0..BF 80..BF + U+1000..U+CFFF E1..EC 80..BF 80..BF + U+D000..U+D7FF ED 80..9F 80..BF + U+D800..U+DFFF + U+E000..U+FFFF EE..EF 80..BF 80..BF + */ + for(i = 0 ; i < r.size / 2 ; i++) { + unsigned short int unic = 0; + if (swap) + unic = prev_adr[i*2] + (prev_adr[i*2 + 1] << 8); + else + unic = (prev_adr[i*2] << 8) + prev_adr[i*2 + 1]; + if (unic <= 0x7F) { + /* U+0000..U+007F 00..7F */ + CH_ADD_RNG0( unic, 0x00 ); + } else if (unic <= 0x07FF) { + /* U+0080..U+07FF C2..DF 80..BF */ + unic -= 0x0080; + CH_ADD_RNG1( unic, 0xbf - 0x80 + 1, 0xc2 ); + CH_ADD_RNG0( unic, 0x80 ); + } else if (unic <= 0x0FFF) { + /* U+0800..U+0FFF E0 A0..BF 80..BF */ + unic -= 0x0800; + CH_ADD_RNG2( unic, 0xbf - 0x80 + 1, 0xbf - 0xa0 + 1, 0xe0 ); + CH_ADD_RNG1( unic, 0xbf - 0x80 + 1, 0xa0 ); + CH_ADD_RNG0( unic, 0x80 ); + } else if (unic <= 0xCFFF) { + /* U+1000..U+CFFF E1..EC 80..BF 80..BF */ + unic -= 0x1000; + CH_ADD_RNG2( unic, 0xbf - 0x80 + 1, 0xbf - 0x80 + 1, 0xe1 ); + CH_ADD_RNG1( unic, 0xbf - 0x80 + 1, 0x80 ); + CH_ADD_RNG0( unic, 0x80 ); + } else if (unic <= 0xD7FF) { + /* U+D000..U+D7FF ED 80..9F 80..BF */ + unic -= 0xD000; + CH_ADD_RNG2( unic, 0xbf - 0x80 + 1, 0x9f - 0x80 + 1, 0xed ); + CH_ADD_RNG1( unic, 0xbf - 0x80 + 1, 0x80 ); + CH_ADD_RNG0( unic, 0x80 ); + } else if (unic <= 0xDFFF) { + /* U+D800..U+DFFF */ + CH_ADD('?'); + /* ill-formed */ + } else if (unic <= 0xFFFF) { + /* U+E000..U+FFFF EE..EF 80..BF 80..BF */ + unic -= 0xE000; + CH_ADD_RNG2( unic, 0xbf - 0x80 + 1, 0xbf - 0x80 + 1, 0xee ); + CH_ADD_RNG1( unic, 0xbf - 0x80 + 1, 0x80 ); + CH_ADD_RNG0( unic, 0x80 ); + } + } + if (opt.errlog) { + fspc(opt.errlog,"warning"); fprintf(opt.errlog,"File %s%s converted from UCS2 to UTF-8 (old size: %d bytes, new size: %d bytes)"LF, urladr, urlfil, (int)r.size, new_offs); + test_flush; + } + freet(r.adr); + r.adr = NULL; + r.size = new_offs; + CH_ADD(0); + r.adr = (char*) new_adr; +#undef CH_ADD +#undef CH_ADD_RNG0 +#undef CH_ADD_RNG1 +#undef CH_ADD_RNG2 + } else if ((nspec > r.size / 100) && (nspec > 10)) { // too many special characters + strcpybuff(r.contenttype,"application/octet-stream"); + if (opt.errlog) { + fspc(opt.errlog,"warning"); fprintf(opt.errlog,"File not parsed, looks like binary: %s%s"LF,urladr,urlfil); + test_flush; + } + } + + /* This hack allows to avoid problems with parsing '\0' characters */ + for(i = 0 ; i < r.size ; i++) { + if (r.adr[i] == '\0') r.adr[i] = ' '; + } } @@ -1189,45 +1243,49 @@ int httpmirror(char* url1,httrackp* ptropt) { } } + // MOVED IN back_finalize() + // // -------------------- // REAL MEDIA HACK // Check if we have to load locally the file // -------------------- - if (!error) { - if (r.statuscode == 200) { // OK (ou 304 en backing) - if (r.adr==NULL) { // Written file - if (may_be_hypertext_mime(r.contenttype, urlfil)) { // to parse! - LLint sz; - sz=fsize(savename); - if (sz>0) { // ok, exists! - if (sz < 8192) { // ok, small file --> to parse! - FILE* fp=fopen(savename,"rb"); - if (fp) { - r.adr=malloct((int)sz + 2); - if (r.adr) { - if (fread(r.adr,1,(INTsys)sz,fp) == sz) { - r.size=sz; - } else { - freet(r.adr); - r.size=0; - r.adr = NULL; - r.statuscode=-1; - strcpybuff(r.msg, ".RAM read error"); - } - fclose(fp); - fp=NULL; - // remove (temporary) file! - remove(savename); - } - if (fp) - fclose(fp); - } - } - } - } - } - } - } + //if (!error) { + // if (r.statuscode == 200) { // OK (ou 304 en backing) + // if (r.adr==NULL) { // Written file + // if (may_be_hypertext_mime(r.contenttype, urlfil)) { // to parse! + // LLint sz; + // sz=fsize(savename); + // if (sz>0) { // ok, exists! + // if (sz < 8192) { // ok, small file --> to parse! + // FILE* fp=fopen(savename,"rb"); + // if (fp) { + // r.adr=malloct((int)sz + 2); + // if (r.adr) { + // if (fread(r.adr,1,(INTsys)sz,fp) == sz) { + // r.size=sz; + // r.adr[sz] = '\0'; + // r.is_write = 0; + // } else { + // freet(r.adr); + // r.size=0; + // r.adr = NULL; + // r.statuscode=STATUSCODE_INVALID; + // strcpybuff(r.msg, ".RAM read error"); + // } + // fclose(fp); + // fp=NULL; + // // remove (temporary) file! + // remove(savename); + // } + // if (fp) + // fclose(fp); + // } + // } + // } + // } + // } + // } + //} // EN OF REAL MEDIA HACK @@ -1271,8 +1329,7 @@ int httpmirror(char* url1,httrackp* ptropt) { /* */ str.liens = liens; str.opt = &opt; - str.back = back; - str.back_max = back_max; + str.sback = sback; str.cache = &cache; str.hashptr = hashptr; str.numero_passe = numero_passe; @@ -1335,6 +1392,29 @@ int httpmirror(char* url1,httrackp* ptropt) { } #endif + /* Load file if necessary */ + if ( + is_hypertext_mime(r.contenttype, urlfil) /* Is HTML or Js, .. */ + && (liens[ptr]->depth>0) /* Depth > 0 (recurse depth) */ + && (r.adr==NULL) /* HTML Data exists */ + && (!store_errpage) /* Not an html error page */ + && (savename[0]!='\0') /* Output filename exists */ + ) + { + r.adr = readfile2(savename, &r.size); + (void) unlink(fconv(savename)); + if (r.adr != NULL) { + if ( (opt.debug>0) && (opt.log!=NULL) ) { + fspc(opt.log,"info"); fprintf(opt.log,"File successfully loaded for parsing: %s%s (%d bytes)"LF,urladr,urlfil,(int)r.size); + test_flush; + } + } else { + if ( opt.log != NULL ) { + fspc(opt.log,"error"); fprintf(opt.log,"File could not be loaded for parsing: %s%s"LF,urladr,urlfil); + test_flush; + } + } + } // ------------------------------------------------------ // ok, fichier chargé localement @@ -1344,7 +1424,7 @@ int httpmirror(char* url1,httrackp* ptropt) { #if DEBUG_CHECKINT { int i; - for(i=0;i<back_max;i++) { + for(i = 0 ; i < sback->count ; i++) { char si[256]; sprintf(si,"Test global après back_wait, index %d",i); _CHECKINT(&back[i],si) @@ -1384,7 +1464,7 @@ int httpmirror(char* url1,httrackp* ptropt) { // traiter if ( ( (is_hypertext_mime(r.contenttype, urlfil)) /* Is HTML or Js, .. */ - || (may_be_hypertext_mime(r.contenttype, urlfil) && (r.adr) ) /* Is real media, .. */ + || (may_be_hypertext_mime(r.contenttype, urlfil) && r.adr != NULL ) /* Is real media, .. */ ) && (liens[ptr]->depth>0) /* Depth > 0 (recurse depth) */ && (r.adr!=NULL) /* HTML Data exists */ @@ -1418,8 +1498,7 @@ int httpmirror(char* url1,httrackp* ptropt) { /* */ str.liens = liens; str.opt = &opt; - str.back = back; - str.back_max = back_max; + str.sback = sback; str.cache = &cache; str.hashptr = hashptr; str.numero_passe = numero_passe; @@ -1590,7 +1669,10 @@ int httpmirror(char* url1,httrackp* ptropt) { while(is_realspace(*a)) a++; // sauter espace(s) if (strnotempty(a)) { - if (strcmp(a,"/") != 0 || opt.robots >= 3) { /* ignoring disallow: / */ +#ifdef IGNORE_RESTRICTIVE_ROBOTS + if (strcmp(a,"/") != 0 || opt.robots >= 3) +#endif + { /* ignoring disallow: / */ if ( (strlen(buff) + strlen(a) + 8) < sizeof(buff)) { strcatbuff(buff,a); strcatbuff(buff,"\n"); @@ -1599,12 +1681,15 @@ int httpmirror(char* url1,httrackp* ptropt) { strcatbuff(infobuff,a); } } - } else { + } +#ifdef IGNORE_RESTRICTIVE_ROBOTS + else { if (opt.errlog!=NULL) { fspc(opt.errlog,"info"); fprintf(opt.errlog,"Note: %s robots.txt rules are too restrictive, ignoring /"LF,urladr); test_flush; } } +#endif } } } @@ -1697,9 +1782,11 @@ int httpmirror(char* url1,httrackp* ptropt) { // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!! if (r.adr) { + file_notify(urladr,urlfil, savename, 1, 1, r.notmodified); if (filesave(&opt,r.adr,(int)r.size,savename,urladr,urlfil)!=0) { int fcheck; if ((fcheck=check_fatal_io_errno())) { + fspc(opt.log,"error"); fprintf(opt.log,"Mirror aborted: disk full or filesystem problems"LF); test_flush; exit_xh=-1; /* fatal error */ } if (opt.errlog) { @@ -1764,8 +1851,7 @@ int httpmirror(char* url1,httrackp* ptropt) { /* */ str.liens = liens; str.opt = &opt; - str.back = back; - str.back_max = back_max; + str.sback = sback; str.cache = &cache; str.hashptr = hashptr; str.numero_passe = numero_passe; @@ -1799,9 +1885,7 @@ int httpmirror(char* url1,httrackp* ptropt) { usercommand(&opt, 0, NULL, savename, urladr, urlfil); } - } // if !error - jump_if_done: // libérer les liens @@ -2114,7 +2198,7 @@ int engine_stats(void) { if (!debug_fp) debug_fp=fopen("esstat.txt","wb"); #endif - HTS_STAT.stat_nsocket=HTS_STAT.stat_errors=HTS_STAT.nbk==0; + HTS_STAT.stat_nsocket=HTS_STAT.stat_errors=HTS_STAT.nbk=0; HTS_STAT.nb=0; if (HTS_STAT.HTS_TOTAL_RECV>2048) { TStamp cdif=mtime_local(); @@ -2162,7 +2246,9 @@ fprintf(debug_fp,"resync timer 1\n"); fflush(debug_fp); #define _ROBOTS ((robots_wizard*)opt->robotsptr) // bannir host (trop lent etc) -void host_ban(httrackp* opt,lien_url** liens,int ptr,int lien_tot,lien_back* back,int back_max,char* host) { +void host_ban(httrackp* opt,lien_url** liens,int ptr,int lien_tot,struct_back* sback,char* host) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; //int l; int i; @@ -2211,9 +2297,10 @@ void host_ban(httrackp* opt,lien_url** liens,int ptr,int lien_tot,lien_back* bac DEBUG_W("host control: deletehttp\n"); #endif back[i].status=0; // terminé + back_set_finished(sback, i); if (back[i].r.soc!=INVALID_SOCKET) deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=-2; // timeout (peu importe si c'est un traffic jam) + back[i].r.statuscode=STATUSCODE_TIMEOUT; // timeout (peu importe si c'est un traffic jam) strcpybuff(back[i].r.msg,"Link Cancelled by host control"); if ((opt->debug>1) && (opt->log!=NULL)) { @@ -2237,10 +2324,6 @@ void host_ban(httrackp* opt,lien_url** liens,int ptr,int lien_tot,lien_back* bac fprintf(opt->log,"Cancel: %s%s"LF,liens[i]->adr,liens[i]->fil); test_flush; } strcpybuff(liens[i]->adr,"!"); // cancel (invalide hash) -#if HTS_HASH -#else - liens[i]->sav_len=-1; // taille invalide -#endif // on efface pas le hash, because si on rencontre le lien, reverif sav.. } } else { @@ -2419,7 +2502,6 @@ int filesave(httrackp* opt,char* adr,int len,char* s,char* url_adr,char* url_fil nl=(int) fwrite(adr,1,(INTsys)len,fp); } fclose(fp); - //xxusercommand(opt,0,NULL,fconv(s),url_adr,url_fil); if (nl!=len) // erreur return -1; } else @@ -2487,6 +2569,42 @@ FILE* filecreate(char* s) { return fp; } +// ouvrir un fichier (avec chemin Un*x) +FILE* fileappend(char* s) { + char BIGSTK fname[HTS_URLMAXSIZE*2]; + FILE* fp; + fname[0]='\0'; + + // noter lst + filenote(s,NULL); + + // if (*s=='/') strcpybuff(fname,s+1); else strcpybuff(fname,s); // pas de / (root!!) // ** SIIIIIII!!! à cause de -O <path> + strcpybuff(fname,s); + +#if HTS_DOSNAME + // remplacer / par des slash arrière + { + int i=0; + while(fname[i]) { + if (fname[i]=='/') + fname[i]='\\'; + i++; + } + } + // a partir d'ici le slash devient antislash +#endif + + // ouvrir + fp=fopen(fname,"ab"); + +#if HTS_WIN==0 + if (fp!=NULL) chmod(fname,HTS_ACCESS_FILE); +#endif + + return fp; +} + + // create an empty file int filecreateempty(char* filename) { FILE* fp; @@ -2528,6 +2646,14 @@ int filenote(char* s,filecreate_params* params) { return 1; } +void file_notify(char* adr,char* fil,char* save,int create,int modify,int not_updated) { +#if HTS_ANALYSTE + if (hts_htmlcheck_filesave2 != NULL) { + hts_htmlcheck_filesave2(adr, fil, save, create, modify, not_updated); + } +#endif +} + // executer commande utilisateur static void postprocess_file(httrackp* opt,char* save, char* adr, char* fil); typedef struct { @@ -2551,9 +2677,10 @@ HTS_INLINE void usercommand(httrackp* opt,int _exe,char* _cmd,char* file,char* a postprocess_file(opt, file, adr, fil); #if HTS_ANALYSTE - if (hts_htmlcheck_filesave != NULL) - if (file != NULL && strnotempty(file)) - hts_htmlcheck_filesave(file); + if (hts_htmlcheck_filesave != NULL) { + if (file != NULL && strnotempty(file)) + hts_htmlcheck_filesave(file); + } #endif if (strc->exe) { @@ -2755,9 +2882,10 @@ void check_rate(TStamp stat_timestart,int maxrate) { // --- // sous routines liées au moteur et au backing -// supplemental links ready (done) after ptr -int backlinks_done(lien_url** liens,int lien_tot,int ptr) { +// supplemental links ready (done) after ptr or ready in background +int backlinks_done(struct_back* sback,lien_url** liens,int lien_tot,int ptr) { int n=0; +#if 0 int i; //Links done and stored in cache for(i=ptr+1;i<lien_tot;i++) { @@ -2767,21 +2895,28 @@ int backlinks_done(lien_url** liens,int lien_tot,int ptr) { } } } +#else + // finalized in background + n+=HTS_STAT.stat_background; +#endif + n+=back_done_incache(sback); return n; } // remplir backing si moins de max_bytes en mémoire -HTS_INLINE int back_fillmax(lien_back* back,int back_max,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot) { +HTS_INLINE int back_fillmax(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot) { if (!opt->state.stop) { - if (back_incache(back,back_max)<opt->maxcache) { // pas trop en mémoire? - return back_fill(back,back_max,opt,cache,liens,ptr,numero_passe,lien_tot); + if (back_incache(sback)<opt->maxcache) { // pas trop en mémoire? + return back_fill(sback,opt,cache,liens,ptr,numero_passe,lien_tot); } } return -1; /* plus de place */ } -int back_pluggable_sockets_strict(lien_back* back, int back_max, httrackp* opt) { - int n = opt->maxsoc - back_nsoc(back, back_max); +int back_pluggable_sockets_strict(struct_back* sback, httrackp* opt) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + int n = opt->maxsoc - back_nsoc(sback); // connect limiter if (n > 0 && opt->maxconn > 0 && HTS_STAT.last_connect > 0) { @@ -2800,25 +2935,31 @@ int back_pluggable_sockets_strict(lien_back* back, int back_max, httrackp* opt) return n; } -int back_pluggable_sockets(lien_back* back, int back_max, httrackp* opt) { +int back_pluggable_sockets(struct_back* sback, httrackp* opt) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; int n; // ajouter autant de socket qu'on peut ajouter - n=back_pluggable_sockets_strict(back, back_max, opt); + n=back_pluggable_sockets_strict(sback, opt); // vérifier qu'il restera assez de place pour les tests ensuite (en théorie, 1 entrée libre restante suffirait) - n=min( n, back_available(back,back_max) - 8 ); + n=min( n, back_available(sback) - 8 ); // no space left on backing stack - do not back anymore - if (back_stack_available(back,back_max) <= 2) + if (back_stack_available(sback) <= 2) n=0; return n; } // remplir backing -int back_fill(lien_back* back,int back_max,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot) { - int n = back_pluggable_sockets(back, back_max, opt); +int back_fill(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot) { + lien_back* const back = sback->lnk; + const int back_max = sback->count; + int n = back_pluggable_sockets(sback, opt); + if (opt->savename_delayed == 2 && !opt->delayed_cached) /* cancel (always delayed) */ + return 0; if (n>0) { int p; @@ -2846,8 +2987,9 @@ int back_fill(lien_back* back,int back_max,httrackp* opt,cache_back* cache,lien_ // note: si un backing est fini, il reste en mémoire jusqu'à ce que // le ptr l'atteigne if (ok) { - if (!back_exist(back,back_max,liens[p]->adr,liens[p]->fil,liens[p]->sav)) { - if (back_add(back,back_max,opt,cache,liens[p]->adr,liens[p]->fil,liens[p]->sav,liens[liens[p]->precedent]->adr,liens[liens[p]->precedent]->fil,liens[p]->testmode,&liens[p]->pass2)==-1) { + int index = back_index(sback, liens[p]->adr,liens[p]->fil,liens[p]->sav); + if (index < 0) { + if (back_add(sback,opt,cache,liens[p]->adr,liens[p]->fil,liens[p]->sav,liens[liens[p]->precedent]->adr,liens[liens[p]->precedent]->fil,liens[p]->testmode,&liens[p]->pass2)==-1) { if ( (opt->debug>1) && (opt->errlog!=NULL) ) { fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: unable to add more links through back_add for back_fill"LF); test_flush; @@ -2862,7 +3004,9 @@ int back_fill(lien_back* back,int back_max,httrackp* opt,cache_back* cache,lien_ printf("backing: %s%s\n",liens[p]->adr,liens[p]->fil); #endif } - } + } else { + back_set_passe2_ptr(opt,cache,sback,index,&liens[p]->pass2); + } } p++; } // while @@ -3328,7 +3472,12 @@ HTSEXT_API int copy_htsopt(httrackp* from,httrackp* to) { if (from->maxtime > -1) to->maxtime = from->maxtime; - + +#if HTS_USEMMS + if (from->mms_maxtime > -1) + to->mms_maxtime = from->mms_maxtime; +#endif + if (from->maxrate > -1) to->maxrate = from->maxrate; @@ -3369,20 +3518,7 @@ HTSEXT_API int copy_htsopt(httrackp* from,httrackp* to) { /* External modules callback */ int htsAddLink(htsmoduleStruct* str, char* link) { if (link != NULL && str != NULL && link[0] != '\0') { - lien_url** liens = (lien_url**) str->liens; - httrackp* opt = (httrackp*) str->opt; - lien_back* back = (lien_back*) str->back; - cache_back* cache = (cache_back*) str->cache; - hash_struct* hashptr = (hash_struct*) str->hashptr; - int back_max = str->back_max; - int numero_passe = str->numero_passe; - int add_tab_alloc = str->add_tab_alloc; - /* */ - int lien_tot = * ( (int*) (str->lien_tot_) ); - int ptr = * ( (int*) (str->ptr_) ); - int lien_size = * ( (int*) (str->lien_size_) ); - char* lien_buffer = * ( (char**) (str->lien_buffer_) ); - /* */ + ENGINE_LOAD_CONTEXT_BASE(); /* */ char BIGSTK adr[HTS_URLMAXSIZE*2], fil[HTS_URLMAXSIZE*2], @@ -3402,13 +3538,20 @@ int htsAddLink(htsmoduleStruct* str, char* link) { // #if HTS_ANALYSTE - if (!hts_htmlcheck_linkdetected(link) || !hts_htmlcheck_linkdetected2(link, NULL)) { - if (opt->errlog) { - fspc(opt->errlog,"error"); fprintf(opt->errlog,"Link %s refused by external wrapper"LF, link); - test_flush; + if (hts_htmlcheck_linkdetected != NULL && !hts_htmlcheck_linkdetected(link)) { + if (opt->errlog) { + fspc(opt->errlog,"error"); fprintf(opt->errlog,"Link %s refused by external wrapper"LF, link); + test_flush; + } + return 0; + } + if (hts_htmlcheck_linkdetected2 != NULL && !hts_htmlcheck_linkdetected2(link, NULL)) { + if (opt->errlog) { + fspc(opt->errlog,"error"); fprintf(opt->errlog,"Link %s refused by external wrapper(2)"LF, link); + test_flush; + } + return 0; } - return 0; - } #endif // adr = c'est la même @@ -3481,10 +3624,22 @@ int htsAddLink(htsmoduleStruct* str, char* link) { opt->savename_type=0; opt->savename_83=0; // note: adr,fil peuvent être patchés - r=url_savename(adr,fil,save,NULL,NULL,NULL,NULL,opt,liens,lien_tot,back,back_max,cache,hashptr,ptr,numero_passe); + r=url_savename(adr,fil,save,NULL,NULL,NULL,NULL,opt,liens,lien_tot,sback,cache,hashptr,ptr,numero_passe,NULL); + // resolve unresolved type + if (r != -1 + && forbidden_url == 0 + && IS_DELAYED_EXT(save) + ) + { // pas d'erreur, on continue + char BIGSTK former_adr[HTS_URLMAXSIZE*2]; + char BIGSTK former_fil[HTS_URLMAXSIZE*2]; + former_adr[0] = former_fil[0] = '\0'; + r = hts_wait_delayed(str, adr, fil, save, former_adr, former_fil, &forbidden_url); + } + // end resolve unresolved type opt->savename_type=a; opt->savename_83=b; - if (r != -1) { + if (r != -1 && !forbidden_url) { if (savename) { if (lienrelatif(tempo,save,savename)==0) { if ((opt->debug>1) && (opt->log!=NULL)) { @@ -3497,7 +3652,9 @@ int htsAddLink(htsmoduleStruct* str, char* link) { } } } - } else { + } + + if (forbidden_url) { if ((opt->debug>1) && (opt->log!=NULL)) { fspc(opt->log,"debug"); fprintf(opt->log,"(module): file not caught: %s"LF,lien); test_flush; } @@ -3510,6 +3667,7 @@ int htsAddLink(htsmoduleStruct* str, char* link) { } r=-1; } + // if (r != -1) { if ((opt->debug>1) && (opt->log!=NULL)) { @@ -3523,30 +3681,13 @@ int htsAddLink(htsmoduleStruct* str, char* link) { // au fichier est la plus grande des deux priorités // // On part de la fin et on essaye de se presser (économise temps machine) -#if HTS_HASH { int i=hash_read(hashptr,save,"",0,opt->urlhack); // lecture type 0 (sav) if (i>=0) { liens[i]->depth=maximum(liens[i]->depth,prio_fix); dejafait=1; } - } -#else - { - int l; - int i; - l=strlen(save); - for(i=lien_tot-1;(i>=0) && (dejafait==0);i--) { - if (liens[i]->sav_len==l) { // même taille de chaîne - if (strcmp(liens[i]->sav,save)==0) { // existe déja - liens[i]->depth=maximum(liens[i]->depth,prio_fix); - dejafait=1; - } - } - } - } -#endif - + } if (!dejafait) { // @@ -3600,10 +3741,8 @@ int htsAddLink(htsmoduleStruct* str, char* link) { } /* Apply changes */ - * ( (int*) (str->lien_tot_) ) = lien_tot; - * ( (int*) (str->ptr_) ) = ptr; - * ( (int*) (str->lien_size_) ) = lien_size; - * ( (char**) (str->lien_buffer_) ) = lien_buffer; + ENGINE_SAVE_CONTEXT_BASE(); + return (forbidden_url == 0); } return 0; diff --git a/src/htscore.h b/src/htscore.h index 97c0127..e1966d3 100644 --- a/src/htscore.h +++ b/src/htscore.h @@ -94,11 +94,7 @@ typedef struct lien_url { char* former_adr; // adresse initiale (avant éventuel moved), peut être nulle char* former_fil; // nom du fichier distant initial (avant éventuel moved), peut être nul // pour optimisation: -#if HTS_HASH int hash_next[3]; // prochain lien avec même valeur hash -#else - int sav_len; // taille de sav -#endif } lien_url; // chargement de fichiers en 'arrière plan' @@ -116,6 +112,7 @@ typedef struct lien_back { char tmpfile_buffer[HTS_URLMAXSIZE*2]; // buffer pour le nom à sauver temporairement char send_too[1024]; // données à envoyer en même temps que le header int status; // status (-1=non utilisé, 0: prêt, >0: opération en cours) + int locked; // locked (to be used soon) int testmode; // mode de test int timeout; // gérer des timeouts? (!=0 : nombre de secondes) TStamp timeout_refresh; // si oui, time refresh @@ -146,6 +143,12 @@ typedef struct lien_back { #endif } lien_back; +typedef struct struct_back { + lien_back* lnk; + int count; + void* ready; +} struct_back; + typedef struct cache_back_zip_entry cache_back_zip_entry; // cache @@ -183,11 +186,6 @@ typedef struct hash_struct { int hash[3][HTS_HASH_SIZE]; // tables pour sav/adr-fil/former_adr-former_fil } hash_struct; -#if HTS_HASH -#else -#define hash_write(A,B) -#endif - typedef struct filecreate_params { FILE* lst; char path[HTS_URLMAXSIZE*2]; @@ -277,10 +275,12 @@ typedef int (* t_hts_htmlcheck)(char* html,int len,char* url_adresse,char* url typedef char* (* t_hts_htmlcheck_query)(char* question); typedef char* (* t_hts_htmlcheck_query2)(char* question); typedef char* (* t_hts_htmlcheck_query3)(char* question); -typedef int (* t_hts_htmlcheck_loop)(lien_back* back,int back_max,int back_index,int lien_tot,int lien_ntot,int stat_time,hts_stat_struct* stats); +typedef int (* t_hts_htmlcheck_loop)(struct_back* sback,int back_index,int lien_tot,int lien_ntot,int stat_time,hts_stat_struct* stats); typedef int (* t_hts_htmlcheck_check)(char* adr,char* fil,int status); +typedef int (* t_hts_htmlcheck_check_mime)(char* adr,char* fil,char* mime,int status); typedef void (* t_hts_htmlcheck_pause)(char* lockfile); typedef void (* t_hts_htmlcheck_filesave)(char* file); +typedef void (* t_hts_htmlcheck_filesave2)(char* hostname,char* filename,char* localfile,int is_new,int is_modified, int not_updated); typedef int (* t_hts_htmlcheck_linkdetected)(char* link); typedef int (* t_hts_htmlcheck_linkdetected2)(char* link, char* tag_start); typedef int (* t_hts_htmlcheck_xfrstatus)(lien_back* back); @@ -306,8 +306,10 @@ extern t_hts_htmlcheck_query2 hts_htmlcheck_query2; extern t_hts_htmlcheck_query3 hts_htmlcheck_query3; extern t_hts_htmlcheck_loop hts_htmlcheck_loop; extern t_hts_htmlcheck_check hts_htmlcheck_check; +extern t_hts_htmlcheck_check_mime hts_htmlcheck_check_mime; extern t_hts_htmlcheck_pause hts_htmlcheck_pause; extern t_hts_htmlcheck_filesave hts_htmlcheck_filesave; +extern t_hts_htmlcheck_filesave2 hts_htmlcheck_filesave2; extern t_hts_htmlcheck_linkdetected hts_htmlcheck_linkdetected; extern t_hts_htmlcheck_linkdetected2 hts_htmlcheck_linkdetected2; extern t_hts_htmlcheck_xfrstatus hts_htmlcheck_xfrstatus; @@ -356,10 +358,12 @@ int httpmirror(char* url1,httrackp* opt); int filesave(httrackp* opt,char* adr,int len,char* s,char* url_adr /* = NULL */,char* url_fil /* = NULL */); int check_fatal_io_errno(void); int engine_stats(void); -void host_ban(httrackp* opt,lien_url** liens,int ptr,int lien_tot,lien_back* back,int back_max,char* host); +void host_ban(httrackp* opt,lien_url** liens,int ptr,int lien_tot,struct_back* sback,char* host); FILE* filecreate(char* s); +FILE* fileappend(char* s); int filecreateempty(char* filename); int filenote(char* s,filecreate_params* params); +void file_notify(char* adr,char* fil,char* save,int create,int modify,int wasupdated); HTS_INLINE void usercommand(httrackp* opt,int exe,char* cmd,char* file,char* adr,char* fil); void usercommand_exe(char* cmd,char* file); //void* structcheck_init(int init); @@ -371,6 +375,7 @@ HTS_INLINE int fspc(FILE* fp,char* type); char* next_token(char* p,int flag); // char* readfile(char* fil); +char* readfile2(char* fil, LLint* size); char* readfile_or(char* fil,char* defaultdata); #if 0 void check_rate(TStamp stat_timestart,int maxrate); @@ -381,11 +386,11 @@ int liens_record(char* adr,char* fil,char* save,char* former_adr,char* former_fi // backing, routines externes -int back_pluggable_sockets(lien_back* back, int back_max, httrackp* opt); -int back_pluggable_sockets_strict(lien_back* back, int back_max, httrackp* opt); -int back_fill(lien_back* back,int back_max,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot); -int backlinks_done(lien_url** liens,int lien_tot,int ptr); -int back_fillmax(lien_back* back,int back_max,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot); +int back_pluggable_sockets(struct_back* sback, httrackp* opt); +int back_pluggable_sockets_strict(struct_back* sback, httrackp* opt); +int back_fill(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot); +int backlinks_done(struct_back* sback,lien_url** liens,int lien_tot,int ptr); +int back_fillmax(struct_back* sback,httrackp* opt,cache_back* cache,lien_url** liens,int ptr,int numero_passe,int lien_tot); // cancel file #if HTS_ANALYSTE diff --git a/src/htscoremain.c b/src/htscoremain.c index bd90593..bdebfb0 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -49,6 +49,11 @@ Please visit our Website: http://www.httrack.com #include "htszlib.h" #include <ctype.h> +#if USE_BEGINTHREAD +#if HTS_WIN +#include <process.h> +#endif +#endif #if HTS_WIN #else #ifndef HTS_DO_NOT_USE_UID @@ -120,8 +125,10 @@ void set_wrappers(void) { hts_htmlcheck_query3 = (t_hts_htmlcheck_query3) htswrap_read("query3"); hts_htmlcheck_loop = (t_hts_htmlcheck_loop) htswrap_read("loop"); hts_htmlcheck_check = (t_hts_htmlcheck_check) htswrap_read("check-link"); + hts_htmlcheck_check_mime = (t_hts_htmlcheck_check_mime) htswrap_read("check-mime"); hts_htmlcheck_pause = (t_hts_htmlcheck_pause) htswrap_read("pause"); hts_htmlcheck_filesave = (t_hts_htmlcheck_filesave) htswrap_read("save-file"); + hts_htmlcheck_filesave2 = (t_hts_htmlcheck_filesave2) htswrap_read("save-file2"); hts_htmlcheck_linkdetected = (t_hts_htmlcheck_linkdetected) htswrap_read("link-detected"); hts_htmlcheck_linkdetected2 = (t_hts_htmlcheck_linkdetected2) htswrap_read("link-detected2"); hts_htmlcheck_xfrstatus = (t_hts_htmlcheck_xfrstatus) htswrap_read("transfer-status"); @@ -204,6 +211,8 @@ int main(int argc, char **argv) { strcpybuff(httrack.from, ""); httrack.savename_83=0; // noms longs par défaut httrack.savename_type=0; // avec structure originale + httrack.savename_delayed=2;// hard delayed type (default) + httrack.delayed_cached=1; // cached delayed type (default) httrack.mimehtml=0; // pas MIME-html httrack.parsejava=1; // parser classes httrack.hostcontrol=0; // PAS de control host pour timeout et traffic jammer @@ -254,7 +263,10 @@ int main(int argc, char **argv) { httrack.maxcache=1048576*32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT -- //httrack.maxcache_anticipate=256; // maximum de liens à anticiper httrack.maxtime=-1; // temps max en secondes - httrack.maxrate=25000; // default max rate +#if HTS_USEMMS + httrack.mms_maxtime = 60*3600; // max time for mms streams (one hour) +#endif + httrack.maxrate=25000; // taux maxi httrack.maxconn=5.0; // nombre connexions/s httrack.waittime=-1; // wait until.. hh*3600+mm*60+ss // @@ -521,44 +533,42 @@ int main(int argc, char **argv) { htsmain_free(); return -1; } else { - char* a; + int i, j; + int inQuote; + char* path; + int noDbl = 0; + if (com[1] == '1') { /* only 1 arg */ + com++; + noDbl = 1; + } na++; - strcpybuff(httrack.path_html,""); - strcpybuff(httrack.path_log,""); - a=strstr(argv[na],"\",\""); // rechercher en premier, au cas ou -O "c:\pipo,test","c:\test" - if (!a) - a=strchr(argv[na],','); // 2 path - else - a++; // position , - if (a) { - strncatbuff(httrack.path_html,argv[na],(int) (a-argv[na])); - strcatbuff(httrack.path_log,a+1); - } else { - strcpybuff(httrack.path_log,argv[na]); - strcpybuff(httrack.path_html,argv[na]); - } - // Eliminer les cas comme -O "C:\mirror\" - if (httrack.path_log[0]=='"') { // Guillemets - char tmp[256]; - strcpybuff(tmp,httrack.path_log+1); - if (tmp[strlen(tmp)-1]=='"') - tmp[strlen(tmp)-1]='\0'; - strcpybuff(httrack.path_log,tmp); - } - if (httrack.path_html[0]=='"') { - char tmp[256]; - strcpybuff(tmp,httrack.path_html+1); - if (tmp[strlen(tmp)-1]=='"') - tmp[strlen(tmp)-1]='\0'; - strcpybuff(httrack.path_html,tmp); - } + httrack.path_html[0] = '\0'; + httrack.path_log[0] = '\0'; + for(i = 0, j = 0, inQuote = 0, path = httrack.path_html ; argv[na][i] != 0 ; i++) { + if (argv[na][i] == '"') { + if (inQuote) + inQuote = 0; + else + inQuote = 1; + } else if (!inQuote && !noDbl && argv[na][i] == ',') { + path[j++] = '\0'; + j = 0; + path = httrack.path_log; + } else { + path[j++] = argv[na][i]; + } + } + path[j++] = '\0'; + if (httrack.path_log[0] == '\0') { + strcpybuff(httrack.path_log, httrack.path_html); + } + check_path(httrack.path_log,argv_firsturl); if (check_path(httrack.path_html,argv_firsturl)) { httrack.dir_topindex=1; // rebuilt top index } - //printf("-->%s\n%s\n",httrack.path_html,httrack.path_log); - + //printf("-->%s\n%s\n",httrack.path_html,httrack.path_log); } break; } // switch @@ -1231,7 +1241,12 @@ int main(int argc, char **argv) { case 'u': httrack.urlhack=1; if (*(com+1)=='0') { httrack.urlhack=0; com++; } break; // url hack case 'v': httrack.verbosedisplay=2; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&httrack.verbosedisplay); while(isdigit((unsigned char)*(com+1))) com++; } break; case 'i': httrack.dir_topindex = 1; if (*(com+1)=='0') { httrack.dir_topindex=0; com++; } break; + case 'N': httrack.savename_delayed = 2; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&httrack.savename_delayed); while(isdigit((unsigned char)*(com+1))) com++; } break; + case 'D': httrack.delayed_cached=1; if (*(com+1)=='0') { httrack.delayed_cached=0; com++; } break; // url hack case '!': httrack.bypass_limits = 1; if (*(com+1)=='0') { httrack.bypass_limits=0; com++; } break; +#if HTS_USEMMS + case 'm': sscanf(com+1,"%d",&httrack.mms_maxtime); while(isdigit((unsigned char)*(com+1))) com++; break; +#endif // preserve: no footer, original links case 'p': @@ -1307,7 +1322,6 @@ int main(int argc, char **argv) { htsmain_free(); return -1; } else{ - char* a; na++; if ( (strlen(argv[na]) + strlen(httrack.mimedefs) + 4) >= sizeof(httrack.mimedefs)) { HTS_PANIC_PRINTF("Mime definition string too long"); @@ -1315,22 +1329,24 @@ int main(int argc, char **argv) { return -1; } // --assume standard - if (strcmp(argv[na],"standard") == 0) { + if (strcmp(argv[na], "standard") == 0) { strcpybuff(httrack.mimedefs,"\n"); strcatbuff(httrack.mimedefs,HTS_ASSUME_STANDARD); strcatbuff(httrack.mimedefs,"\n"); } else { - strcatbuff(httrack.mimedefs,argv[na]); - strcatbuff(httrack.mimedefs,"\n"); - } - a=httrack.mimedefs; - while(*a) { - switch(*a) { - case ',': case ' ': case '\r': case ';': case '\t': - *a='\n'; - break; + char* a; + char* b = httrack.mimedefs + strlen(httrack.mimedefs); + for(a = argv[na] ; *a != '\0' ; a++) { + if (*a == ';') { /* next one */ + *b++ = '\n'; + } else if (*a == ',' || *a == '\n' || *a == '\r' || *a == '\t') { + *b++ = ' '; + } else { + *b++ = *a; + } } - a++; + *b++ = '\n'; /* next def */ + *b++ = '\0'; } } break; @@ -1661,8 +1677,8 @@ int main(int argc, char **argv) { fprintf(stdout, "X-URL: %s%s%s\r\n", (link_has_authority(adr)) ? "" : "http://", adr, fil); - if (url_savename(adr, fil, sav, NULL, NULL, NULL, NULL, - &httrack, NULL, 0, NULL, 0, &cache, NULL, 0, 0)!=-1) { + if (url_savename(adr, fil, sav, /*former_adr*/NULL, /*former_fil*/NULL, /*referer_adr*/NULL, /*referer_fil*/NULL, + /*opt*/&httrack, /*liens*/NULL, /*lien_tot*/0, /*sback*/NULL, /*cache*/&cache, /*hash*/NULL, /*ptr*/0, /*numero_passe*/0, /*mime_type*/NULL)!=-1) { if (fexist(sav)) { fprintf(stdout, "Content-location: %s\r\n", sav); } @@ -1828,7 +1844,7 @@ int main(int argc, char **argv) { case '1': /* test #1 : fil_simplifie */ if (na+1>=argc) { HTS_PANIC_PRINTF("Option #1 needs to be followed by an URL"); - printf("Example: '-#0' ./foo/bar/../foobar\n"); + printf("Example: '-#1' ./foo/bar/../foobar\n"); htsmain_free(); return -1; } else { @@ -1838,6 +1854,34 @@ int main(int argc, char **argv) { return 0; } break; + case '2': // mimedefs + if (na+1>=argc) { + HTS_PANIC_PRINTF("Option #1 needs to be followed by an URL"); + printf("Example: '-#2' /foo/bar.php\n"); + htsmain_free(); + return -1; + } else { + char mime[256]; + // initialiser mimedefs + get_userhttptype(1,httrack.mimedefs,NULL); + // check + mime[0] = '\0'; + get_httptype(mime, argv[na+1], 0); + if (mime[0] != '\0') { + char ext[256]; + printf("%s is '%s'\n", argv[na+1], mime); + ext[0] = '\0'; + give_mimext(ext, mime); + if (ext[0]) { + printf("and its local type is '.%s'\n", ext); + } + } else { + printf("%s is of an unknown MIME type\n", argv[na+1]); + } + htsmain_free(); + return 0; + } + break; case '!': if (na+1>=argc) { HTS_PANIC_PRINTF("Option #! needs to be followed by a commandline"); @@ -1863,6 +1907,9 @@ int main(int argc, char **argv) { } break; case 'O': // output path + while(isdigit(com[1])) { + com++; + } na++; // sauter, déja traité break; case 'P': // proxy @@ -2232,6 +2279,14 @@ int main(int argc, char **argv) { } fprintf(fp,LF); fprintf(fp, "To pause the engine: create an empty file named 'hts-stop.lock'"LF); +#if USE_BEGINTHREAD + fprintf(fp, "PID=%d\n", (int)getpid()); +#ifndef _WIN32 + fprintf(fp, "UID=%d\n", (int)getuid()); + fprintf(fp, "GID=%d\n", (int)getuid()); +#endif + fprintf(fp, "START=%d\n", (int)time(NULL)); +#endif fclose(fp); fp=NULL; } @@ -2298,7 +2353,9 @@ int main(int argc, char **argv) { fspc(httrack.log,"info"); fprintf(httrack.log,"engine: init"LF); } #if HTS_ANALYSTE - hts_htmlcheck_init(); + if (hts_htmlcheck_init != NULL) { + hts_htmlcheck_init(); + } set_wrappers(); // init() is allowed to set other wrappers #endif @@ -2373,7 +2430,9 @@ deprecated - see SIGCHLD fspc(httrack.log,"info"); fprintf(httrack.log,"engine: free"LF); } #if HTS_ANALYSTE - hts_htmlcheck_uninit(); + if (hts_htmlcheck_uninit != NULL) { + hts_htmlcheck_uninit(); + } #endif if (httrack_logmode!=1) { diff --git a/src/htscoremain.h b/src/htscoremain.h index 548c7f6..db781eb 100644 --- a/src/htscoremain.h +++ b/src/htscoremain.h @@ -42,7 +42,7 @@ Please visit our Website: http://www.httrack.com // --assume standard #define HTS_ASSUME_STANDARD \ - "php2,php3,php4,php,cgi,asp,jsp,pl,cfm,nsf=text/html" + "php2 php3 php4 php cgi asp jsp pl cfm nsf=text/html" #include "htsglobal.h" diff --git a/src/htsdefines.h b/src/htsdefines.h index e91b5b4..9f1de79 100644 --- a/src/htsdefines.h +++ b/src/htsdefines.h @@ -50,8 +50,10 @@ typedef char* (* t_hts_htmlcheck_query2)(char* question); typedef char* (* t_hts_htmlcheck_query3)(char* question); typedef int (* t_hts_htmlcheck_loop)(lien_back* back,int back_max,int back_index,int lien_tot,int lien_ntot,int stat_time,hts_stat_struct* stats); typedef int (* t_hts_htmlcheck_check)(char* adr,char* fil,int status); +typedef int (* t_hts_htmlcheck_check_mime)(char* adr,char* fil,char* mime,int status); typedef void (* t_hts_htmlcheck_pause)(char* lockfile); typedef void (* t_hts_htmlcheck_filesave)(char* file); +typedef void (* t_hts_htmlcheck_filesave2)(char* hostname,char* filename,char* localfile,int is_new,int is_modified,int not_updated); typedef int (* t_hts_htmlcheck_linkdetected)(char* link); typedef int (* t_hts_htmlcheck_linkdetected2)(char* link, char* tag_start); typedef int (* t_hts_htmlcheck_xfrstatus)(lien_back* back); @@ -77,8 +79,10 @@ extern t_hts_htmlcheck_query2 hts_htmlcheck_query2; extern t_hts_htmlcheck_query3 hts_htmlcheck_query3; extern t_hts_htmlcheck_loop hts_htmlcheck_loop; extern t_hts_htmlcheck_check hts_htmlcheck_check; +extern t_hts_htmlcheck_check_mime hts_htmlcheck_check_mime; extern t_hts_htmlcheck_pause hts_htmlcheck_pause; extern t_hts_htmlcheck_filesave hts_htmlcheck_filesave; +extern t_hts_htmlcheck_filesave2 hts_htmlcheck_filesave2; extern t_hts_htmlcheck_linkdetected hts_htmlcheck_linkdetected; extern t_hts_htmlcheck_linkdetected2 hts_htmlcheck_linkdetected2; extern t_hts_htmlcheck_xfrstatus hts_htmlcheck_xfrstatus; diff --git a/src/htsfilters.c b/src/htsfilters.c index 681b506..cd7abdd 100644 --- a/src/htsfilters.c +++ b/src/htsfilters.c @@ -59,7 +59,7 @@ Please visit our Website: http://www.httrack.com // optionnel: taille à contrôller (ou numéro, etc) en pointeur // (en de détection de *size, la taille limite est écrite par dessus *size) // exemple: +-*.gif*[<5] == supprimer GIF si <5KB -int fa_strjoker(char** filters,int nfil,char* nom,LLint* size,int* size_flag,int* depth) { +int fa_strjoker(int type,char** filters,int nfil,char* nom,LLint* size,int* size_flag,int* depth) { int verdict = 0; // on sait pas int i; LLint sizelimit=0; @@ -67,9 +67,18 @@ int fa_strjoker(char** filters,int nfil,char* nom,LLint* size,int* size_flag,int sizelimit=*size; for(i=0;i<nfil;i++) { LLint sz; + int filteroffs = 1; + if (strncmp(filters[i] + filteroffs, "mime:", 5) == 0) { + if (type == 0) // regular filters + continue; + filteroffs += 5; // +mime:text/html + } else { // mime filters + if (type != 0) + continue; + } if (size) sz=*size; - if (strjoker(nom,filters[i] + 1,&sz,size_flag)) { // reconnu + if (strjoker(nom, filters[i] + filteroffs, &sz, size_flag)) { // reconnu if (size) if (sz != *size) sizelimit=sz; @@ -208,6 +217,9 @@ HTS_INLINE char* strjoker(char* chaine,char* joker,LLint* size,int* size_flag) { } else err=1; i+=3; } else { // 1 car, ex: *[ ] + if (joker[i+2]=='\\' && joker[i+3] != 0) { // escaped char, such as *[\[] or *[\]] + i++; + } pass[(int) (unsigned char) joker[i]]=1; i++; } @@ -225,7 +237,7 @@ HTS_INLINE char* strjoker(char* chaine,char* joker,LLint* size,int* size_flag) { int i; for(i=0;i<256;i++) pass[i]=1; // tout autoriser jmp=1; - if (joker[2]==LEFT) jmp=3; // permet de recher *<crochet ouvrant> + ////if (joker[2]==LEFT) jmp=3; // permet de recher *<crochet ouvrant> } { diff --git a/src/htsfilters.h b/src/htsfilters.h index f963322..a1ba329 100644 --- a/src/htsfilters.h +++ b/src/htsfilters.h @@ -44,7 +44,7 @@ Please visit our Website: http://www.httrack.com /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -int fa_strjoker(char** filters,int nfil,char* nom,LLint* size,int* size_flag,int* depth); +int fa_strjoker(int type,char** filters,int nfil,char* nom,LLint* size,int* size_flag,int* depth); HTS_INLINE char* strjoker(char* chaine,char* joker,LLint* size,int* size_flag); char* strjokerfind(char* chaine,char* joker); #endif diff --git a/src/htsftp.c b/src/htsftp.c index 7b04052..1084558 100644 --- a/src/htsftp.c +++ b/src/htsftp.c @@ -276,7 +276,7 @@ int run_launch_ftp(lien_back* back) { } else { strcpybuff(back->r.msg,"Unexpected PORT error"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } } @@ -310,7 +310,7 @@ int run_launch_ftp(lien_back* back) { if (hp == NULL) { strcpybuff(back->r.msg,"Unable to get server's address"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-5; + back->r.statuscode=STATUSCODE_NON_FATAL; _HALT_FTP return 0; } @@ -327,7 +327,7 @@ int run_launch_ftp(lien_back* back) { if (soc_ctl==INVALID_SOCKET) { strcpybuff(back->r.msg,"Unable to create a socket"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; _HALT_FTP return 0; } @@ -345,7 +345,7 @@ int run_launch_ftp(lien_back* back) { #endif strcpybuff(back->r.msg,"Unable to connect to the server"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; _HALT_FTP return 0; #if HTS_WIN @@ -385,7 +385,7 @@ int run_launch_ftp(lien_back* back) { } else { strcpybuff(back->r.msg,"TYPE I error"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } #if 0 // --CWD-- @@ -412,34 +412,34 @@ int run_launch_ftp(lien_back* back) { } else { strcpybuff(back->r.msg,"TYPE I error"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } } else { sprintf(back->r.msg,"CWD error: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } // sinon on est prêts } else { strcpybuff(back->r.msg,"Unexpected ftp error"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } #endif } else { sprintf(back->r.msg,"Bad password: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } } else { sprintf(back->r.msg,"Bad user name: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } } else { sprintf(back->r.msg,"Connection refused: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } // ok, si on est prêts on écoute sur un port et on demande la sauce @@ -498,7 +498,7 @@ int run_launch_ftp(lien_back* back) { } else { sprintf(back->r.msg,"PASV incorrect: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } // sinon on est prêts } else { /* @@ -530,12 +530,12 @@ int run_launch_ftp(lien_back* back) { } else { sprintf(back->r.msg,"EPSV incorrect: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } } else { sprintf(back->r.msg,"PASV/EPSV error: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } // sinon on est prêts } #else @@ -651,7 +651,7 @@ int run_launch_ftp(lien_back* back) { // sprintf(back->r.msg,"RETR command errror: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } // sinon on est prêts } else { #if FTP_DEBUG @@ -661,22 +661,22 @@ int run_launch_ftp(lien_back* back) { // strcpybuff(back->r.msg,"Unable to connect"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } // sinon on est prêts } else { strcpybuff(back->r.msg,"Unable to create a socket"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } // sinon on est prêts } else { sprintf(back->r.msg,"Unable to resolve IP %s",adr_ip); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } // sinon on est prêts } else { sprintf(back->r.msg,"PASV incorrect: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } // sinon on est prêts #else //T_SOC soc_servdat; @@ -699,17 +699,17 @@ int run_launch_ftp(lien_back* back) { if ( (soc_dat=accept(soc_servdat,&dummyaddr,&dummylen)) == INVALID_SOCKET) { strcpybuff(back->r.msg,"Unable to accept connection"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } } else { sprintf(back->r.msg,"RETR command errror: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } } else { sprintf(back->r.msg,"PORT command error: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } #if HTS_WIN closesocket(soc_servdat); @@ -719,7 +719,7 @@ int run_launch_ftp(lien_back* back) { } else { strcpybuff(back->r.msg,"Unable to listen to a port"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } #endif @@ -728,10 +728,12 @@ int run_launch_ftp(lien_back* back) { // if (soc_dat != INVALID_SOCKET) { if (rest_understood) { // REST envoyée et comprise - filenote(back->url_sav,NULL); - back->r.fp = fopen(fconv(back->url_sav),"ab"); - } else + file_notify(back->url_adr, back->url_fil, back->url_sav, 0, 1, 0); + back->r.fp = fileappend(back->url_sav); + } else { + file_notify(back->url_adr, back->url_fil, back->url_sav, 1, 1, 0); back->r.fp = filecreate(back->url_sav); + } strcpybuff(back->info,"receiving"); if (back->r.fp != NULL) { char BIGSTK buff[1024]; @@ -746,13 +748,13 @@ int run_launch_ftp(lien_back* back) { case -1: strcpybuff(back->r.msg,"FTP read error"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; len=0; // fin break; case 0: sprintf(back->r.msg,"Time out (%d)",timeout); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; len=0; // fin break; } @@ -773,19 +775,19 @@ int run_launch_ftp(lien_back* back) { */ strcpybuff(back->r.msg,"Write error"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; len=0; // error } } else { strcpybuff(back->r.msg,"Unexpected write error"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } } else { // Erreur ou terminé // back->status=FTP_STATUS_READY; // fini back->r.statuscode=0; if (back->r.totalsize > 0 && back->r.size != back->r.totalsize) { - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; strcpybuff(back->r.msg,"FTP file incomplete"); } } @@ -800,7 +802,7 @@ int run_launch_ftp(lien_back* back) { } else { strcpybuff(back->r.msg,"Unable to write file"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } #if HTS_WIN closesocket(soc_dat); @@ -820,12 +822,12 @@ int run_launch_ftp(lien_back* back) { } else { sprintf(back->r.msg,"RETR incorrect: %s",linejmp(line)); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } } else { strcpybuff(back->r.msg,"FTP read error"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; } } @@ -1140,7 +1142,7 @@ int stop_ftp(lien_back* back) { if (back->stop_ftp) { strcpybuff(back->r.msg,"Cancelled by User"); // back->status=FTP_STATUS_READY; // fini - back->r.statuscode=-1; + back->r.statuscode=STATUSCODE_INVALID; return 1; } return 0; diff --git a/src/htsglobal.h b/src/htsglobal.h index d045f14..eef3ab3 100644 --- a/src/htsglobal.h +++ b/src/htsglobal.h @@ -40,8 +40,8 @@ Please visit our Website: http://www.httrack.com #define HTTRACK_GLOBAL_DEFH // Version -#define HTTRACK_VERSION "3.33-2" -#define HTTRACK_VERSIONID "3.33.16" +#define HTTRACK_VERSION "3.40-2" +#define HTTRACK_VERSIONID "3.40.4" #define HTTRACK_AFF_VERSION "3.x" //#define HTTRACK_AFF_WARNING "This is a BETA release of WinHTTrack Website Copier ("HTTRACK_VERSION")\nPlease report any crashes, bugs or problems" @@ -211,6 +211,11 @@ Please visit our Website: http://www.httrack.com #define HTS_USEOPENSSL 1 #endif +// utiliser mms://? +#ifndef HTS_USEMMS +#define HTS_USEMMS 1 +#endif + #ifndef HTS_DLOPEN #define HTS_DLOPEN 1 #endif @@ -287,7 +292,7 @@ Please visit our Website: http://www.httrack.com #endif /* Copyright (C) Xavier Roche and other contributors */ -#define HTTRACK_AFF_AUTHORS "[XR&CO'2005]" +#define HTTRACK_AFF_AUTHORS "[XR&CO'2006]" #define HTS_DEFAULT_FOOTER "<!-- Mirrored from %s%s by HTTrack Website Copier/"HTTRACK_AFF_VERSION" "HTTRACK_AFF_AUTHORS", %s -->" #define HTTRACK_WEB "http://www.httrack.com" #define HTS_UPDATE_WEBSITE "http://www.httrack.com/update.php3?Product=HTTrack&Version="HTTRACK_VERSIONID"&VersionStr="HTTRACK_VERSION"&Platform=%d&Language=%s" diff --git a/src/htshash.c b/src/htshash.c index 38a2d64..67d34d0 100644 --- a/src/htshash.c +++ b/src/htshash.c @@ -59,7 +59,6 @@ Please visit our Website: http://www.httrack.com // hash[no_enregistrement][pos]->hash est un index dans le tableau général liens // #define HTS_HASH_SIZE 8191 (premier si possible!) // type: numero enregistrement - 0 est case insensitive (sav) 1 (adr+fil) 2 (former_adr+former_fil) -#if HTS_HASH // recherche dans la table selon nom1,nom2 et le no d'enregistrement // retour: position ou -1 si non trouvé int hash_read(hash_struct* hash,char* nom1,char* nom2,int type,int normalized) { @@ -311,6 +310,5 @@ int* hash_calc_chaine(hash_struct* hash,int type,int pos) { #endif return &(hash->liens[pos]->hash_next[type]); } -#endif // FIN GESTION DES TABLES DE HACHAGE diff --git a/src/htshelp.c b/src/htshelp.c index af6f742..d1557ac 100644 --- a/src/htshelp.c +++ b/src/htshelp.c @@ -406,7 +406,7 @@ void help(char* app,int more) { #ifdef HTTRACK_AFF_WARNING infomsg("NOTE: "HTTRACK_AFF_WARNING); #endif - sprintf(info,"\tusage: %s <URLs> [-option] [+<FILTERs>] [-<FILTERs>]",app); + sprintf(info,"\tusage: %s <URLs> [-option] [+<URL_FILTER>] [-<URL_FILTER>] [+<mime:MIME_FILTER>] [-<mime:MIME_FILTER>]",app); infomsg(info); infomsg("\twith options listed below: (* is the default value)"); infomsg(""); @@ -439,6 +439,9 @@ void help(char* app,int more) { infomsg(" AN maximum transfer rate in bytes/seconds (1000=1KB/s max)"); infomsg(" %cN maximum number of connections/seconds (*%c10)"); infomsg(" GN pause transfer if N bytes reached, and wait until lock file is deleted"); +#if HTS_USEMMS + infomsg(" %mN maximum mms stream download time in seconds (60=1 minute, 3600=1 hour)"); +#endif infomsg(""); infomsg("Flow control:"); infomsg(" cN number of multiple connections (*c8)"); @@ -457,6 +460,8 @@ void help(char* app,int more) { infomsg("Build options:"); infomsg(" NN structure type (0 *original structure, 1+: see below)"); infomsg(" or user defined structure (-N \"%h%p/%n%q.%t\")"); + infomsg(" %N delayed type check, don't make any link test but wait for files download to start instead (experimental) (%N0 don't use, %N1 use for unknown extensions, * %N2 always use)"); + infomsg(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)"); infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)"); infomsg(" LN long names (L1 *long names / L0 8-3 conversion / L2 ISO9660 compatible)"); infomsg(" KN keep original links (e.g. http://www.adr/link) (K0 *relative link, K absolute links, K4 original links, K3 absolute URI links)"); @@ -525,6 +530,8 @@ void help(char* app,int more) { infomsg("Guru options: (do NOT use if possible)"); infomsg(" #X *use optimized engine (limited memory boundary checks)"); infomsg(" #0 filter test (-#0 '*.gif' 'www.bar.com/foo.gif')"); + infomsg(" #1 simplify test (-#1 ./foo/bar/../foobar)"); + infomsg(" #2 type test (-#2 /foo/bar.php)"); infomsg(" #C cache list (-#C '*.com/spider*.gif'"); infomsg(" #R cache repair (damaged cache)"); infomsg(" #d debug parser"); @@ -586,9 +593,9 @@ void help(char* app,int more) { infomsg(" '%q' small query string MD5 (16 bits, 4 ascii bytes)"); infomsg(" '%s?' Short name version (ex: %sN)"); infomsg(" '%[param]' param variable in query string"); - infomsg(" '%[param:before:after:notfound:empty]' advanced variable extraction"); + infomsg(" '%[param:before:after:empty:notfound]' advanced variable extraction"); infomsg("Details: User-defined option N and advanced variable extraction"); - infomsg(" %[param:before:after:notfound:empty]"); + infomsg(" %[param:before:after:empty:notfound]"); infomsg(" param : parameter name"); infomsg(" before : string to prepend if the parameter was found"); infomsg(" after : string to append if the parameter was found"); @@ -635,6 +642,7 @@ void help(char* app,int more) { infomsg("'check-link' : int (* myfunction)(char* adr,char* fil,int status);"); infomsg("'pause' : void (* myfunction)(char* lockfile);"); infomsg("'save-file' : void (* myfunction)(char* file);"); + infomsg("'save-file2' : void (* myfunction)(char* hostname,char* filename,char* localfile,int is_new,int is_modified);"); infomsg("'link-detected' : int (* myfunction)(char* link);"); infomsg("'link-detected2' : int (* myfunction)(char* link, char* start_tag);"); infomsg("'transfer-status' : int (* myfunction)(lien_back* back);"); @@ -645,7 +653,7 @@ void help(char* app,int more) { infomsg("example: httrack www.someweb.com/bob/"); infomsg("means: mirror site www.someweb.com/bob/ and only this site"); infomsg(""); - infomsg("example: httrack www.someweb.com/bob/ www.anothertest.com/mike/ +*.com/*.jpg"); + infomsg("example: httrack www.someweb.com/bob/ www.anothertest.com/mike/ +*.com/*.jpg -mime:application/*"); infomsg("means: mirror the two sites together (with shared links) and accept any .jpg files on .com sites"); infomsg(""); infomsg("example: httrack www.someweb.com/bob/bobby.html +* -r6"); diff --git a/src/htsindex.c b/src/htsindex.c index af87396..0546b2f 100644 --- a/src/htsindex.c +++ b/src/htsindex.c @@ -180,6 +180,10 @@ int index_keyword(const char* html_data,LLint size,const char* mime,const char* (strfield2(mime,"image/svg+xml")) || (strfield2(mime,"image/svg-xml")) +#if HTS_USEMMS + || + strfield2(mime,"video/x-ms-asf") +#endif ) { inscript=0; } diff --git a/src/htsinthash.c b/src/htsinthash.c index eb155cb..e81a74f 100644 --- a/src/htsinthash.c +++ b/src/htsinthash.c @@ -42,11 +42,16 @@ Please visit our Website: http://www.httrack.com /* specific definitions */ #include "htsbase.h" -#include "htsglobal.h" #include "htsmd5.h" /* END specific definitions */ /* Specific macros */ +#ifdef NO_MALLOCT +#undef malloct +#undef freet +#undef calloct +#undef strcpybuff +#endif #ifndef malloct #define malloct malloc #define freet free @@ -54,21 +59,55 @@ Please visit our Website: http://www.httrack.com #define strcpybuff strcpy #endif +// static functions + +static void inthash_delchain(inthash_chain* hash,t_inthash_freehandler free_handler); +static void inthash_default_free_handler(void* value); +static unsigned long int inthash_key(const char* value); +static void inthash_init(inthash hashtable); + + // inthash -- simple hash table, using a key (char[]) and a value (ulong int) -unsigned long int inthash_key(char* value) { +static unsigned long int inthash_key(const char* value) { return md5sum32(value); } +int inthash_read_pvoid(inthash hashtable,const char* name, void** pvalue) { + inthash_value value = INTHASH_VALUE_NULL; + int ret = inthash_read_value(hashtable, name, (pvalue != NULL) ? &value : NULL); + if (pvalue != NULL) + *pvalue = value.ptr; + return ret; +} + +int inthash_write_pvoid(inthash hashtable,const char* name, void* pvalue) { + inthash_value value = INTHASH_VALUE_NULL; + value.ptr = pvalue; + return inthash_write_value(hashtable, name, value); +} + +void inthash_add_pvoid(inthash hashtable, const char* name, void* pvalue) { + inthash_value value = INTHASH_VALUE_NULL; + value.ptr = pvalue; + inthash_add_value(hashtable, name, value); +} + // Check for duplicate entry (==1 : added) -int inthash_write(inthash hashtable,char* name,long int value) { +int inthash_write(inthash hashtable,const char* name,long int intvalue) { + inthash_value value = INTHASH_VALUE_NULL; + value.intg = intvalue; + return inthash_write_value(hashtable, name, value); +} + +int inthash_write_value(inthash hashtable,const char* name,inthash_value value) { int pos = (inthash_key(name) % hashtable->hash_size); inthash_chain* h=hashtable->hash[pos]; while (h) { if (strcmp(h->name,name)==0) { /* Delete element */ if (hashtable->flag_valueismalloc) { - void* ptr = (void*)h->value.intg; + void* ptr = h->value.ptr; if (ptr != NULL) { if (hashtable->free_handler) hashtable->free_handler(ptr); @@ -77,19 +116,19 @@ int inthash_write(inthash hashtable,char* name,long int value) { } } /* Insert */ - h->value.intg=value; + h->value=value; return 0; } h=h->next; } // Not found, add it! - inthash_add(hashtable,name,value); + inthash_add_value(hashtable,name,value); return 1; } // Increment pos value, create one if necessary (=0) // (==1 : created) -int inthash_inc(inthash hashtable,char* name) { +int inthash_inc(inthash hashtable,const char* name) { long int value=0; int r=0; if (inthash_read(hashtable,name,&value)) { @@ -105,7 +144,14 @@ int inthash_inc(inthash hashtable,char* name) { // Does not check for duplicate entry -void inthash_add(inthash hashtable,char* name,long int value) { +void inthash_add(inthash hashtable, const char* name, long int intvalue) { + inthash_value value = INTHASH_VALUE_NULL; + memset(&value, 0, sizeof(value)); + value.intg = intvalue; + inthash_add_value(hashtable, name, value); +} + +void inthash_add_value(inthash hashtable, const char* name, inthash_value value) { int pos = (inthash_key(name) % hashtable->hash_size); inthash_chain** h=&hashtable->hash[pos]; @@ -120,11 +166,12 @@ void inthash_add(inthash hashtable,char* name,long int value) { (*h)->name=((char*)(*h)) + sizeof(inthash_chain); (*h)->next=NULL; strcpybuff((*h)->name,name); - (*h)->value.intg=value; + (*h)->value=value; + hashtable->nitems++; } } -void* inthash_addblk(inthash hashtable,char* name,int blksize) { +void* inthash_addblk(inthash hashtable,const char* name,int blksize) { int pos = (inthash_key(name) % hashtable->hash_size); inthash_chain** h=&hashtable->hash[pos]; @@ -141,19 +188,28 @@ void* inthash_addblk(inthash hashtable,char* name,int blksize) { (*h)->name = ((char*)(*h)) + sizeof(inthash_chain); (*h)->next=NULL; strcpybuff((*h)->name,name); - (*h)->value.intg = (unsigned long) (char*) ((char*)(*h)) + sizeof(inthash_chain) + strlen(name) + 2; - return (void*)(*h)->value.intg; + (*h)->value.ptr = (void*) ( ((char*)(*h)) + sizeof(inthash_chain) + strlen(name) + 2 ); + hashtable->nitems++; + return (*h)->value.ptr; } return NULL; } -int inthash_read(inthash hashtable,char* name,long int* value) { +int inthash_read(inthash hashtable,const char* name,long int* intvalue) { + inthash_value value = INTHASH_VALUE_NULL; + int ret = inthash_read_value(hashtable, name, (intvalue != NULL) ? &value : NULL); + if (intvalue != NULL) + *intvalue = value.intg; + return ret; +} + +int inthash_read_value(inthash hashtable,const char* name,inthash_value* value) { int pos = (inthash_key(name) % hashtable->hash_size); inthash_chain* h=hashtable->hash[pos]; while (h) { if (strcmp(h->name,name)==0) { if (value != NULL) - *value=h->value.intg; + *value=h->value; return 1; } h=h->next; @@ -161,7 +217,45 @@ int inthash_read(inthash hashtable,char* name,long int* value) { return 0; } -int inthash_readptr(inthash hashtable,char* name,long int* value) { +int inthash_exists(inthash hashtable, const char* name) { + return inthash_read_value(hashtable, name, NULL); +} + +int inthash_remove(inthash hashtable,const char* name) { + int pos = (inthash_key(name) % hashtable->hash_size); + inthash_chain** h=&hashtable->hash[pos]; + t_inthash_freehandler free_handler=NULL; + if ( hashtable->flag_valueismalloc ) { + if ( hashtable->free_handler ) + free_handler=hashtable->free_handler; + else + free_handler=inthash_default_free_handler; + } + while (*h) { + if (strcmp((*h)->name,name)==0) { + inthash_chain* next; + if (free_handler) { + if ((*h)->value.ptr) { + void* ptr = (*h)->value.ptr; + if (free_handler) + free_handler(ptr); + else + freet(ptr); + (*h)->value.ptr=0; + } + } + next=(*h)->next; + freet(*h); + *h=next; + hashtable->nitems--; + return 1; + } + h=&((*h)->next); + } + return 0; +} + +int inthash_readptr(inthash hashtable,const char* name,long int* value) { int ret; *value = 0; ret = inthash_read(hashtable, name, value); @@ -170,31 +264,32 @@ int inthash_readptr(inthash hashtable,char* name,long int* value) { return ret; } -void inthash_init(inthash hashtable) { +static void inthash_init(inthash hashtable) { unsigned int i; for(i=0;i<hashtable->hash_size;i++) { hashtable->hash[i]=NULL; } } -void inthash_delchain(inthash_chain* hash,t_inthash_freehandler free_handler) { - if (hash) { - inthash_delchain(hash->next,free_handler); +static void inthash_delchain(inthash_chain* hash,t_inthash_freehandler free_handler) { + while(hash != NULL) { + inthash_chain* next=hash->next; if (free_handler) { // pos is a malloc() block, delete it! - if (hash->value.intg) { - void* ptr = (void*)hash->value.intg; + if (hash->value.ptr) { + void* ptr = hash->value.ptr; if (free_handler) free_handler(ptr); else freet(ptr); - hash->value.intg=0; + hash->value.ptr=0; } } freet(hash); + hash=next; } } -void inthash_default_free_handler(void* value) { +static void inthash_default_free_handler(void* value) { if (value) freet(value); } @@ -210,6 +305,7 @@ inthash inthash_new(int size) { hashtable->hash_size=size; inthash_init(hashtable); } + hashtable->nitems = 0; } return hashtable; } @@ -229,6 +325,12 @@ void inthash_value_set_free_handler(inthash hashtable, t_inthash_freehandler fre hashtable->free_handler = free_handler; } +unsigned int inthash_nitems(inthash hashtable) { + if (hashtable!= NULL) + return hashtable->nitems; + return 0; +} + void inthash_delete(inthash* hashtable) { if (hashtable) { if (*hashtable) { @@ -253,3 +355,29 @@ void inthash_delete(inthash* hashtable) { } } } + +// Enumerators + +struct_inthash_enum inthash_enum_new(inthash hashtable) { + struct_inthash_enum e; + memset(&e, 0, sizeof(e)); + e.index = 0; + e.item = NULL; + e.table = hashtable; + return e; +} + +inthash_chain* inthash_enum_next(struct_inthash_enum* e) { + inthash_chain* item = NULL; + if (e != NULL) { + while(e->item == NULL && e->index < (int) e->table->hash_size) { + e->item = e->table->hash[e->index]; + e->index++; + } + if (e->item != NULL) { + item = e->item; + e->item = e->item->next; + } + } + return item; +} diff --git a/src/htsinthash.h b/src/htsinthash.h index 5d7b992..b11b7ac 100644 --- a/src/htsinthash.h +++ b/src/htsinthash.h @@ -42,13 +42,18 @@ Please visit our Website: http://www.httrack.com // inthash -- simple hash table, using a key (char[]) and a value (ulong int) +// value +typedef union inthash_value { + unsigned long int intg; /* integer value */ + void* ptr; /* ptr value */ +} inthash_value; + +#define INTHASH_VALUE_NULL { 0 } + // simple hash table for other routines typedef struct inthash_chain { char* name; /* key (name) */ - union { - unsigned long int intg; /* integer value */ - void* ptr; /* ptr value */ - } value; + inthash_value value; /* value */ struct inthash_chain* next; /* next element */ } inthash_chain; @@ -56,6 +61,7 @@ typedef struct inthash_chain { typedef void (* t_inthash_freehandler)(void* value); typedef struct struct_inthash { inthash_chain** hash; + unsigned int nitems; t_inthash_freehandler free_handler; unsigned int hash_size; unsigned short flag_valueismalloc; @@ -64,31 +70,47 @@ typedef struct struct_inthash { // main inthash type typedef struct_inthash* inthash; +// enumeration +typedef struct struct_inthash_enum { + inthash table; + int index; + inthash_chain* item; +} struct_inthash_enum; + /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -// subfunctions -unsigned long int inthash_key(char* value); -void inthash_init(inthash hashtable); -void inthash_delchain(inthash_chain* hash,t_inthash_freehandler free_handler); -void inthash_default_free_handler(void* value); // main functions: /* Hash functions: */ -inthash inthash_new(int size); /* Create a new hash table */ -int inthash_created(inthash hashtable); /* Test if the hash table was successfully created */ -void inthash_delete(inthash* hashtable); /* Delete an hash table */ -void inthash_value_is_malloc(inthash hashtable,int flag); /* Is the 'value' member a value that needs to be free()'ed ? */ -void inthash_value_set_free_handler(inthash hashtable, /* value free() handler (default one is 'free') */ - t_inthash_freehandler free_handler); +inthash inthash_new(int size); /* Create a new hash table */ +int inthash_created(inthash hashtable); /* Test if the hash table was successfully created */ +unsigned int inthash_nitems(inthash hashtable); /* Number of items */ +void inthash_delete(inthash* hashtable); /* Delete an hash table */ +void inthash_value_is_malloc(inthash hashtable,int flag); /* Is the 'value' member a value that needs to be free()'ed ? */ +void inthash_value_set_free_handler(inthash hashtable, /* value free() handler (default one is 'free') */ + t_inthash_freehandler free_handler); +/* */ +int inthash_read(inthash hashtable,const char* name,long int* intvalue); /* Read entry from the hash table */ +int inthash_readptr(inthash hashtable,const char* name,long int* intvalue); /* Same function, but returns 0 upon null ptr */ +int inthash_exists(inthash hashtable, const char* name); /* Is the key existing ? */ +/* */ +int inthash_read_value(inthash hashtable,const char* name,inthash_value* value); +int inthash_write_value(inthash hashtable,const char* name,inthash_value value); +void inthash_add_value(inthash hashtable, const char* name, inthash_value value); +/* */ +int inthash_read_pvoid(inthash hashtable,const char* name, void** value); +int inthash_write_pvoid(inthash hashtable,const char* name, void* value); +void inthash_add_pvoid(inthash hashtable, const char* name, void* value); /* */ -int inthash_read(inthash hashtable,char* name,long int* value); /* Read entry from the hash table */ -int inthash_readptr(inthash hashtable,char* name,long int* value); /* Same function, but returns 0 upon null ptr */ +void inthash_add(inthash hashtable,const char* name,long int value); /* Add entry in the hash table */ +void* inthash_addblk(inthash hashtable,const char* name,int blksize); /* Add entry in the hash table and set value to a new memory block */ +int inthash_write(inthash hashtable,const char* name,long int value); /* Overwrite/add entry in the hash table */ +int inthash_inc(inthash hashtable,const char* name); /* Increment entry in the hash table */ +int inthash_remove(inthash hashtable,const char* name); /* Remove an entry from the hashtable */ /* */ -void inthash_add(inthash hashtable,char* name,long int value); /* Add entry in the hash table */ -void* inthash_addblk(inthash hashtable,char* name,int blksize); /* Add entry in the hash table and set value to a new memory block */ -int inthash_write(inthash hashtable,char* name,long int value); /* Overwrite/add entry in the hash table */ -int inthash_inc(inthash hashtable,char* name); /* Increment entry in the hash table */ +struct_inthash_enum inthash_enum_new(inthash hashtable); /* Start a new enumerator */ +inthash_chain* inthash_enum_next(struct_inthash_enum* e); /* Fetch an item in the enumerator */ /* End of hash functions: */ #endif diff --git a/src/htslib.c b/src/htslib.c index 9c389c8..93119df 100644 --- a/src/htslib.c +++ b/src/htslib.c @@ -624,7 +624,7 @@ htsblk httpget(char* url) { retour.adr=NULL; retour.size=0; retour.msg[0]='\0'; - retour.statuscode=-1; + retour.statuscode=STATUSCODE_INVALID; strcpybuff(retour.msg,"Error invalid URL"); return retour; } @@ -655,7 +655,7 @@ int http_xfopen(int mode,int treat,int waitconnect,char* xsend,char* adr,char* f retour->adr=NULL; retour->size=0; retour->msg[0]='\0'; - retour->statuscode=-5; // a priori erreur non fatale + retour->statuscode=STATUSCODE_NON_FATAL; // a priori erreur non fatale } #if HDEBUG @@ -697,7 +697,11 @@ int http_xfopen(int mode,int treat,int waitconnect,char* xsend,char* adr,char* f if (retour) { if (retour->msg) { if (!strnotempty(retour->msg)) { - strcpybuff(retour->msg,"Connect error"); +#ifdef _WIN32 + sprintf(retour->msg,"Connect error: %s", strerror(WSAGetLastError())); +#else + sprintf(retour->msg,"Connect error: %s", strerror(errno)); +#endif } } } @@ -1189,11 +1193,11 @@ void treatfirstline(htsblk* retour,char* rcvd) { // type MIME par défaut2 strcpybuff(retour->contenttype,HTS_HYPERTEXT_DEFAULT_MIME); } else { // pas de code! - retour->statuscode=-1; + retour->statuscode=STATUSCODE_INVALID; strcpybuff(retour->msg,"Unknown response structure"); } } else { // euhh?? - retour->statuscode=-1; + retour->statuscode=STATUSCODE_INVALID; strcpybuff(retour->msg,"Unknown response structure"); } } else { @@ -1204,7 +1208,7 @@ void treatfirstline(htsblk* retour,char* rcvd) { strcpybuff(retour->msg, "Unknown, assuming junky server"); strcpybuff(retour->contenttype,HTS_HYPERTEXT_DEFAULT_MIME); } else if (strnotempty(a)) { - retour->statuscode=-1; + retour->statuscode=STATUSCODE_INVALID; strcpybuff(retour->msg,"Unknown (not HTTP/xx) response structure"); } else { /* This is dirty .. */ @@ -1216,7 +1220,7 @@ void treatfirstline(htsblk* retour,char* rcvd) { } } else { // vide! /* - retour->statuscode=-1; + retour->statuscode=STATUSCODE_INVALID; strcpybuff(retour->msg,"Empty reponse or internal error"); */ /* This is dirty .. */ @@ -1483,19 +1487,32 @@ void treathead(t_cookie* cookie,char* adr,char* fil,htsblk* retour,char* rcvd) { //if (*a==';') { // finit par un ; // vérifier débordements if ( (((int) (token_end - token_st))<200) && (((int) (value_end - value_st))<8000) - && (((int) (token_end - token_st))>0) && (((int) (value_end - value_st))>0) ) { + && (((int) (token_end - token_st))>0) && (((int) (value_end - value_st))>0) ) + { + int name_len = (int) (token_end - token_st); + int value_len = (int) (value_end - value_st); name[0]='\0'; value[0]='\0'; - strncatbuff(name,token_st,(int) (token_end - token_st)); - strncatbuff(value,value_st,(int) (value_end - value_st)); + strncatbuff(name,token_st,name_len); + strncatbuff(value,value_st,value_len); #if DEBUG_COOK printf("detected cookie-av: name=\"%s\" value=\"%s\"\n",name,value); #endif if (strfield2(name,"domain")) { - strcpybuff(domain,value); + if (value_len < sizeof(domain) - 1) { + strcpybuff(domain,value); + } else { + cook_name[0] = 0; + break; + } } else if (strfield2(name,"path")) { - strcpybuff(path,value); + if (value_len < sizeof(path) - 1) { + strcpybuff(path,value); + } else { + cook_name[0] = 0; + break; + } } else if (strfield2(name,"max-age")) { // ignoré.. @@ -1513,12 +1530,17 @@ void treathead(t_cookie* cookie,char* adr,char* fil,htsblk* retour,char* rcvd) { // ignoré } else { - if (strnotempty(cook_name)==0) { // noter premier: nom et valeur cookie - strcpybuff(cook_name,name); - strcpybuff(cook_value,value); - } else { // prochain cookie - a=start_loop; // on devra recommencer à cette position - next=1; // enregistrer + if (value_len < sizeof(cook_value) - 1 && name_len < sizeof(cook_name) - 1) { + if (strnotempty(cook_name)==0) { // noter premier: nom et valeur cookie + strcpybuff(cook_name,name); + strcpybuff(cook_value,value); + } else { // prochain cookie + a=start_loop; // on devra recommencer à cette position + next=1; // enregistrer + } + } else { + cook_name[0] = 0; + break; } } } @@ -1791,7 +1813,7 @@ LLint http_xfread1(htsblk* r,int bufl) { if (nl > 0) { r->size+=nl; if ((INTsys)fwrite(buff,1,nl,r->out)!=nl) { - r->statuscode=-1; + r->statuscode=STATUSCODE_INVALID; strcpybuff(r->msg,"Write error on disk"); nl=READ_ERROR; } @@ -1994,7 +2016,7 @@ htsblk http_test(char* adr,char* fil,char* loc) { if (retour.adr!=NULL) { freet(retour.adr); retour.adr=NULL; } } } else { - retour.statuscode=-2; + retour.statuscode=STATUSCODE_TIMEOUT; strcpybuff(retour.msg,"Timeout While Testing"); } @@ -2081,7 +2103,11 @@ int newhttp(char* _iadr,htsblk* retour,int port,int waitconnect) { #endif if (retour) if (retour->msg) - strcpybuff(retour->msg,"Unable to get server's address"); +#ifdef _WIN32 + sprintf(retour->msg,"Unable to get server's address: %s", strerror(WSAGetLastError())); +#else + sprintf(retour->msg,"Unable to get server's address: %s", strerror(errno)); +#endif return INVALID_SOCKET; } // copie adresse @@ -2108,7 +2134,11 @@ int newhttp(char* _iadr,htsblk* retour,int port,int waitconnect) { if (soc==INVALID_SOCKET) { if (retour) if (retour->msg) - strcpybuff(retour->msg,"Unable to create a socket"); +#ifdef _WIN32 + sprintf(retour->msg,"Unable to create a socket: %s", strerror(WSAGetLastError())); +#else + sprintf(retour->msg,"Unable to create a socket: %s", strerror(errno)); +#endif return INVALID_SOCKET; // erreur création socket impossible } @@ -2120,7 +2150,11 @@ int newhttp(char* _iadr,htsblk* retour,int port,int waitconnect) { bind(soc, (struct sockaddr *)hp->h_addr_list[0], hp->h_length) != 0) { if (retour) if (retour->msg) - strcpybuff(retour->msg,"Unable to bind the specificied server address"); +#ifdef _WIN32 + sprintf(retour->msg,"Unable to bind the specificied server address: %s", strerror(WSAGetLastError())); +#else + sprintf(retour->msg,"Unable to bind the specificied server address: %s", strerror(errno)); +#endif deletesoc(soc); return INVALID_SOCKET; } @@ -2164,7 +2198,11 @@ int newhttp(char* _iadr,htsblk* retour,int port,int waitconnect) { #endif if (retour) if (retour->msg) - strcpybuff(retour->msg,"Unable to connect to the server"); +#ifdef _WIN32 + sprintf(retour->msg,"Unable to connect to the server: %s", strerror(WSAGetLastError())); +#else + sprintf(retour->msg,"Unable to connect to the server: %s", strerror(errno)); +#endif /* Close the socket and notify the error!!! */ deletesoc(soc); return INVALID_SOCKET; @@ -2231,6 +2269,10 @@ int ident_url_absolute(char* url,char* adr,char* fil) { } else if (SSL_is_available && (pos=strfield(url,"https:"))) { // HTTPS strcpybuff(adr,"https://"); #endif +#if HTS_USEMMS + } else if ((pos = strfield(url,"mms:"))) { // mms + strcpybuff(adr,"mms://"); +#endif } else if (scheme) { return -1; // erreur non reconnu } else @@ -2323,22 +2365,29 @@ void fil_simplifie(char* f) { int rollid = 0; char lc = '/'; int query = 0; + int wasAbsolute = (*f == '/'); for(a = b = f ; *a != '\0' ; ) { if (*a == '?') query = 1; if (query == 0 && lc == '/' && a[0] == '.' && a[1] == '/') { /* foo/./bar or ./foo */ a += 2; } - else if (query == 0 && lc == '/' && a[0] == '.' && a[1] == '.' && a[2] == '/') { /* foo/../bar or ../foo */ - a += 3; + else if (query == 0 && lc == '/' && a[0] == '.' && a[1] == '.' && ( a[2] == '/' || a[2] == '\0' ) ) { /* foo/../bar or ../foo or .. */ + if (a[2] == '\0') + a += 2; + else + a += 3; if (rollid > 1) { rollid--; b = rollback[rollid - 1]; - } else { + } else { /* too many ../ */ rollid = 0; b = f; + if (wasAbsolute) + b++; /* after the / */ } - } else { + } + else { *b++ = lc = *a; if (*a == '/') { rollback[rollid++] = b; @@ -2352,9 +2401,14 @@ void fil_simplifie(char* f) { } *b = '\0'; if (*f == '\0') { - f[0] = '.'; - f[1] = '/'; - f[2] = '\0'; + if (wasAbsolute) { + f[0] = '/'; + f[1] = '\0'; + } else { + f[0] = '.'; + f[1] = '/'; + f[2] = '\0'; + } } } @@ -2633,8 +2687,10 @@ int get_filetime_rfc822(char* file,char* date) { A=gmtime(&tt); if (A==NULL) A=localtime(&tt); - time_rfc822(date, A); - return 1; + if (A != NULL) { + time_rfc822(date, A); + return 1; + } } return 0; } @@ -2804,7 +2860,7 @@ int binput(char* buff, char* s, int max) { int destCount = 0; // Note: \0 will return 1 - while(count < max && buff != NULL && buff[count] != '\0' && buff[count] != '\n') { + while(destCount < max && buff != NULL && buff[count] != '\0' && buff[count] != '\n') { if (buff[count] != '\r') { s[destCount++] = buff[count]; } @@ -3024,28 +3080,50 @@ void map_characters(unsigned char* buffer, unsigned int size, unsigned int* map) // -1 : on sait pas // -2 : on sait pas, pas d'extension int ishtml(const char* fil) { - const char *a; - - // patch pour les truc.html?Choix=toto - if ( (a=strchr(fil,'?')) ) // paramètres? - a--; // pointer juste avant le ? - else - a=fil+strlen(fil)-1; // pointer sur le dernier caractère - - if (*a=='/') return -1; // répertoire, on sait pas!! - //if (*a=='/') return 1; // ok répertoire, html + /* User-defined MIME types (overrides ishtml()) */ + char BIGSTK fil_noquery[HTS_URLMAXSIZE*2]; + char mime[256]; + char* a; + strcpybuff(fil_noquery, fil); + if ((a = strchr(fil_noquery, '?')) != NULL) { + *a = '\0'; + } + if (get_userhttptype(0, mime, fil_noquery)) { + if (strfield2(mime, "text/html")) { + return 1; + } else { + return 0; + } + } - while ( (*a!='.') && (*a!='/') && ( a > fil)) a--; - if (*a=='.') { // a une extension + /* Search for known ext */ + for (a = fil_noquery + strlen(fil_noquery) - 1 ; (*a!='.') && (*a!='/') && ( a > fil_noquery ) ; a-- ); + if (*a == '.') { // a une extension char BIGSTK fil_noquery[HTS_URLMAXSIZE*2]; char* b; + int ret; + char* dotted = a; fil_noquery[0]='\0'; a++; // pointer sur extension strncatbuff(fil_noquery,a,HTS_URLMAXSIZE); b=strchr(fil_noquery,'?'); if (b) *b='\0'; - return ishtml_ext(fil_noquery); // retour + ret = ishtml_ext(fil_noquery); // retour + if (ret == -1) { + switch(is_knowntype(dotted)) { + case 1: + ret = 0; // connu, non html + break; + case 2: + ret = 1; // connu, html + break; + default: + ret = -1; // inconnu.. + break; + } + } + return ret; } else return -2; // indéterminé, par exemple /truc } @@ -3064,7 +3142,11 @@ int ishtml_ext(const char* a) { // // insuccès.. else { - switch(is_knowntype(a)) { +#if 1 + html = -1; // inconnu.. +#else + // XXXXXX not suitable (ext) + switch(is_knownext(a)) { case 1: html = 0; // connu, non html break; @@ -3075,6 +3157,7 @@ int ishtml_ext(const char* a) { html = -1; // inconnu.. break; } +#endif } return html; } @@ -3259,6 +3342,10 @@ HTS_INLINE char* jump_protocol(char* source) { source+=p; else if ((p=strfield(source,"file:"))) source+=p; +#if HTS_USEMMS + else if ((p=strfield(source,"mms:"))) + source+=p; +#endif // net_path if (strncmp(source,"//",2)==0) source+=2; @@ -3596,6 +3683,7 @@ HTSEXT_API char* unescape_http(char* s) { } // unescape in URL/URI ONLY what has to be escaped, to form a standard URL/URI +// DOES NOT DECODE %25 HTSEXT_API char* unescape_http_unharm(char* s, int no_high) { char* tempo; int i,j=0; @@ -3605,7 +3693,7 @@ HTSEXT_API char* unescape_http_unharm(char* s, int no_high) { int nchar=(char) ehex(s+i+1); int test = ( CHAR_RESERVED(nchar) - || CHAR_DELIM(nchar) + || ( nchar != '%' && CHAR_DELIM(nchar) ) || CHAR_UNWISE(nchar) || CHAR_LOW(nchar) /* CHAR_SPECIAL */ || CHAR_XXAVOID(nchar) @@ -3870,33 +3958,40 @@ HTS_INLINE int is_realspace(char c) { // deviner type d'un fichier local.. // ex: fil="toto.gif" -> s="image/gif" void guess_httptype(char *s,const char *fil) { - get_httptype(s,fil,1); + get_httptype(s, fil, 1); } // idem // flag: 1 si toujours renvoyer un type void get_httptype(char *s,const char *fil,int flag) { - if (ishtml(fil)==1) + // userdef overrides get_httptype + if (get_userhttptype(0, s, fil)) { + return ; + } + // regular tests + if (ishtml(fil) == 1) { strcpybuff(s,"text/html"); - else { - const char *a=fil+strlen(fil)-1; - while ( (*a!='.') && (*a!='/') && (a>fil)) a--; + } else { + /* Check html -> text/html */ + const char* a = fil + strlen(fil) - 1; + while ( (*a!='.') && (*a!='/') && (a>fil)) a--; if (*a=='.' && strlen(a) < 32) { - int ok=0; int j=0; a++; - while( (!ok) && (strnotempty(hts_mime[j][1])) ) { + while(strnotempty(hts_mime[j][1])) { if (strfield2(hts_mime[j][1],a)) { if (hts_mime[j][0][0]!='*') { // Une correspondance existe strcpybuff(s,hts_mime[j][0]); - ok=1; + return; } } j++; } - if (!ok) if (flag) sprintf(s,"application/%s",a); + if (flag) + sprintf(s,"application/%s",a); } else { - if (flag) strcpybuff(s,"application/octet-stream"); + if (flag) + strcpybuff(s,"application/octet-stream"); } } } @@ -3904,7 +3999,7 @@ void get_httptype(char *s,const char *fil,int flag) { // get type of fil (php) // s: buffer (text/html) or NULL // return: 1 if known by user -int get_userhttptype(int setdefs,char *s,const char *ext) { +int get_userhttptype(int setdefs, char *s, const char *fil) { char** buffer=NULL; NOSTATIC_RESERVE(buffer, char*, 1); if (setdefs) { @@ -3913,11 +4008,72 @@ int get_userhttptype(int setdefs,char *s,const char *ext) { } else { if (s) s[0]='\0'; - if (!ext) + if (fil == NULL || *fil == '\0') return 0; +#if 1 + if (*buffer) { + + /* Check --assume foooo/foo/bar.cgi=text/html, then foo/bar.cgi=text/html, then bar.cgi=text/html */ + /* also: --assume baz,bar,foooo/foo/bar.cgi=text/html */ + /* start from path begining */ + do { + char* next; + char* mimedefs = *buffer; /* loop through mime definitions : \nfoo=bar\nzoo=baz\n.. */ + while(*mimedefs != '\0') { + const char* segment = fil + 1; + if (*mimedefs == '\n') { + mimedefs++; + } + /* compare current segment with user's definition */ + do { + int i; + /* check current item */ + for(i = 0 ; + mimedefs[i] != '\0' /* end of all defs */ + && mimedefs[i] != ' ' /* next item in left list */ + && mimedefs[i] != '=' /* end of left list */ + && mimedefs[i] != '\n' /* end of this def (?) */ + && mimedefs[i] == segment[i] /* same item */ + ; i++); + /* success */ + if ( ( mimedefs[i] == '=' || mimedefs[i] == ' ' ) && segment[i] == '\0') { + int i2; + while(mimedefs[i] != 0 && mimedefs[i] != '\n' && mimedefs[i] != '=') i++; + if (mimedefs[i] == '=') { + i++; + for(i2 = 0 ; mimedefs[i + i2] != '\n' && mimedefs[i + i2] != '\0' ; i2++) { + s[i2] = mimedefs[i + i2]; + } + s[i2] = '\0'; + return 1; /* SUCCESS! */ + } + } + /* next item in list */ + for(mimedefs += i ; + *mimedefs != '\0' && *mimedefs != '\n' && *mimedefs != '=' && *mimedefs != ' ' ; + mimedefs++); + if (*mimedefs == ' ') { + mimedefs++; + } + } while(*mimedefs != '\0' && *mimedefs != '\n' && *mimedefs != '='); + /* next user-def */ + for( ; *mimedefs != '\0' && *mimedefs != '\n' ; mimedefs++); + } + /* shorten segment */ + next = strchr(fil + 1, '/'); + if (next == NULL) { + /* ext tests */ + next = strchr(fil + 1, '.'); + } + fil = next; + } while(fil != NULL); + } +#else if (*buffer) { char BIGSTK search[1024]; char* detect; + + sprintf(search,"\n%s=",ext); // php=text/html detect=strstr(*buffer,search); if (!detect) { @@ -3939,12 +4095,13 @@ int get_userhttptype(int setdefs,char *s,const char *ext) { } } } +#endif } return 0; } // renvoyer extesion d'un type mime.. // ex: "image/gif" -> gif -void give_mimext(char *s,char *st) { +void give_mimext(char *s,const char *st) { int ok=0; int j=0; s[0]='\0'; @@ -3963,7 +4120,7 @@ void give_mimext(char *s,char *st) { // application/mp3 if (!ok) { int p; - char* a=NULL; + const char* a=NULL; if ((p=strfield(st,"application/x-"))) a=st+p; else if ((p=strfield(st,"application/"))) @@ -3983,12 +4140,14 @@ void give_mimext(char *s,char *st) { // 1 : oui // 2 : html int is_knowntype(const char *fil) { + const char *ext; int j=0; if (!fil) return 0; + ext = get_ext(fil); while(strnotempty(hts_mime[j][1])) { - if (strfield2(hts_mime[j][1],fil)) { - if (strfield2(hts_mime[j][0],"text/html")) + if (strfield2(hts_mime[j][1], ext)) { + if (strfield2(hts_mime[j][0], "text/html")) return 2; else return 1; @@ -4899,6 +5058,7 @@ HTSEXT_API int hts_init(void) { htswrap_add("check-link",htsdefault_check); htswrap_add("pause",htsdefault_pause); htswrap_add("save-file",htsdefault_filesave); + htswrap_add("save-file2",htsdefault_filesave2); htswrap_add("link-detected",htsdefault_linkdetected); htswrap_add("link-detected2",htsdefault_linkdetected2); htswrap_add("transfer-status",htsdefault_xfrstatus); @@ -4985,6 +5145,9 @@ char* __cdecl htsdefault_query3(char* question) { int __cdecl htsdefault_check(char* adr,char* fil,int status) { return -1; } +int __cdecl htsdefault_check_mime(char* adr,char* fil,char* mime,int status) { + return -1; +} void __cdecl htsdefault_pause(char* lockfile) { while (fexist(lockfile)) { Sleep(1000); @@ -4992,6 +5155,8 @@ void __cdecl htsdefault_pause(char* lockfile) { } void __cdecl htsdefault_filesave(char* file) { } +void __cdecl htsdefault_filesave2(char* adr, char* file, char* sav, int is_new, int is_modified, int not_updated) { +} int __cdecl htsdefault_linkdetected(char* link) { return 1; } diff --git a/src/htslib.h b/src/htslib.h index 23a8400..2a720da 100644 --- a/src/htslib.h +++ b/src/htslib.h @@ -122,6 +122,11 @@ typedef struct htsblk { /* ANCIENNE STURCTURE pour cache 1.0 */ typedef struct { + int active; + char name[1024]; + int port; +} OLD_t_proxy; +typedef struct { int statuscode; // ANCIENNE STURCTURE - status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945) int notmodified; // ANCIENNE STURCTURE - page ou fichier NON modifié (transféré) int is_write; // ANCIENNE STURCTURE - sortie sur disque (out) ou en mémoire (adr) @@ -135,7 +140,7 @@ typedef struct { int is_file; // ANCIENNE STURCTURE - ce n'est pas une socket mais un descripteur de fichier si 1 T_SOC soc; // ANCIENNE STURCTURE - ID socket FILE* fp; // ANCIENNE STURCTURE - fichier pour file:// - t_proxy proxy; // ANCIENNE STURCTURE - proxy + OLD_t_proxy proxy; // ANCIENNE STURCTURE - proxy int user_agent_send; // ANCIENNE STURCTURE - user agent (ex: httrack/1.0 [sun]) char user_agent[64]; int http11; // ANCIENNE STURCTURE - l'en tête doit être signé HTTP/1.1 et non HTTP/1.0 @@ -249,8 +254,8 @@ int ishtml_ext(const char* a); int ishttperror(int err); void guess_httptype(char *s,const char *fil); void get_httptype(char *s,const char *fil,int flag); -int get_userhttptype(int setdefs,char *s,const char *ext); -void give_mimext(char *s,char *st); +int get_userhttptype(int setdefs,char *s,const char *fil); +void give_mimext(char *s,const char *st); int is_knowntype(const char *fil); int is_userknowntype(const char *fil); int is_dyntype(const char *fil); @@ -362,8 +367,10 @@ char* __cdecl htsdefault_query(char* question); char* __cdecl htsdefault_query2(char* question); char* __cdecl htsdefault_query3(char* question); int __cdecl htsdefault_check(char* adr,char* fil,int status); +int __cdecl htsdefault_check_mime(char* adr,char* fil,char* mime,int status); void __cdecl htsdefault_pause(char* lockfile); void __cdecl htsdefault_filesave(char*); +void __cdecl htsdefault_filesave2(char* adr, char* file, char* sav, int is_new, int is_modified,int not_updated); int __cdecl htsdefault_linkdetected(char* link); int __cdecl htsdefault_linkdetected2(char* link, char* tag_start); int __cdecl htsdefault_xfrstatus(void* back); @@ -392,6 +399,18 @@ extern void clearCallbacks(htscallbacks* chain); //HTS_INLINE int is_space(char); //HTS_INLINE int is_realspace(char); +#define HTTP_IS_REDIRECT(code) ( \ + (code) == 301 \ + || (code) == 302 \ + || (code) == 303 \ + || (code) == 307 \ + ) +#define HTTP_IS_NOTMODIFIED(code) ( \ + (code) == 304 \ + ) +#define HTTP_IS_OK(code) ( ( (code) / 100 ) == 2 ) +#define HTTP_IS_ERROR(code) ( !HTTP_IS_OK(code) && !HTTP_IS_REDIRECT(code) && !HTTP_IS_NOTMODIFIED(code) ) + // compare le début de f avec s et retourne la position de la fin // 'A=a' (case insensitive) static int strfield(const char* f,const char* s) { @@ -422,6 +441,12 @@ static int strcmpnocase(char* a,char* b) { // is this MIME an hypertext MIME (text/html), html/js-style or other script/text type? #define HTS_HYPERTEXT_DEFAULT_MIME "text/html" + +#if HTS_USEMMS +#define OPT_MMS(a) (strfield2((a), "video/x-ms-asf") != 0) +#else +#define OPT_MMS(a) (false) +#endif #define is_hypertext_mime__(a) \ ( (strfield2((a),"text/html")!=0)\ || (strfield2((a),"application/x-javascript")!=0) \ @@ -436,6 +461,7 @@ static int strcmpnocase(char* a,char* b) { (\ (strfield2((a),"audio/x-pn-realaudio")!=0) \ || (strfield2((a),"audio/x-mpegurl")!=0) \ + || OPT_MMS(a) \ ) diff --git a/src/htsmd5.c b/src/htsmd5.c index 92aec5e..adbdb67 100644 --- a/src/htsmd5.c +++ b/src/htsmd5.c @@ -47,7 +47,7 @@ Please visit our Website: http://www.httrack.com #include <string.h> #include <stdio.h> -int domd5mem(unsigned char * buf, int len, +int domd5mem(const unsigned char * buf, int len, unsigned char * digest, int asAscii) { int endian = 1; unsigned char bindigest[16]; @@ -85,7 +85,7 @@ int domd5mem(unsigned char * buf, int len, return 0; } -unsigned long int md5sum32(char* buff) { +unsigned long int md5sum32(const char* buff) { unsigned char md5digest[16]; unsigned char* md5digest_ = md5digest; domd5mem(buff,strlen(buff),md5digest,0); diff --git a/src/htsmd5.h b/src/htsmd5.h index 3e3b00c..8892895 100644 --- a/src/htsmd5.h +++ b/src/htsmd5.h @@ -44,9 +44,9 @@ Please visit our Website: http://www.httrack.com /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -int domd5mem(unsigned char * buf, int len, - unsigned char * digest, int asAscii); -unsigned long int md5sum32(char* buff); +int domd5mem(const unsigned char * buf, int len, + unsigned char * digest, int asAscii); +unsigned long int md5sum32(const char* buff); #endif #endif diff --git a/src/htsmms.c b/src/htsmms.c new file mode 100644 index 0000000..3d76cda --- /dev/null +++ b/src/htsmms.c @@ -0,0 +1,245 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) Xavier Roche and other contributors + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +Important notes: + +- We hereby ask people using this source NOT to use it in purpose of grabbing +emails addresses, or collecting any other private information on persons. +This would disgrace our work, and spoil the many hours we spent on it. + + +Please visit our Website: http://www.httrack.com +*/ + + +/* ------------------------------------------------------------ */ +/* File: basic mms protocol manager */ +/* Author: Xavier Roche */ +/* */ +/* The mms routines were written by Nicolas BENOIT, */ +/* based on the work of SDP Multimedia and Major MMS */ +/* Thanks to all of them! */ +/* ------------------------------------------------------------ */ + +/* Internal engine bytecode */ +#define HTS_INTERNAL_BYTECODE + +// Gestion protocole mms + +#include "htsglobal.h" + +#if HTS_USEMMS + +#include "htsbase.h" +#include "htsnet.h" +#include "htsthread.h" + +#include "htsmms.h" +#include "mmsrip/mms.h" + +#define FTP_STATUS_READY 1001 + +static int run_launch_mms(MMSDownloadStruct* back); +static PTHREAD_TYPE PTHREAD_TYPE_FNC back_launch_mms( void* pP ) { + MMSDownloadStruct *pStruct = (MMSDownloadStruct*)pP; + if (pStruct == NULL) + return PTHREAD_RETURN; + + /* Initialize */ + hts_init(); + + /* Run */ + run_launch_mms(pStruct); + + /* Set as ready */ + { + lien_back* back = pStruct->pBack; + back->status=FTP_STATUS_READY; + } + + /* Delete structure */ + free(pP); + + /* Uninitialize */ + hts_uninit(); + return PTHREAD_RETURN; +} + +/* download cancelled */ +static int stop_mms(lien_back* back) { + if (back->stop_ftp) { + strcpybuff(back->r.msg, "Cancelled by User"); + back->r.statuscode = STATUSCODE_INVALID; + return 1; + } + return 0; +} + +/* Background launch */ +void launch_mms(const MMSDownloadStruct* pStruct) { + MMSDownloadStruct *pCopy = calloc(sizeof(MMSDownloadStruct), 1); + memcpy(pCopy, pStruct, sizeof(*pCopy)); + (void) hts_newthread(back_launch_mms, 0, (void*) pCopy); +} + +/* Code mainly written by Nicolas BENOIT */ +static int run_launch_mms(MMSDownloadStruct* pStruct) { + lien_back* back = pStruct->pBack; + httrackp* opt = pStruct->pOpt; + /* */ + char url[HTS_URLMAXSIZE*2]; + MMS *mms; + FILE *f; + ssize_t len_written; + uint64_t total_len_written; + int delay = opt->mms_maxtime; + time_t end = time(NULL) + delay; + short checkPending = 0; + INTsys existingSize = fsize(back->url_sav); + + // effacer + strcpybuff(back->r.msg,""); + back->status=1000; + back->r.statuscode=200; + back->r.size=0; + + /* Create file */ + if (existingSize > 0) { + /* back->r.out = fileappend(back->url_sav); + */ + (void) unlink(fconcat(back->url_sav, ".old")); + if (rename(fconcat(back->url_sav, ""), fconcat(back->url_sav, ".old")) == 0) { + checkPending = 1; + } + back->r.out = filecreate(back->url_sav); + } else { + back->r.out = filecreate(back->url_sav); + } + if ((f = back->r.out) != NULL) { + // create mms resource + strcpybuff(url, back->url_adr); /* mms:// */ + strcatbuff(url, back->url_fil); + if ( ( mms = mms_create( url, f, NULL, 0, 1 ) ) != NULL ) { + if ( mms_connect ( mms ) == 0 ) { + if ( mms_handshake ( mms ) == 0 ) { + if ( ( len_written = mms_write_stream_header ( mms ) ) != -1 ) { + total_len_written = len_written; + HTS_STAT.HTS_TOTAL_RECV += len_written; + + /* not modified */ + if (checkPending) { + if (mms->is_live != MMS_LIVE + && mms->expected_file_size == existingSize + 50 /* Why 50 additional bytes declared ?? */ + ) // abort download + { + fclose(back->r.out); + f = back->r.out = NULL; + if (unlink(fconcat(back->url_sav, "")) == 0 + && rename(fconcat(back->url_sav, ".old"), fconcat(back->url_sav, "")) == 0) + { + back->r.notmodified = 1; + back->r.statuscode = 200; + strcpybuff(back->r.msg, "Not modified"); + } else { + back->r.statuscode = 500; + strcpybuff(back->r.msg, "Unable to rename previous file (not updated)"); + } + } else { + (void) unlink(fconcat(back->url_sav, ".old")); + } + } + + /* begin rip */ + if ( f != NULL && mms_begin_rip ( mms ) == 0 ) { + if ( mms->is_live != MMS_LIVE ) { + back->r.totalsize = mms->expected_file_size; + back->r.totalsize -= 50; /* Why 50 additional bytes declared ?? */ + } else + back->r.totalsize = -1; + + /* Start download */ + while ( !stop_mms(back) ) { + len_written = mms_write_stream_data ( mms ); + if ( len_written == 0 ) { + break; + } else if ( len_written == -1 ) { + back->r.statuscode = -1; + back->r.statuscode = 500; + strcpybuff(back->r.msg, "Unable to write stream data"); + break; + } + + total_len_written += len_written; + back->r.size = total_len_written; + HTS_STAT.HTS_TOTAL_RECV += len_written; + + fflush ( f ); + + if ( delay != 0 && end <= time(NULL) ) { + delay = -1; + back->r.statuscode = 200; + strcpybuff(back->r.msg, "Download interrupted"); + break; + } + } // while + + back->r.statuscode = 0; /* Finished */ + } else if (f != NULL) { + back->r.statuscode = -1; + back->r.statuscode = 500; + strcpybuff(back->r.msg, "Can not begin ripping"); + } + } else { + back->r.statuscode = -1; + back->r.statuscode = 500; + strcpybuff(back->r.msg, "Can not write stream header"); + } + } else { + back->r.statuscode = -1; + back->r.statuscode = 500; + strcpybuff(back->r.msg, "Can not handshake"); + } + mms_disconnect ( mms ); + } else { + back->r.statuscode = -1; + back->r.statuscode = 500; + strcpybuff(back->r.msg, "Can not connect"); + } + mms_destroy ( mms ); + } else { + back->r.statuscode = -1; + back->r.statuscode = 500; + strcpybuff(back->r.msg, "Can not create mms resource"); + } + } else { + back->r.statuscode = -1; + back->r.statuscode = 500; + strcpybuff(back->r.msg, "Unable to open local output file"); + } + + // End + if (back->r.statuscode != -1) { + back->r.statuscode=200; + strcpybuff(back->r.msg, "OK"); + } + return 0; +} + +#endif diff --git a/src/htsmms.h b/src/htsmms.h new file mode 100644 index 0000000..43a6c1e --- /dev/null +++ b/src/htsmms.h @@ -0,0 +1,64 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) Xavier Roche and other contributors + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +Important notes: + +- We hereby ask people using this source NOT to use it in purpose of grabbing +emails addresses, or collecting any other private information on persons. +This would disgrace our work, and spoil the many hours we spent on it. + + +Please visit our Website: http://www.httrack.com +*/ + + +/* ------------------------------------------------------------ */ +/* File: basic mms protocol manager .h */ +/* Author: Xavier Roche */ +/* */ +/* The mms routines were written by Nicolas BENOIT, */ +/* based on the work of SDP Multimedia and Major MMS */ +/* Thanks to all of them! */ +/* ------------------------------------------------------------ */ + + +#ifndef HTSMMS_DEFH +#define HTSMMS_DEFH + +#include "htsglobal.h" + +#if HTS_USEMMS + +#include "htsbase.h" +#include "htsbasenet.h" +#include "htsthread.h" + +// lien_back +#include "htscore.h" + +typedef struct MMSDownloadStruct { + lien_back *pBack; + httrackp *pOpt; +} MMSDownloadStruct; + +void launch_mms(const MMSDownloadStruct* pStruct); +#endif + +#endif diff --git a/src/htsmodules.c b/src/htsmodules.c index 3299c41..ba3927e 100644 --- a/src/htsmodules.c +++ b/src/htsmodules.c @@ -307,8 +307,11 @@ void htspe_init() { #ifdef _WIN32 handle = LoadLibraryA((char*)"ssleay32"); #else - /* We are compatible with 0.9.6/7 and potentially above */ - handle = dlopen("libssl.so.0.9.7", RTLD_LAZY); + /* We are compatible with 0.9.6/7/8 and potentially above */ + handle = dlopen("libssl.so.0.9.8", RTLD_LAZY); + if (handle == NULL) { + handle = dlopen("libssl.so.0.9.7", RTLD_LAZY); + } if (handle == NULL) { handle = dlopen("libssl.so.0.9.6", RTLD_LAZY); } diff --git a/src/htsmodules.h b/src/htsmodules.h index 5d2b989..6c4a305 100644 --- a/src/htsmodules.h +++ b/src/htsmodules.h @@ -83,8 +83,7 @@ struct htsmoduleStruct { /* Internal use - please don't touch */ void* liens; void* opt; - void* back; - int back_max; + void* sback; void* cache; void* hashptr; int numero_passe; diff --git a/src/htsname.c b/src/htsname.c index 8af2062..0176c5c 100644 --- a/src/htsname.c +++ b/src/htsname.c @@ -81,21 +81,21 @@ static const char *hts_tbdev[] = #define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET() do { \ int prev = _hts_in_html_parsing; \ - while(back_pluggable_sockets_strict(back, back_max, opt) <= 0) { \ + while(back_pluggable_sockets_strict(sback, opt) <= 0) { \ _hts_in_html_parsing = 6; \ /* Wait .. */ \ - back_wait(back,back_max,opt,cache,0); \ + back_wait(sback,opt,cache,0); \ /* Transfer rate */ \ engine_stats(); \ /* Refresh various stats */ \ - HTS_STAT.stat_nsocket=back_nsoc(back,back_max); \ + HTS_STAT.stat_nsocket=back_nsoc(sback); \ HTS_STAT.stat_errors=fspc(NULL,"error"); \ HTS_STAT.stat_warnings=fspc(NULL,"warning"); \ HTS_STAT.stat_infos=fspc(NULL,"info"); \ - HTS_STAT.nbk=backlinks_done(liens,lien_tot,ptr); \ - HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,back,back_max); \ + HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr); \ + HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback); \ /* Check */ \ - if (!hts_htmlcheck_loop(back,back_max,-1,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \ + if (!hts_htmlcheck_loop(sback->lnk, sback->count,-1,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \ return -1; \ } \ } \ @@ -105,12 +105,23 @@ static const char *hts_tbdev[] = // forme le nom du fichier à sauver (save) à partir de fil et adr // système intelligent, qui renomme en cas de besoin (exemple: deux INDEX.HTML et index.html) -int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_adr,char* former_fil,char* referer_adr,char* referer_fil,httrackp* opt,lien_url** liens,int lien_tot,lien_back* back,int back_max,cache_back* cache,hash_struct* hash,int ptr,int numero_passe) { +int url_savename(char* adr_complete, char* fil_complete, char* save, + char* former_adr, char* former_fil, + char* referer_adr, char* referer_fil, + httrackp* opt, + lien_url** liens, int lien_tot, + struct_back* sback, cache_back* cache, hash_struct* hash, + int ptr, int numero_passe, const lien_back* headers) { + const char* mime_type = headers ? headers->r.contenttype : NULL; + lien_back* const back = sback->lnk; + const int back_max = sback->count; + /* */ char BIGSTK newfil[HTS_URLMAXSIZE*2]; /* ="" */ /*char BIGSTK normadr_[HTS_URLMAXSIZE*2];*/ char BIGSTK normadr_[HTS_URLMAXSIZE*2], normfil_[HTS_URLMAXSIZE*2]; - int protocol = 0; - static const char* protocol_str[] = {"http", "https", "ftp", "file", "unknown"}; + enum { PROTOCOL_HTTP, PROTOCOL_HTTPS, PROTOCOL_FTP, PROTOCOL_FILE, PROTOCOL_MMS, PROTOCOL_UNKNOWN }; + static const char* protocol_str[] = {"http", "https", "ftp", "file", "mms", "unknown"}; + int protocol = PROTOCOL_HTTP; char* normadr; char* normfil; char* fil; @@ -118,7 +129,8 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a char* print_adr; char *start_pos=NULL,*nom_pos=NULL,*dot_pos=NULL; // Position nom et point // pour changement d'extension ou de nom (content-disposition) - int ext_chg=0; + int ext_chg=0, ext_chg_delayed=0; + int is_html=0; char ext[256]; int max_char=0; //CLEAR @@ -175,13 +187,15 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a // à afficher sans ftp:// print_adr=jump_protocol(adr); if (strfield(adr_complete, "https:")) { - protocol = 1; + protocol = PROTOCOL_HTTPS; } else if (strfield(adr_complete, "ftp:")) { - protocol = 2; + protocol = PROTOCOL_FTP; } else if (strfield(adr_complete, "file:")) { - protocol = 3; + protocol = PROTOCOL_FILE; + } else if (strfield(adr_complete, "mms:")) { + protocol = PROTOCOL_MMS; } else { - protocol = 0; + protocol = PROTOCOL_HTTP; } // court-circuit pour lien primaire @@ -199,7 +213,6 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a if (liens!=NULL) { int i; -#if HTS_HASH i=hash_read(hash,normadr,normfil,1,opt->urlhack); // recherche table 1 (adr+fil) if (i>=0) { // ok, trouvé strcpybuff(save,liens[i]->sav); @@ -214,34 +227,6 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a strcpybuff(save,liens[i]->sav); // copier (formé à partir du nouveau lien!) return 0; } -#else - for(i=lien_tot-1;i>=0;i--) { -#if HTS_CASSE - if ((strcmp(liens[i]->adr,normadr)==0) && (strcmp(liens[i]->fil,normfil)==0)) -#else - if ((strfield2(liens[i]->adr,normadr)) && (strfield2(liens[i]->fil,normfil))) -#endif - { // ok c'est le même lien, adresse déja définie - strcpybuff(save,liens[i]->sav); - return 0; - } - if (liens[i]->former_adr) { // tester ancienne loc? -#if HTS_CASSE - if ((strcmp(liens[i]->former_adr,normadr)==0) && (strcmp(liens[i]->former_fil,normfil)==0)) -#else - if ((strfield2(liens[i]->former_adr,normadr)) && (strfield2(liens[i]->former_fil,normfil))) -#endif - { - // copier location moved! - strcpybuff(adr_complete,liens[i]->adr); - strcpybuff(fil_complete,liens[i]->fil); - // et save - strcpybuff(save,liens[i]->sav); // copier (formé à partir du nouveau lien!) - return 0; - } - } - } -#endif // chercher sans / ou avec / dans former { @@ -252,7 +237,6 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a fil_complete_patche[strlen(fil_complete_patche)-1]='\0'; else strcatbuff(fil_complete_patche,"/"); -#if HTS_HASH i=hash_read(hash,normadr,fil_complete_patche,2,opt->urlhack); // recherche table 2 (former_adr+former_fil) if (i>=0) { // écraser fil et adr (pas former_fil?????) @@ -262,26 +246,6 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a strcpybuff(save,liens[i]->sav); return 0; } -#else - // même boucle en gros - for(i=lien_tot-1;i>=0;i--) { - if (liens[i]->former_adr) { // former-adr? -#if HTS_CASSE - if ((strcmp(liens[i]->former_adr,normadr)==0) && (strcmp(liens[i]->former_fil,fil_complete_patche)==0)) -#else - if ((strfield2(liens[i]->former_adr,normadr)) && (strfield2(liens[i]->former_fil,fil_complete_patche))) -#endif - { // ok c'est le même lien, adresse déja définie - // écraser fil et adr (pas former_fil?????) - strcpybuff(adr_complete,liens[i]->adr); - strcpybuff(fil_complete,liens[i]->fil); - // écrire save - strcpybuff(save,liens[i]->sav); - return 0; - } - } - } -#endif } } @@ -298,47 +262,93 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a } fil=newfil; } - // décoder % + // Decode remaining % strcpybuff(fil,unescape_http(fil)); - + // , BUT do not decode high chars + //strcpybuff(fil,unescape_http_unharm(fil, 1)); + // YES (not server side, but fs/client side) + +#if HTS_USEMMS + /* .asx hack */ + if (headers != NULL && headers->r.cdispo[0] != 0 + && strfield(headers->r.contenttype, "video/") + && strfield2(get_ext(headers->r.cdispo), "asx") == 0) + { + ext_chg = 1; + strcpybuff(ext, "asx"); + } + else if (headers != NULL && headers->r.contenttype[0] != 0 + && strfield2(headers->r.contenttype, "video/x-ms-asf")) + { + char *exts = get_ext(headers->url_fil); + if (strfield2(exts, "wmv") == 0) + { + ext_chg = 1; + strcpybuff(ext, "wmv"); + } + else if (strfield2(exts, "asf") == 0) + { + ext_chg = 1; + strcpybuff(ext, "asf"); + } + else if (strfield2(exts, "avi") == 0) + { + ext_chg = 1; + strcpybuff(ext, "avi"); + } + else if (strfield2(exts, "asx") == 0) + { + ext_chg = 1; + strcpybuff(ext, "asx"); + } + } +#endif + /* replace shtml to html.. */ - switch (ishtml(fil)) { /* .html,.shtml,.. */ + if (opt->savename_delayed == 2) + is_html = -1; /* ALWAYS delay type */ + else + is_html = ishtml(fil); + switch ( is_html ) { /* .html,.shtml,.. */ case 1: if ( (strfield2(get_ext(fil),"html") == 0) && (strfield2(get_ext(fil),"htm") == 0) - ) { + ) + { strcpybuff(ext,"html"); ext_chg=1; } break; - case 0: - if (!strnotempty(ext)) { - if (is_userknowntype(get_ext(fil))) { // mime known by user - char BIGSTK mime[1024]; - mime[0]=ext[0]='\0'; - get_userhttptype(0,mime,get_ext(fil)); - if (strnotempty(mime)) { - give_mimext(ext,mime); - if (strnotempty(ext)) { - ext_chg=1; - } + case 0: + if (!strnotempty(ext)) { + if (is_userknowntype(fil)) { // mime known by user + char BIGSTK mime[1024]; + mime[0]=ext[0]='\0'; + get_userhttptype(0,mime,fil); + if (strnotempty(mime)) { + give_mimext(ext,mime); + if (strnotempty(ext)) { + ext_chg=1; } } } - break; + } + break; } - // si option check_type activée - if ((opt->check_type) && (!ext_chg)) { - int ishtest; + if (is_html < 0 && opt->check_type && !ext_chg) { + int ishtest=0; if ( (!strfield(adr_complete,"file://")) && (!strfield(adr_complete,"ftp://")) +#if HTS_USEMMS + && (!strfield(adr_complete,"mms://")) +#endif ) { // tester type avec requète HEAD si on ne connait pas le type du fichier if (!( (opt->check_type==1) && (fil[strlen(fil)-1]=='/') )) // slash doit être html? - if ((ishtest=ishtml(fil)) < 0) { // on ne sait pas si c'est un html ou un fichier.. + if (opt->savename_delayed == 2 || (ishtest=ishtml(fil)) < 0) { // on ne sait pas si c'est un html ou un fichier.. // lire dans le cache htsblk r = cache_read(opt,cache,adr,fil,NULL,NULL); // test uniquement if (r.statuscode != -1) { // pas d'erreur de lecture cache @@ -359,11 +369,11 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a } } // - } else if (is_userknowntype(fil)) { /* PATCH BY BRIAN SCHRÖDER. - Lookup mimetype not only by extension, - but also by filename */ + } else if (opt->savename_delayed != 2 && is_userknowntype(fil)) { /* PATCH BY BRIAN SCHRÖDER. + Lookup mimetype not only by extension, + but also by filename */ /* Note: "foo.cgi => text/html" means that foo.cgi shall have the text/html MIME file type, - that is, ".html" */ + that is, ".html" */ char BIGSTK mime[1024]; mime[0]=ext[0]='\0'; get_userhttptype(0, mime, fil); @@ -373,7 +383,26 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a ext_chg=1; } } - } else { // test imposible dans le cache, faire une requête + } + // note: if savename_delayed is enabled, the naming will be temporary (and slightly invalid!) + else if (opt->savename_delayed != 0) { + if (mime_type != NULL) { + ext[0] = '\0'; + if (*mime_type) { + give_mimext(ext, mime_type); + } + if (strnotempty(ext)) { + ext_chg = 1; + } + } else { + /* Avoid collisions (no collisionning detection) */ + sprintf(ext, "%x.%s", opt->state.delayedId++, DELAYED_EXT); + ext_chg = 1; + ext_chg_delayed = 1; /* due to naming system */ + } + } + // test imposible dans le cache, faire une requête + else { // #if HTS_ANALYSTE int hihp=_hts_in_html_parsing; @@ -400,9 +429,9 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a strcpybuff(curr_fil,fil_complete); // ajouter dans le backing le fichier en mode test // savename: rien car en mode test - if (back_add(back,back_max,opt,cache,curr_adr,curr_fil,BACK_ADD_TEST,referer_adr,referer_fil,1,NULL)!=-1) { + if (back_add(sback,opt,cache,curr_adr,curr_fil,BACK_ADD_TEST,referer_adr,referer_fil,1,NULL)!=-1) { int b; - b=back_index(back,back_max,curr_adr,curr_fil,BACK_ADD_TEST); + b=back_index(sback,curr_adr,curr_fil,BACK_ADD_TEST); if (b>=0) { int stop_looping=0; int petits_tours=0; @@ -410,10 +439,10 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a do { // temps à attendre, et remplir autant que l'on peut le cache (backing) if (back[b].status>0) { - back_wait(back,back_max,opt,cache,0); + back_wait(sback,opt,cache,0); } if (ptr>=0) { - back_fillmax(back,back_max,opt,cache,liens,ptr,numero_passe,lien_tot); + back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot); } // on est obligé d'appeler le shell pour le refresh.. @@ -424,17 +453,17 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a engine_stats(); // Refresh various stats - HTS_STAT.stat_nsocket=back_nsoc(back,back_max); + HTS_STAT.stat_nsocket=back_nsoc(sback); HTS_STAT.stat_errors=fspc(NULL,"error"); HTS_STAT.stat_warnings=fspc(NULL,"warning"); HTS_STAT.stat_infos=fspc(NULL,"info"); - HTS_STAT.nbk=backlinks_done(liens,lien_tot,ptr); - HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,back,back_max); + HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr); + HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback); - if (!hts_htmlcheck_loop(back,back_max,b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { + if (!hts_htmlcheck_loop(sback->lnk, sback->count,b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { return -1; } else if (_hts_cancel || !back_checkmirror(opt)) { // cancel 2 ou 1 (cancel parsing) - back_delete(opt,cache,back,b); // cancel test + back_delete(opt,cache,sback,b); // cancel test stop_looping = 1; } } @@ -443,11 +472,7 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a // traitement des 304,303.. if (back[b].status<=0) { - if ( (back[b].r.statuscode==301) - || (back[b].r.statuscode==302) - || (back[b].r.statuscode==303) - || (back[b].r.statuscode==307) - ) { // agh moved.. un tit tour de plus + if (HTTP_IS_REDIRECT(back[b].r.statuscode)) { // agh moved.. un tit tour de plus if ((petits_tours<5) && (former_adr) && (former_fil)) { // on va pas tourner en rond non plus! if ((int) strnotempty(back[b].r.location)) { // location existe! char BIGSTK mov_url[HTS_URLMAXSIZE*2],mov_adr[HTS_URLMAXSIZE*2],mov_fil[HTS_URLMAXSIZE*2]; @@ -480,7 +505,7 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a NULL) == 1) { /* forbidden */ has_been_moved = 1; - back_maydelete(opt,cache,back,b); // ok + back_maydelete(opt,cache,sback,b); // ok strcpybuff(curr_adr,mov_adr); strcpybuff(curr_fil,mov_fil); mov_url[0]='\0'; @@ -489,9 +514,14 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a } // ftp: stop! - if (strfield(mov_url,"ftp://")) { // ftp, ok on arrête + if (strfield(mov_url,"ftp://") +#if HTS_USEMMS + || strfield(mov_url,"mms://") +#endif + ) + { // ftp, ok on arrête has_been_moved = 1; - back_maydelete(opt,cache,back,b); // ok + back_maydelete(opt,cache,sback,b); // ok strcpybuff(curr_adr,mov_adr); strcpybuff(curr_fil,mov_fil); stop_looping = 1; @@ -508,17 +538,17 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a } // Ajouter URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET(); - if (back_add(back,back_max,opt,cache,mov_adr,mov_fil,methode,referer_adr,referer_fil,1,NULL)!=-1) { // OK + if (back_add(sback,opt,cache,mov_adr,mov_fil,methode,referer_adr,referer_fil,1,NULL)!=-1) { // OK if ( (opt->debug>1) && (opt->errlog!=NULL) ) { fspc(opt->errlog,"warning"); fprintf(opt->errlog,"(during prefetch) %s (%d) to link %s at %s%s"LF,back[b].r.msg,back[b].r.statuscode,back[b].r.location,curr_adr,curr_fil); test_flush; } // libérer emplacement backing actuel et attendre le prochain - back_maydelete(opt,cache,back,b); + back_maydelete(opt,cache,sback,b); strcpybuff(curr_adr,mov_adr); strcpybuff(curr_fil,mov_fil); - b=back_index(back,back_max,curr_adr,curr_fil,methode); + b=back_index(sback,curr_adr,curr_fil,methode); if (!get_test_request) has_been_moved = 1; // sinon ne pas forcer has_been_moved car non déplacé petits_tours++; @@ -560,7 +590,7 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a fspc(opt->errlog,0); fprintf(opt->errlog,"Error: (during prefetch) %s (%d) to link %s at %s%s"LF,back[b].r.msg,back[b].r.statuscode,back[b].r.location,curr_adr,curr_fil); test_flush; } - back_delete(opt,cache,back,b); + back_delete(opt,cache,sback,b); return -1; // ERREUR (404 par exemple) */ } @@ -584,7 +614,7 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a // FIN Si non déplacé, forcer type? // libérer emplacement backing - back_maydelete(opt,cache,back,b); + back_maydelete(opt,cache,sback,b); // --- --- --- // oops, a été déplacé.. on recalcule en récursif (osons!) @@ -594,7 +624,7 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a strcpybuff(fil_complete,curr_fil); // copier adr, fil - return url_savename(curr_adr,curr_fil,save,NULL,NULL,referer_adr,referer_fil,opt,liens,lien_tot,back,back_max,cache,hash,ptr,numero_passe); + return url_savename(curr_adr,curr_fil,save,NULL,NULL,referer_adr,referer_fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL); } // --- --- --- @@ -626,9 +656,18 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a // Donner nom par défaut? if (fil[strlen(fil)-1]=='/') { - if (!strfield(adr_complete,"ftp://")) + if (!strfield(adr_complete,"ftp://") +#if HTS_USEMMS + && !strfield(adr_complete,"mms://") +#endif + ) + { strcatbuff(fil,DEFAULT_HTML); // nommer page par défaut!! - else { +#if HTS_USEMMS + } else if (strfield(adr_complete,"mms://")) { + strcatbuff(fil,DEFAULT_MMS); +#endif + } else { if (!opt->proxy.active) strcatbuff(fil,DEFAULT_FTP); // nommer page par défaut (texte) else @@ -770,11 +809,15 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a strcpybuff(b, name[4]); /* not found replacement if any */ b += strlen(b); } - } + } else { + strcpybuff(b, name[4]); /* not found replacement if any */ + b += strlen(b); + } } break; case '%': *b++='%'; break; case 'n': // nom sans ext + *b='\0'; if (dot_pos) { if (!short_ver) // Noms longs strncatbuff(b,nom_pos,(int) (dot_pos - nom_pos)); @@ -812,9 +855,9 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a strncatbuff(b,dot_pos+1,3); } else { if (!short_ver) // Noms longs - strcpybuff(b,DEFAULT_EXT); // pas de.. + strcpybuff(b,DEFAULT_EXT + 1); // pas de.. else - strcpybuff(b,DEFAULT_EXT_SHORT); // pas de.. + strcpybuff(b,DEFAULT_EXT_SHORT + 1); // pas de.. } b+=strlen(b); // pointer à la fin // @@ -828,9 +871,9 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a strncatbuff(b,dot_pos+1,3); } else { if (!short_ver) // Noms longs - strcpybuff(b,DEFAULT_EXT); // pas de.. + strcpybuff(b,DEFAULT_EXT + 1); // pas de.. else - strcpybuff(b,DEFAULT_EXT_SHORT); // pas de.. + strcpybuff(b,DEFAULT_EXT_SHORT + 1); // pas de.. } b+=strlen(b); // pointer à la fin break; @@ -889,6 +932,19 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a strcatbuff(b, protocol_str[protocol]); b+=strlen(b); // pointer à la fin break; + + /* Patch by Juan Fco Rodriguez to get the full query string */ + case 'k': + { + char *d = strchr(fil_complete,'?'); + if( d != NULL ) + { + strcatbuff(b, d); + b+=strlen(b); + } + } + break; + } } else *b++=*a++; @@ -928,9 +984,7 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a } } -#if HTS_CASSE==0 hts_lowcase(save); -#endif /* // ne sert à rien car a déja été filtré normalement @@ -981,13 +1035,13 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a } // si un html à coup sûr - if ( (ext_chg!=0) ? (ishtml_ext(ext)==1) : (ishtml(fil)==1) ) { + if ( (ext_chg!=0) ? (ishtml_ext(ext) == 1) : (ishtml(fil)==1) ) { if (opt->savename_type%100==2) { // html/ - strcatbuff(save,"html/"); + strcatbuff(save, "html/"); } } else { if ((opt->savename_type%100==1) || (opt->savename_type%100==2)) { // html & images - strcatbuff(save,"images/"); + strcatbuff(save, "images/"); } } @@ -1062,9 +1116,7 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a break; } -#if HTS_CASSE==0 hts_lowcase(save); -#endif if (save[strlen(save)-1]=='/') strcatbuff(save,DEFAULT_HTML); // nommer page par défaut!! @@ -1246,6 +1298,24 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a } #endif + /* Ensure that the MANDATORY "temporary" extension is set */ + if (ext_chg_delayed) { + char *ptr; + char *lastDot = NULL; + for(ptr = save ; *ptr != 0 ; ptr++) { + if (*ptr == '.') { + lastDot = ptr; + } else if (*ptr == '/' || *ptr == '\\') { + lastDot = NULL; + } + } + if (lastDot == NULL) { + strcatbuff(save, "." DELAYED_EXT); + } else if (!IS_DELAYED_EXT(save)) { + strcatbuff(lastDot, "." DELAYED_EXT); + } + } + // chemin primaire éventuel A METTRE AVANT if (strnotempty(opt->path_html)) { char BIGSTK tempo[HTS_URLMAXSIZE*2]; @@ -1269,29 +1339,17 @@ int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_a printf("\nStart search\n"); #endif -#if HTS_HASH i=hash_read(hash,save,"",0,0); // lecture type 0 (sav) if (i>=0) -#else - for(i=lien_tot-1;i>=0;i--) { -#if DEBUG_SAVENAME -printf("%cParse: %d",13,i); -#endif - - if (liens[i]->sav_len==len) { // même taille de chaîne -#if HTS_CASSE - if (strcmp(liens[i]->sav,save)==0) // existe déja -#else - if (strfield2(liens[i]->sav,save)) // un tel nom existe déja -#endif -#endif { -#if HTS_CASSE - if ((strcmp(liens[i]->adr,adr)==0) && (strcmp(liens[i]->fil,fil_complete)==0)) -#else - if ((strfield2(liens[i]->adr, normadr)) && (strfield2(liens[i]->fil, normfil))) - //if ((strfield2(liens[i]->adr,adr)) && (strfield2(liens[i]->fil,fil_complete))) -#endif + int sameAdr = ( strfield2(liens[i]->adr, normadr) != 0 ); + int sameFil; + // NO - URL hack is only for stripping // and www. + //if (opt->urlhack != 0) + // sameFil = ( strfield2(liens[i]->fil, normfil) != 0); + //else + sameFil = ( strcmp(liens[i]->fil, normfil) == 0); + if (sameAdr && sameFil) { // ok c'est le même lien, adresse déja définie /* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */ if (strcmp(liens[i]->sav, save) != 0) { @@ -1301,13 +1359,12 @@ printf("%cParse: %d",13,i); #if DEBUG_SAVENAME printf("\nOK ALREADY DEFINED\n",13,i); #endif -#if HTS_CASSE -#endif } else { // utilisé par un AUTRE, changer de nom char BIGSTK tempo[HTS_URLMAXSIZE*2]; char* a=save+strlen(save)-1; char* b; int n=2; + char collisionSeparator = ( (opt->savename_83 != 2) ? '-' : '_'); tempo[0]='\0'; #if DEBUG_SAVENAME @@ -1325,7 +1382,7 @@ printf("\nWRONG CASE UNMATCH : \n%s\n%s, REDEFINE\n",liens[i]->fil,fil_complete) // tester la présence d'un -xx (ex: index-2.html -> index-3.html) b=tempo+strlen(tempo)-1; while (isdigit((unsigned char)*b)) b--; - if (*b=='-') { + if (*b == collisionSeparator) { sscanf(b+1,"%d",&n); *b='\0'; // couper n++; // plus un @@ -1343,7 +1400,7 @@ printf("\nWRONG CASE UNMATCH : \n%s\n%s, REDEFINE\n",liens[i]->fil,fil_complete) } // ajouter -xx (ex: index.html -> index-2.html) - sprintf(tempo+strlen(tempo),"-%d",n); + sprintf(tempo+strlen(tempo),"%c%d", collisionSeparator, n); // ajouter extension if (*a=='.') @@ -1354,13 +1411,7 @@ printf("\nWRONG CASE UNMATCH : \n%s\n%s, REDEFINE\n",liens[i]->fil,fil_complete) //printf("switched: %s\n",save); } // if -#if HTS_HASH } -#else - } // if - } // if sav_len - } // for -#endif #if DEBUG_SAVENAME printf("\nEnd search, %s\n",fil_complete); #endif @@ -1399,9 +1450,9 @@ void standard_name(char* b,char* dot_pos,char* nom_pos,char* fil_complete,int sh strncatbuff(b,dot_pos+1,3); } else { if (!short_ver) // Noms longs - strcatbuff(b,DEFAULT_EXT); // pas de.. + strcatbuff(b,DEFAULT_EXT + 1); // pas de.. else - strcatbuff(b,DEFAULT_EXT_SHORT); // pas de.. + strcatbuff(b,DEFAULT_EXT_SHORT + 1); // pas de.. } } diff --git a/src/htsname.h b/src/htsname.h index 61ed1de..908e8e6 100644 --- a/src/htsname.h +++ b/src/htsname.h @@ -42,9 +42,24 @@ Please visit our Website: http://www.httrack.com #include "htscore.h" +#define DELAYED_EXT "delayed" +#define IS_DELAYED_EXT(a) ( ((a) != NULL) && ((a)[0] != 0) && strendwith_(a, "." DELAYED_EXT) ) +static int strendwith_(const char* a, const char* b) { + int i, j; + for(i = 0 ; a[i] != 0 ; i++); + for(j = 0 ; b[j] != 0 ; j++); + while(i >= 0 && j >= 0 && a[i] == b[j]) { + i--; + j--; + } + return (j == -1); +} + + /* Library internal definictions */ #ifdef HTS_INTERNAL_BYTECODE -int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_adr,char* former_fil,char* referer_adr,char* referer_fil,httrackp* opt,lien_url** liens,int lien_tot,lien_back* back,int back_max,cache_back* cache,hash_struct* hash,int ptr,int numero_passe); +// note: 'headers' can either be null, or incomplete (only r member filled) +int url_savename(char* adr_complete,char* fil_complete,char* save,char* former_adr,char* former_fil,char* referer_adr,char* referer_fil,httrackp* opt,lien_url** liens,int lien_tot,struct_back* sback,cache_back* cache,hash_struct* hash,int ptr,int numero_passe,const lien_back* headers); void standard_name(char* b,char* dot_pos,char* nom_pos,char* fil_complete,int short_ver); void url_savename_addstr(char* d,char* s); char* url_md5(char* fil_complete); diff --git a/src/htsnet.h b/src/htsnet.h index 7b7cc1a..4880721 100644 --- a/src/htsnet.h +++ b/src/htsnet.h @@ -118,10 +118,12 @@ typedef struct SOCaddr { if (hpsize == sizeof(struct sockaddr_in)) { \ server_len=sizeof(struct sockaddr_in); \ SOCaddr_sinfamily(server) = (*(struct sockaddr_in*)(hpaddr)).sin_family; \ + SOCaddr_sinport(server) = (*(struct sockaddr_in*)(hpaddr)).sin_port; \ memcpy(&SOCaddr_sinaddr(server), &(*(struct sockaddr_in*)(hpaddr)).sin_addr, sizeof(SOCaddr_sinaddr(server))); \ } else if (hpsize == 4) {\ server_len=sizeof(struct sockaddr_in); \ SOCaddr_sinfamily(server) = AF_INET; \ + SOCaddr_sinport(server) = 0; \ memcpy(&SOCaddr_sinaddr(server), (hpaddr), sizeof(SOCaddr_sinaddr(server))); \ } else if ((hpsize > 0) && (hpsize <= sizeof(server))) { \ server_len=hpsize; \ @@ -194,22 +196,27 @@ typedef struct SOCaddr { if (hpsize == sizeof(struct sockaddr_in6)) { \ server_len=sizeof(struct sockaddr_in6); \ SOCaddr_sinfamily(server) = (*(struct sockaddr_in6*)(hpaddr)).sin6_family; \ + SOCaddr_sinport(server) = (*(struct sockaddr_in6*)(hpaddr)).sin6_port; \ SOCaddr_sinflowinfo(server) = (*(struct sockaddr_in6*)(hpaddr)).sin6_flowinfo; \ memcpy(&SOCaddr_sinaddr(server), &(*(struct sockaddr_in6*)(hpaddr)).sin6_addr, sizeof(SOCaddr_sinaddr(server))); \ } else if (hpsize > sizeof(struct sockaddr_in6)) { \ server_len=hpsize; \ + SOCaddr_sinport(server) = 0; \ memcpy(&(server), hpaddr, hpsize); \ } else if (hpsize == sizeof(struct sockaddr_in)) { \ server_len=sizeof(struct sockaddr_in); \ (*(struct sockaddr_in*)(&server)).sin_family = AF_INET; \ + SOCaddr_sinport(server) = (*(struct sockaddr_in*)(hpaddr)).sin_port; \ memcpy(&(*(struct sockaddr_in*)&(server)).sin_addr, &(*(struct sockaddr_in*)(hpaddr)).sin_addr, sizeof((*(struct sockaddr_in*)(hpaddr)).sin_addr)); \ } else if (hpsize == 4) {\ server_len=sizeof(struct sockaddr_in); \ (*(struct sockaddr_in*)(&server)).sin_family = AF_INET; \ + SOCaddr_sinport(server) = 0; \ memcpy(&(*(struct sockaddr_in*)&(server)).sin_addr, hpaddr, 4); \ } else if (hpsize == 16) {\ server_len=sizeof(struct sockaddr_in6); \ SOCaddr_sinfamily(server) = AF_INET6; \ + SOCaddr_sinport(server) = 0; \ memcpy(&SOCaddr_sinaddr(server), (hpaddr), 16); \ } else if ((hpsize > 0) && (hpsize <= sizeof(server))) { \ server_len=hpsize; \ diff --git a/src/htsopt.h b/src/htsopt.h index 3328ce0..bf62c72 100644 --- a/src/htsopt.h +++ b/src/htsopt.h @@ -77,6 +77,7 @@ typedef struct htsoptstate { int mimehtml_created; char mimemid[256]; FILE* mimefp; + int delayedId; /* */ htscallbacks callbacks; } htsoptstate; @@ -108,6 +109,9 @@ typedef struct httrackp { int rateout; // nombre d'octets minium pour le transfert int maxtime; // temps max en secondes int maxrate; // taux de transfert max +#if HTS_USEMMS + int mms_maxtime; // max duration of a mms file +#endif float maxconn; // nombre max de connexions/s int waittime; // démarrage programmé int cache; // génération d'un cache @@ -117,6 +121,8 @@ typedef struct httrackp { int savename_83; // conversion 8-3 pour les noms de fichiers int savename_type; // type de noms: structure originale/html-images en un seul niveau char savename_userdef[256]; // structure userdef (ex: %h%p/%n%q.%t) + int savename_delayed; // delayed type check + int delayed_cached; // delayed type check can be cached to speedup updates int mimehtml; // MIME-html int user_agent_send; // user agent (ex: httrack/1.0 [sun]) char user_agent[128]; @@ -197,6 +203,7 @@ typedef struct hts_stat_struct { // int stat_files; // nombre de fichiers écrits int stat_updated_files; // nombre de fichiers mis à jour + int stat_background; // nombre de fichiers écrits en arrière plan // int stat_nrequests; // nombre de requêtes sur socket int stat_sockid; // nombre de sockets allouées au total diff --git a/src/htsparse.c b/src/htsparse.c index 79cc1cc..4aa1b7e 100644 --- a/src/htsparse.c +++ b/src/htsparse.c @@ -63,7 +63,7 @@ Please visit our Website: http://www.httrack.com // parser #include "htsparse.h" - +#include "htsback.h" // specific defines #define urladr (liens[ptr]->adr) @@ -150,66 +150,72 @@ Please visit our Website: http://www.httrack.com #define HT_ADD_END { \ int ok=0;\ if (ht_buff) { \ - char digest[32+2];\ - digest[0]='\0';\ - domd5mem(ht_buff,ht_len,digest,1);\ - if (fsize(fconv(savename))==ht_len) { \ - int mlen = 0;\ - char* mbuff;\ - cache_readdata(cache,"//[HTML-MD5]//",savename,&mbuff,&mlen);\ - if (mlen) mbuff[mlen]='\0';\ - if ((mlen == 32) && (strcmp(((mbuff!=NULL)?mbuff:""),digest)==0)) {\ - ok=1;\ - if ( (opt->debug>1) && (opt->log!=NULL) ) {\ - fspc(opt->log,"debug"); fprintf(opt->log,"File not re-written (md5): %s"LF,savename);\ - test_flush;\ - }\ - } else {\ - ok=0;\ - } \ - }\ - if (!ok) { \ - fp=filecreate(savename); \ - if (fp) { \ - if (ht_len>0) {\ - if ((INTsys)fwrite(ht_buff,1,ht_len,fp) != ht_len) { \ - int fcheck;\ - if ((fcheck=check_fatal_io_errno())) {\ - opt->state.exit_xh=-1;\ - }\ - if (opt->errlog) { \ - fspc(opt->errlog,"error"); fprintf(opt->errlog,"Unable to write HTML file %s: %s"LF, savename, strerror(errno));\ - if (fcheck) {\ - fspc(opt->errlog,"error");\ - fprintf(opt->errlog,"* * Fatal write error, giving up"LF);\ - }\ - test_flush;\ - }\ - }\ - }\ - fclose(fp); fp=NULL; \ - if (strnotempty(r->lastmodified)) \ - set_filetime_rfc822(savename,r->lastmodified); \ - } else {\ - int fcheck;\ - if ((fcheck=check_fatal_io_errno())) {\ - opt->state.exit_xh=-1;\ - }\ - if (opt->errlog) { \ - fspc(opt->errlog,"error");\ - fprintf(opt->errlog,"Unable to save file %s : %s"LF, savename, strerror(errno));\ - if (fcheck) {\ - fspc(opt->errlog,"error");\ - fprintf(opt->errlog,"* * Fatal write error, giving up"LF);\ - }\ - test_flush;\ - }\ - }\ - } else {\ - filenote(savename,NULL); \ - }\ - if (cache->ndx)\ - cache_writedata(cache->ndx,cache->dat,"//[HTML-MD5]//",savename,digest,(int)strlen(digest));\ + char digest[32+2];\ + INTsys fsize_old=fsize(fconv(savename));\ + digest[0]='\0';\ + domd5mem(ht_buff,ht_len,digest,1);\ + if (fsize_old==ht_len) { \ + int mlen = 0;\ + char* mbuff;\ + cache_readdata(cache,"//[HTML-MD5]//",savename,&mbuff,&mlen);\ + if (mlen) \ + mbuff[mlen]='\0';\ + if ((mlen == 32) && (strcmp(((mbuff!=NULL)?mbuff:""),digest)==0)) {\ + ok=1;\ + if ( (opt->debug>1) && (opt->log!=NULL) ) {\ + fspc(opt->log,"debug"); fprintf(opt->log,"File not re-written (md5): %s"LF,savename);\ + test_flush;\ + }\ + } else {\ + ok=0;\ + } \ + }\ + if (!ok) { \ + file_notify(urladr, urlfil, savename, 1, 1, r->notmodified); \ + fp=filecreate(savename); \ + if (fp) { \ + if (ht_len>0) {\ + if ((INTsys)fwrite(ht_buff,1,ht_len,fp) != ht_len) { \ + int fcheck;\ + if ((fcheck=check_fatal_io_errno())) {\ + opt->state.exit_xh=-1;\ + }\ + if (opt->errlog) { \ + fspc(opt->errlog,"error"); fprintf(opt->errlog,"Unable to write HTML file %s: %s"LF, savename, strerror(errno));\ + if (fcheck) {\ + fspc(opt->errlog,"error");\ + fprintf(opt->errlog,"* * Fatal write error, giving up"LF);\ + }\ + test_flush;\ + }\ + }\ + }\ + fclose(fp); fp=NULL; \ + if (strnotempty(r->lastmodified)) \ + set_filetime_rfc822(savename,r->lastmodified); \ + } else {\ + int fcheck;\ + if ((fcheck=check_fatal_io_errno())) {\ + fspc(opt->log,"error"); fprintf(opt->log,"Mirror aborted: disk full or filesystem problems"LF); \ + test_flush; \ + opt->state.exit_xh=-1;\ + }\ + if (opt->errlog) { \ + fspc(opt->errlog,"error");\ + fprintf(opt->errlog,"Unable to save file %s : %s"LF, savename, strerror(errno));\ + if (fcheck) {\ + fspc(opt->errlog,"error");\ + fprintf(opt->errlog,"* * Fatal write error, giving up"LF);\ + }\ + test_flush;\ + }\ + }\ + } else {\ + file_notify(urladr, urlfil, savename, 0, 0, r->notmodified); \ + filenote(savename,NULL); \ + }\ + if (cache->ndx)\ + cache_writedata(cache->ndx,cache->dat,"//[HTML-MD5]//",savename,digest,(int)strlen(digest));\ } \ freet(ht_buff); ht_buff=NULL; \ } @@ -243,13 +249,10 @@ Please visit our Website: http://www.httrack.com // enfin on écrit à l'adresse courante du buffer, qu'on incrémente. on décrémente la taille dispo d'autant ensuite // codebase: si non nul et si .class stockee on le note pour chemin primaire pour classes // FA,FS: former_adr et former_fil, lien original -#if HTS_HASH #define liens_record_sav_len(A) -#else -#define liens_record_sav_len(A) (A)->sav_len=strlen((A)->sav) -#endif // COPIE DE HTSCORE.C + #define liens_record(A,F,S,FA,FF) { \ int notecode=0; \ int lienurl_len=((sizeof(lien_url)+HTS_ALIGN-1)/HTS_ALIGN)*HTS_ALIGN,\ @@ -260,58 +263,56 @@ Please visit our Website: http://www.httrack.com former_adr_len=strlen(FA),\ former_fil_len=strlen(FF); \ if (former_adr_len>0) {\ - former_adr_len=(former_adr_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \ - former_fil_len=(former_fil_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \ - } else former_adr_len=former_fil_len=0;\ - if (strlen(F)>6) if (strnotempty(codebase)) if (strfield(F+strlen(F)-6,".class")) { notecode=1; \ - cod_len=strlen(codebase); cod_len=(cod_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; } \ - adr_len=(adr_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; fil_len=(fil_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; sav_len=(sav_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \ - if ((int) lien_size < (int) (adr_len+fil_len+sav_len+cod_len+former_adr_len+former_fil_len+lienurl_len)) { \ - lien_buffer=(char*) ((void*) calloct(add_tab_alloc,1)); \ - lien_size=add_tab_alloc; \ - if (lien_buffer!=NULL) { \ - liens[lien_tot]=(lien_url*) (void*) lien_buffer; lien_buffer+=lienurl_len; lien_size-=lienurl_len; \ - liens[lien_tot]->firstblock=1; \ + former_adr_len=(former_adr_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \ + former_fil_len=(former_fil_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \ + } else \ + former_adr_len=former_fil_len=0;\ + if (strlen(F)>6) if (strnotempty(codebase)) if (strfield(F+strlen(F)-6,".class")) {\ + notecode=1; \ + cod_len=strlen(codebase); \ + cod_len=(cod_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \ } \ + adr_len=(adr_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \ + fil_len=(fil_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \ + sav_len=(sav_len/HTS_ALIGN)*HTS_ALIGN+HTS_ALIGN*2; \ + if ((int) lien_size < (int) (adr_len+fil_len+sav_len+cod_len+former_adr_len+former_fil_len+lienurl_len)) { \ + lien_buffer=(char*) ((void*) calloct(add_tab_alloc,1)); \ + lien_size=add_tab_alloc; \ + if (lien_buffer!=NULL) { \ + liens[lien_tot]=(lien_url*) (void*) lien_buffer; lien_buffer+=lienurl_len; lien_size-=lienurl_len; \ + liens[lien_tot]->firstblock=1; \ + } \ } else { \ - liens[lien_tot]=(lien_url*) (void*) lien_buffer; lien_buffer+=lienurl_len; lien_size-=lienurl_len; \ - liens[lien_tot]->firstblock=0; \ + liens[lien_tot]=(lien_url*) (void*) lien_buffer; lien_buffer+=lienurl_len; lien_size-=lienurl_len; \ + liens[lien_tot]->firstblock=0; \ } \ if (liens[lien_tot]!=NULL) { \ - liens[lien_tot]->adr=lien_buffer; lien_buffer+=adr_len; lien_size-=adr_len; \ - liens[lien_tot]->fil=lien_buffer; lien_buffer+=fil_len; lien_size-=fil_len; \ - liens[lien_tot]->sav=lien_buffer; lien_buffer+=sav_len; lien_size-=sav_len; \ - liens[lien_tot]->cod=NULL; \ - if (notecode) { liens[lien_tot]->cod=lien_buffer; lien_buffer+=cod_len; lien_size-=cod_len; strcpybuff(liens[lien_tot]->cod,codebase); } \ - if (former_adr_len>0) {\ - liens[lien_tot]->former_adr=lien_buffer; lien_buffer+=former_adr_len; lien_size-=former_adr_len; \ - liens[lien_tot]->former_fil=lien_buffer; lien_buffer+=former_fil_len; lien_size-=former_fil_len; \ - strcpybuff(liens[lien_tot]->former_adr,FA); \ - strcpybuff(liens[lien_tot]->former_fil,FF); \ - }\ - strcpybuff(liens[lien_tot]->adr,A); \ - strcpybuff(liens[lien_tot]->fil,F); \ - strcpybuff(liens[lien_tot]->sav,S); \ - liens_record_sav_len(liens[lien_tot]); \ - hash_write(hashptr,lien_tot,opt->urlhack); \ + liens[lien_tot]->adr=lien_buffer; lien_buffer+=adr_len; lien_size-=adr_len; \ + liens[lien_tot]->fil=lien_buffer; lien_buffer+=fil_len; lien_size-=fil_len; \ + liens[lien_tot]->sav=lien_buffer; lien_buffer+=sav_len; lien_size-=sav_len; \ + liens[lien_tot]->cod=NULL; \ + if (notecode) { \ + liens[lien_tot]->cod=lien_buffer; \ + lien_buffer+=cod_len; \ + lien_size-=cod_len; \ + strcpybuff(liens[lien_tot]->cod,codebase); \ + } \ + if (former_adr_len>0) {\ + liens[lien_tot]->former_adr=lien_buffer; lien_buffer+=former_adr_len; lien_size-=former_adr_len; \ + liens[lien_tot]->former_fil=lien_buffer; lien_buffer+=former_fil_len; lien_size-=former_fil_len; \ + strcpybuff(liens[lien_tot]->former_adr,FA); \ + strcpybuff(liens[lien_tot]->former_fil,FF); \ + }\ + strcpybuff(liens[lien_tot]->adr,A); \ + strcpybuff(liens[lien_tot]->fil,F); \ + strcpybuff(liens[lien_tot]->sav,S); \ + liens_record_sav_len(liens[lien_tot]); \ + hash_write(hashptr,lien_tot,opt->urlhack); \ } \ } #define ENGINE_LOAD_CONTEXT() \ - lien_url** liens = (lien_url**) str->liens; \ - httrackp* opt = (httrackp*) str->opt; \ - lien_back* back = (lien_back*) str->back; \ - cache_back* cache = (cache_back*) str->cache; \ - hash_struct* hashptr = (hash_struct*) str->hashptr; \ - int back_max = str->back_max; \ - int numero_passe = str->numero_passe; \ - int add_tab_alloc = str->add_tab_alloc; \ - /* */ \ - int lien_tot = * ( (int*) (str->lien_tot_) ); \ - int ptr = * ( (int*) (str->ptr_) ); \ - int lien_size = * ( (int*) (str->lien_size_) ); \ - char* lien_buffer = * ( (char**) (str->lien_buffer_) ); \ - /* */ \ + ENGINE_LOAD_CONTEXT_BASE(); \ /* */ \ htsblk* r = stre->r_; \ hash_struct* hash = stre->hash_; \ @@ -336,11 +337,7 @@ Please visit our Website: http://www.httrack.com FILE* makestat_fp = stre->makestat_fp #define ENGINE_SAVE_CONTEXT() \ - /* Apply changes */ \ - * ( (int*) (str->lien_tot_) ) = lien_tot; \ - * ( (int*) (str->ptr_) ) = ptr; \ - * ( (int*) (str->lien_size_) ) = lien_size; \ - * ( (char**) (str->lien_buffer_) ) = lien_buffer; \ + ENGINE_SAVE_CONTEXT_BASE(); \ /* */ \ * stre->error_ = error; \ * stre->store_errpage_ = store_errpage; \ @@ -404,7 +401,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { char* adr=r->adr; // pointeur (on parcourt) char* lastsaved; // adresse du dernier octet sauvé + 1 if ( (opt->debug>1) && (opt->log!=NULL) ) { - fspc(opt->log,"debug"); fprintf(opt->log,"scan file.."LF); test_flush; + fspc(opt->log,"debug"); fprintf(opt->log,"scanning file %s%s (%s).."LF, urladr, urlfil, savename); test_flush; } @@ -459,13 +456,14 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { int inscriptgen=0; // on est dans un code générant, ex après obj.write(".. //int inscript_check_comments=0, inscript_in_comments=0; // javascript comments char scriptgen_q='\0'; // caractère faisant office de guillemet (' ou ") - int no_esc_utf=0; // ne pas echapper chars > 127 + //int no_esc_utf=0; // ne pas echapper chars > 127 int nofollow=0; // ne pas scanner // int parseall_lastc='\0'; // dernier caractère parsé pour parseall //int parseall_incomment=0; // dans un /* */ (exemple: a = /* URL */ "img.gif";) // - char* intag_start=adr; + char* intag_start = adr; + char* intag_name = NULL; char* intag_startattr=NULL; int intag_start_valid=0; int intag_ctype=0; @@ -513,6 +511,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { if (ptr == 0) { intag=1; intag_start_valid=0; + intag_name = NULL; } /* Check is the file is a .js file */ else if ( @@ -557,14 +556,14 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { } // Detect UTF8 format - if (is_unicode_utf8((unsigned char*) r->adr, (unsigned int) r->size) == 1) { - no_esc_utf=1; - } else { - no_esc_utf=0; - } - // Hack to prevent any problems with ram files of other files - * ( r->adr + r->size ) = '\0'; + //if (is_unicode_utf8((unsigned char*) r->adr, (unsigned int) r->size) == 1) { + // no_esc_utf=1; + //} else { + // no_esc_utf=0; + //} + // Hack to prevent any problems with ram files of other files + * ( r->adr + r->size ) = '\0'; // ------------------------------------------------------------ // analyser ce qu'il y a en mémoire (fichier html) @@ -627,6 +626,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { if (p) { // ok center if (makeindex_fp==NULL) { + file_notify("", "", fconcat(opt->path_html,"index.html"), 1, 1, 0); verif_backblue(opt,opt->path_html); // générer gif makeindex_fp=filecreate(fconcat(opt->path_html,"index.html")); if (makeindex_fp!=NULL) { @@ -705,11 +705,31 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { && (!in_media) /* Not in media */ ) { intag=1; - intag_ctype=0; - //parseall_incomment=0; - //inquote=0; // effacer quote - intag_start=adr; intag_start_valid=1; - codebase[0]='\0'; // effacer éventuel codebase + intag_ctype=0; + //parseall_incomment=0; + //inquote=0; // effacer quote + intag_start = adr; + for(intag_name = adr + 1 ; is_realspace(*intag_name) ; intag_name++ ); + intag_start_valid = 1; + codebase[0]='\0'; // effacer éventuel codebase + + /* Meta ? */ + if (check_tag(intag_start, "meta")) { + int pos; + // <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + if ((pos = rech_tageq_all(adr, "http-equiv"))) { + const char* token = NULL; + int len = rech_endtoken(adr + pos, &token); + if (len > 0) { + if (strfield(token, "content-type")) { + intag_ctype=1; + } + else if (strfield(token, "refresh")) { + intag_ctype=2; + } + } + } + } if (opt->getmode & 1) { // sauver html p=strfield(adr,"</html"); @@ -719,17 +739,19 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { char* eol="\n"; if (strchr(r->adr,'\r')) eol="\r\n"; - if (strnotempty(opt->footer)) { - char BIGSTK tempo[1024+HTS_URLMAXSIZE*2]; - char gmttime[256]; - tempo[0]='\0'; - time_gmt_rfc822(gmttime); - strcatbuff(tempo,eol); - sprintf(tempo+strlen(tempo),opt->footer,jump_identification(urladr),urlfil,gmttime,HTTRACK_VERSIONID,"","","","","","",""); - strcatbuff(tempo,eol); - //fwrite(tempo,1,strlen(tempo),fp); - HT_ADD(tempo); - if (r->charset[0]) { + if (strnotempty(opt->footer) || opt->urlmode != 4) { /* != preserve */ + if (strnotempty(opt->footer)) { + char BIGSTK tempo[1024+HTS_URLMAXSIZE*2]; + char gmttime[256]; + tempo[0]='\0'; + time_gmt_rfc822(gmttime); + strcatbuff(tempo,eol); + sprintf(tempo+strlen(tempo),opt->footer,jump_identification(urladr),urlfil,gmttime,HTTRACK_VERSIONID,"","","","","","",""); + strcatbuff(tempo,eol); + //fwrite(tempo,1,strlen(tempo),fp); + HT_ADD(tempo); + } + if (strnotempty(r->charset)) { HT_ADD("<!-- Added by HTTrack --><meta http-equiv=\"content-type\" content=\"text/html;charset="); HT_ADD(r->charset); HT_ADD("\"><!-- /Added by HTTrack -->"); @@ -758,6 +780,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { intag=0; incomment=0; intag_start_valid=0; + intag_name = NULL; if (opt->parsedebug) { HT_ADD("<@@ /inscript @@>"); } } else if (!incomment) { intag=0; //inquote=0; @@ -792,6 +815,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { intag=0; incomment=0; intag_start_valid=0; + intag_name = NULL; } #if GT_ENDS_COMMENT /* wrong comment ending */ @@ -804,6 +828,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { intag=0; incomment=0; intag_start_valid=0; + intag_name = NULL; } } #endif @@ -858,7 +883,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { p=0; valid_p=1; } - else if (strcmp(in_media,"AAM")==0) { // AAM + else if (strcmp(in_media, "AAM")==0) { // AAM if (is_space((unsigned char)adr[0]) && ! is_space((unsigned char)adr[1])) { char* a = adr + 1; int n = 0; @@ -994,7 +1019,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { // <A HREF=.. pour les liens HTML p=rech_tageq(adr,"href"); if (p) { // href.. tester si c'est une bas href! - if ((intag_start_valid) && check_tag(intag_start,"base")) { // oui! + if ((intag_start_valid) && check_tag(intag_start, "base")) { // oui! // ** note: base href et codebase ne font pas bon ménage.. p_type=2; // c'est un chemin } @@ -1206,27 +1231,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { } } - // charset meta tags - if (p==0) { - if ((intag_start_valid) && check_tag(intag_start,"meta")) { - int pos; - // <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> - if ((pos=rech_tageq(adr, "http-equiv"))) { - const char* token = NULL; - int len = rech_endtoken(adr + pos, &token); - if (len > 0) { - if (strfield(token, "content-type")) { - intag_ctype=1; - } - else if (strfield(token, "refresh")) { - intag_ctype=2; - } - } - } - } - } - - // entrée dans une applet javascript + // entrée dans une applet javascript /*if (!inscript) { // sinon on est dans un obj.write(".. if (p==0) if (rech_sampletag(adr,"script")) @@ -1294,6 +1299,12 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { nc = strfield(adr,".src"); // nom.src="image"; if (!nc) nc = strfield(adr,".location"); // document.location="doc" if (!nc) nc = strfield(adr,":location"); // javascript:location="doc" + if (!nc) { // location="doc" + if ( ( nc = strfield(adr,"location") ) + && !isspace(*(adr - 1)) + ) + nc = 0; + } if (!nc) nc = strfield(adr,".href"); // document.location="doc" if (!nc) if ( (nc = strfield(adr,".open")) ) { // window.open("doc",.. expected='('; // parenthèse @@ -1506,7 +1517,6 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { char* a; #endif // "Comparisons of scheme names MUST be case-insensitive" (RFC2616) - //if ((strncmp(tempo,"http://",7)==0) || (strncmp(tempo,"ftp://",6)==0)) // ok pas de problème if ( (strfield(tempo,"http:")) || (strfield(tempo,"ftp:")) @@ -1516,6 +1526,9 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { (strfield(tempo,"https:")) ) #endif +#if HTS_USEMMS + || strfield(tempo,"mms:") +#endif ) // ok pas de problème url_ok=1; else if (tempo[strlen(tempo)-1]=='/') { // un slash: ok.. @@ -1611,6 +1624,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { char quote='\0'; int quoteinscript=0; int noquote=0; + char *tag_attr_start = adr; // si nofollow ou un stop a été déclenché, réécrire tous les liens en externe if ((nofollow) || (opt->state.stop)) @@ -1739,6 +1753,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { if (!inscript && !in_media) { intag=0; // PLUS dans un tag! intag_start_valid=0; + intag_name = NULL; } ok=0; } @@ -1813,7 +1828,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { // 1: interdit (patcher tout de même adresse) if ((opt->debug>1) && (opt->log!=NULL)) { - fspc(opt->log,"debug"); fprintf(opt->log,"link detected in html: %s"LF,lien); test_flush; + fspc(opt->log,"debug"); fprintf(opt->log,"link detected in html (tag): %s"LF,lien); test_flush; } // external check @@ -1905,7 +1920,9 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { unescape_amp(query); // décoder l'inutile (%2E par exemple) et coder espaces // XXXXXXXXXXXXXXXXX strcpybuff(lien,unescape_http(lien)); - strcpybuff(lien,unescape_http_unharm(lien, (no_esc_utf)?0:1)); + //strcpybuff(lien,unescape_http_unharm(lien, (no_esc_utf)?0:1)); + /* Never unescape high-chars (we don't know the encoding!!) */ + strcpybuff(lien,unescape_http_unharm(lien, 1)); /* note: '%' is still escaped */ escape_remove_control(lien); escape_spc_url(lien); strcatbuff(lien,query); /* restore */ @@ -2076,10 +2093,18 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { if (!error) { // pas d'erreur? if (p_type==2) { // code ET PAS codebase char* a=lien+strlen(lien)-1; + char* start_of_filename = jump_identification(lien); + if (start_of_filename != NULL + && (start_of_filename = strchr(start_of_filename, '/')) != NULL) + start_of_filename++; + if (start_of_filename == NULL) + strcatbuff(lien, "/"); while( (a > lien) && (*a) && (*a!='/')) a--; - if (*a=='/') // ok on a repéré le dernier / - *(a+1)='\0'; // couper - else { + if (*a=='/') { // ok on a repéré le dernier / + if (start_of_filename != NULL && a >= start_of_filename) { + *(a+1)='\0'; // couper + } + } else { *lien='\0'; // éliminer error=1; // erreur, ne pas poursuivre } @@ -2278,7 +2303,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { } forbidden_url=hts_acceptlink(opt,ptr,lien_tot,liens, adr,fil, - NULL, NULL, + intag_name ? intag_name : NULL, intag_name ? tag_attr_start : NULL, &set_prio_to, &just_test_it); if ((opt->debug>1) && (opt->log!=NULL)) { @@ -2292,8 +2317,6 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { // calculer meme_adresse meme_adresse=strfield2(jump_identification(adr),jump_identification(urladr)); - - // Début partie sauvegarde // ici on forme le nom du fichier à sauver, et on patche l'URL @@ -2321,7 +2344,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { //char last_fil[HTS_URLMAXSIZE*2]=""; strcpybuff(last_adr,adr); // ancienne adresse //strcpybuff(last_fil,fil); // ancien chemin - r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,back,back_max,cache,hash,ptr,numero_passe); + r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL); if (strcmp(jump_identification(last_adr),jump_identification(adr)) != 0) { // a changé // 2e test si moved @@ -2338,7 +2361,7 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { } forbidden_url=hts_acceptlink(opt,ptr,lien_tot,liens, adr,fil, - NULL, NULL, + intag_name ? intag_name : NULL, intag_name ? tag_attr_start : NULL, &set_prio_to, &just_test_it); if ((opt->debug>1) && (opt->log!=NULL)) { @@ -2356,6 +2379,18 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { strcpybuff(save,""); // dummy } } + + // resolve unresolved type + if (r_sv!=-1 + && p_type != 2 && p_type != -2 + && forbidden_url == 0 + && IS_DELAYED_EXT(save) + ) + { // pas d'erreur, on continue + r_sv = hts_wait_delayed(str, adr, fil, save, former_adr, former_fil, &forbidden_url); + } + + // record! if (r_sv!=-1) { // pas d'erreur, on continue /* log */ if ((opt->debug>1) && (opt->log!=NULL)) { @@ -2495,10 +2530,12 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { strcpybuff(save,opt->path_html); strcatbuff(save,cat_name); if (lienrelatif(tempo,save, relativesavename)==0) { - if (!no_esc_utf) - escape_uri(tempo); // escape with %xx - else - escape_uri_utf(tempo); // escape with %xx + /* Never escape high-chars (we don't know the encoding!!) */ + escape_uri_utf(tempo); // escape with %xx + //if (!no_esc_utf) + // escape_uri(tempo); // escape with %xx + //else + // escape_uri_utf(tempo); // escape with %xx HT_ADD_HTMLESCAPED(tempo); // page externe if (add_url) { HT_ADD("?link="); // page externe @@ -2664,15 +2701,17 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { if (lienrelatif(tempo,save,relativesavename)==0) { if (!in_media) { // In media (such as real audio): don't patch - if (!no_esc_utf) - escape_uri(tempo); // escape with %xx - else { - /* No escaping at all - remaining upper chars will be escaped below */ - /* FIXME - Should be done in all local cases */ - //x_escape_html(tempo); - //escape_uri_utf(tempo); // FIXME - escape with %xx - //escape_uri(tempo); // escape with %xx - } + /* Never escape high-chars (we don't know the encoding!!) */ + escape_uri_utf(tempo); + //if (!no_esc_utf) + // escape_uri(tempo); // escape with %xx + //else { + // /* No escaping at all - remaining upper chars will be escaped below */ + // /* FIXME - Should be done in all local cases */ + // //x_escape_html(tempo); + // //escape_uri_utf(tempo); // FIXME - escape with %xx + // //escape_uri(tempo); // escape with %xx + //} } if ((opt->debug>1) && (opt->log!=NULL)) { fspc(opt->log,"debug"); fprintf(opt->log,"relative link at %s build with %s and %s: %s"LF,adr,save,relativesavename,tempo); @@ -2817,7 +2856,6 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { // au fichier est la plus grande des deux priorités // // On part de la fin et on essaye de se presser (économise temps machine) -#if HTS_HASH { int i=hash_read(hash,save,"",0,opt->urlhack); // lecture type 0 (sav) if (i>=0) { @@ -2834,21 +2872,6 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { dejafait=1; } } -#else - { - int l; - int i; - l=strlen(save); // opti - for(i=lien_tot-1;(i>=0) && (dejafait==0);i--) { - if (liens[i]->sav_len==l) { // même taille de chaîne - if (strcmp(liens[i]->sav,save)==0) { // existe déja - liens[i]->depth=maximum(liens[i]->depth,liens[ptr]->depth - 1); - dejafait=1; - } - } - } - } -#endif // le lien n'a jamais été créé. // cette fois ci, on le crée! @@ -2860,64 +2883,68 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { //liens[lien_tot]->adr[0]=liens[lien_tot]->fil[0]=liens[lien_tot]->sav[0]='\0'; // même adresse: l'objet père est l'objet père de l'actuel - // DEBUT ROBOTS.TXT AJOUT - if (!just_test_it) { - if ( - (!strfield(adr,"ftp://")) // non ftp - && (!strfield(adr,"file://")) ) { // non file - if (opt->robots) { // récupérer robots - if (ishtml(fil)!=0) { // pas la peine pour des fichiers isolés - if (checkrobots(_ROBOTS,adr,"") != -1) { // robots.txt ? - checkrobots_set(_ROBOTS ,adr,""); // ajouter entrée vide - if (checkrobots(_ROBOTS,adr,"") == -1) { // robots.txt ? - // enregistrer robots.txt (MACRO) - liens_record(adr,"/robots.txt","","",""); - if (liens[lien_tot]==NULL) { // erreur, pas de place réservée - printf("PANIC! : Not enough memory [%d]\n",__LINE__); - if (opt->errlog) { - fprintf(opt->errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url))); - test_flush; - } - if ((opt->getmode & 1) && (ptr>0)) { if (fp) { fclose(fp); fp=NULL; } } - XH_uninit; // désallocation mémoire & buffers - return -1; - } - liens[lien_tot]->testmode=0; // pas mode test - liens[lien_tot]->link_import=0; // pas mode import - liens[lien_tot]->premier=lien_tot; - liens[lien_tot]->precedent=ptr; - liens[lien_tot]->depth=0; - liens[lien_tot]->pass2=max(0,numero_passe); - liens[lien_tot]->retry=0; - lien_tot++; // UN LIEN DE PLUS + // DEBUT ROBOTS.TXT AJOUT + if (!just_test_it) { + if ((!strfield(adr,"ftp://")) // non ftp + && (!strfield(adr,"file://")) +#if HTS_USEMMS + && (!strfield(adr,"mms://")) +#endif + ) + { // non file + if (opt->robots) { // récupérer robots + if (ishtml(fil)!=0) { // pas la peine pour des fichiers isolés + if (checkrobots(_ROBOTS,adr,"") != -1) { // robots.txt ? + checkrobots_set(_ROBOTS ,adr,""); // ajouter entrée vide + if (checkrobots(_ROBOTS,adr,"") == -1) { // robots.txt ? + // enregistrer robots.txt (MACRO) + liens_record(adr,"/robots.txt","","",""); + if (liens[lien_tot]==NULL) { // erreur, pas de place réservée + printf("PANIC! : Not enough memory [%d]\n",__LINE__); + if (opt->errlog) { + fprintf(opt->errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url))); + test_flush; + } + if ((opt->getmode & 1) && (ptr>0)) { if (fp) { fclose(fp); fp=NULL; } } + XH_uninit; // désallocation mémoire & buffers + return -1; + } + liens[lien_tot]->testmode=0; // pas mode test + liens[lien_tot]->link_import=0; // pas mode import + liens[lien_tot]->premier=lien_tot; + liens[lien_tot]->precedent=ptr; + liens[lien_tot]->depth=0; + liens[lien_tot]->pass2=max(0,numero_passe); + liens[lien_tot]->retry=0; + lien_tot++; // UN LIEN DE PLUS #if DEBUG_ROBOTS - printf("robots.txt: added file robots.txt for %s\n",adr); + printf("robots.txt: added file robots.txt for %s\n",adr); #endif - if ((opt->debug>1) && (opt->log!=NULL)) { - fspc(opt->log,"debug"); fprintf(opt->log,"robots.txt added at %s"LF,adr); - test_flush; - } - } else { - if (opt->errlog) { - fprintf(opt->errlog,"Unexpected robots.txt error at %d"LF,__LINE__); - test_flush; - } - } - } - } - } - } - } - // FIN ROBOTS.TXT AJOUT - - // enregistrer (MACRO) - liens_record(adr,fil,save,former_adr,former_fil); - if (liens[lien_tot]==NULL) { // erreur, pas de place réservée - printf("PANIC! : Not enough memory [%d]\n",__LINE__); - if (opt->errlog) { - fprintf(opt->errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url))); - test_flush; - } + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"robots.txt added at %s"LF,adr); + test_flush; + } + } else { + if (opt->errlog) { + fprintf(opt->errlog,"Unexpected robots.txt error at %d"LF,__LINE__); + test_flush; + } + } + } + } + } + } + } + // FIN ROBOTS.TXT AJOUT + + // enregistrer (MACRO) + liens_record(adr,fil,save,former_adr,former_fil); + if (liens[lien_tot]==NULL) { // erreur, pas de place réservée + printf("PANIC! : Not enough memory [%d]\n",__LINE__); + if (opt->errlog) { + fprintf(opt->errlog,"Not enough memory, can not re-allocate %d bytes"LF,(int)((add_tab_alloc+1)*sizeof(lien_url))); + test_flush; + } if ((opt->getmode & 1) && (ptr>0)) { if (fp) { fclose(fp); fp=NULL; } } XH_uninit; // désallocation mémoire & buffers return -1; @@ -3062,21 +3089,21 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { if (_hts_in_html_poll) { _hts_in_html_poll=0; // temps à attendre, et remplir autant que l'on peut le cache (backing) - back_wait(back,back_max,opt,cache,HTS_STAT.stat_timestart); - back_fillmax(back,back_max,opt,cache,liens,ptr,numero_passe,lien_tot); + back_wait(sback,opt,cache,HTS_STAT.stat_timestart); + back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot); // Transfer rate engine_stats(); // Refresh various stats - HTS_STAT.stat_nsocket=back_nsoc(back,back_max); + HTS_STAT.stat_nsocket=back_nsoc(sback); HTS_STAT.stat_errors=fspc(NULL,"error"); HTS_STAT.stat_warnings=fspc(NULL,"warning"); HTS_STAT.stat_infos=fspc(NULL,"info"); - HTS_STAT.nbk=backlinks_done(liens,lien_tot,ptr); - HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,back,back_max); + HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr); + HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback); - if (!hts_htmlcheck_loop(back,back_max,0,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { + if (!hts_htmlcheck_loop(sback->lnk, sback->count, 0,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { if (opt->errlog) { fspc(opt->errlog,"info"); fprintf(opt->errlog,"Exit requested by shell or user"LF); test_flush; @@ -3094,8 +3121,8 @@ int htsparse(htsmoduleStruct* str, htsmoduleStructExtended* stre) { // refresh the backing system each 2 seconds if (engine_stats()) { - back_wait(back,back_max,opt,cache,HTS_STAT.stat_timestart); - back_fillmax(back,back_max,opt,cache,liens,ptr,numero_passe,lien_tot); + back_wait(sback,opt,cache,HTS_STAT.stat_timestart); + back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot); } #endif } while(( ((int) (adr - r->adr)) ) < r->size); @@ -3160,11 +3187,7 @@ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre) // ------------------------------------------------------------ // Rattrapage des 301,302,307 (moved) et 412,416 - les 304 le sont dans le backing // ------------------------------------------------------------ - if ( (r->statuscode==301) - || (r->statuscode==302) - || (r->statuscode==303) - || (r->statuscode==307) - ) { + if (HTTP_IS_REDIRECT(r->statuscode)) { //if (r->adr!=NULL) { // adr==null si fichier direct. [catch: davename normalement si cgi] //int i=0; char *rn=NULL; @@ -3256,10 +3279,6 @@ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre) // canceller lien actuel error=1; strcpybuff(liens[ptr]->adr,"!"); // caractère bidon (invalide hash) -#if HTS_HASH -#else - liens[ptr]->sav_len=-1; // taille invalide -#endif // noter NOUVEAU lien //xxc xxc // set_prio_to=0+1; // protection if the moved URL is an html page!! @@ -3267,7 +3286,7 @@ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre) { char BIGSTK mov_sav[HTS_URLMAXSIZE*2]; // calculer lien et éventuellement modifier addresse/fichier - if (url_savename(mov_adr,mov_fil,mov_sav,NULL,NULL,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil,opt,liens,lien_tot,back,back_max,cache,hash,ptr,numero_passe)!=-1) { + if (url_savename(mov_adr,mov_fil,mov_sav,NULL,NULL,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL)!=-1) { if (hash_read(hash,mov_sav,"",0,0)<0) { // n'existe pas déja // enregistrer lien (MACRO) avec SAV IDENTIQUE liens_record(mov_adr,mov_fil,liens[ptr]->sav,"",""); @@ -3390,10 +3409,6 @@ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre) // canceller lien actuel error=1; strcpybuff(liens[ptr]->adr,"!"); // caractère bidon (invalide hash) -#if HTS_HASH -#else - liens[ptr]->sav_len=-1; // taille invalide -#endif // } else { // oups erreur, plus de mémoire!! printf("PANIC! : Not enough memory [%d]\n",__LINE__); @@ -3421,43 +3436,45 @@ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre) int can_retry=0; // cas où l'on peut reessayer - // -2=timeout -3=rateout (interne à httrack) switch(r->statuscode) { //case -1: can_retry=1; break; - case -2: if (opt->hostcontrol) { // timeout et retry épuisés - if ((opt->hostcontrol & 1) && (liens[ptr]->retry<=0)) { - if ((opt->debug>1) && (opt->log!=NULL)) { - fspc(opt->log,"debug"); fprintf(opt->log,"Link banned: %s%s"LF,urladr,urlfil); test_flush; - } - host_ban(opt,liens,ptr,lien_tot,back,back_max,jump_identification(urladr)); - if ((opt->debug>1) && (opt->log!=NULL)) { - fspc(opt->log,"debug"); fprintf(opt->log,"Info: previous log - link banned: %s%s"LF,urladr,urlfil); test_flush; - } + case STATUSCODE_TIMEOUT: + if (opt->hostcontrol) { // timeout et retry épuisés + if ((opt->hostcontrol & 1) && (liens[ptr]->retry<=0)) { + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"Link banned: %s%s"LF,urladr,urlfil); test_flush; + } + host_ban(opt,liens,ptr,lien_tot,sback,jump_identification(urladr)); + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"Info: previous log - link banned: %s%s"LF,urladr,urlfil); test_flush; + } + } else can_retry=1; } else can_retry=1; - } else can_retry=1; break; - case -3: if ((opt->hostcontrol) && (liens[ptr]->retry<=0)) { // too slow - if (opt->hostcontrol & 2) { - if ((opt->debug>1) && (opt->log!=NULL)) { - fspc(opt->log,"debug"); fprintf(opt->log,"Link banned: %s%s"LF,urladr,urlfil); test_flush; - } - host_ban(opt,liens,ptr,lien_tot,back,back_max,jump_identification(urladr)); - if ((opt->debug>1) && (opt->log!=NULL)) { - fspc(opt->log,"debug"); fprintf(opt->log,"Info: previous log - link banned: %s%s"LF,urladr,urlfil); test_flush; - } + case STATUSCODE_SLOW: + if ((opt->hostcontrol) && (liens[ptr]->retry<=0)) { // too slow + if (opt->hostcontrol & 2) { + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"Link banned: %s%s"LF,urladr,urlfil); test_flush; + } + host_ban(opt,liens,ptr,lien_tot,sback,jump_identification(urladr)); + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"Info: previous log - link banned: %s%s"LF,urladr,urlfil); test_flush; + } + } else can_retry=1; } else can_retry=1; - } else can_retry=1; break; - case -4: // connect closed + case STATUSCODE_CONNERROR: // connect closed can_retry=1; break; - case -5: // other (non fatal) error + case STATUSCODE_NON_FATAL: // other (non fatal) error can_retry=1; break; - case -6: // bad SSL handskake + case STATUSCODE_SSL_HANDSHAKE: // bad SSL handskake can_retry=1; break; - case 408: case 409: case 500: case 502: case 504: can_retry=1; + case 408: case 409: case 500: case 502: case 504: + can_retry=1; break; } @@ -3468,7 +3485,7 @@ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre) fspc(opt->errlog,"error"); fprintf(opt->errlog,"\"%s\" (%d) after %d retries at link %s%s (from %s%s)"LF,r->msg,r->statuscode,opt->retry,urladr,urlfil,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil); } else { - if (r->statuscode==-10) { // test OK + if (r->statuscode==STATUSCODE_TEST_OK) { // test OK if ((opt->debug>0) && (opt->errlog!=NULL)) { fspc(opt->errlog,"info"); fprintf(opt->errlog,"Test OK at link %s%s (from %s%s)"LF,urladr,urlfil,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil); @@ -3612,26 +3629,26 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* if ( (opt->debug>0) && (opt->log!=NULL) ) { fspc(opt->log,"info"); fprintf(opt->log,"engine: pause requested.."LF); } - while (back_nsoc(back,back_max)>0) { // attendre fin des transferts - back_wait(back,back_max,opt,cache,HTS_STAT.stat_timestart); + while (back_nsoc(sback)>0) { // attendre fin des transferts + back_wait(sback,opt,cache,HTS_STAT.stat_timestart); Sleep(200); #if HTS_ANALYSTE { - back_wait(back,back_max,opt,cache,HTS_STAT.stat_timestart); + back_wait(sback,opt,cache,HTS_STAT.stat_timestart); // Transfer rate engine_stats(); // Refresh various stats - HTS_STAT.stat_nsocket=back_nsoc(back,back_max); + HTS_STAT.stat_nsocket=back_nsoc(sback); HTS_STAT.stat_errors=fspc(NULL,"error"); HTS_STAT.stat_warnings=fspc(NULL,"warning"); HTS_STAT.stat_infos=fspc(NULL,"info"); - HTS_STAT.nbk=backlinks_done(liens,lien_tot,ptr); - HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,back,back_max); + HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr); + HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback); b=0; - if (!hts_htmlcheck_loop(back,back_max,b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT) + if (!hts_htmlcheck_loop(sback->lnk, sback->count, b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT) || !back_checkmirror(opt)) { if (opt->errlog) { fspc(opt->errlog,"info"); fprintf(opt->errlog,"Exit requested by shell or user"LF); @@ -3664,7 +3681,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* hts_htmlcheck_pause(fconcat(opt->path_log,"hts-paused.lock")); #else while (fexist(fconcat(opt->path_log,"hts-paused.lock"))) { - //back_wait(back,back_max,opt,cache,HTS_STAT.stat_timestart); inutile!! (plus de sockets actives) + //back_wait(sback,opt,cache,HTS_STAT.stat_timestart); inutile!! (plus de sockets actives) Sleep(1000); } #endif @@ -3695,7 +3712,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* // noter NOUVEAU lien char BIGSTK add_sav[HTS_URLMAXSIZE*2]; // calculer lien et éventuellement modifier addresse/fichier - if (url_savename(add_adr,add_fil,add_sav,NULL,NULL,NULL,NULL,opt,liens,lien_tot,back,back_max,cache,hash,ptr,numero_passe)!=-1) { + if (url_savename(add_adr,add_fil,add_sav,NULL,NULL,NULL,NULL,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,NULL)!=-1) { if (hash_read(hash,add_sav,"",0,0)<0) { // n'existe pas déja // enregistrer lien (MACRO) liens_record(add_adr,add_fil,add_sav,"",""); @@ -3744,27 +3761,27 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* _hts_addurl=NULL; // libérer _hts_addurl } // si une pause a été demandée - if (_hts_setpause || back_pluggable_sockets_strict(back, back_max, opt) <= 0) { + if (_hts_setpause || back_pluggable_sockets_strict(sback, opt) <= 0) { // index du lien actuel - int b=back_index(back,back_max,urladr,urlfil,savename); + int b=back_index(sback,urladr,urlfil,savename); int prev = _hts_in_html_parsing; if (b<0) b=0; // forcer pour les stats - while(_hts_setpause || back_pluggable_sockets_strict(back, back_max, opt) <= 0) { // on fait la pause.. + while(_hts_setpause || back_pluggable_sockets_strict(sback, opt) <= 0) { // on fait la pause.. _hts_in_html_parsing = 6; - back_wait(back,back_max,opt,cache,HTS_STAT.stat_timestart); + back_wait(sback,opt,cache,HTS_STAT.stat_timestart); // Transfer rate engine_stats(); // Refresh various stats - HTS_STAT.stat_nsocket=back_nsoc(back,back_max); + HTS_STAT.stat_nsocket=back_nsoc(sback); HTS_STAT.stat_errors=fspc(NULL,"error"); HTS_STAT.stat_warnings=fspc(NULL,"warning"); HTS_STAT.stat_infos=fspc(NULL,"info"); - HTS_STAT.nbk=backlinks_done(liens,lien_tot,ptr); - HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,back,back_max); + HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr); + HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback); - if (!hts_htmlcheck_loop(back,back_max,b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { + if (!hts_htmlcheck_loop(sback->lnk, sback->count, b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { if (opt->errlog) { fspc(opt->errlog,"info"); fprintf(opt->errlog,"Exit requested by shell or user"LF); test_flush; @@ -3780,11 +3797,11 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* #endif // si le fichier n'est pas en backing, le mettre.. - if (!back_exist(back,back_max,urladr,urlfil,savename)) { + if (!back_exist(sback,urladr,urlfil,savename)) { #if BDEBUG==1 printf("crash backing: %s%s\n",liens[ptr]->adr,liens[ptr]->fil); #endif - if (back_add(back,back_max,opt,cache,urladr,urlfil,savename,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil,liens[ptr]->testmode,&liens[ptr]->pass2)==-1) { + if (back_add(sback,opt,cache,urladr,urlfil,savename,liens[liens[ptr]->precedent]->adr,liens[liens[ptr]->precedent]->fil,liens[ptr]->testmode,&liens[ptr]->pass2)==-1) { printf("PANIC! : Crash adding error, unexpected error found.. [%d]\n",__LINE__); #if BDEBUG==1 printf("error while crash adding\n"); @@ -3802,7 +3819,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* #endif // ajouter autant de socket qu'on peut ajouter - n=opt->maxsoc-back_nsoc(back,back_max); + n=opt->maxsoc-back_nsoc(sback); #if BDEBUG==1 printf("%d sockets available for backing\n",n); #endif @@ -3813,12 +3830,12 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* if (n>0) { // si sockets libre #endif // remplir autant que l'on peut le cache (backing) - back_fillmax(back,back_max,opt,cache,liens,ptr,numero_passe,lien_tot); + back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot); } // index du lien actuel /* - b=back_index(back,back_max,urladr,urlfil,savename); + b=back_index(sback,urladr,urlfil,savename); if (b>=0) */ @@ -3828,22 +3845,22 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* do { // index du lien actuel - b=back_index(back,back_max,urladr,urlfil,savename); + b=back_index(sback,urladr,urlfil,savename); #if BDEBUG==1 printf("back index %d, waiting\n",b); #endif // Continue to the loop if link still present if (b<0) - continue; + break; // Receive data if (back[b].status>0) - back_wait(back,back_max,opt,cache,HTS_STAT.stat_timestart); + back_wait(sback,opt,cache,HTS_STAT.stat_timestart); // Continue to the loop if link still present - b=back_index(back,back_max,urladr,urlfil,savename); + b=back_index(sback,urladr,urlfil,savename); if (b<0) - continue; + break; // Stop the mirror if (!back_checkmirror(opt)) { @@ -3854,12 +3871,12 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* // And fill the backing stack if (back[b].status>0) - back_fillmax(back,back_max,opt,cache,liens,ptr,numero_passe,lien_tot); + back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot); // Continue to the loop if link still present - b=back_index(back,back_max,urladr,urlfil,savename); + b=back_index(sback,urladr,urlfil,savename); if (b<0) - continue; + break; // autres occupations de HTTrack: statistiques, boucle d'attente, etc. if ((opt->makestat) || (opt->maketrack)) { @@ -3876,7 +3893,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* int i; fspc(stre->maketrack_fp,"info"); fprintf(stre->maketrack_fp,LF); for(i=0;i<back_max;i++) { - back_info(back,i,3,stre->maketrack_fp); + back_info(sback,i,3,stre->maketrack_fp); } fprintf(stre->maketrack_fp,LF); fflush(stre->maketrack_fp); @@ -3900,9 +3917,10 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* #endif if (back[i].r.soc!=INVALID_SOCKET) deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; - back[i].r.statuscode=-1; + back[i].r.statuscode=STATUSCODE_INVALID; strcpybuff(back[i].r.msg,"Cancelled by User"); back[i].status=0; // terminé + back_set_finished(sback, i); } else // cancel ftp.. flag à 1 back[i].stop_ftp = 1; } @@ -3916,14 +3934,14 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* engine_stats(); // Refresh various stats - HTS_STAT.stat_nsocket=back_nsoc(back,back_max); + HTS_STAT.stat_nsocket=back_nsoc(sback); HTS_STAT.stat_errors=fspc(NULL,"error"); HTS_STAT.stat_warnings=fspc(NULL,"warning"); HTS_STAT.stat_infos=fspc(NULL,"info"); - HTS_STAT.nbk=backlinks_done(liens,lien_tot,ptr); - HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,back,back_max); + HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr); + HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback); - if (!hts_htmlcheck_loop(back,back_max,b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { + if (!hts_htmlcheck_loop(sback->lnk, sback->count, b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { if (opt->errlog) { fspc(opt->errlog,"info"); fprintf(opt->errlog,"Exit requested by shell or user"LF); test_flush; @@ -3980,7 +3998,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* fprintf(fp,"TIME %d"LF,(int) (tl-HTS_STAT.stat_timestart)); fprintf(fp,"TOTAL %d"LF,(int) HTS_STAT.stat_bytes); fprintf(fp,"RATE %d"LF,(int) (HTS_STAT.HTS_TOTAL_RECV/(tl-HTS_STAT.stat_timestart))); - fprintf(fp,"SOCKET %d"LF,back_nsoc(back,back_max)); + fprintf(fp,"SOCKET %d"LF,back_nsoc(sback)); fprintf(fp,"LINK %d"LF,lien_tot); { LLint mem=0; @@ -3991,7 +4009,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* } for(j=0;j<2;j++) { // passes pour ready et wait for(i=0;i<back_max;i++) { - back_info(back,i,j+1,stdout); // maketrack_fp a la place de stdout ?? // ** + back_info(sback,i,j+1,stdout); // maketrack_fp a la place de stdout ?? // ** } } fprintf(fp,LF); @@ -4032,7 +4050,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* back[b].r = cache_read_ro(opt,cache,back[b].url_adr,back[b].url_fil,back[b].url_sav, back[b].location_buffer); /* ensure correct location buffer set */ back[b].r.location=back[b].location_buffer; - if (back[b].r.statuscode == -1) { + if (back[b].r.statuscode == STATUSCODE_INVALID) { if (opt->errlog) { fspc(opt->errlog,"error"); fprintf(opt->errlog,"Unexpected error: %s%s not found anymore in cache"LF,back[b].url_adr,back[b].url_fil); test_flush; @@ -4056,10 +4074,14 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* fflush(stdout); } } else if (opt->verbosedisplay==1) { - if (back[b].r.statuscode==200) - printf("%d/%d: %s%s ("LLintP" bytes) - OK\33[K\r",ptr,lien_tot,back[b].url_adr,back[b].url_fil,(LLint)back[b].r.size); - else - printf("%d/%d: %s%s ("LLintP" bytes) - %d\33[K\r",ptr,lien_tot,back[b].url_adr,back[b].url_fil,(LLint)back[b].r.size,back[b].r.statuscode); + if (b >= 0) { + if (back[b].r.statuscode==200) + printf("%d/%d: %s%s ("LLintP" bytes) - OK\33[K\r",ptr,lien_tot,back[b].url_adr,back[b].url_fil,(LLint)back[b].r.size); + else + printf("%d/%d: %s%s ("LLintP" bytes) - %d\33[K\r",ptr,lien_tot,back[b].url_adr,back[b].url_fil,(LLint)back[b].r.size,back[b].r.statuscode); + } else { + fspc(opt->errlog,"error"); fprintf(opt->errlog,"Link disappeared"); + } fflush(stdout); } //} @@ -4079,14 +4101,16 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* #endif // copier structure réponse htsblk - memcpy(r, &(back[b].r), sizeof(htsblk)); - r->location=stre->loc_; // ne PAS copier location!! adresse, pas de buffer - if (back[b].r.location) - strcpybuff(r->location,back[b].r.location); - back[b].r.adr=NULL; // ne pas faire de desalloc ensuite - - // libérer emplacement backing - back_maydelete(opt,cache,back,b); + if (b >= 0) { + memcpy(r, &(back[b].r), sizeof(htsblk)); + r->location=stre->loc_; // ne PAS copier location!! adresse, pas de buffer + if (back[b].r.location) + strcpybuff(r->location,back[b].r.location); + back[b].r.adr=NULL; // ne pas faire de desalloc ensuite + + // libérer emplacement backing + back_maydelete(opt,cache,sback,b); + } // progression #if 0 @@ -4165,13 +4189,367 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* } */ + ENGINE_SAVE_CONTEXT(); + return 0; +} +/* Wait for delayed types */ +int hts_wait_delayed(htsmoduleStruct* str, + char* adr, char* fil, char* save, + char* former_adr, char* former_fil, + int* forbidden_url) { + ENGINE_LOAD_CONTEXT_BASE(); + hash_struct* hash = hashptr; + + int r_sv=0; + + // resolve unresolved type + if (opt->savename_delayed != 0 + && *forbidden_url == 0 + && IS_DELAYED_EXT(save) + && !opt->state.stop + ) + { + int loops=0; + int continue_loop = 1; + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"Waiting for type to be known: %s%s"LF, adr, fil); + test_flush; + } - ENGINE_SAVE_CONTEXT(); + /* Follow while type is unknown and redirects occurs */ + while(IS_DELAYED_EXT(save) && continue_loop && loops++ < 7) { + continue_loop = 0; + + /* + Wait for an available slot + */ + WAIT_FOR_AVAILABLE_SOCKET(); + + /* We can lookup directly in the cache to speedup this mess */ + if (opt->delayed_cached) { + lien_back back; + memset(&back, 0, sizeof(back)); + back.r = cache_read(opt, cache, adr, fil, NULL, NULL); // test uniquement + if (back.r.statuscode == 200 && strnotempty(back.r.contenttype)) { // cache found, and aswer is 'OK' + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"Direct type lookup in cache (-%%D1): %s"LF, back.r.contenttype); + test_flush; + } - return 0; + /* Recompute filename with MIME type */ + save[0] = '\0'; + r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&back); + /* Recompute authorization with MIME type */ + { + int new_forbidden_url = hts_acceptmime(opt, ptr, lien_tot, liens, adr,fil, back.r.contenttype); + if (new_forbidden_url != -1) { + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"result for wizard mime test: %s"LF,new_forbidden_url); + test_flush; + } + if (new_forbidden_url == 1) { + *forbidden_url = new_forbidden_url; + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"link forbidden because of MIME types restrictions: %s%s"LF, adr, fil); + test_flush; + } + break; // exit loop + } + } + } -} + /* And exit loop */ + break; + } + } + + /* Add in backing (back_index() will respond correctly) */ + if (back_add_if_not_exists(sback,opt,cache,adr,fil,save,NULL,NULL,0,NULL) != -1) { + int b; + b=back_index(sback,adr,fil,save); + if (b<0) { + printf("PANIC! : Crash adding error, unexpected error found.. [%d]\n",__LINE__); + XH_uninit; // désallocation mémoire & buffers + return -1; + } + + /* Cache read failed because file does not exists (bad delayed name!) + Just re-add with the correct name, as we know the MIME now! + */ + if (back[b].r.statuscode == STATUSCODE_INVALID && back[b].r.adr == NULL) { + lien_back delayed_back; + //char BIGSTK delayed_ctype[128]; + // delayed_ctype[0] = '\0'; + // strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1); // copier content-type + back_copy_static(&back[b], &delayed_back); + + /* Delete entry */ + back_delete(opt,cache,sback,b); // cancel + b = -1; + + /* Recompute filename with MIME type */ + save[0] = '\0'; + r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&delayed_back); + + /* Recompute authorization with MIME type */ + { + int new_forbidden_url = hts_acceptmime(opt, ptr, lien_tot, liens, adr,fil, delayed_back.r.contenttype); + if (new_forbidden_url != -1) { + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"result for wizard mime test: %d"LF,*forbidden_url); + test_flush; + } + if (new_forbidden_url == 1) { + *forbidden_url = new_forbidden_url; + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"link forbidden because of MIME types restrictions: %s%s"LF, adr, fil); + test_flush; + } + break; // exit loop + } + } + } + + /* Re-Add wiht correct type */ + if (back_add_if_not_exists(sback,opt,cache,adr,fil,save,NULL,NULL,0,NULL) != -1) { + b=back_index(sback,adr,fil,save); + } + if (b<0) { + printf("PANIC! : Crash adding error, unexpected error found.. [%d]\n",__LINE__); + XH_uninit; // désallocation mémoire & buffers + return -1; + } + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"Type immediately loaded from cache: %s"LF, delayed_back.r.contenttype); + test_flush; + } + } + + /* Wait for headers to be received */ + do { + if (b < 0) + break; + + // temps à attendre, et remplir autant que l'on peut le cache (backing) + if (back[b].status>0) { + back_wait(sback,opt,cache,0); + } + if (ptr>=0) { + back_fillmax(sback,opt,cache,liens,ptr,numero_passe,lien_tot); + } + + // on est obligé d'appeler le shell pour le refresh.. +#if HTS_ANALYSTE + { + + // Transfer rate + engine_stats(); + + // Refresh various stats + HTS_STAT.stat_nsocket=back_nsoc(sback); + HTS_STAT.stat_errors=fspc(NULL,"error"); + HTS_STAT.stat_warnings=fspc(NULL,"warning"); + HTS_STAT.stat_infos=fspc(NULL,"info"); + HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr); + HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback); + + if (!hts_htmlcheck_loop(sback->lnk, sback->count, b,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { + return -1; + } else if (_hts_cancel || !back_checkmirror(opt)) { // cancel 2 ou 1 (cancel parsing) + back_delete(opt,cache,sback,b); // cancel test + break; + } + } +#endif + } while( + /* dns/connect/request */ + ( back[b].status >= 99 && back[b].status <= 101 ) + || + /* For redirects, wait for request to be terminated */ + ( HTTP_IS_REDIRECT(back[b].r.statuscode) && back[b].status > 0 ) + || + /* Same for errors */ + ( HTTP_IS_ERROR(back[b].r.statuscode) && back[b].status > 0 ) + ); + /* ready (chunked) or ready (regular download) or ready (completed) */ + + // Note: filename NOT in hashtable yet - liens_record will do it, with the correct ext! + if (b >= 0) { + lien_back delayed_back; + //char BIGSTK delayed_ctype[128]; + //delayed_ctype[0] = '\0'; + //strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1); // copier content-type + back_copy_static(&back[b], &delayed_back); + + /* Error */ + if (HTTP_IS_ERROR(back[b].r.statuscode)) + { + /* 'no error page' selected or file discarded by size rules! */ + if (!opt->errpage || ( back[b].r.statuscode == STATUSCODE_TOO_BIG ) ) { + /* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */ + *forbidden_url = 1; /* Forbidden! */ + if (opt->log != NULL) { + if (back[b].r.statuscode == STATUSCODE_TOO_BIG) { + fspc(opt->log,"error"); fprintf(opt->log,"link not taken because of its size (%d bytes) at %s%s"LF,(int)back[b].r.totalsize,adr,fil); + } else { + fspc(opt->log,"error"); fprintf(opt->log,"link not taken because of error (%d '%s') at %s%s"LF,back[b].r.statuscode,back[b].r.msg,adr,fil); + } + test_flush; + } + break; + } + } + /* Moved! */ + else if (HTTP_IS_REDIRECT(back[b].r.statuscode)) + { + char BIGSTK mov_url[HTS_URLMAXSIZE*2]; + mov_url[0] = '\0'; + strcpybuff(mov_url, back[b].r.location); // copier URL + + /* Remove (temporarily created) file if it was created */ + unlink(fconv(back[b].url_sav)); + + /* Remove slot! */ + if (back[b].status == 0) { + back_maydelete(opt, cache, sback, b); + } else { /* should not happend */ + back_delete(opt, cache, sback, b); + } + b = -1; + /* Handle redirect */ + if ((int) strnotempty(mov_url)) { // location existe! + char BIGSTK mov_adr[HTS_URLMAXSIZE*2],mov_fil[HTS_URLMAXSIZE*2]; + mov_adr[0]=mov_fil[0]='\0'; + // + if (ident_url_relatif(mov_url,adr,fil,mov_adr,mov_fil)>=0) { + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"Redirect while resolving type: %s%s -> %s%s"LF, adr, fil, mov_adr, mov_fil); + test_flush; + } + // si non bouclage sur soi même, ou si test avec GET non testé + if (strcmp(mov_adr,adr) != 0 || strcmp(mov_fil,fil) != 0) { + + // recopier former_adr/fil? + if ((former_adr) && (former_fil)) { + if (strnotempty(former_adr)==0) { // Pas déja noté + strcpybuff(former_adr,adr); + strcpybuff(former_fil,fil); + } + } + + // check explicit forbidden - don't follow 3xx in this case + { + int set_prio_to=0; + robots_wizard* robots = (robots_wizard*) opt->robotsptr; + if (hts_acceptlink(opt,ptr,lien_tot,liens, + mov_adr,mov_fil, + NULL, NULL, + &set_prio_to, + NULL) == 1) + { /* forbidden */ + /* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */ + *forbidden_url = 1; /* Forbidden! */ + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"link forbidden because of redirect beyond the mirror scope at %s%s"LF,adr,fil); + test_flush; + } + strcpybuff(adr,mov_adr); + strcpybuff(fil,mov_fil); + mov_url[0]='\0'; + break; + } + } + + // ftp: stop! + if (strfield(mov_url,"ftp://") +#if HTS_USEMMS + || strfield(mov_url,"mms://") +#endif + ) { + strcpybuff(adr,mov_adr); + strcpybuff(fil,mov_fil); + break; + } + + /* ok, continue */ + strcpybuff(adr,mov_adr); + strcpybuff(fil,mov_fil); + continue_loop = 1; + } else { + if ( opt->errlog!=NULL ) { + fspc(opt->errlog,"warning"); fprintf(opt->errlog,"Unable to test %s%s (loop to same filename)"LF,adr,fil); + test_flush; + } + } // loop to same location + } // ident_url_relatif() + } // location + } // redirect + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"Final type for %s%s: '%s'"LF, adr, fil, delayed_back.r.contenttype); + test_flush; + } + + /* Recompute filename with MIME type */ + save[0] = '\0'; + r_sv=url_savename(adr,fil,save,former_adr,former_fil,liens[ptr]->adr,liens[ptr]->fil,opt,liens,lien_tot,sback,cache,hash,ptr,numero_passe,&delayed_back); + + /* Recompute authorization with MIME type */ + { + int new_forbidden_url = hts_acceptmime(opt, ptr, lien_tot, liens, adr,fil, delayed_back.r.contenttype); + if (new_forbidden_url != -1) { + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"result for wizard mime test: %d"LF,forbidden_url); + test_flush; + } + if (new_forbidden_url == 1) { + *forbidden_url = new_forbidden_url; + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"link forbidden because of MIME types restrictions: %s%s"LF, adr, fil); + test_flush; + } + break; // exit loop + } + } + } + + /* Still have a back reference */ + if (b >= 0) { + /* Finalize now as we have the type */ + if (back[b].status == 0) { + if (!back[b].finalized) { + back_finalize(opt,cache,sback,b); + } + } + /* Patch destination filename for direct-to-disk mode */ + strcpybuff(back[b].url_sav, save); + } + + } // b >= 0 + } else { + printf("PANIC! : Crash adding error, unexpected error found.. [%d]\n",__LINE__); + XH_uninit; // désallocation mémoire & buffers + return -1; + } + + } // while(IS_DELAYED_EXT(save)) + + // error + if (*forbidden_url != 1 + && IS_DELAYED_EXT(save)) { + *forbidden_url = 1; + if (opt->log!=NULL) { + fspc(opt->log,"warning"); fprintf(opt->log,"link is probably looping, type unknown, aborting: %s%s"LF, adr, fil); + test_flush; + } + } + + } // delayed type check ? + + ENGINE_SAVE_CONTEXT_BASE(); + + return 0; +} diff --git a/src/htsparse.h b/src/htsparse.h index d36217c..561ae68 100644 --- a/src/htsparse.h +++ b/src/htsparse.h @@ -108,4 +108,63 @@ int hts_mirror_check_moved(htsmoduleStruct* str, htsmoduleStructExtended* stre); */ int hts_mirror_wait_for_next_file(htsmoduleStruct* str, htsmoduleStructExtended* stre); +/* + Wair for (adr, fil, save) to be started, that is, + to be ready for naming, having its header MIME type + If the final URL is to be forbidden, sets 'forbidden_url' to the corresponding value +*/ +int hts_wait_delayed(htsmoduleStruct* str, + char* adr, char* fil, char* save, + char* former_adr, char* former_fil, + int* forbidden_url); + + +/* Context state */ + +#define ENGINE_LOAD_CONTEXT_BASE() \ + lien_url** liens = (lien_url**) str->liens; \ + httrackp* opt = (httrackp*) str->opt; \ + struct_back* sback = (struct_back*) str->sback; \ + lien_back* const back = sback->lnk; \ + const int back_max = sback->count; \ + cache_back* cache = (cache_back*) str->cache; \ + hash_struct* hashptr = (hash_struct*) str->hashptr; \ + int numero_passe = str->numero_passe; \ + int add_tab_alloc = str->add_tab_alloc; \ + /* */ \ + int lien_tot = * ( (int*) (str->lien_tot_) ); \ + int ptr = * ( (int*) (str->ptr_) ); \ + int lien_size = * ( (int*) (str->lien_size_) ); \ + char* lien_buffer = * ( (char**) (str->lien_buffer_) ) + +#define ENGINE_SAVE_CONTEXT_BASE() \ + /* Apply changes */ \ + * ( (int*) (str->lien_tot_) ) = lien_tot; \ + * ( (int*) (str->ptr_) ) = ptr; \ + * ( (int*) (str->lien_size_) ) = lien_size; \ + * ( (char**) (str->lien_buffer_) ) = lien_buffer + +#define WAIT_FOR_AVAILABLE_SOCKET() do { \ + int prev = _hts_in_html_parsing; \ + while(back_pluggable_sockets_strict(sback, opt) <= 0) { \ + _hts_in_html_parsing = 6; \ + /* Wait .. */ \ + back_wait(sback,opt,cache,0); \ + /* Transfer rate */ \ + engine_stats(); \ + /* Refresh various stats */ \ + HTS_STAT.stat_nsocket=back_nsoc(sback); \ + HTS_STAT.stat_errors=fspc(NULL,"error"); \ + HTS_STAT.stat_warnings=fspc(NULL,"warning"); \ + HTS_STAT.stat_infos=fspc(NULL,"info"); \ + HTS_STAT.nbk=backlinks_done(sback,liens,lien_tot,ptr); \ + HTS_STAT.nb=back_transfered(HTS_STAT.stat_bytes,sback); \ + /* Check */ \ + if (!hts_htmlcheck_loop(sback->lnk, sback->count, -1,ptr,lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \ + return -1; \ + } \ + } \ + _hts_in_html_parsing = prev; \ +} while(0) + #endif diff --git a/src/htsserver.c b/src/htsserver.c index 0906b39..bb30640 100644 --- a/src/htsserver.c +++ b/src/htsserver.c @@ -127,19 +127,24 @@ static int linput(FILE* fp,char* s,int max); // 0- Init the URL catcher with standard port // smallserver_init(&port,&return_host); -T_SOC smallserver_init_std(int* port_prox,char* adr_prox) { - T_SOC soc; - int try_to_listen_to[]={8080,8081,8082,8083,8084,8085,8086,8087,8088,8089, - 32000,32001,32002,32003,32004,32005,32006,32007,32008,32009, - 42000,42001,42002,42003,42004,42005,42006,42007,42008,42009, - 0,-1}; - int i=0; - do { - soc=smallserver_init(&try_to_listen_to[i],adr_prox); - *port_prox=try_to_listen_to[i]; - i++; - } while( (soc == INVALID_SOCKET) && (try_to_listen_to[i]>=0)); - return soc; +T_SOC smallserver_init_std(int* port_prox, char* adr_prox, int defaultPort) { + T_SOC soc; + if (defaultPort <= 0) { + int try_to_listen_to[]={8080,8081,8082,8083,8084,8085,8086,8087,8088,8089, + 32000,32001,32002,32003,32004,32006,32006,32007,32008,32009, + 42000,42001,42002,42003,42004,42006,42006,42007,42008,42009, + 0,-1}; + int i=0; + do { + soc=smallserver_init(&try_to_listen_to[i],adr_prox); + *port_prox=try_to_listen_to[i]; + i++; + } while( (soc == INVALID_SOCKET) && (try_to_listen_to[i]>=0)); + } else { + soc=smallserver_init(&defaultPort, adr_prox); + *port_prox = defaultPort; + } + return soc; } @@ -293,7 +298,7 @@ int smallserver(T_SOC soc,char* url,char* method,char* data, char* path) { }; initStrElt initStr[] = { { "user", "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" }, - { "footer", "<!-- Mirrored from %s%s by HTTrack Website Copier/3.x [XR&CO'2005], %s -->" }, + { "footer", "<!-- Mirrored from %s%s by HTTrack Website Copier/3.x [XR&CO'2006], %s -->" }, { "url2", "+*.png +*.gif +*.jpg +*.css +*.js -ad.doubleclick.net/*" }, { NULL, NULL } }; @@ -1613,22 +1618,7 @@ static char* LANGINTKEY(char* name) { -static int check_readinput_t(T_SOC soc, int timeout) { - if (soc != INVALID_SOCKET) { - fd_set fds; // poll structures - struct timeval tv; // structure for select - FD_ZERO(&fds); - FD_SET(soc,&fds); - tv.tv_sec=timeout; - tv.tv_usec=0; - select(soc + 1,&fds,NULL,NULL,&tv); - if (FD_ISSET(soc,&fds)) - return 1; - else - return 0; - } else - return 0; -} + static int recv_bl(T_SOC soc, void* buffer, size_t len, int timeout) { if (check_readinput_t(soc, timeout)) { @@ -1647,28 +1637,6 @@ static int recv_bl(T_SOC soc, void* buffer, size_t len, int timeout) { return -1; } -static int linputsoc(T_SOC soc, char* s, int max) { - int c; - int j=0; - do { - unsigned char ch; - if (recv(soc, &ch, 1, 0) == 1) { - c = ch; - } else { - c = EOF; - } - if (c!=EOF) { - switch(c) { - case 13: break; // sauter CR - case 10: c=-1; break; - case 9: case 12: break; // sauter ces caractères - default: s[j++]=(char) c; break; - } - } - } while((c!=-1) && (c!=EOF) && (j<(max-1))); - s[j]='\0'; - return j; -} // check if data is available static int check_readinput(htsblk* r) { @@ -1688,12 +1656,6 @@ static int check_readinput(htsblk* r) { return 0; } -static int linputsoc_t(T_SOC soc, char* s, int max, int timeout) { - if (check_readinput_t(soc, timeout)) { - return linputsoc(soc, s, max); - } - return -1; -} /*int strfield(const char* f,const char* s) { int r=0; diff --git a/src/htsserver.h b/src/htsserver.h index acac908..9a633cb 100644 --- a/src/htsserver.h +++ b/src/htsserver.h @@ -39,7 +39,7 @@ Please visit our Website: http://www.httrack.com #ifndef HTS_SERVER_DEFH #define HTS_SERVER_DEFH -#include "htsbasenet.h" +#include "htsnet.h" /* String */ #include "htsstrings.h" @@ -47,7 +47,7 @@ Please visit our Website: http://www.httrack.com // Fonctions void socinput(T_SOC soc,char* s,int max); -T_SOC smallserver_init_std(int* port_prox,char* adr_prox); +T_SOC smallserver_init_std(int* port_prox,char* adr_prox,int defaultPort); T_SOC smallserver_init(int* port,char* adr); int smallserver(T_SOC soc,char* url,char* method,char* data, char* path); @@ -116,7 +116,53 @@ static int linput_trim(FILE* fp,char* s,int max); static char* concat(const char* a,const char* b); static int fexist(char* s); static int linput(FILE* fp,char* s,int max); -static int linputsoc_t(T_SOC soc, char* s, int max, int timeout); + +static int linputsoc(T_SOC soc, char* s, int max) { + int c; + int j=0; + do { + unsigned char ch; + if (recv(soc, &ch, 1, 0) == 1) { + c = ch; + } else { + c = EOF; + } + if (c!=EOF) { + switch(c) { + case 13: break; // sauter CR + case 10: c=-1; break; + case 9: case 12: break; // sauter ces caractères + default: s[j++]=(char) c; break; + } + } + } while((c!=-1) && (c!=EOF) && (j<(max-1))); + s[j]='\0'; + return j; +} + +static int check_readinput_t(T_SOC soc, int timeout) { + if (soc != INVALID_SOCKET) { + fd_set fds; // poll structures + struct timeval tv; // structure for select + FD_ZERO(&fds); + FD_SET(soc,&fds); + tv.tv_sec=timeout; + tv.tv_usec=0; + select(soc + 1,&fds,NULL,NULL,&tv); + if (FD_ISSET(soc,&fds)) + return 1; + else + return 0; + } else + return 0; +} + +static int linputsoc_t(T_SOC soc, char* s, int max, int timeout) { + if (check_readinput_t(soc, timeout)) { + return linputsoc(soc, s, max); + } + return -1; +} static char* gethomedir(void) { char* home = getenv( "HOME" ); diff --git a/src/htsstrings.h b/src/htsstrings.h index 0c4998a..14e851f 100755 --- a/src/htsstrings.h +++ b/src/htsstrings.h @@ -94,7 +94,7 @@ static void* StringAcquire(String* blk) { blk->len = 0; return buff; } -#define StringStrcat(blk, str) StringMemcat(blk, str, ((str) != NULL) ? strlen(str) : 0) +#define StringStrcat(blk, str) StringMemcat(blk, str, ((str) != NULL) ? (int)strlen(str) : 0) #define StringStrcpy(blk, str) do { \ StringClear(blk); \ StringStrcat(blk, str); \ diff --git a/src/htsthread.h b/src/htsthread.h index cac48de..f62c39c 100644 --- a/src/htsthread.h +++ b/src/htsthread.h @@ -58,7 +58,6 @@ Please visit our Website: http://www.httrack.com #define PTHREAD_TYPE_FNC __cdecl #define PTHREAD_LOCK_TYPE HANDLE #define PTHREAD_HANDLE HANDLE -#define PTHREAD_WAIT_THREAD(A) do { WaitForSingleObject(A, INFINITE); CloseHandle(A); } while(0) /* Useless - see '__declspec( thread )' */ @@ -82,7 +81,6 @@ Please visit our Website: http://www.httrack.com #define PTHREAD_KEY_SET(key, val, ptrtype) pthread_setspecific(key, (void*)val) #define PTHREAD_KEY_GET(key, ptrval, ptrtype) do { *(ptrval)=(ptrtype)pthread_getspecific(key); } while(0) #define PTHREAD_HANDLE pthread_t -#define PTHREAD_WAIT_THREAD(A) do { pthread_join(A, NULL); CloseHandle(A); } while(0) #endif diff --git a/src/htstools.c b/src/htstools.c index 389bd3a..ee83b09 100644 --- a/src/htstools.c +++ b/src/htstools.c @@ -91,6 +91,12 @@ int ident_url_relatif(char *lien,char* origin_adr,char* origin_fil,char* adr,cha } else { ok=-2; // non supporté } +#if HTS_USEMMS + } else if (strfield(lien,"mms://")) { + if (ident_url_absolute(lien,adr,fil)==-1) { + ok=-1; // erreur URL + } +#endif #if HTS_USEOPENSSL } else if (strfield(lien,"https://")) { if (SSL_is_available) { @@ -106,6 +112,9 @@ int ident_url_relatif(char *lien,char* origin_adr,char* origin_fil,char* adr,cha (!strfield(lien,"http:")) && (!strfield(lien,"https:")) && (!strfield(lien,"ftp:")) +#if HTS_USEMMS + && (!strfield(lien,"mms:")) +#endif )) { ok=-1; // unknown scheme } else { // c'est un lien relatif @@ -129,6 +138,12 @@ int ident_url_relatif(char *lien,char* origin_adr,char* origin_fil,char* adr,cha lien+=4; strcpybuff(adr, "ftp://"); // même adresse forcée en ftp strcatbuff(adr, jump_protocol(origin_adr)); +#if HTS_USEMMS + } else if (strfield(lien,"mms:")) { + lien+=4; + strcpybuff(adr, "mms://"); // même adresse forcée en ftp + strcatbuff(adr, jump_protocol(origin_adr)); +#endif } else { strcpybuff(adr,origin_adr); // même adresse ; et même éventuel protocole } @@ -235,11 +250,7 @@ int lienrelatif(char* s,char* link,char* curr_fil) { l=link; c=curr; // couper ce qui est commun -#if HTS_CASSE - while ((*link==*curr) && (*link!=0)) {link++; curr++; } -#else while ((streql(*link,*curr)) && (*link!=0)) {link++; curr++; } -#endif // mais on veut un répertoirer entier! // si on a /toto/.. et /toto2/.. on ne veut pas sauter /toto ! while(((*link!='/') || (*curr!='/')) && ( link > l)) { link--; curr--; } @@ -391,7 +402,7 @@ void longfile_to_83(int mode,char* n83,char* save) { } // corriger vers 8-3 n83[0]='\0'; - strncatbuff(n83,nom,8); + strncatbuff(n83,nom,max); if (strnotempty(ext)) { strcatbuff(n83,"."); strncatbuff(n83,ext,3); @@ -481,6 +492,40 @@ HTS_INLINE int __rech_tageq(const char* adr,const char* s) { return 0; } +HTS_INLINE int rech_tageq_all(const char* adr, const char* s) { + int p; + char quot = 0; + const char *token = NULL; + int s_len = (int) strlen(s); + if (adr == NULL) { + return 0; + } + for(p = 0 ; adr[p] != 0 ; p++) { + if (quot == 0) { + if (adr[p] == '"' || adr[p] == '\'' ) { + quot = adr[p]; + } else if (adr[p] == '=' || is_realspace(adr[p]) ) { + token = NULL; + } else if (adr[p] == '>') { + break; + } else { /* note: bogus for bogus foo = bar */ + if (token == NULL) { + if (strncasecmp(&adr[p], s, s_len) == 0 + && (is_realspace(adr[p + s_len]) || adr[p + s_len] == '=') + ) { + for( p += s_len ; is_realspace(adr[p]) || adr[p] == '=' ; p++ ); + return p; + } + token = &adr[p]; + } + } + } else if (adr[p] == quot) { + quot = 0; + } + } + return 0; +} + HTS_INLINE int rech_endtoken(const char* adr, const char** start) { char quote = '\0'; int length = 0; diff --git a/src/htstools.h b/src/htstools.h index c75d74d..90d5d7b 100644 --- a/src/htstools.h +++ b/src/htstools.h @@ -87,6 +87,7 @@ void long_to_83(int mode,char* n83,char* save); void longfile_to_83(int mode,char* n83,char* save); HTS_INLINE int __rech_tageq(const char* adr,const char* s); HTS_INLINE int __rech_tageqbegdigits(const char* adr,const char* s); +HTS_INLINE int rech_tageq_all(const char* adr, const char* s); #define rech_tageq(adr,s) \ ( \ ( (*((adr)-1)=='<') || (is_space(*((adr)-1))) ) ? \ diff --git a/src/htsweb.c b/src/htsweb.c index 8e33e34..51c85df 100644 --- a/src/htsweb.c +++ b/src/htsweb.c @@ -81,7 +81,7 @@ Please visit our Website: http://www.httrack.com static PTHREAD_LOCK_TYPE refreshMutex; -static int help_server(char* dest_path); +static int help_server(char* dest_path, int defaultPort); extern int commandRunning; extern int commandEnd; extern int commandReturn; @@ -97,6 +97,7 @@ int main(int argc, char* argv[]) { int i; int ret = 0; + int defaultPort = 0; printf("Initialzing the server..\n"); #ifdef _WIN32 @@ -119,7 +120,7 @@ int main(int argc, char* argv[]) if (argc < 2 || (argc % 2) != 0) { fprintf(stderr, "** Warning: use the webhttrack frontend if available\n"); - fprintf(stderr, "usage: %s <path-to-html-root-dir> [key value [key value]..]\n", argv[0]); + fprintf(stderr, "usage: %s [--port <port>] <path-to-html-root-dir> [key value [key value]..]\n", argv[0]); fprintf(stderr, "example: %s /usr/share/httrack/\n", argv[0]); return 1; } @@ -188,8 +189,15 @@ int main(int argc, char* argv[]) } /* set commandline keys */ - for(i = 2 ; i < argc ; i += 2) { - smallserver_setkey(argv[i], argv[i + 1]); + for(i = 2 ; i < argc ; i += 2) { + if (strcmp(argv[i], "--port") == 0) { + if (sscanf(argv[i + 1], "%d", &defaultPort) != 1 || defaultPort < 0 || defaultPort >= 65535 ) { + fprintf(stderr, "couldn't set the port number to %d\n", argv[i + 1]); + return -1; + } + } else { + smallserver_setkey(argv[i], argv[i + 1]); + } } /* sigpipe */ @@ -198,7 +206,7 @@ int main(int argc, char* argv[]) #endif /* launch */ - ret = help_server(argv[1]); + ret = help_server(argv[1], defaultPort); htsthread_wait(); hts_uninit(); @@ -292,8 +300,10 @@ static int webhttrack_runmain(int argc, char** argv) { htswrap_add("query2",htsshow_query2); htswrap_add("query3",htsshow_query3); htswrap_add("check-link",htsshow_check); + htswrap_add("check-mime",htsshow_check_mime); htswrap_add("pause",htsshow_pause); htswrap_add("save-file",htsshow_filesave); + htswrap_add("save-file2",htsshow_filesave2); htswrap_add("link-detected",htsshow_linkdetected); htswrap_add("link-detected2",htsshow_linkdetected2); htswrap_add("transfer-status",htsshow_xfrstatus); @@ -304,11 +314,11 @@ static int webhttrack_runmain(int argc, char** argv) { } -static int help_server(char* dest_path) { +static int help_server(char* dest_path, int defaultPort) { int returncode = 0; char adr_prox[HTS_URLMAXSIZE*2]; int port_prox; - T_SOC soc=smallserver_init_std(&port_prox,adr_prox); + T_SOC soc=smallserver_init_std(&port_prox, adr_prox, defaultPort); if (soc!=INVALID_SOCKET) { char url[HTS_URLMAXSIZE*2]; char method[32]; @@ -327,7 +337,7 @@ static int help_server(char* dest_path) { fflush(stderr); // if (!smallserver(soc,url,method,data,dest_path)) { - fprintf(stderr, "Unable to create the server\n"); + fprintf(stderr, "Unable to create the server: %s\n", strerror(errno)); #ifdef _WIN32 closesocket(soc); #else @@ -500,17 +510,27 @@ int __cdecl htsshow_loop(lien_back* back,int back_max,int back_index,int lien_n, strcpybuff(StatsBuffer[index].state,"request"); ok=1; } else if (back[i].status==100) { - strcpybuff(StatsBuffer[index].state,"connect"); ok=1; - } - else if (back[i].status==101) { - strcpybuff(StatsBuffer[index].state,"search"); ok=1; - } - else if (back[i].status==1000) { // ohh le beau ftp - sprintf(StatsBuffer[index].state,"ftp: %s",back[i].info); ok=1; - } - break; - default: - if (back[i].status==0) { // prêt + strcpybuff(StatsBuffer[index].state,"connect"); ok=1; + } + else if (back[i].status==101) { + strcpybuff(StatsBuffer[index].state,"search"); ok=1; + } + else if (back[i].status==1000) { // ohh le beau ftp + char proto[] = "ftp"; + if (back[i].url_adr[0]) { + char* ep = strchr(back[i].url_adr, ':'); + char* eps = strchr(back[i].url_adr, '/'); + int count; + if (ep != NULL && ep < eps && (count = (ep - back[i].url_adr) ) < 4) { + proto[0] = '\0'; + strncat(proto, back[i].url_adr, count); + } + } + sprintf(StatsBuffer[index].state,"%s: %s",proto,back[i].info); ok=1; + } + break; + default: + if (back[i].status==0) { // prêt if ((back[i].r.statuscode==200)) { strcpybuff(StatsBuffer[index].state,"ready"); ok=1; } @@ -655,10 +675,15 @@ char* __cdecl htsshow_query3(char* question) { int __cdecl htsshow_check(char* adr,char* fil,int status) { return -1; } +int __cdecl htsshow_check_mime(char* adr,char* fil,char* mime,int status) { + return -1; +} void __cdecl htsshow_pause(char* lockfile) { } void __cdecl htsshow_filesave(char* file) { } +void __cdecl htsshow_filesave2(char* adr, char* fil, char* save, int is_new, int is_modified,int not_updated) { +} int __cdecl htsshow_linkdetected(char* link) { return 1; } diff --git a/src/htsweb.h b/src/htsweb.h index 82293b4..4f9439d 100644 --- a/src/htsweb.h +++ b/src/htsweb.h @@ -92,8 +92,10 @@ char* __cdecl htsshow_query(char* question); char* __cdecl htsshow_query2(char* question); char* __cdecl htsshow_query3(char* question); int __cdecl htsshow_check(char* adr,char* fil,int status); +int __cdecl htsshow_check_mime(char* adr,char* fil,char* mime,int status); void __cdecl htsshow_pause(char* lockfile); void __cdecl htsshow_filesave(char* file); +void __cdecl htsshow_filesave2(char* adr, char* fil, char* save, int is_new, int is_modified, int not_updated); int __cdecl htsshow_linkdetected(char* link); int __cdecl htsshow_linkdetected2(char* link, char* start_tag); int __cdecl htsshow_xfrstatus(lien_back* back); diff --git a/src/htswizard.c b/src/htswizard.c index 366a23a..ab851bf 100644 --- a/src/htswizard.c +++ b/src/htswizard.c @@ -67,7 +67,32 @@ Please visit our Website: http://www.httrack.com assertf((*opt->filters.filptr) < opt->maxfilter); \ } while(0) +typedef struct htspair_t { + char *tag; + char *attr; +} htspair_t; +/* "embedded" */ +htspair_t hts_detect_embed[] = { + { "img", "src" }, + { "link", "href" }, + + /* embedded script hack */ + { "script", ".src" }, + + /* style */ + { "style", "import" }, + + { NULL, NULL } +}; + +/* Internal */ +static int hts_acceptlink_(httrackp* opt, + int ptr,int lien_tot,lien_url** liens, + char* adr,char* fil, + char* tag, char* attribute, + int* set_prio_to, + int* just_test_it); /* httrackp opt bloc d'options @@ -84,19 +109,53 @@ int* set_prio_to int* just_test_it callback optionnel "ne faire que tester ce lien éventuellement" retour: - 0 accepté - 1 refusé - -1 pas d'avis +0 accepté +1 refusé +-1 pas d'avis */ + int hts_acceptlink(httrackp* opt, - int ptr,int lien_tot,lien_url** liens, - char* adr,char* fil, - char* tag, char* attribute, - int* set_prio_to, - int* just_test_it) { - + int ptr,int lien_tot,lien_url** liens, + char* adr,char* fil, + char* tag, char* attribute, + int* set_prio_to, + int* just_test_it) +{ + int forbidden_url = hts_acceptlink_(opt, ptr, lien_tot, liens, + adr, fil, tag, attribute, set_prio_to, just_test_it); + int prev_prio = set_prio_to ? *set_prio_to : 0; + + // -------------------- PHASE 6 -------------------- +#if HTS_ANALYSTE + if (hts_htmlcheck_check != NULL) { + int test_url = hts_htmlcheck_check(adr, fil, forbidden_url); + if (test_url != -1) { + forbidden_url = test_url; + if (set_prio_to) + *set_prio_to = prev_prio; + } + } +#endif + + return forbidden_url; +} + +static int cmp_token(const char *tag, const char *cmp) { + int p; + return (strncasecmp(tag, cmp, ( p = (int) strlen(cmp) ) ) == 0 + && !isalnum((unsigned char) tag[p])); +} + +static int hts_acceptlink_(httrackp* opt, + int ptr,int lien_tot,lien_url** liens, + char* adr,char* fil, + char* tag, char* attribute, + int* set_prio_to, + int* just_test_it) +{ int forbidden_url=-1; int meme_adresse; + int embedded_triggered = 0; #define _FILTERS (*opt->filters.filters) #define _FILTERS_PTR (opt->filters.filptr) #define _ROBOTS ((robots_wizard*)opt->robotsptr) @@ -119,6 +178,23 @@ int hts_acceptlink(httrackp* opt, return 0; /* Yokai */ } + // -------------------- PRELUDE OF PHASE 3-BIS -------------------- + + /* Built-in known tags (<img src=..>, ..) */ + if (forbidden_url != 0 && opt->nearlink && tag != NULL && attribute != NULL) { + int i; + for(i = 0 ; hts_detect_embed[i].tag != NULL ; i++) { + if (cmp_token(tag, hts_detect_embed[i].tag) + && cmp_token(attribute, hts_detect_embed[i].attr) + ) + { + embedded_triggered = 1; + break; + } + } + } + + // -------------------- PHASE 1 -------------------- /* Doit-on traiter les non html? */ @@ -136,7 +212,7 @@ int hts_acceptlink(httrackp* opt, /* Niveau 1: ne pas parser suivant! */ if (ptr>0) { - if (liens[ptr]->depth <= 1) { + if ( ( liens[ptr]->depth <= 0 ) || ( liens[ptr]->depth <= 1 && !embedded_triggered ) ) { forbidden_url=1; // interdire récupération du lien if ((opt->debug>1) && (opt->log!=NULL)) { fspc(opt->log,"debug"); fprintf(opt->log,"file from too far level ignored at %s : %s"LF,adr,fil); @@ -146,7 +222,7 @@ int hts_acceptlink(httrackp* opt, } /* en cas d'échec en phase 1, retour immédiat! */ - if (forbidden_url==1) { + if (forbidden_url == 1) { return forbidden_url; } @@ -394,7 +470,7 @@ int hts_acceptlink(httrackp* opt, // -------------------- PHASE 3 -------------------- // récupérer les liens à côtés d'un lien (nearlink) (nvelle pos) - if (opt->nearlink) { + if (forbidden_url != 0 && opt->nearlink) { if (!ishtml(fil)) { // non html //printf("ok %s%s\n",ad,fil); forbidden_url=0; // autoriser @@ -405,7 +481,20 @@ int hts_acceptlink(httrackp* opt, } } } - + + // -------------------- PHASE 3-BIS -------------------- + + /* Built-in known tags (<img src=..>, ..) */ + if (forbidden_url != 0 && embedded_triggered) { + forbidden_url=0; // autoriser + may_set_prio_to=1+1; // set prio to 1 (parse but skip urls) if near is the winner + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"near link authorized (friendly tag): %s%s"LF,adr,fil); + test_flush; + } + } + + // -------------------- PHASE 4 -------------------- // ------------------------------------------------------ @@ -479,8 +568,8 @@ int hts_acceptlink(httrackp* opt, { int jokDepth1=0,jokDepth2=0; int jok1=0,jok2=0; - jok1 = fa_strjoker(_FILTERS,*_FILTERS_PTR,lfull,NULL,NULL,&jokDepth1); - jok2 = fa_strjoker(_FILTERS,*_FILTERS_PTR,l, NULL,NULL,&jokDepth2); + jok1 = fa_strjoker(/*url*/0, _FILTERS,*_FILTERS_PTR,lfull,NULL,NULL,&jokDepth1); + jok2 = fa_strjoker(/*url*/0, _FILTERS,*_FILTERS_PTR,l, NULL,NULL,&jokDepth2); if (jok2 == 0) { // #2 doesn't know jok = jok1; // then, use #1 mdepth = _FILTERS[jokDepth1]; @@ -816,7 +905,11 @@ int hts_acceptlink(httrackp* opt, if (just_test_it) { if (forbidden_url==1) { if (opt->travel&256) { // tester tout de même - if (strfield(adr,"ftp://")==0) { // PAS ftp! + if (strfield(adr,"ftp://")==0 +#if HTS_USEMMS + && strfield(adr,"mms://")==0 +#endif + ) { // PAS ftp! forbidden_url=1; // oui oui toujours interdit (note: sert à rien car ==1 mais c pour comprendre) *just_test_it=1; // mais on teste if ((opt->debug>1) && (opt->log!=NULL)) { @@ -828,17 +921,6 @@ int hts_acceptlink(httrackp* opt, //adr[0]='\0'; // cancel } - // -------------------- PHASE 6 -------------------- -#if HTS_ANALYSTE - { - int test_url=hts_htmlcheck_check(adr,fil,forbidden_url); - if (test_url!=-1) { - forbidden_url=test_url; - may_set_prio_to=0; // clear may-set flag - } - } -#endif - // -------------------- FINAL PHASE -------------------- // Test if the "Near" test won if (may_set_prio_to && forbidden_url == 0) { @@ -846,6 +928,55 @@ int hts_acceptlink(httrackp* opt, } return forbidden_url; +#undef _FILTERS +#undef _FILTERS_PTR +#undef _ROBOTS +} + +int hts_acceptmime(httrackp* opt, + int ptr,int lien_tot,lien_url** liens, + char* adr,char* fil, + char* mime) +{ +#define _FILTERS (*opt->filters.filters) +#define _FILTERS_PTR (opt->filters.filptr) +#define _ROBOTS ((robots_wizard*)opt->robotsptr) + int forbidden_url = -1; + char* mdepth=""; + int jokDepth = 0; + int jok = 0; + + /* Authorized ? */ + jok = fa_strjoker(/*mime*/1, _FILTERS, *_FILTERS_PTR, mime, NULL, NULL, &jokDepth); + if (jok != 0) { + mdepth = _FILTERS[jokDepth]; + if (jok == 1) { // autorisé + forbidden_url=0; // URL autorisée + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"(wizard) explicit authorized (%s) link %s%s: mime '%s'"LF,mdepth,adr,fil,mime); + test_flush; + } + } else if (jok == -1) { // forbidden + forbidden_url=1; // URL interdite + if ((opt->debug>1) && (opt->log!=NULL)) { + fspc(opt->log,"debug"); fprintf(opt->log,"(wizard) explicit forbidden (%s) link %s%s: mime '%s'"LF,mdepth,adr,fil,mime); + test_flush; + } + } // sinon on touche à rien + } + /* userdef test */ +#if HTS_ANALYSTE + if (hts_htmlcheck_check_mime != NULL) { + int test_url=hts_htmlcheck_check_mime(adr,fil,mime,forbidden_url); + if (test_url!=-1) { + forbidden_url=test_url; + } + } +#endif + return forbidden_url; +#undef _FILTERS +#undef _FILTERS_PTR +#undef _ROBOTS } // tester taille @@ -873,17 +1004,14 @@ int hts_testlinksize(httrackp* opt, if (*fil!='/') strcatbuff(l,"/"); strcatbuff(lfull,fil); - // tester filtres (taille) - // jok = fa_strjoker(opt->filters.filters,*opt->filters.filptr,l,&sz,&size_flag,NULL); - // filters, 0=sait pas 1=ok -1=interdit { int jokDepth1=0,jokDepth2=0; int jok1=0,jok2=0; LLint sz1=size,sz2=size; int size_flag1=0,size_flag2=0; - jok1 = fa_strjoker(*opt->filters.filters,*opt->filters.filptr,lfull,&sz1,&size_flag1,&jokDepth1); - jok2 = fa_strjoker(*opt->filters.filters,*opt->filters.filptr,l, &sz2,&size_flag2,&jokDepth2); + jok1 = fa_strjoker(/*url*/0, *opt->filters.filters,*opt->filters.filptr,lfull,&sz1,&size_flag1,&jokDepth1); + jok2 = fa_strjoker(/*url*/0, *opt->filters.filters,*opt->filters.filptr,l, &sz2,&size_flag2,&jokDepth2); if (jok2 == 0) { // #2 doesn't know jok = jok1; // then, use #1 sz = sz1; diff --git a/src/htswizard.h b/src/htswizard.h index a36940d..7236573 100644 --- a/src/htswizard.h +++ b/src/htswizard.h @@ -52,6 +52,10 @@ int hts_acceptlink(httrackp* opt, int hts_testlinksize(httrackp* opt, char* adr,char* fil, LLint size); +int hts_acceptmime(httrackp* opt, + int ptr,int lien_tot,lien_url** liens, + char* adr,char* fil, + char* mime); #endif #endif diff --git a/src/httrack.c b/src/httrack.c index b3accb1..e2b6729 100644 --- a/src/httrack.c +++ b/src/httrack.c @@ -166,12 +166,14 @@ Log: "check-html: <url>" hts_htmlcheck_query3 = (t_hts_htmlcheck_query3) htswrap_read("query3"); hts_htmlcheck_loop = (t_hts_htmlcheck_loop) htswrap_read("loop"); hts_htmlcheck_check = (t_hts_htmlcheck_check) htswrap_read("check-link"); + hts_htmlcheck_check_mime = (t_hts_htmlcheck_check_mime) htswrap_read("check-mime"); Log: none hts_htmlcheck_pause = (t_hts_htmlcheck_pause) htswrap_read("pause"); Log: "pause: <lockfile>" hts_htmlcheck_filesave = (t_hts_htmlcheck_filesave) htswrap_read("save-file"); + hts_htmlcheck_filesave2 = (t_hts_htmlcheck_filesave2) htswrap_read("save-file2"); hts_htmlcheck_linkdetected = (t_hts_htmlcheck_linkdetected) htswrap_read("link-detected"); hts_htmlcheck_linkdetected2 = (t_hts_htmlcheck_linkdetected2) htswrap_read("link-detected2"); Log: none @@ -200,8 +202,10 @@ Log: htswrap_add("query2",htsshow_query2); htswrap_add("query3",htsshow_query3); htswrap_add("check-link",htsshow_check); + htswrap_add("check-mime",htsshow_check_mime); htswrap_add("pause",htsshow_pause); htswrap_add("save-file",htsshow_filesave); + htswrap_add("save-file2",htsshow_filesave2); htswrap_add("link-detected",htsshow_linkdetected); htswrap_add("link-detected2",htsshow_linkdetected2); htswrap_add("transfer-status",htsshow_xfrstatus); @@ -592,6 +596,9 @@ static char* __cdecl htsshow_query3(char* question) { static int __cdecl htsshow_check(char* adr,char* fil,int status) { return -1; } +static int __cdecl htsshow_check_mime(char* adr,char* fil,char* mime,int status) { + return -1; +} static void __cdecl htsshow_pause(char* lockfile) { while (fexist(lockfile)) { Sleep(1000); @@ -599,6 +606,8 @@ static void __cdecl htsshow_pause(char* lockfile) { } static void __cdecl htsshow_filesave(char* file) { } +static void __cdecl htsshow_filesave2(char* adr, char* fil, char* save, int is_new, int is_modified,int not_updated) { +} static int __cdecl htsshow_linkdetected(char* link) { return 1; } diff --git a/src/httrack.h b/src/httrack.h index a3c82a4..229251c 100644 --- a/src/httrack.h +++ b/src/httrack.h @@ -42,7 +42,7 @@ Please visit our Website: http://www.httrack.com #include "htsglobal.h" #include "htscore.h" -typedef struct { +typedef struct t_StatsBuffer { char name[1024]; char file[1024]; char state[256]; @@ -58,7 +58,7 @@ typedef struct { int actived; // pour disabled } t_StatsBuffer; -typedef struct { +typedef struct t_InpInfo { int ask_refresh; int refresh; LLint stat_bytes; @@ -92,8 +92,10 @@ static char* __cdecl htsshow_query(char* question); static char* __cdecl htsshow_query2(char* question); static char* __cdecl htsshow_query3(char* question); static int __cdecl htsshow_check(char* adr,char* fil,int status); +static int __cdecl htsshow_check_mime(char* adr,char* fil,char* mime,int status); static void __cdecl htsshow_pause(char* lockfile); static void __cdecl htsshow_filesave(char* file); +static void __cdecl htsshow_filesave2(char* adr, char* fil, char* save, int is_new, int is_modified,int not_updated); static int __cdecl htsshow_linkdetected(char* link); static int __cdecl htsshow_linkdetected2(char* link, char* start_tag); static int __cdecl htsshow_xfrstatus(lien_back* back); @@ -206,32 +208,6 @@ extern FILE* ioinfo; } \ } \ } while(0) -#define strncpybuff(A, B, N) do { \ - assertf( (A) != NULL ); \ - if ( ! (B) ) { assertf( 0 ); } \ - if (htsMemoryFastXfr) { \ - if (sizeof(A) != sizeof(char*)) { \ - (A)[sizeof(A) - 1] = '\0'; \ - } \ - strncpy(A, B, N); \ - if (sizeof(A) != sizeof(char*)) { \ - assertf((A)[sizeof(A) - 1] == '\0'); \ - } \ - } else { \ - unsigned int szf = (unsigned int) strlen(B); \ - if (szf > (unsigned int) (N)) szf = (unsigned int) (N); \ - if (sizeof(A) != sizeof(char*)) { \ - assertf(szf + 1 < sizeof(A)); \ - if (szf > 0) { \ - if (szf + 1 < sizeof(A)) { \ - memcpy((A), (B), szf); \ - } \ - } \ - } else { \ - memcpybuff((A), (B), szf); \ - } \ - } \ -} while(0) // emergency log typedef void (*t_abortLog)(char* msg, char* file, int line); diff --git a/src/mmsrip/AUTHORS b/src/mmsrip/AUTHORS new file mode 100644 index 0000000..530a73f --- /dev/null +++ b/src/mmsrip/AUTHORS @@ -0,0 +1,27 @@ +Nicolas BENOIT (main developper) +nbenoit@tuxfamily.org +http://nbenoit.tuxfamily.org + +Major MMS (author of mmsclient) +http://www.geocities.com/majormms + +SDP Multimedia (author of the only doc about MMS) +http://get.to/sdp + +Aurelien REQUIEM (author of mmsrip's ebuild script) +aurelien@menfin.net + +Luis COSTA (initial idea and patch for the -d/--delay option) +izhirahider@gmail.com + +Jeff FULMER (Solaris port) +jeff@joedog.org + +Federico SIMONCELLI (RPMs for Fedora Core 4) +federico.simoncelli@gmail.com + +Kyuzz (Fix for Cygwin compilation) +kyuzz.org + +Xavier ROCHE (Initial port to Win32) +roche@httrack.com diff --git a/src/mmsrip/COPYING b/src/mmsrip/COPYING new file mode 100644 index 0000000..5b6e7c6 --- /dev/null +++ b/src/mmsrip/COPYING @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/src/mmsrip/ChangeLog b/src/mmsrip/ChangeLog new file mode 100644 index 0000000..da17d27 --- /dev/null +++ b/src/mmsrip/ChangeLog @@ -0,0 +1,156 @@ +2006-01-24 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * README, NEWS: getting ready for 0.7.0 release. + * gentoo/mmsrip-0.7.0.ebuild: ebuild script for 0.7.0 release. + * spec/mmsrip.spec: updated for 0.7.0 release. + + +2006-01-23 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * src/mms.h, src/mms.c, src/main.c, doc/mmsrip.1: renamed compatibility mode to trick enabled mode (-c/--compat switches are now -k/--trick). + * src/mms.c, src/main.c: fixed parsing of urls that look like 'stream.asf?digest=7Q2bjXo&provider=lala'. + * src/mms.c, src/mms.h, src/main.c, doc/mmsrip.1, configure.ac: replaced configure's --enable-debug switch by a -gFILE/--debug=FILE runtime switch. + * doc/mmsrip.1: improved presentation. + + +2006-01-22 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * src/mms.c: fixed presentation in debug output. + * AUTHORS: updated. + * src/mms.c: improved debug ouput for header interpretation. + * src/mms.c: fixed expected file size. + * src/mms.c: added a few calls to mms_get_32() for code lisibility. + * src/mms.c: added mms_get_64() function (improves lisibility in mms_interp_header()). + * src/mms.c: improved ASF header interpretation. + * src/mms.h, src/mms.c, src/main.c, doc/mmsrip.1: added compatibility mode and -c/--compat switches. + * src/main.c: minor presentation improvements. + * src/mms.c: added an entry for mmsh:// protocol though we don't support it. + + +2006-01-21 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * src/mms.h, src/mms.c, src/error.c, src/error.h, src/main.c, configure.ac: fixed compilation on Win32 (thanks to Xavier ROCHE for the initial port). + * AUTHORS: updated. + * src/mms.c: fixed URL parsing (mmst:// is now ok). + * src/mms.c, doc/mmsrip.1: the user should read the manpage when the server sends a no auth error. + * configure.ac: added a switch to enable debug output to stdout. + + +2006-01-20 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * src/mms.h, src/main.c, configure.ac: fixed compilation on Solaris (from Jeff FULMER's Solaris port). + * AUTHORS: updated. + * gentoo/mmsrip-0.6.6.ebuild: ebuild script for 0.6.6 release. + * spec/mmsrip.spec: spec file for RPM building (thanks to Federico SIMONCELLI). + * src/mms.c, configure.ac: added the display of the ripping speed. + + +2006-01-17 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * src/mms.h: fixed compilation on Cygwin (greetings to Kyuzz for bug reporting). + + +2006-01-06 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * configure.ac, src/common.h: 0.6.5 release. + + +2006-01-06 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * src/mms.c: fixed a call to error(). + * src/mms.h: reordered some error codes. + * src/main.c: improved the error code returned by the program. + * doc/mmsrip.1: added some documentation about the program's returned value. + * gentoo/mmsrip-0.6.5.ebuild: ebuild script for 0.6.5 release. + * README, NEWS: getting ready for 0.6.5 release. + + +2005-11-23 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * src/mms.h, src/mms.c, src/main.c: replaced a few values with #defines. + * src/mms.c: added the support for no auth errors (this happens sometimes, for example on canalplus.fr). + + +2005-07-09 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * configure.ac, src/common.h: 0.6.4 release. + + +2005-07-09 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * doc/mmsrip.1: added man page. + * doc/Makefile.am, Makefile.am, configure.ac: added the man page to the package. + * src/main.c: fixed a compilation warning about variable `end` initialization. + * src/mms.c: fixed a potential security issue in the handling of files containing more than 20 streams (patch ported from MPlayer). + * src/mms.c: fixed a bug in the media stream MBR selection that prevented ASF files from being ripped properly (bug reported by Jozef RIHA). + * gentoo/mmsrip-0.6.4.ebuild: ebuild script for 0.6.4 release. + * README, NEWS: getting ready for 0.6.4 release. + + +2005-06-05 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * configure.ac, src/common.h: 0.6.2 release. + + +2005-06-05 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * Makefile.am: ebuild script moved in a dedicated gentoo directory. + * gentoo/mmsrip-0.6.2.ebuild: ebuild script for 0.6.2 release. + * src/main.c: added '-d' switch which makes mmsrip exit after the specified delay (idea and patch by Luis COSTA). + * src/main.c: fixed bug that made mmsrip attempt to use invalid URLs. + * AUTHORS: updated. + * src/main.c: added support for long options. + * README, NEWS: getting ready for 0.6.2 release. + + +2005-05-29 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * configure.ac: 0.6.0 release. + + +2005-05-29 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * src/main.c: added '-q' switch which makes mmsrip quiet. + * src/mms.c, src/mms.h: added support for quiet mode. + * src/main.c: added '-t' switch which makes mmsrip check stream availability only. + * src/main.c: added '-o' switch which makes mmsrip output stream to specified file. + * src/main.c: fixed a bug in args handling. + * README, NEWS, src/common.h: getting ready for 0.6.0 release. + + +2005-05-28 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * mmsrip-0.6.0.ebuild: added Aurelien REQUIEM's ebuild. + * README: updated. + * AUTHORS: updated. + * Makefile.am: added ebuild script to EXTRA_DIST. + * configure.ac: added CVS infos. + + +2005-02-21 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * configure.ac, src/common.h: 0.4.2 release. + + +2005-02-21 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * src/mms.c: Removed bad EOF handling in mms_recv_packet(). + * README, NEWS: Getting ready for 0.4.2. + + +2005-02-20 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * configure.ac, src/common.h: 0.4.1 release. + + +2005-02-20 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * src/main.c: Fixed bug with multiple streams (mmsrip used to download the same stream every time). + * configure.ac: Removed unnecessary checks. + * src/mms.c: Removed strndup() because a lot of people don't have it. + * README, NEWS: Getting ready for 0.4.1. + + +2005-02-20 Nicolas BENOIT <nbenoit@tuxfamily.org> + + * configure.ac, src/common.h: 0.4.0 release. diff --git a/src/mmsrip/NEWS b/src/mmsrip/NEWS new file mode 100644 index 0000000..43b54aa --- /dev/null +++ b/src/mmsrip/NEWS @@ -0,0 +1,68 @@ + MMS Ripper v0.7.0: +==================== + + - Added '-k' and '--trick' switches that activate a trick for some recalcitrant servers. + - Added the display of the ripping speed. + - Added '-gFILE' and '--debug=FILE' switches which activate debug output to specified file. + - Fixed parsing of urls that include options after file name (stream.asf?dig=7QXo&pr=lala). + - Fixed protocol checking in URL parsing. + - Fixed expected file size calculation. + - Fixed compilation on Solaris, Cygwin and Win32 (check AUTHORS file for greetings). + - Improved debug output. + - Improved ASF header interpretation. + - Improved presentation in manpage. + - Minor code cleanups. + - A spec file is now distributed. + + + MMS Ripper v0.6.5: +==================== + + - Added the handling of the no auth error during streaming. + (this should fix mmsrip behavior on some servers such as canalplus.fr) + - Improved the diagnostic value returned by mmsrip in case of error (please read the manpage). + + + MMS Ripper v0.6.4: +==================== + + - Added a man page. + - Fixed a compilation warning. + - Fixed a potential security issue. + - Fixed the ripping of some ASF files (bug reported by Jozef RIHA). + + + MMS Ripper v0.6.2: +==================== + + - Added '-d' switch which makes mmsrip exit after DELAY seconds. + - Added support for long options. + - Fixed bug that made mmsrip attempt to use invalid URLs. + + + MMS Ripper v0.6.0: +==================== + + - Added '-q' switch which enables quiet mode. + - Added '-t' switch which makes mmsrip check stream availability only. + - Added '-o' switch so the user can choose the output file for every stream. + - Fixed a bug in arguments handling. + + + MMS Ripper v0.4.2: +==================== + + - Fixed a serious bug in packet reception (mmsrip went crazy). + + + MMS Ripper v0.4.1: +==================== + + - Fixed a bug that made mmsrip download the same stream every time. + - Compilation fixes. + + + MMS Ripper v0.4.0: +==================== + + - First Release of mmsrip. diff --git a/src/mmsrip/README b/src/mmsrip/README new file mode 100644 index 0000000..bf7f7d1 --- /dev/null +++ b/src/mmsrip/README @@ -0,0 +1,37 @@ + MMS Ripper release 0.7.0 + +These are the release notes for mmsrip version 0.7.0 +Read them carefully, as they tell you what this is all about. + + +WHAT IS MMSRIP ? + + MMSRIP allows you to save on your hard-disk the content being streamed + by an MMS server. This program has been written for personnal use, so + don't blame me if you think I am stupid doing such tool for the others. + + It should run on every POSIX compliant Operating System, but I can't + give you any complete list. + + +HOW TO INSTALL ? + + Read the file INSTALL in order to get the answer of this question... ;) + If you use a gentoo based distribution, enjoy the ebuild script contributed by Aurelien REQUIEM. + If you use a RPM based distribution, you may use the spec file provided by Frederico SIMONCELLI. + + +HOW TO RUN ? + + Once you have compiled & installed, you should be able to run the program. + + +IMPORTANT NOTE + + All the credits go to SDP Multimedia and Major MMS. + + +-- + +$RCSfile: README,v $ +$Date: 2006/01/24 18:13:07 $ - $Revision: 1.13 $ diff --git a/src/mmsrip/common.h b/src/mmsrip/common.h new file mode 100644 index 0000000..47f5567 --- /dev/null +++ b/src/mmsrip/common.h @@ -0,0 +1,50 @@ +/* + * $RCSfile: common.h,v $ + * $Date: 2006/01/17 19:59:27 $ - $Revision: 1.11 $ + * + * This file is distributed as a part of MMSRIP ( MMS Ripper ). + * Copyright (c) 2005-2006 Nicolas BENOIT + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + + +#ifndef __COMMON_H__ +#define __COMMON_H__ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#else +#define PROGRAM_SHORT_NAME "mmsrip" +#define PROGRAM_FULL_NAME "MMS Ripper" +#define PROGRAM_VERSION "0.7.0-rc1" +#endif + +#define PROGRAM_AUTHORS "Nicolas BENOIT <nbenoit@tuxfamily.org>" +#define PROGRAM_HELPERS "SDP Multimedia and Major MMS" + + +/* + * quick struct in order to make list of streams + */ +typedef struct +{ + void *next; + char *stream; + char *output; +} STREAM_LIST; + +#endif diff --git a/src/mmsrip/error.c b/src/mmsrip/error.c new file mode 100644 index 0000000..9d2f4c9 --- /dev/null +++ b/src/mmsrip/error.c @@ -0,0 +1,117 @@ +/* + * $RCSfile: error.c,v $ + * $Date: 2006/01/21 20:09:57 $ - $Revision: 1.7 $ + * + * This file is distributed as a part of MMSRIP ( MMS Ripper ). + * Copyright (c) 2005-2006 Nicolas BENOIT + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + + +#define _GNU_SOURCE + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "common.h" +#include "error.h" + + +/* + * Prints A Warning + */ +#ifdef HAVE_VSNPRINTF +void +warning ( const char *prod, + const char *format, + ... ) +{ + char *message; + va_list ap; + + if ( format == NULL ) + return; + + message = ( char * ) malloc ( ERROR_MSG_LEN ); + va_start ( ap, format ); + vsnprintf ( message, ERROR_MSG_LEN, format, ap ); + va_end ( ap ); + + if ( prod != NULL ) + fprintf ( stderr, "warning in %s(): %s.\n", prod, message ); + else + fprintf ( stderr, "warning: %s.\n", message ); + + free ( message ); +} +#else +void +warning ( const char *prod, + const char *message ) +{ + if ( message == NULL ) + return; + + if ( prod != NULL ) + fprintf ( stderr, "warning in %s(): %s.\n", prod, message ); + else + fprintf ( stderr, "warning: %s.\n", message ); +} +#endif + + +/* + * Prints An Error Message + */ +#ifdef HAVE_VSNPRINTF +void +error ( const char *prod, + const char *format, + ... ) +{ + char *message; + va_list ap; + + if ( format == NULL ) + return; + + message = ( char * ) malloc ( ERROR_MSG_LEN ); + va_start ( ap, format ); + vsnprintf ( message, ERROR_MSG_LEN, format, ap ); + va_end ( ap ); + + if ( prod != NULL ) + fprintf ( stderr, "error in %s(): %s.\n", prod, message ); + else + fprintf ( stderr, "error: %s.\n", message ); + + free ( message ); +} +#else +void +error ( const char *prod, + const char *message ) +{ + if ( message == NULL ) + return; + + if ( prod != NULL ) + fprintf ( stderr, "error in %s(): %s.\n", prod, message ); + else + fprintf ( stderr, "error: %s.\n", message ); +} +#endif diff --git a/src/mmsrip/error.h b/src/mmsrip/error.h new file mode 100644 index 0000000..d0e379c --- /dev/null +++ b/src/mmsrip/error.h @@ -0,0 +1,40 @@ +/* + * $RCSfile: error.h,v $ + * $Date: 2006/01/21 20:09:57 $ - $Revision: 1.4 $ + * + * This file is distributed as a part of MMSRIP ( MMS Ripper ). + * Copyright (c) 2005-2006 Nicolas BENOIT + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + + +#ifndef __ERROR_H__ +#define __ERROR_H__ + +#ifdef HAVE_VSNPRINTF +#include <stdarg.h> + +#define ERROR_MSG_LEN 128 + +void warning ( const char *, const char *, ... ); +void error ( const char *, const char *, ... ); +#else +void warning ( const char *, const char * ); +void error ( const char *, const char * ); +#endif + +#endif diff --git a/src/mmsrip/main.c b/src/mmsrip/main.c new file mode 100644 index 0000000..a575b4a --- /dev/null +++ b/src/mmsrip/main.c @@ -0,0 +1,753 @@ +/* + * $RCSfile: main.c,v $ + * $Date: 2006/01/23 20:30:42 $ - $Revision: 1.32 $ + * + * This file is distributed as a part of MMSRIP ( MMS Ripper ). + * Copyright (c) 2005-2006 Nicolas BENOIT + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + + +#define _GNU_SOURCE + +#include <stdlib.h> +#include <stdio.h> +#include <unistd.h> +#include <string.h> +#include <time.h> +#include <sys/time.h> +#include "common.h" +#include "mms.h" +#include "error.h" + +#ifdef HAVE_GETOPT_H +#include <getopt.h> +#endif + +/* + * options + */ +#if defined(SOLARIS) || defined(sun) +static char * options = "ahvtqko:d:g:"; +#else +static char * options = "-ahvtqko:d:g:"; +#endif + +#ifdef HAVE_GETOPT_LONG +static struct option long_options[] = { + {"about", 0, NULL, 'a'}, + {"version", 0, NULL, 'v'}, + {"help", 0, NULL, 'h'}, + {"test", 0, NULL, 't'}, + {"quiet", 0, NULL, 'q'}, + {"trick", 0, NULL, 'k'}, + {"output", 1, NULL, 'o'}, + {"delay", 1, NULL, 'd'}, + {"debug", 1, NULL, 'g'}, + {NULL, 0, NULL, 0 } +}; +#endif + +/* + * usage + */ +void +usage ( void ) +{ + fprintf ( stderr, "%s (%s) version %s\n\n", PROGRAM_SHORT_NAME, PROGRAM_FULL_NAME, PROGRAM_VERSION ); + fprintf ( stderr, "usage: %s <[-oFILE] stream url> ...\n\n", PROGRAM_SHORT_NAME ); + +#ifdef HAVE_GETOPT_LONG + fprintf ( stderr, "General Options:\n\t" \ + "-a, --about\t\tshow information about %s\n\t" \ + "-h, --help\t\tshow this help\n\t" \ + "-v, --version\t\tshow version number\n\n" \ + "Program Behaviour:\n\t" \ + "-oFILE, --output=FILE\toutput to specified file (can be repeated)\n\t" \ + "-gFILE, --debug=FILE\toutput debug info to specified file\n\t" \ + "-q, --quiet\t\tquiet mode (can be repeated)\n\t" \ + "-dDELAY, --delay=DELAY\tsave the stream during DELAY seconds and exit\n\t" \ + "-k, --trick\t\tattempt to trick recalcitrant servers\n\t" \ + "-t, --test\t\ttest mode (check stream availability)\n\n", PROGRAM_SHORT_NAME ); +#else + fprintf ( stderr, "General Options:\n\t" \ + "-a\tshow information about %s\n\t" \ + "-h\tshow this help\n\t" \ + "-v\tshow version number\n\n" \ + "Program Behaviour:\n\t" \ + "-oFILE\toutput to specified file (can be repeated)\n\t" \ + "-gFILE\toutput debug info to specified file\n\t" \ + "-q\tquiet mode (can be repeated)\n\t" \ + "-dDELAY\tsave the stream during DELAY seconds and exit\n\t" \ + "-k\tattempt to trick recalcitrant servers\n\t" \ + "-t\ttest mode (check stream availability)\n\n", PROGRAM_SHORT_NAME ); +#endif + + return; +} + + +/* + * main + */ +int +main ( int argc, + char *argv[] ) +{ + MMS *mms; + FILE *f; + FILE *stddebug = NULL; + char *output_filename; + STREAM_LIST *stream_list; + STREAM_LIST *block; + char c; + int ret = MMS_RET_SUCCESS; + int quiet = 0; + int trick = MMS_TRICK_DISABLED; + int retry = 0; + int test = 0; + int delay = 0; + time_t end = 0; + float speed = 0.0f; + struct timeval speed_last_update; + struct timeval now; + float elapsed_time; + ssize_t len_written; + uint64_t total_len_written = 0; + uint64_t old_total_len_written = 0; + + if ( ( stream_list = (STREAM_LIST *) malloc(sizeof(STREAM_LIST)) ) == NULL ) + { + error ( "main", "early initialization failed" ); + return 1; + } + + stream_list->next = NULL; + stream_list->stream = NULL; + stream_list->output = NULL; + + block = stream_list; + +#ifdef HAVE_GETOPT_LONG + while ( ( c = getopt_long(argc, argv, options, long_options, NULL) ) != -1 ) +#elif defined(SOLARIS) || defined(sun) + /* Implementation of getopt in Solaris is a bit strange, it returns -1 even if there are still args to parse... */ + while ( optind < argc ) +#else + while ( ( c = getopt(argc, argv, options) ) != -1 ) +#endif + { +#if defined(SOLARIS) || defined(sun) + c = getopt ( argc, argv, options ); +#endif + switch (c) + { + case 'h': + { + fprintf ( stderr, "\n" ); + usage(); + return 0; + } + + case 'v': + { + fprintf ( stderr, "%s version %s\n", PROGRAM_SHORT_NAME, PROGRAM_VERSION); + return 0; + } + + case 'a': + { + fprintf ( stderr, "\n" ); + fprintf ( stderr, "%s (%s) version %s\n\n", PROGRAM_SHORT_NAME, PROGRAM_FULL_NAME, PROGRAM_VERSION); + fprintf ( stderr, "Written by %s.\n", PROGRAM_AUTHORS ); + fprintf ( stderr, "With a lot of help from %s.\n\n", PROGRAM_HELPERS ); + fprintf ( stderr, "This program is free software; you can redistribute it and/or\nmodify it under the terms " \ + "of the GNU General Public License\nas published by the Free Software Foundation; either version 2" \ + ",\nor (at your option) any later version.\n\n" ); + return 0; + } + + case 't': + { + test = 1; + break; + } + + case 'q': + { + quiet += 1; + break; + } + + case 'k': + { + trick = MMS_TRICK_ENABLED; + break; + } + + case 'o': + { + if ( optarg != NULL ) + { + if ( block->stream != NULL ) + { + if ( ( block->next = malloc ( sizeof(STREAM_LIST) ) ) == NULL ) + { + error ( "main", "early initialization failed" ); + ret = MMS_RET_ERROR; + goto clean; + } + + block = block->next; + block->next = NULL; + block->stream = NULL; + } + + if ( block->output != NULL ) + free ( block->output ); + + block->output = (char *) strdup ( optarg ); + } + + break; + } + + case 'g': + { + if ( optarg != NULL ) + { + if ( stddebug != NULL ) + fclose ( stddebug ); + + if ( ( stddebug = fopen ( optarg, "w" ) ) == NULL ) + { + if ( quiet < 2 ) +#ifdef HAVE_VSNPRINTF + warning ( NULL, "unable to write debug info in \'%s\'", optarg ); +#else + warning ( NULL, "unable to write debug info in specified file" ); +#endif + } + else + { + fprintf ( stddebug, "%s (%s) version %s\n\n", PROGRAM_SHORT_NAME, PROGRAM_FULL_NAME, PROGRAM_VERSION); + fprintf ( stddebug, "--> debug log begins now\n" ); + } + } + + break; + } + + case 'd': + { + if ( optarg != NULL ) + { + delay = atoi( optarg ); + + if ( delay < 0 ) + delay = 0; + } + + break; + } + + +#if defined(SOLARIS) || defined(sun) + case -1: + { + if ( optind >= argc ) + break; +#else + case 1: + { + if ( optarg != NULL ) + { +#endif + if ( block->stream != NULL ) + { + if ( ( block->next = malloc ( sizeof(STREAM_LIST) ) ) == NULL ) + { + error ( "main", "early initialization failed" ); + ret = MMS_RET_ERROR; + goto clean; + } + + block = block->next; + block->next = NULL; + block->output = NULL; + } +#if defined(SOLARIS) || defined(sun) + /* optind is not incremented when meeting something else than an option, so we do that... */ + block->stream = (char *) strdup ( argv[optind++] ); +#else + block->stream = (char *) strdup ( optarg ); + } +#endif + break; + } + } + } + + if ( stream_list->stream == NULL ) + { + usage ( ); + ret = MMS_RET_ERROR; + goto clean; + } + + if ( !quiet ) + { + fprintf ( stderr, "\n" ); + fprintf ( stderr, "%s (%s) version %s\n\n", PROGRAM_SHORT_NAME, PROGRAM_FULL_NAME, PROGRAM_VERSION ); + } + + for ( block=stream_list; block!=NULL; block=(STREAM_LIST*)block->next ) + { + if ( block->stream == NULL ) + { + if ( quiet < 2 ) + { + if ( block->output == NULL ) +#ifdef HAVE_VSNPRINTF + error ( "main", "invalid invocation of %s", PROGRAM_SHORT_NAME ); +#else + error ( "main", "invalid invocation of mmsrip" ); +#endif + else +#ifdef HAVE_VSNPRINTF + error ( "main", "output to \'%s\' is not attached to any stream", block->output ); +#else + error ( "main", "one of the -o output is not attached to any stream" ); +#endif + } + + ret = MMS_RET_ERROR; + goto clean; + } + + if ( block->output == NULL ) + { + char *tmp = strchr ( block->stream, '/' ); + char *interro_ptr = strchr ( block->stream, '?' ); + + if ( interro_ptr == NULL ) + output_filename = strrchr ( block->stream, '/' ); + else + { + do /* we look for the last '/' before the first '?' */ + { + output_filename = tmp; + tmp = strchr ( tmp+1, '/' ); + } + while ( ( tmp < interro_ptr ) && ( tmp != NULL ) ); + } + + if ( output_filename == NULL ) + { + if ( quiet < 2 ) +#ifdef HAVE_VSNPRINTF + error ( "main", "malformed url: \'%s\'", block->stream ); +#else + error ( "main", "malformed url" ); +#endif + + ret = MMS_RET_ERROR; + continue; + } + + ++output_filename; + + if ( strlen ( output_filename ) == 0 ) + { + if ( quiet < 2 ) +#ifdef HAVE_VSNPRINTF + error ( "main", "malformed url: \'%s\'", block->stream ); +#else + error ( "main", "malformed url" ); +#endif + + ret = MMS_RET_ERROR; + continue; + } + + block->output = (char *) strdup ( output_filename ); + + /* we clean filenames that look like 'stream.asf?digest=7Q2bjXo&provider=lala' */ + if ( ( interro_ptr = strchr(block->output,'?') ) != NULL ) + *interro_ptr = '\0'; + } + } + + if ( ret != MMS_RET_SUCCESS ) + goto clean; + + if ( delay != 0 ) + end = time(NULL) + delay; + + for ( block=stream_list; block!=NULL; block=(STREAM_LIST*)(retry?block:block->next) ) + { + output_filename = block->output; + retry = 0; + + if ( !test ) + { + if ( ( f = fopen ( output_filename, "w" ) ) == NULL ) + { + if ( quiet < 2 ) +#ifdef HAVE_VSNPRINTF + error ( "main", "unable to write in \'%s\'", output_filename ); +#else + error ( "main", "unable to write in output file" ); +#endif + + ret = MMS_RET_ERROR; + continue; + } + } + else + f = NULL; + + if ( ( mms = mms_create ( block->stream, f, stddebug, trick, test?1:(quiet>>1) ) ) == NULL ) + { + if ( quiet < 2 ) + error ( "main", "unable to create mms struct" ); + + if ( !test ) + { + fclose ( f ); + remove ( output_filename ); + } + + ret = MMS_RET_ERROR; + continue; + } + + if ( !quiet ) + fprintf ( stderr, "Connecting ...\r" ); + + if ( mms_connect ( mms ) != MMS_RET_SUCCESS ) + { + if ( quiet < 2 ) + error ( "main", "unable to connect" ); + + mms_destroy ( mms ); + + if ( !test ) + { + fclose ( f ); + remove ( output_filename ); + } + + ret = MMS_RET_ERROR; + continue; + } + + if ( !quiet ) + fprintf ( stderr, "Handshaking ...\r" ); + + if ( mms_handshake ( mms ) != MMS_RET_SUCCESS ) + { + if ( ( quiet < 2 ) && ( !test ) ) + error ( "main", "unable to handshake" ); + + mms_disconnect ( mms ); + mms_destroy ( mms ); + + if ( !test ) + { + fclose ( f ); + remove ( output_filename ); + } + + if ( !quiet ) + { + if ( !test ) + fprintf ( stderr, "Stream \'%s\' is not good.\n\n", block->stream ); + else + fprintf ( stderr, "Stream \'%s\' is not good.\n", block->stream ); + } + + ret = MMS_RET_ERROR; + continue; + } + + if ( test ) + { + if ( !quiet ) + fprintf ( stderr, "Stream \'%s\' is available.\n", block->stream ); + + mms_disconnect ( mms ); + mms_destroy ( mms ); + continue; + } + + if ( !quiet ) + fprintf ( stderr, "Getting header ...\r" ); + + if ( ( len_written = mms_write_stream_header ( mms ) ) == MMS_RET_ERROR ) + { + if ( quiet < 2 ) + error ( "main", "unable to write stream header" ); + + mms_disconnect ( mms ); + mms_destroy ( mms ); + fclose ( f ); + remove ( output_filename ); + ret = MMS_RET_ERROR; + continue; + } + + total_len_written = len_written; + + if ( !quiet ) + fprintf ( stderr, "Rip is about to start ...\r" ); + + if ( mms_begin_rip ( mms ) != MMS_RET_SUCCESS ) + { + if ( quiet < 2 ) + error ( "main", "unable to begin the rip" ); + + mms_disconnect ( mms ); + mms_destroy ( mms ); + fclose ( f ); + remove ( output_filename ); + ret = -1; + continue; + } + + if ( mms->is_live == MMS_LIVE ) + { + if ( !quiet ) + { + + fprintf ( stderr, " \r" ); + fprintf ( stderr, "%s: %d bytes written (--.- kbps)\r", output_filename, (ssize_t)total_len_written ); + gettimeofday ( &speed_last_update, NULL ); + } + + while ( 1 ) + { + len_written = mms_write_stream_data ( mms ); + + if ( len_written == 0 ) + break; + else if ( len_written == MMS_RET_ERROR ) + { + if ( quiet < 2 ) + error ( "main", "unable to write stream data" ); + + ret = MMS_RET_ERROR; + break; + } + else if ( len_written == MMS_RET_NO_AUTH ) + { + if ( trick == MMS_TRICK_DISABLED ) + { + /* we retry with the trick enabled */ + if ( quiet < 2 ) + fprintf ( stderr, "\r*** retrying with the anti-recalcitrant servers trick enabled ***\n" ); + + trick = MMS_TRICK_ENABLED; + retry = 1; + } + else + { + /* it's definitely not working */ + if ( quiet < 2 ) + error ( "main", "unable to write stream data" ); + + ret = MMS_RET_NO_AUTH; + } + + break; + } + + total_len_written += len_written; + + if ( !quiet ) + { + gettimeofday ( &now, NULL ); + + if ( now.tv_sec > speed_last_update.tv_sec ) + { + elapsed_time = (now.tv_sec - speed_last_update.tv_sec) + ((now.tv_usec - speed_last_update.tv_usec) / 1000000.0f); + + if ( elapsed_time >= 1.0f ) + { + speed = ( ( ((float) (total_len_written-old_total_len_written)) / elapsed_time) / 1024.0f ); + old_total_len_written = total_len_written; + gettimeofday ( &speed_last_update, NULL ); + } + } + + if ( speed > 0.0f ) + { + if ( (speed / 1024.0f) >= 1.0f ) + fprintf ( stderr, "%s: %d bytes written (%.1f mbps) \r", output_filename, (ssize_t)total_len_written, speed/1024.0f ); + else + fprintf ( stderr, "%s: %d bytes written (%.1f kbps) \r", output_filename, (ssize_t)total_len_written, speed ); + } + else + fprintf ( stderr, "%s: %d bytes written (--.- kbps) \r", output_filename, (ssize_t)total_len_written ); + } + + fflush ( f ); + + if ( delay != 0 ) + { + if ( end <= time(NULL) ) + { + delay = -1; + break; + } + } + } + + /* for a live, we should rewrite the header */ + + if ( ( ret == MMS_RET_SUCCESS ) && ( !quiet ) && ( !retry ) ) + fprintf ( stderr, "%s: %d bytes written \r", output_filename, (ssize_t)total_len_written ); + } + else + { + register const double coef = 100.0 / (double) mms->expected_file_size; + + if ( !quiet ) + { + fprintf ( stderr, " \r" ); + fprintf ( stderr, "%s: %.2f%% (--.- kbps)\r", output_filename, (double) total_len_written * coef ); + gettimeofday ( &speed_last_update, NULL ); + } + + while ( 1 ) + { + len_written = mms_write_stream_data ( mms ); + + if ( len_written == 0 ) + break; + else if ( len_written == MMS_RET_ERROR ) + { + if ( quiet < 2 ) + error ( "main", "unable to write stream data" ); + + ret = MMS_RET_ERROR; + break; + } + else if ( len_written == MMS_RET_NO_AUTH ) + { + if ( trick == MMS_TRICK_DISABLED ) + { + /* we retry with the trick enabled */ + if ( quiet < 2 ) + fprintf ( stderr, "\r*** retrying with the anti-recalcitrant servers trick enabled ***\n" ); + + trick = MMS_TRICK_ENABLED; + retry = 1; + } + else + { + /* it's definitely not working */ + if ( quiet < 2 ) + error ( "main", "unable to write stream data" ); + + ret = MMS_RET_NO_AUTH; + } + + break; + } + + total_len_written += len_written; + + if ( !quiet ) + { + gettimeofday ( &now, NULL ); + + if ( now.tv_sec > speed_last_update.tv_sec ) + { + elapsed_time = (now.tv_sec - speed_last_update.tv_sec) + ((now.tv_usec - speed_last_update.tv_usec) / 1000000.0f); + + if ( elapsed_time >= 1.0f ) + { + speed = ( ( ((float) (total_len_written-old_total_len_written)) / elapsed_time) / 1024.0f ); + old_total_len_written = total_len_written; + gettimeofday ( &speed_last_update, NULL ); + } + } + + if ( speed > 0.0f ) + { + if ( (speed / 1024.0f) >= 1.0f ) + fprintf ( stderr, "%s: %.2f%% (%.1f mbps) \r", output_filename, (double) total_len_written * coef, speed/1024.0f ); + else + fprintf ( stderr, "%s: %.2f%% (%.1f kbps) \r", output_filename, (double) total_len_written * coef, speed ); + } + else + fprintf ( stderr, "%s: %.2f%% (--.- kbps) \r", output_filename, (double) total_len_written * coef ); + } + + fflush ( f ); + + if ( delay != 0 ) + { + if ( end <= time(NULL) ) + { + delay = -1; + break; + } + } + } + + if ( ( ret == MMS_RET_SUCCESS ) && ( !quiet ) && ( delay != -1 ) && ( !retry ) ) + fprintf ( stderr, "%s: 100.00%% \r", output_filename ); + } + + mms_disconnect ( mms ); + mms_destroy ( mms ); + + fclose ( f ); + + if ( delay == -1 ) + break; + } + + clean: + if ( !quiet ) + fprintf ( stderr, "\n" ); + + if ( stddebug != NULL ) + { + fprintf ( stddebug, "\n\n--> debug log ends now\n" ); + fclose ( stddebug ); + } + + for ( block=stream_list; block!=NULL; ) + { + STREAM_LIST *old; + old = block; + block = (STREAM_LIST *) block->next; + + if ( old->stream != NULL ) + free ( old->stream ); + + if ( old->output != NULL ) + free ( old->output ); + + free ( old ); + } + + return (ret<0) ? (ret*-1) : ret; +} diff --git a/src/mmsrip/mms.c b/src/mmsrip/mms.c new file mode 100644 index 0000000..022f7ae --- /dev/null +++ b/src/mmsrip/mms.c @@ -0,0 +1,1239 @@ +/* + * $RCSfile: mms.c,v $ + * $Date: 2006/01/23 20:30:43 $ - $Revision: 1.33 $ + * + * This file is distributed as a part of MMSRIP ( MMS Ripper ). + * Copyright (c) 2005-2006 Nicolas BENOIT + * + * It is highly based on the work of SDP Multimedia and Major MMS. + * They deserve all the credits for it. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + + +#define _GNU_SOURCE + +#ifdef _WIN32 +#include <winsock2.h> +#else +#include <unistd.h> +#include <strings.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <netdb.h> +#endif + +#include <stdio.h> +#include <string.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <stdlib.h> +#include <errno.h> + +#include "common.h" +#include "mms.h" +#include "error.h" + + +/* + * Put 32 bits inside a command buffer. + */ +static void +mms_put_32 ( MMS_PACKET *pak, + uint32_t value ) +{ + pak->buf[pak->num_bytes ] = value % 256; + value = value >> 8; + pak->buf[pak->num_bytes+1] = value % 256 ; + value = value >> 8; + pak->buf[pak->num_bytes+2] = value % 256 ; + value = value >> 8; + pak->buf[pak->num_bytes+3] = value % 256 ; + + pak->num_bytes += 4; + return; +} + + +/* + * Returns the 32 bits located at specified offset. + */ +static uint32_t +mms_get_32 ( const unsigned char *buf, + const int offset ) +{ + return ( (((uint32_t)buf[offset+0]) << 0) | + (((uint32_t)buf[offset+1]) << 8) | + (((uint32_t)buf[offset+2]) << 16) | + (((uint32_t)buf[offset+3]) << 24) ); +} + + +/* + * Returns the 64 bits located at specified offset. + */ +static uint64_t +mms_get_64 ( const unsigned char *buf, + const int offset ) +{ + return ( (((uint64_t)buf[offset+0]) << 0) | + (((uint64_t)buf[offset+1]) << 8) | + (((uint64_t)buf[offset+2]) << 16) | + (((uint64_t)buf[offset+3]) << 24) | + (((uint64_t)buf[offset+4]) << 32) | + (((uint64_t)buf[offset+5]) << 40) | + (((uint64_t)buf[offset+6]) << 48) | + (((uint64_t)buf[offset+7]) << 56) ); +} + + +/* + * Converts a string into a UTF-16 string. + */ +static void +mms_string_utf16 ( uint8_t *dest, + unsigned char *src, + const ssize_t len ) +{ + ssize_t i; + memset ( dest, 0, len ); + + for ( i=0; i<len; ++i ) + { + dest[i*2] = src[i]; + dest[i*2+1] = 0; + } + + dest[i*2] = 0; + dest[i*2+1] = 0; + return; +} + + +/* + * Checks a URL for a correct MMS protocol. + * Returns the host name index if the protocol is valid, else it returns MMS_RET_ERROR. + */ +static int +mms_check_protocol ( const char *url ) +{ + static const char *proto[] = { "mms://", "mmsu://", "mmst://", "mmsh://", NULL }; + static const int proto_len[] = { 6, 7, 7, 7, 0 }; + register const char *p = proto[0]; + int i = 0; + + while ( p != NULL ) + { + if ( strncmp ( url, p, proto_len[i] ) == 0 ) + return proto_len[i]; + + p = proto[++i]; + } + + return MMS_RET_ERROR; +} + + +/* + * Prints out a command packet (debug purpose). + */ +static void +mms_print_packet ( FILE *stddebug, + const MMS_PACKET *pak, + const ssize_t length, + const int orig ) +{ + ssize_t i; + unsigned char c; + + fprintf ( stddebug, "\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n" ); + + if ( orig == MMS_SERVER ) + fprintf ( stddebug, " command from server (%d bytes)\n", length ); + else + fprintf ( stddebug, " command from client (%d bytes)\n", length ); + + fprintf ( stddebug, " start sequence: 0x%08x\n", mms_get_32(pak->buf, 0) ); + fprintf ( stddebug, " command id: 0x%08x\n", mms_get_32(pak->buf, 4) ); + fprintf ( stddebug, " length: 0x%8x \n", mms_get_32(pak->buf, 8) ); + fprintf ( stddebug, " len8: 0x%8x \n", mms_get_32(pak->buf, 16) ); + fprintf ( stddebug, " sequence #: 0x%08x\n", mms_get_32(pak->buf, 20) ); + fprintf ( stddebug, " len8 (II): 0x%8x \n", mms_get_32(pak->buf, 32) ); + fprintf ( stddebug, " dir | comm: 0x%08x\n", mms_get_32(pak->buf, 36) ); + fprintf ( stddebug, " switches: 0x%08x\n", mms_get_32(pak->buf, 40) ); + + fprintf ( stddebug, "\nascii contents:\n" ); + + for ( i=48; i<length; i+=2 ) + { + c = pak->buf[i]; + + if ( ( c>=32 ) && ( c<=128 ) ) + fprintf ( stddebug, "%c", c ); + else + fprintf ( stddebug, "." ); + } + + fprintf ( stddebug, "\n\npackage hexdump:\n " ); + + i = 0; + + while ( 1 ) + { + c = pak->buf[i]; + fprintf ( stddebug, "%02x", c ); + + ++i; + + if ( i < length ) + { + if ( (i % 16) == 0 ) + fprintf ( stddebug, "\n" ); + + if ( (i % 2) == 0 ) + fprintf ( stddebug, " " ); + } + else + break; + } + + fprintf ( stddebug, "\n\n" ); + fflush ( stddebug ); + return; +} + + +/* + * Sends a command packet to the server. + * Returns MMS_RET_SUCCESS if success, MMS_RET_ERROR either. + */ +static int +mms_send_packet ( MMS *mms, + const int command, + const uint32_t switches, + const uint32_t extra, + const ssize_t length, + const uint8_t *data ) +{ + MMS_PACKET pak; + ssize_t written_len; + + const ssize_t len8 = ( length + (length%8) ) / 8; + + pak.num_bytes = 0; + + mms_put_32 ( &pak, 0x00000001 ); /* start sequence */ + mms_put_32 ( &pak, 0xB00BFACE ); /* #-)) */ + mms_put_32 ( &pak, length + 32 ); + mms_put_32 ( &pak, 0x20534d4d ); /* protocol type "MMS " */ + mms_put_32 ( &pak, len8 + 4 ); + mms_put_32 ( &pak, mms->seq_num++ ); + mms_put_32 ( &pak, 0x0 ); /* unknown */ + mms_put_32 ( &pak, 0x0 ); + mms_put_32 ( &pak, 2+len8 ); + mms_put_32 ( &pak, 0x00030000 | command ); /* dir | command */ + mms_put_32 ( &pak, switches ); + mms_put_32 ( &pak, extra ); + + memcpy ( &pak.buf[48], data, length ); + + written_len = write ( mms->socket, pak.buf, length+48 ); + + if ( written_len == -1 ) + { + if ( !mms->quiet ) +#ifdef HAVE_VSNPRINTF + error ( "mms_send_packet", "write() said: %s", strerror(errno) ); +#else + error ( "mms_send_packet", "write() failed" ); +#endif + + return MMS_RET_ERROR; + } + else if ( written_len != (length+48) ) + { + if ( !mms->quiet ) + error ( "mms_send_packet", "did not write enough bytes" ); + + return MMS_RET_ERROR; + } + + if ( mms->stddebug != NULL ) + mms_print_packet ( mms->stddebug, &pak, length+48, MMS_CLIENT ); + + return MMS_RET_SUCCESS; +} + + +/* + * Reads some bytes from the socket. + * This is blocking until we don't have the required amount of bytes. + * Returns MMS_RET_SUCCESS if success, MMS_RET_ERROR either. + */ +static int +mms_recv_packet ( const int s, + MMS_PACKET *pak, + size_t count, + const int quiet ) +{ + ssize_t read_len, total; + total = 0; + + if ( MMS_BUF_SIZE < count ) + { + if ( !quiet ) + warning ( "mms_recv_packet", "caller is too greedy" ); + + count = MMS_BUF_SIZE; + } + + while ( total < count ) + { + read_len = read ( s, &pak->buf[total], count-total ); + + if ( read_len == -1 ) + { + if ( !quiet ) +#ifdef HAVE_VSNPRINTF + error ( "mms_recv_packet", "read() said: %s", strerror(errno) ); +#else + error ( "mms_recv_packet", "read() failed" ); +#endif + + return MMS_RET_ERROR; + } + + total += read_len; + } + + return MMS_RET_SUCCESS; +} + + +/* + * Gets server's command. + * The first 8 bytes can be skipped setting offset to 8. + * It writes the packet length in the provided storage. + * Returns command ID, MMS_CMD_INVALID if error. + */ +static int +mms_recv_cmd_packet ( const int s, + MMS_PACKET *pak, + ssize_t *packet_len, + const int offset, + const int quiet ) +{ + MMS_PACKET tmp; + + if ( ( offset > 8 ) || ( offset < 0 ) ) + { + if ( !quiet ) + error ( "mms_recv_cmd_packet", "provided offset is invalid" ); + + return MMS_RET_ERROR; + } + + if ( mms_recv_packet ( s, &tmp, 12-offset, quiet ) != 0 ) + { + if ( !quiet ) + error ( "mms_recv_cmd_packet", "unable to get packet header" ); + + return MMS_RET_ERROR; + } + + memcpy ( &pak->buf[offset], tmp.buf, 12-offset ); + + if ( offset == 0 ) + { + if ( mms_get_32(pak->buf, 4) != 0xb00bface ) /* probably a coincidence... */ + { + if ( !quiet ) + error ( "mms_recv_cmd_packet", "answer should have been a cmd packet" ); + + return MMS_RET_ERROR; + } + } + + memcpy ( packet_len, &pak->buf[8], 4 ); + *packet_len = mms_get_32 ( (unsigned char *) packet_len, 0 ) + 4; + + if ( ( *packet_len + 12 ) > MMS_BUF_SIZE ) + { + if ( !quiet ) + error ( "mms_recv_cmd_packet", "incoming packet is too big for me" ); + + return MMS_RET_ERROR; + } + + if ( mms_recv_packet ( s, &tmp, *packet_len, quiet ) != 0 ) + { + if ( !quiet ) + error ( "mms_recv_cmd_packet", "unable to get packet body" ); + + return MMS_RET_ERROR; + } + + memcpy ( &pak->buf[12], tmp.buf, *packet_len ); + + return ( mms_get_32(pak->buf, 36) & 0xFFFF ); +} + + +/* + * Gets the header of the ASF/WMV stream/file. + * Returns the amount of bytes received, MMS_RET_ERROR if we encounter an error. + */ +static ssize_t +mms_recv_header_packet ( MMS *mms, + MMS_PACKET *pak ) +{ + MMS_PACKET pre_header; + MMS_PACKET tmp; + ssize_t header_len; + ssize_t packet_len; + + header_len = 0; + + while ( 1 ) + { + if ( mms_recv_packet ( mms->socket, &pre_header, 8, mms->quiet ) != 0 ) + { + if ( !mms->quiet ) + error ( "mms_recv_header_packet", "unable to get pre-header" ); + + return MMS_RET_ERROR; + } + + if ( pre_header.buf[4] == 0x02 ) + { + if ( mms->stddebug != NULL ) + { + fprintf ( mms->stddebug, "\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n" ); + fprintf ( mms->stddebug, " getting header packet from server\n\n" ); + } + + if ( mms->stddebug != NULL ) + { + static int i; /* static, so we don't recreate it for each call */ + + for ( i=0; i<8; ++i ) + fprintf ( mms->stddebug, "pre_header.buf[%d] = 0x%02x (%d)\n", i, pre_header.buf[i], pre_header.buf[i] ); + } + + packet_len = ( pre_header.buf[7] << 8 | pre_header.buf[6] ) - 8; + + if ( mms->stddebug != NULL ) + fprintf ( mms->stddebug, "\nASF Header Packet (%d bytes)\n", packet_len ); + + if ( mms_recv_packet ( mms->socket, &tmp, packet_len, mms->quiet ) != 0 ) + { + if ( !mms->quiet ) + error ( "mms_recv_header_packet", "unable to get header" ); + + return MMS_RET_ERROR; + } + + if ( ( header_len + packet_len ) > MMS_BUF_SIZE ) + { + if ( !mms->quiet ) + error ( "mms_recv_header_packet", "total header length is too big for me" ); + + return MMS_RET_ERROR; + } + + memcpy ( &pak->buf[header_len], tmp.buf, packet_len ); + + header_len += packet_len; + + if ( mms->stddebug != NULL ) + fprintf ( mms->stddebug, "\n" ); + + /* test if end of header is reached */ + if ( ( pak->buf[header_len-1] == 1) && + ( pak->buf[header_len-2] == 1 ) ) + return header_len; + } + else /* we might receive an ack or an error */ + { + int command = mms_recv_cmd_packet ( mms->socket, &tmp, &packet_len, 8, mms->quiet ); + + if ( command == -1 ) + { + if ( !mms->quiet ) + error ( "mms_recv_header_packet", "unable to get cmd packet" ); + + return MMS_RET_ERROR; + } + + memcpy ( tmp.buf, pre_header.buf, 8 ); + + if ( mms->stddebug != NULL ) + mms_print_packet ( mms->stddebug, &tmp, packet_len, MMS_SERVER ); + + if ( command == MMS_CMD_PING ) + mms_send_packet ( mms, MMS_CMD_PONG, 0, 0, 0, tmp.buf ); + else if ( command != MMS_CMD_HEADER_DATA ) + { + if ( !mms->quiet ) +#ifdef HAVE_VSNPRINTF + error ( "mms_recv_header_packet", "unknown command 0x%02x\n", command ); +#else + error ( "mms_recv_header_packet", "unknown command\n" ); +#endif + + return MMS_RET_ERROR; + } + } + } +} + + +/* + * Interprates the header (media packet size, etc.). + * Returns the size of a media packet (should be != 0 ). + */ +static ssize_t +mms_interp_header ( MMS *mms, + const uint8_t *header, + const ssize_t header_len ) +{ + int i; + ssize_t packet_length = 0; + + mms->expected_file_size = header_len; + + if ( mms->stddebug != NULL ) + { + fprintf ( mms->stddebug, "\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n" ); + fprintf ( mms->stddebug, " header interpretation\n\n" ); + } + + /* parse header */ + mms->num_stream_ids = 0; + i = 30; + + while ( i < header_len ) + { + uint64_t guid_1, guid_2, length; + + if ( mms->stddebug != NULL ) + { + /* this is MS GUID format */ + fprintf ( mms->stddebug, "{%02X%02X%02X%02X-%02X%02X-%02X%02X-%02X%02X-%02X%02X%02X%02X%02X%02X}\n", + header[i+3], header[i+2], header[i+1], header[i+0], + header[i+5], header[i+4], header[i+7], header[i+6], + header[i+8], header[i+9], header[i+10], header[i+11], + header[i+12], header[i+13], header[i+14], header[i+15] ); + } + + guid_2 = mms_get_64 ( header, i ); + i += 8; + + guid_1 = mms_get_64 ( header, i ); + i += 8; + + length = mms_get_64 ( header, i ); + i += 8; + + if ( ( guid_1 == 0x6553200cc000e48eULL ) && ( guid_2 == 0x11cfa9478cabdca1ULL ) ) + { + packet_length = mms_get_32 ( header, i+92-24 ); + + if ( packet_length != mms_get_32(header, i+72) ) + warning ( NULL, "size of media packets is not constant, it should not happen" ); + + if ( mms->stddebug != NULL ) + { + fprintf ( mms->stddebug, "File Properties Object (%lld bytes)\n", length ); + fprintf ( mms->stddebug, " -> broadcast bit: %d\n", mms_get_32(header,i+64)&1 ); + fprintf ( mms->stddebug, " -> min packet length: %d\n", packet_length ); + fprintf ( mms->stddebug, " -> max packet length: %d\n", mms_get_32(header, i+72) ); + fprintf ( mms->stddebug, " -> number of media packets: %d\n\n", mms_get_32(header, i+32) ); + } + + if ( mms_get_32(header,i+64)&1 ) + { + if ( ( !mms->quiet ) && ( !mms->is_live ) ) + warning ( NULL, "stream seems to be live, an error may occur" ); + + mms->is_live = MMS_LIVE; + } + } + else if ( ( guid_1 == 0x6553200cc000e68eULL ) && ( guid_2 == 0x11cfa9b7b7dc0791ULL ) ) + { + int stream_id = header[i+48] | header[i+49] << 8; + + if ( mms->num_stream_ids < 20 ) + { + mms->stream_ids[mms->num_stream_ids] = stream_id; + ++mms->num_stream_ids; + } + else + { + if ( !mms->quiet ) +#ifdef HAVE_VSNPRINTF + warning ( "mms_interp_header", "too many streams, stream \'%d\' skipped", stream_id ); +#else + warning ( "mms_interp_header", "too many streams, skipping.." ); +#endif + } + + if ( mms->stddebug != NULL ) + { + fprintf ( mms->stddebug, "Stream Object (%lld bytes)\n", length ); + fprintf ( mms->stddebug, " -> stream id: %d\n\n", stream_id ); + } + } + else if ( ( guid_1 == 0x6cce6200aa00d9a6ULL ) && ( guid_2 == 0x11cf668e75b22636ULL ) ) + { + mms->expected_file_size += ( length - 50 ); /* valid values are at least 50 bytes (why?) */ + + if ( mms->stddebug != NULL ) + fprintf ( mms->stddebug, "Data Object (%lld bytes)\n\n", length ); + } + else if ( mms->stddebug != NULL ) + { + if ( ( guid_1 == 0x6cce6200aa00d9a6ULL ) && ( guid_2 == 0x11cf668e75b22630ULL ) ) + fprintf ( mms->stddebug, "Header Object (%lld bytes)\n\n", length ); + else if ( ( guid_1 == 0xcb4903c9a000f489ULL ) && ( guid_2 == 0x11cfe5b133000890ULL) ) + fprintf ( mms->stddebug, "Simple Index Object (%lld bytes)\n\n", length ); + else if ( ( guid_1 == 0xbe4903c9a0003490ULL ) && ( guid_2 == 0x11d135dad6e229d3ULL) ) + fprintf ( mms->stddebug, "Index Object (%lld bytes)\n\n", length ); + else if ( ( guid_1 == 0x6553200cc000e38eULL ) && ( guid_2 == 0x11cfa92e5fbf03b5ULL) ) + fprintf ( mms->stddebug, "Header Extension Object (%lld bytes)\n\n", length ); + else if ( ( guid_1 == 0x6cce6200aa00d9a6ULL ) && ( guid_2 == 0x11cf668e75b22633ULL) ) + fprintf ( mms->stddebug, "Content Description Object (%lld bytes)\n\n", length ); + else if ( ( guid_1 == 0x50a85ec9a000f097ULL ) && ( guid_2 == 0x11d2e307d2d0a440ULL) ) + fprintf ( mms->stddebug, "Extended Content Description Object (%lld bytes)\n\n", length ); + else if ( ( guid_1 == 0xf64803c9a000a4a3ULL ) && ( guid_2 == 0x11d0311d86d15240ULL) ) + fprintf ( mms->stddebug, "Codec List Object (%lld bytes)\n\n", length ); + else if ( ( guid_1 == 0xb2a2c9976000828dULL ) && ( guid_2 == 0x11d1468d7bf875ceULL) ) + fprintf ( mms->stddebug, "Stream Bitrate Properties Object (%lld bytes)\n\n", length ); + else + fprintf ( mms->stddebug, "Unknown Object (%lld bytes)\n\n", length ); + } + + i += length-24; + } + + return packet_length; +} + + +/* + * Gets a media packet. + * Returns the amount of bytes in the media packet (0 if EOF), MMS_RET_ACKED if ACKed, + * MMS_RET_NO_AUTH if authorization failed, MMS_RET_ERROR either. + */ +static ssize_t +mms_recv_media_packet ( MMS *mms, + MMS_PACKET *pak ) +{ + MMS_PACKET pre_header; + ssize_t packet_len; + + if ( mms_recv_packet ( mms->socket, &pre_header, 8, mms->quiet ) != 0 ) + { + if ( !mms->quiet ) + error ( "mms_recv_media_packet", "unable to get pre-header" ); + + return MMS_RET_ERROR; + } + + if ( pre_header.buf[4] == 0x04 ) + { + if ( mms->stddebug != NULL ) + { + fprintf ( mms->stddebug, "\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n" ); + fprintf ( mms->stddebug, " getting media packet from server\n\n" ); + } + + if ( mms->stddebug != NULL ) + { + static int i; /* static, so we don't recreate it for each call */ + + for ( i=0; i<8; i++ ) + fprintf ( mms->stddebug, "pre_header[%d] = 0x%02x (%d)\n", i, pre_header.buf[i], pre_header.buf[i] ); + } + + packet_len = (pre_header.buf[7] << 8 | pre_header.buf[6]) - 8; + + if ( mms->stddebug != NULL ) + fprintf ( mms->stddebug, "\nASF Media Packet (%d bytes)\n", packet_len ); + + memset ( pak->buf, 0, mms->media_packet_len ); + + if ( mms_recv_packet (mms->socket, pak, packet_len, mms->quiet) != 0 ) + { + if ( !mms->quiet ) + error ( "mms_recv_media_packet", "unable to get media packet" ); + + return MMS_RET_ERROR; + } + } + else + { + int command = mms_recv_cmd_packet ( mms->socket, pak, &packet_len, 8, mms->quiet ); + + if ( command == MMS_RET_ERROR ) + { + if ( !mms->quiet ) + error ( "mms_recv_media_packet", "unable to get cmd packet" ); + + return MMS_RET_ERROR; + } + + memcpy ( pak->buf, pre_header.buf, 8 ); + + if ( mms->stddebug != NULL ) + mms_print_packet ( mms->stddebug, pak, packet_len, MMS_SERVER ); + + if ( command == MMS_CMD_PING ) + { + mms_send_packet ( mms, MMS_CMD_PONG, 0, 0, 0, pak->buf ); + return MMS_RET_ACKED; + } + else if ( command == MMS_CMD_END_OF_STREAM ) + return 0; + else if ( command == MMS_CMD_STREAM_SELECT_ACK ) /* it happens sometimes */ + return MMS_RET_ACKED; + else if ( command == MMS_CMD_READY_TO_STREAM ) /* it always happen before the first media packet */ + { + /* this happens on some server for some unknown reason */ + if ( mms_get_32(pak->buf, 40) == 0x80070005 ) + { + if ( !mms->quiet ) + error ( "mms_recv_media_packet", "streaming denied (read manpage & retry later)" ); + + return MMS_RET_NO_AUTH; + } + + return MMS_RET_ACKED; + } + else + { + if ( !mms->quiet ) +#ifdef HAVE_VSNPRINTF + error ( "mms_recv_media_packet", "unknown command 0x%02x\n", command ); +#else + error ( "mms_recv_media_packet", "unknown command\n" ); +#endif + + return MMS_RET_ERROR; + } + } + + if ( mms->stddebug != NULL ) + fprintf ( mms->stddebug, "\n" ); + + return packet_len; +} + + +/* + * Creates An MMS Struct + * Returns an initialized MMS Struct if successful, NULL if an error occurs. + * url must start with mms:// or mmst:// (mmsu:// and mmsh:// are OK too, but we won't use UDP or do MMS over HTTP) + * trick must be MMS_TRICK_DISABLED or MMS_TRICK_ENABLED + */ +MMS * +mms_create ( const char *url, + FILE *out, + FILE *stddebug, + const int trick, + const int quiet ) +{ + MMS *mms; + register const char *sep; + register const int host_idx = mms_check_protocol ( url ); + register const int url_len = strlen ( url ); + + if ( stddebug != NULL ) + { + fprintf ( stddebug, "\n\n********************************************************************************\n\n" ); + fprintf ( stddebug, "Url -> \'%s\'\n", url ); + } + + if ( host_idx == MMS_RET_ERROR ) + { + if ( !quiet ) + error ( "mms_create", "bad protocol (mms:// was expected)" ); + + return NULL; + } + + sep = strchr ( &url[host_idx], '/' ); + + if ( sep == NULL ) + { + if ( !quiet ) + error ( "mms_create", "url seems to be malformed" ); + + return NULL; + } + + if ( ( mms = ( MMS * ) malloc ( sizeof(MMS) ) ) == NULL ) + { + if ( !quiet ) + error ( "mms_create", "unable to allocate memory" ); + + return NULL; + } + + mms->host = malloc ( ( sep - &url[host_idx] ) + 1 ); + strncpy ( mms->host, &url[host_idx], (sep - &url[host_idx]) ); + mms->host[sep-&url[host_idx]] = '\0'; /* strndup */ + + mms->path = strdup ( sep+1 ); + mms->out = out; + mms->seq_num = 0; + mms->expected_file_size = 0; + mms->is_live = MMS_NO_LIVE; + + /* we try to guess stream type with the filename extension */ + if ( ( sep = strchr(sep,'?') ) != NULL ) + { + if ( ( *(sep-4) == '.' ) && /* this will never segfault because */ + ( *(sep-3) == 'w' ) && /* the url is at least 6 chars long (mms://) */ + ( *(sep-2) == 'm' ) && + ( *(sep-1) == 'v' ) ) + mms->stream_type = MMS_WMV; + else + mms->stream_type = MMS_ASF; + } + else + { + if ( ( url[url_len-4] == '.' ) && + ( url[url_len-3] == 'w' ) && + ( url[url_len-2] == 'm' ) && + ( url[url_len-1] == 'v' ) ) + mms->stream_type = MMS_WMV; + else + mms->stream_type = MMS_ASF; + } + + mms->stddebug = stddebug; + mms->quiet = quiet; + mms->trick = ( ((trick==MMS_TRICK_DISABLED)||(trick==MMS_TRICK_ENABLED)) ? trick : MMS_TRICK_DISABLED ); + + if ( mms->stddebug != NULL ) + { + fprintf ( mms->stddebug, "Host -> \'%s\'\nPath -> \'%s\'\n", mms->host, mms->path ); + fprintf ( mms->stddebug, "Stream type -> %s\n", (mms->stream_type==MMS_WMV)?"MMS_WMV":"MMS_ASF" ); + } + + return mms; +} + + +/* + * Establishes a connection with the MMS server. + * Returns MMS_RET_SUCCESS if success, MMS_RET_ERROR either. + */ +int +mms_connect ( MMS* mms ) +{ + struct sockaddr_in sa; + struct hostent *hp; + + if ( mms == NULL ) + return -1; + + if ( ( hp = gethostbyname(mms->host) ) == NULL ) + { + if ( !mms->quiet ) + error ( "mms_connect", "unable to resolve host name" ); + + return MMS_RET_ERROR; + } + + bcopy ( (char *) hp->h_addr, (char *) &sa.sin_addr, hp->h_length ); + sa.sin_family = hp->h_addrtype; + sa.sin_port = htons ( 1755 ); + + if ( ( mms->socket = socket(hp->h_addrtype, SOCK_STREAM, 0) ) == -1 ) + { + if ( !mms->quiet ) +#ifdef HAVE_VSNPRINTF + error ( "mms_connect", "socket() said: %s", strerror(errno) ); +#else + error ( "mms_connect", "socket() failed" ); +#endif + + return MMS_RET_ERROR; + } + + if ( connect ( mms->socket, (struct sockaddr *) &sa, sizeof sa ) != 0 ) + { + if ( !mms->quiet ) +#ifdef HAVE_VSNPRINTF + error ( "mms_connect", "connect() said: %s", strerror(errno) ); +#else + error ( "mms_connect", "connect failed" ); +#endif + + return MMS_RET_ERROR; + } + + return MMS_RET_SUCCESS; +} + + +/* + * Handshake with the MMS server. + * ( we don't care about server's answers ) + */ +int +mms_handshake ( MMS *mms ) +{ + MMS_PACKET pak; + int cmd = MMS_CMD_PING; + ssize_t packet_len; + char str [ 1024 ]; + uint8_t data [ 2092 ]; + + if ( mms == NULL ) + return MMS_RET_ERROR; + + /* who we are */ + memset ( data, 0, sizeof (data) ); + + if ( mms->trick == MMS_TRICK_DISABLED ) + { +#ifdef HAVE_SNPRINTF + snprintf ( str, sizeof(str), "\034\003NSPlayer/7.0.0.1956; {3300AD50-2C39-46c0-AE0A-60181587CBA}; Host: %s", mms->host ); +#else + if ( (strlen("\034\003NSPlayer/7.0.0.1956; {3300AD50-2C39-46c0-AE0A-60181587CBA}; Host: ")+strlen(mms->host)) < sizeof(str) ) + sprintf ( str, "\034\003NSPlayer/7.0.0.1956; {3300AD50-2C39-46c0-AE0A-60181587CBA}; Host: %s", mms->host ); + else + { + error ( "mms_handshake", "host name is too long" ); + return MMS_RET_ERROR; + } +#endif + } + else + strcpy ( str, "\034\003NSPlayer/4.1.0.3928; {3300AD50-2C39-46c0-AE0A-60181587CBA}" ); + + mms_string_utf16 ( data, (unsigned char *) str, strlen(str)+2 ); + mms_send_packet ( mms, MMS_CMD_HELLO, 0, 0x0004000b, strlen(str)*2 + 6, data ); + + cmd = MMS_CMD_PING; + while ( cmd == MMS_CMD_PING ) + { + cmd = mms_recv_cmd_packet ( mms->socket, &pak, &packet_len, 0, mms->quiet ); + + if ( cmd == MMS_CMD_PING ) + mms_send_packet ( mms, MMS_CMD_PONG, 0, 0, 0, pak.buf ); + } + + if ( cmd == MMS_CMD_INVALID ) + { + if ( !mms->quiet ) + error ( "mms_handshake", "unable to get cmd packet" ); + + return MMS_RET_ERROR; + } + + if ( mms->stddebug != NULL ) + mms_print_packet ( mms->stddebug, &pak, packet_len, MMS_SERVER ); + + /* transport protocol selection */ + memset ( &data, 0, sizeof(data) ); + mms_string_utf16 ( &data[8], (unsigned char *) "\002\000\\\\192.168.0.129\\TCP\\1037\0000", 28 ); + mms_send_packet ( mms, MMS_CMD_PROTOCOL_SELECT, 0, 0, 28*2+8, data ); + + cmd = MMS_CMD_PING; + while ( cmd == MMS_CMD_PING ) + { + cmd = mms_recv_cmd_packet ( mms->socket, &pak, &packet_len, 0, mms->quiet ); + + if ( cmd == MMS_CMD_PING ) + mms_send_packet ( mms, MMS_CMD_PONG, 0, 0, 0, pak.buf ); + } + + if ( cmd == MMS_CMD_INVALID ) + { + if ( !mms->quiet ) + error ( "mms_handshake", "unable to get cmd packet" ); + + return MMS_RET_ERROR; + } + + if ( mms->stddebug != NULL ) + mms_print_packet ( mms->stddebug, &pak, packet_len, MMS_SERVER ); + + /* requested file */ + mms_string_utf16 ( &data[8], (unsigned char *) mms->path, strlen(mms->path) ); + memset ( data, 0, 8 ); + mms_send_packet ( mms, MMS_CMD_FILE_REQUEST, 0, 0, strlen(mms->path)*2+12, data ); + + cmd = MMS_CMD_PING; + while ( cmd == MMS_CMD_PING ) + { + cmd = mms_recv_cmd_packet ( mms->socket, &pak, &packet_len, 0, mms->quiet ); + + if ( cmd == MMS_CMD_PING ) + mms_send_packet ( mms, MMS_CMD_PONG, 0, 0, 0, pak.buf ); + } + + if ( cmd == MMS_CMD_INVALID ) + { + if ( !mms->quiet ) + error ( "mms_handshake", "unable to get cmd packet" ); + + return MMS_RET_ERROR; + } + + if ( cmd == MMS_CMD_STREAM_INFOS ) + { + if ( mms_get_32 ( pak.buf, 48 ) == -1 ) + { + if ( !mms->quiet ) + error ( "mms_handshake", "stream infos are not available" ); + + return MMS_RET_ERROR; + } + + if ( ( mms_get_32(pak.buf,72) == 0 ) || ( mms_get_32(pak.buf,72) == 0xFFFFFFFF ) ) + { + mms->is_live = MMS_LIVE; + + if ( !mms->quiet ) + warning ( NULL, "stream seems to be live, an error may occur" ); + } + } + + if ( mms->stddebug != NULL ) + mms_print_packet ( mms->stddebug, &pak, packet_len, MMS_SERVER ); + + return MMS_RET_SUCCESS; +} + + +/* + * Gets ASF/WMV stream/file header + */ +ssize_t +mms_write_stream_header ( MMS *mms ) +{ + MMS_PACKET pak; + ssize_t len; + + if ( mms == NULL ) + return MMS_RET_ERROR; + + /* give me the stream header please... */ + memset ( pak.buf, 0, 40 ); + pak.buf[32] = 2; + mms_send_packet ( mms, MMS_CMD_HEADER_REQUEST, 1, 0, 40, pak.buf ); + + len = mms_recv_header_packet ( mms, &pak ); + + if ( len == -1 ) + { + if ( !mms->quiet ) + error ( "mms_get_stream_header", "unable to get stream header\n" ); + + return MMS_RET_ERROR; + } + else if ( len == 0 ) + { + if ( !mms->quiet ) + error ( "mms_get_stream_header", "no stream header available, file may not exist" ); + + return MMS_RET_ERROR; + } + + fwrite ( pak.buf, len, 1, mms->out ); + + mms->media_packet_len = mms_interp_header ( mms, pak.buf, len ); + + if ( mms->media_packet_len > MMS_BUF_SIZE ) + { + if ( !mms->quiet ) + error ( "mms_get_stream_header", "media packet length is too big for me" ); + + return MMS_RET_ERROR; + } + + return len; +} + + +/* + * Tells the server we're ready to rip the stream from the beginning (don't know what happens if it's a live) + * Returns 0 if success, -1 either. + */ +int +mms_begin_rip ( MMS *mms ) +{ + int i; + uint8_t data [ 1024 ]; + MMS_PACKET pak; + ssize_t packet_len; + int cmd; + + if ( mms == NULL ) + return MMS_RET_ERROR; + + /* media stream selection */ + memset ( data, 0, 40 ); + + for ( i=1; i<mms->num_stream_ids; ++i ) + { + data [ (i-1) * 6 + 2 ] = 0xFF; + data [ (i-1) * 6 + 3 ] = 0xFF; + data [ (i-1) * 6 + 4 ] = mms->stream_ids[i]; + data [ (i-1) * 6 + 5 ] = 0x00; + } + + /* we add the final bytes of the stream selector, necessary for ASF streams only */ + if ( mms->stream_type == MMS_ASF ) + { + data [ (mms->num_stream_ids-1) * 6 + 0 ] = 0x00; + data [ (mms->num_stream_ids-1) * 6 + 1 ] = 0x00; + data [ (mms->num_stream_ids-1) * 6 + 2 ] = 0x00; + data [ (mms->num_stream_ids-1) * 6 + 3 ] = 0x20; + data [ (mms->num_stream_ids-1) * 6 + 4 ] = 0xac; + data [ (mms->num_stream_ids-1) * 6 + 5 ] = 0x40; + data [ (mms->num_stream_ids-1) * 6 + 6 ] = 0x02; + + mms_send_packet ( mms, MMS_CMD_STREAM_SELECT, mms->num_stream_ids, 0xFFFF | mms->stream_ids[0] << 16, (mms->num_stream_ids-1)*6+2+4, data ); + } + else + { + mms_send_packet ( mms, MMS_CMD_STREAM_SELECT, mms->num_stream_ids, 0xFFFF | mms->stream_ids[0] << 16, (mms->num_stream_ids-1)*6+2, data ); + } + + cmd = MMS_CMD_PING; + while ( cmd == MMS_CMD_PING ) + { + cmd = mms_recv_cmd_packet ( mms->socket, &pak, &packet_len, 0, mms->quiet ); + + if ( cmd == MMS_CMD_PING ) + mms_send_packet ( mms, MMS_CMD_PONG, 0, 0, 0, pak.buf ); + } + + if ( cmd == MMS_CMD_INVALID ) + { + if ( !mms->quiet ) + error ( "mms_begin_rip", "unable to get server\'s confirmation" ); + + return MMS_RET_ERROR; + } + + if ( mms->stddebug != NULL ) + mms_print_packet ( mms->stddebug, &pak, packet_len, MMS_SERVER ); + + + /* ready... */ + memset ( data, 0, 40 ); + + for ( i=8; i<16; ++i ) + data[i] = 0xFF; + + data[20] = 0x04; + + mms_send_packet ( mms, MMS_CMD_START_PACKET, 1, 0xFFFF | mms->stream_ids[0] << 16, 24, data ); + + return MMS_RET_SUCCESS; +} + + +/* + * Writes a media packet. + * Returns the amount of bytes written, 0 if EOF, MMS_RET_NO_AUTH if streaming is denied, MMS_RET_ERROR either. + */ +ssize_t +mms_write_stream_data ( MMS *mms ) +{ + ssize_t len; + MMS_PACKET pak; + + if ( mms == NULL ) + return 0; + + do + { + len = mms_recv_media_packet ( mms, &pak ); + } + while ( len == MMS_RET_ACKED ); /* while we receive ACK packets */ + + if ( len == 0 ) + return 0; + else if ( len == -1 ) + { + if ( !mms->quiet ) + error ( "mms_write_stream_data", "mms_recv_media_packet failed" ); + + return MMS_RET_ERROR; + } + else if ( len == MMS_RET_NO_AUTH ) + { + if ( !mms->quiet ) + error ( "mms_write_stream_data", "mms_recv_media_packet failed" ); + + return MMS_RET_NO_AUTH; + } + + fwrite ( pak.buf, mms->media_packet_len, 1, mms->out ); /* we might have read less than the arbitrary media packet size */ + + return mms->media_packet_len; +} + + +/* + * Closes the connection. + */ +void +mms_disconnect ( MMS *mms ) +{ + uint8_t data [ 1024 ]; + + if ( mms == NULL ) + return; + + mms_send_packet ( mms, MMS_CMD_BYE_BYE, 0, 0, 0, data ); + + if ( ( shutdown(mms->socket, SHUT_RDWR) | close(mms->socket)) != 0 ) + if ( !mms->quiet ) + warning ( "mms_disconnect", "unable to close the socket properly" ); + + return; +} + + +/* + * Destroys An MMS Struct + */ +void +mms_destroy ( MMS *mms ) +{ + if ( mms == NULL ) + return; + + if ( mms->host != NULL ) + free ( mms->host ); + + if ( mms->path != NULL ) + free ( mms->path ); + + free ( mms ); + + return; +} diff --git a/src/mmsrip/mms.h b/src/mmsrip/mms.h new file mode 100644 index 0000000..600d7bd --- /dev/null +++ b/src/mmsrip/mms.h @@ -0,0 +1,129 @@ +/* + * $RCSfile: mms.h,v $ + * $Date: 2006/01/23 20:30:43 $ - $Revision: 1.17 $ + * + * This file is distributed as a part of MMSRIP ( MMS Ripper ). + * Copyright (c) 2005-2006 Nicolas BENOIT + * + * It is highly based on the work of SDP Multimedia and Major MMS. + * They deserve all the credits for it. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + + +#ifndef __MMS_H__ +#define __MMS_H__ + +#ifdef _WIN32 +typedef signed long int ssize_t; +#define bcopy(s, d, l) memcpy(d, s, l) +#include <assert.h> +#define close closesocket +#define read(soc, data, len) recv(soc, data, len, 0) +#define write(soc, data, len) send(soc, data, len, 0) +#define SHUT_RDWR SD_BOTH +#endif + +#if defined(__CYGWIN__) || defined(_WIN32) +typedef unsigned char uint8_t; +#ifndef __uint32_t_defined +#define __uint32_t_defined +typedef unsigned int uint32_t; +#endif +typedef unsigned long long int uint64_t; +#else +#if defined(SOLARIS) || defined(sun) +#include <inttypes.h> +#else +#include <stdint.h> +#endif +#endif + +#define MMS_SERVER 0 +#define MMS_CLIENT 1 + +#define MMS_NO_LIVE 0 +#define MMS_LIVE 1 + +#define MMS_WMV 0 +#define MMS_ASF 1 + +#define MMS_TRICK_DISABLED 0 +#define MMS_TRICK_ENABLED 1 + +#define MMS_CMD_INVALID -1 +#define MMS_CMD_HELLO 0x01 +#define MMS_CMD_PROTOCOL_SELECT 0x02 +#define MMS_CMD_FILE_REQUEST 0x05 +#define MMS_CMD_READY_TO_STREAM 0x05 +#define MMS_CMD_STREAM_INFOS 0x06 +#define MMS_CMD_START_PACKET 0x07 +#define MMS_CMD_STOP_STREAM 0x09 +#define MMS_CMD_BYE_BYE 0x0D +#define MMS_CMD_HEADER_DATA 0x11 +#define MMS_CMD_HEADER_REQUEST 0x15 +#define MMS_CMD_NET_TESTING 0x15 +#define MMS_CMD_PING 0x1B +#define MMS_CMD_PONG 0x1B +#define MMS_CMD_END_OF_STREAM 0x1E +#define MMS_CMD_STREAM_SELECT_ACK 0x21 +#define MMS_CMD_STREAM_SELECT 0x33 + +#define MMS_RET_SUCCESS 0 +#define MMS_RET_ERROR -1 +#define MMS_RET_NO_AUTH -2 +#define MMS_RET_ACKED -3 + + +#define MMS_BUF_SIZE 102400 + +typedef struct +{ + uint8_t buf[MMS_BUF_SIZE]; + int num_bytes; +} MMS_PACKET ; + + +typedef struct +{ + char *host; + char *path; + int socket; + FILE *out; + FILE *stddebug; + ssize_t media_packet_len; + uint64_t expected_file_size; + int is_live; + int stream_type; + int seq_num; + int num_stream_ids; + int stream_ids[20]; + int quiet; + int trick; +} MMS ; + + +MMS * mms_create ( const char *, FILE *, FILE *, const int, const int ); +int mms_connect ( MMS* ); +int mms_handshake ( MMS * ); +ssize_t mms_write_stream_header ( MMS * ); +int mms_begin_rip ( MMS * ); +ssize_t mms_write_stream_data ( MMS * ); +void mms_disconnect ( MMS * ); +void mms_destroy ( MMS * ); + +#endif diff --git a/src/proxy/AUTHORS b/src/proxy/AUTHORS new file mode 100644 index 0000000..66da09f --- /dev/null +++ b/src/proxy/AUTHORS @@ -0,0 +1 @@ +Xavier Roche <roche@httrack.com>
diff --git a/src/proxy/COPYING b/src/proxy/COPYING new file mode 100644 index 0000000..d60c31a --- /dev/null +++ b/src/proxy/COPYING @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/src/proxy/changelog.txt b/src/proxy/changelog.txt new file mode 100644 index 0000000..f5ae48b --- /dev/null +++ b/src/proxy/changelog.txt @@ -0,0 +1,20 @@ +0.4 - Sept 18 2005
+- implemented very limited WebDAV (RFC2518) primitives
+- index enumeration fixes
+- limited access to the proxy server through HTTP in non-proxy mode
+
+0.3 - Sept 10 2005
+- implemented ICPv2 server (tested with Squid Web Proxy Cache) implementing ICP_OP_QUERY and ICP_OP_SECHO
+- redirects for URLs with missing ending '/'
+- fixed htsnet.h macro errors (bogus port during address copy)
+- keep-alive fixes
+
+0.2 - Sept 4 2005
+- hack to fix the "external files stored as absolute references" bug
+- proper locking for indexes (unlocked zFile)
+- added previous httrack .dat/.ndx cache format
+- added catalog as index fallback
+- started to write ICPv2 server (RFC2186), but not yet ready
+
+0.1 - Aug 27 2005
+- initial release: HTTP (RFC2616) proxy and aggregation ready
diff --git a/src/proxy/main.c b/src/proxy/main.c new file mode 100644 index 0000000..e48b51d --- /dev/null +++ b/src/proxy/main.c @@ -0,0 +1,164 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) Xavier Roche and other contributors + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +Please visit our Website: http://www.httrack.com +*/ + +/* ------------------------------------------------------------ */ +/* File: ProxyTrack, httrack cache-based proxy */ +/* Author: Xavier Roche */ +/* ------------------------------------------------------------ */ + +/* Standard includes */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <ctype.h> + +#include "htsbase.h" +#include "htsnet.h" +#include "htslib.h" +#include "store.h" +#include "proxytrack.h" + +#ifndef _WIN32 +#include <signal.h> +static void sig_brpipe( int code ) { + /* ignore */ +} +#endif + +static int scanHostPort(const char* str, char *host, int *port) { + char* pos = strrchr(str, ':'); + if (pos != NULL) { + int n = (int) ( pos - str ); + if (n < 256) { + host[0] = '\0'; + strncat(host, str, n); + if (sscanf(pos + 1, "%d", port) == 1) { + return 1; + } + } + } + return 0; +} + +int main(int argc, char* argv[]) +{ + int i; + int ret = 0; + int proxyPort, icpPort; + char proxyAddr[256 + 1], icpAddr[256 + 1]; + PT_Indexes index; + +#ifdef _WIN32 + { + WORD wVersionRequested; // requested version WinSock API + WSADATA wsadata; // Windows Sockets API data + int stat; + wVersionRequested = 0x0101; + stat = WSAStartup( wVersionRequested, &wsadata ); + if (stat != 0) { + fprintf(stderr, "Winsock not found!\n"); + return -1; + } else if (LOBYTE(wsadata.wVersion) != 1 && HIBYTE(wsadata.wVersion) != 1) { + fprintf(stderr, "WINSOCK.DLL does not support version 1.1\n"); + WSACleanup(); + return -1; + } + } +#endif + + /* Args */ + printf("ProxyTrack %s, build proxies upon HTTrack Website Copier Archives\n", PROXYTRACK_VERSION); + printf("Copyright (C) Xavier Roche and other contributors\n"); + printf("\n"); + printf("This program is free software; you can redistribute it and/or\n"); + printf("modify it under the terms of the GNU General Public License\n"); + printf("as published by the Free Software Foundation; either version 2\n"); + printf("of the License, or any later version.\n"); + printf("\n"); + printf("*** This version is a development release ***\n"); + printf("\n"); + if (argc < 3 + || !scanHostPort(argv[1], proxyAddr, &proxyPort) + || !scanHostPort(argv[2], icpAddr, &icpPort)) + { + fprintf(stderr, "usage: %s <proxy-addr:proxy-port> <ICP-addr:ICP-port> [ ( <new.zip path> | <new.ndx path> | --list <file-list> ) ..]\n", argv[0]); + fprintf(stderr, "\texample:%s proxy:8080 localhost:3130 /home/archives/www-archive-01.zip /home/old-archives/www-archive-02.ndx\n", argv[0]); + return 1; + } + index = PT_New(); + for(i = 3 ; i < argc ; i++) { + if (argv[i][0] == '-') { + if (strcmp(argv[i], "--list") == 0) { + if (i + 1 < argc) { + char line[256 + 1]; + FILE *fp = fopen(argv[++i], "rb"); + if (fp == NULL) { + fprintf(stderr, "error: could not process list %s\n", argv[i]); + exit(1); + } + while(linput(fp, line, 256)) { + int itemsAdded = PT_AddIndex(index, line); + if (itemsAdded > 0) { + fprintf(stderr, "processed: %s (%d items added)\n", line, itemsAdded); + } else if (itemsAdded == 0) { + fprintf(stderr, "processed: %s (no items added)\n", line); + } else { + fprintf(stderr, "error: could not process %s\n", line); + } + } + fclose(fp); + } + } else { + fprintf(stderr, "* bad arg %s\n", argv[i]); + exit(1); + } + } else { + int itemsAdded = PT_AddIndex(index, argv[i]); + if (itemsAdded > 0) { + fprintf(stderr, "processed: %s (%d items added)\n", argv[i], itemsAdded); + } else if (itemsAdded == 0) { + fprintf(stderr, "processed: %s (no items added)\n", argv[i]); + } else { + fprintf(stderr, "error: could not process %s\n", argv[i]); + } + } + } + + /* sigpipe */ +#ifndef _WIN32 + signal( SIGPIPE , sig_brpipe ); // broken pipe (write into non-opened socket) +#endif + + /* Go */ + ret = proxytrack_main(proxyAddr, proxyPort, icpAddr, icpPort, index); + + /* Wipe */ + PT_Delete(index); + +#ifdef _WIN32 + WSACleanup(); +#endif + + return ret; +} + diff --git a/src/proxy/proxystrings.h b/src/proxy/proxystrings.h new file mode 100755 index 0000000..87bcf34 --- /dev/null +++ b/src/proxy/proxystrings.h @@ -0,0 +1,153 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) Xavier Roche and other contributors + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +Please visit our Website: http://www.httrack.com +*/ + + +/* ------------------------------------------------------------ */ +/* File: Strings */ +/* Author: Xavier Roche */ +/* ------------------------------------------------------------ */ + +// Strings a bit safer than static buffers + +#ifndef HTS_STRINGS_DEFSTATIC +#define HTS_STRINGS_DEFSTATIC + +typedef struct String { + char* buff; + int len; + int capa; +} String; + +#define STRING_EMPTY {NULL, 0, 0} +#define STRING_BLK_SIZE 256 +#define StringBuff(blk) ((blk).buff) +#define StringLength(blk) ((blk).len) +#define StringCapacity(blk) ((blk).capa) +#define StringRoom(blk, size) do { \ + if ((blk).len + (int)(size) + 1 > (blk).capa) { \ + (blk).capa = ((blk).len + (size) + 1) * 2; \ + (blk).buff = (char*) realloc((blk).buff, (blk).capa); \ + } \ +} while(0) +#define StringBuffN(blk, size) StringBuffN_(&(blk), size) +static char* StringBuffN_(String* blk, int size) { + StringRoom(*blk, (blk->len) + size); + return StringBuff(*blk); +} +#define StringClear(blk) do { \ + StringRoom(blk, 0); \ + (blk).buff[0] = '\0'; \ + (blk).len = 0; \ +} while(0) +#define StringFree(blk) do { \ + if ((blk).buff != NULL) { \ + free((blk).buff); \ + (blk).buff = NULL; \ + } \ + (blk).capa = 0; \ + (blk).len = 0; \ +} while(0) +#define StringMemcat(blk, str, size) do { \ + StringRoom(blk, size); \ + if ((int)(size) > 0) { \ + memcpy((blk).buff + (blk).len, (str), (size)); \ + (blk).len += (size); \ + } \ + *((blk).buff + (blk).len) = '\0'; \ +} while(0) +#define StringAddchar(blk, c) do { \ + char __c = (c); \ + StringMemcat(blk, &__c, 1); \ +} while(0) +static void* StringAcquire(String* blk) { + void* buff = blk->buff; + blk->buff = NULL; + blk->capa = 0; + blk->len = 0; + return buff; +} +static StringAttach(String* blk, char** str) { + StringFree(*blk); + if (str != NULL && *str != NULL) { + blk->buff = *str; + blk->capa = (int)strlen(blk->buff); + blk->len = blk->capa; + *str = NULL; + } +} +#define StringStrcat(blk, str) StringMemcat(blk, str, ((str) != NULL) ? (int)strlen(str) : 0) +#define StringStrcpy(blk, str) do { \ + StringClear(blk); \ + StringStrcat(blk, str); \ +} while(0) + +/* Tools */ + +static int ehexh(char c) { + if ((c>='0') && (c<='9')) return c-'0'; + if ((c>='a') && (c<='f')) c-=('a'-'A'); + if ((c>='A') && (c<='F')) return (c-'A'+10); + return 0; +} + +static int ehex(const char* s) { + return 16*ehexh(*s)+ehexh(*(s+1)); +} + +static void unescapehttp(const char* s, String* tempo) { + int i; + for (i = 0; s[i] != '\0' ; i++) { + if (s[i]=='%' && s[i+1]=='%') { + i++; + StringAddchar(*tempo, '%'); + } else if (s[i]=='%') { + char hc; + i++; + hc = (char) ehex(s+i); + StringAddchar(*tempo, (char) hc); + i++; // sauter 2 caractères finalement + } + else if (s[i]=='+') { + StringAddchar(*tempo, ' '); + } + else + StringAddchar(*tempo, s[i]); + } +} + +static void escapexml(const char* s, String* tempo) { + int i; + for (i=0 ; s[i] != '\0' ; i++) { + if (s[i] == '&') + StringStrcat(*tempo, "&"); + else if (s[i] == '<') + StringStrcat(*tempo, "<"); + else if (s[i] == '>') + StringStrcat(*tempo, ">"); + else if (s[i] == '\"') + StringStrcat(*tempo, """); + else + StringAddchar(*tempo, s[i]); + } +} + +#endif diff --git a/src/proxy/proxytrack.c b/src/proxy/proxytrack.c new file mode 100644 index 0000000..7604804 --- /dev/null +++ b/src/proxy/proxytrack.c @@ -0,0 +1,1621 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) Xavier Roche and other contributors + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +Please visit our Website: http://www.httrack.com +*/ + +/* ------------------------------------------------------------ */ +/* File: ProxyTrack, httrack cache-based proxy */ +/* Author: Xavier Roche */ +/* ------------------------------------------------------------ */ + + +/* + +/\/\/\/\/\/\/\/\/\/\/\/\/\ PENDING WORK /\/\/\/\/\/\/\/\/\/\/\/\/\ +- Etag update handling +- Other cache archive handling (.arc) +- Live plug/unplug of archives +- Listing +\/\/\/\/\/\/\/\/\/\/\/\/\/ PENDING WORK \/\/\/\/\/\/\/\/\/\/\/\/\/ + +*/ + +/* +Architecture rough draft +Xavier Roche 2005 + +Aim: Building a sub-proxy to be linked with other top level proxies (such as Squid) +Basic design: Classical HTTP/1.0 proxy server, with ICP server support +Internal data design: HTTrack cache indexing in fast hashtables, with 'pluggable' design (add/removal of caches on-the-fly) + + +Index structure organization: +----------------------------- + +foo/hts-cache/new.zip -----> Index[0] \ +bar/hts-cache/new.zip -----> Index[1] > Central Index Lookup (CIL) +baz/hts-cache/new.zip -----> Index[2] / +.. -----> .. + +Indexes are hashtables with URL (STRING) -> INTEGER lookup. + +URL -----> CIL Ask for index ID +URL -----> Index[ID] Ask for index properties (ZIP cache index) + + +Lookup of an entry: +------------------- + +ID = CIL[URL] +If ID is valid Then + return SUCCESS +Else + return FAILURE +EndIf + + +Fetching of an entry: +--------------------- + +RESOURCE = null +ID = CIL[URL] +If ID is valid Then + OFFSET = Index[ID][URL] + If OFFSET is valid Then + RESOURCE = Fetch(ID, OFFSET) + EndIf +EndIf + + +Removal of index N: +------------------- + +For all entries in Index[N] + URL(key) -----> Lookup all other caches + Found: Replace in CIL + Not Found: Delete entry in CIL +Done +Delete Index[N] + + +Adding of index N: +------------------ + +Build Index[N] +For all entries in Index[N] + URL(key) -----> Lookup in CIL + Found: Do nothing if corresponding Cache is newer than this one + Not Found: Add/Replace entry in CIL +Done + +Remark: If no cache newer than the added one is found, all entries can be added without any lookup (optim) + +*/ + +/* HTTrack definitions */ +#include "htsbase.h" +#include "htsnet.h" +#include "htslib.h" +#include "htsglobal.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <fcntl.h> +#if HTS_WIN +#else +#include <arpa/inet.h> +#endif +#ifndef _WIN32 +#include <signal.h> +#endif +/* END specific definitions */ + +/* String */ +#include "proxystrings.h" + +/* Network base */ +#include "htsbasenet.h" + +/* définitions globales */ +#include "htsglobal.h" + +/* htslib */ +/*#include "htslib.h"*/ + +/* HTTrack Website Copier Library */ +#include "httrack-library.h" + +/* htsweb */ +#include "htsinthash.h" + +/* ProxyTrack */ +#include "proxytrack.h" + +/* Store manager */ +#include "../minizip/mztools.h" +#include "store.h" + +/* threads */ +#ifdef _WIN32 +#include <process.h> /* _beginthread, _endthread */ +#else +#include <pthread.h> +#endif + +/* External references */ +// htsErrorCallback htsCallbackErr = NULL; +int htsMemoryFastXfr = 1; /* fast xfr by default */ +void abortLog__fnc(char* msg, char* file, int line); +void abortLog__fnc(char* msg, char* file, int line) { + FILE* fp = fopen("CRASH.TXT", "wb"); + if (!fp) fp = fopen("/tmp/CRASH.TXT", "wb"); + if (!fp) fp = fopen("C:\\CRASH.TXT", "wb"); + if (!fp) fp = fopen("CRASH.TXT", "wb"); + if (fp) { + fprintf(fp, "HTTrack " HTTRACK_VERSIONID " closed at '%s', line %d\r\n", file, line); + fprintf(fp, "Reason:\r\n%s\r\n", msg); + fflush(fp); + fclose(fp); + } +} +// HTSEXT_API t_abortLog abortLog__ = abortLog__fnc; /* avoid VC++ inlining */ +#define webhttrack_lock(A) do{}while(0) + +/* Static definitions */ + +static int linputsoc(T_SOC soc, char* s, int max) { + int c; + int j=0; + do { + unsigned char ch; + if (recv(soc, &ch, 1, 0) == 1) { + c = ch; + } else { + c = EOF; + } + if (c!=EOF) { + switch(c) { + case 13: break; // sauter CR + case 10: c=-1; break; + case 9: case 12: break; // sauter ces caractères + default: s[j++]=(char) c; break; + } + } + } while((c!=-1) && (c!=EOF) && (j<(max-1))); + s[j]='\0'; + return j; +} + +static int check_readinput_t(T_SOC soc, int timeout) { + if (soc != INVALID_SOCKET) { + fd_set fds; // poll structures + struct timeval tv; // structure for select + FD_ZERO(&fds); + FD_SET(soc,&fds); + tv.tv_sec=timeout; + tv.tv_usec=0; + select((int)(soc + 1),&fds,NULL,NULL,&tv); + if (FD_ISSET(soc,&fds)) + return 1; + else + return 0; + } else + return 0; +} + +static int linputsoc_t(T_SOC soc, char* s, int max, int timeout) { + if (check_readinput_t(soc, timeout)) { + return linputsoc(soc, s, max); + } + return -1; +} + +static void unescapeini(char* s, String* tempo) { + int i; + char lastc=0; + for (i=0;i<(int) strlen(s);i++) { + if (s[i]=='%' && s[i+1]=='%') { + i++; + StringAddchar(*tempo, lastc = '%'); + } else if (s[i]=='%') { + char hc; + i++; + hc = (char) ehex(s+i); + if (!is_retorsep(hc) || !is_retorsep(lastc)) { + StringAddchar(*tempo, lastc = (char) hc); + } + i++; // sauter 2 caractères finalement + } + else + StringAddchar(*tempo, lastc = s[i]); + } +} + +static int gethost(const char* hostname, SOCaddr *server, size_t server_size) { + if (hostname != NULL && *hostname != '\0') { +#if HTS_INET6==0 + /* + ipV4 resolver + */ + t_hostent* hp=gethostbyname(hostname); + if (hp!=NULL) { + if ( (hp->h_length) && ( ((unsigned int) hp->h_length) <= buffer->addr_maxlen) ) { + SOCaddr_copyaddr(server, server_size, hp->h_addr_list[0], hp->h_length); + return 1; + } + } +#else + /* + ipV6 resolver + */ + struct addrinfo* res = NULL; + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); +#if 0 + if (IPV6_resolver == 1) // V4 only (for bogus V6 entries) + hints.ai_family = PF_INET; + else if (IPV6_resolver == 2) // V6 only (for testing V6 only) + hints.ai_family = PF_INET6; + else +#endif + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = IPPROTO_TCP; + if (getaddrinfo(hostname, NULL, &hints, &res) == 0) { + if (res) { + if ( (res->ai_addr) && (res->ai_addrlen) ) { + SOCaddr_copyaddr(*server, server_size, res->ai_addr, res->ai_addrlen); + freeaddrinfo(res); + return 1; + } + } + } + if (res) { + freeaddrinfo(res); + } + +#endif + } + return 0; +} + +static String getip(SOCaddr *server, int serverLen) { + String s = STRING_EMPTY; +#if HTS_INET6==0 + unsigned int sizeMax = sizeof("999.999.999.999:65535"); +#else + unsigned int sizeMax = sizeof("ffff:ffff:ffff:ffff:ffff:ffff:ffff:65535"); +#endif + char * dotted = malloc(sizeMax + 1); + unsigned short port = ntohs(SOCaddr_sinport(*server)); + if (dotted == NULL) { + CRITICAL("memory exhausted"); + return s; + } + SOCaddr_inetntoa(dotted, sizeMax, *server, serverLen); + sprintf(dotted + strlen(dotted), ":%d", port); + StringAttach(&s, &dotted); + return s; +} + + +static T_SOC smallserver_init(const char* adr, int port, int family) { + SOCaddr server; + size_t server_size = sizeof(server); + + memset(&server, 0, sizeof(server)); + SOCaddr_initany(server, server_size); + if (gethost(adr, &server, server_size)) { // host name + T_SOC soc = INVALID_SOCKET; + if ( (soc = socket(SOCaddr_sinfamily(server), family, 0)) != INVALID_SOCKET) { + SOCaddr_initport(server, port); + if ( bind(soc,(struct sockaddr*) &server, (int)server_size) == 0 ) { + if (family != SOCK_STREAM + || listen(soc, 10) >=0 ) { + return soc; + } else { +#ifdef _WIN32 + closesocket(soc); +#else + close(soc); +#endif + soc=INVALID_SOCKET; + } + } else { +#ifdef _WIN32 + closesocket(soc); +#else + close(soc); +#endif + soc=INVALID_SOCKET; + } + } + } + return INVALID_SOCKET; +} + +static int proxytrack_start(PT_Indexes indexes, T_SOC soc, T_SOC socICP); +int proxytrack_main(char* proxyAddr, int proxyPort, + char* icpAddr, int icpPort, + PT_Indexes index) { + int returncode = 0; + T_SOC soc = smallserver_init(proxyAddr, proxyPort, SOCK_STREAM); + T_SOC socICP = smallserver_init(proxyAddr, icpPort, SOCK_DGRAM); + if (soc != INVALID_SOCKET + && socICP != INVALID_SOCKET) + { + char url[HTS_URLMAXSIZE*2]; + char method[32]; + char data[32768]; + url[0]=method[0]=data[0]='\0'; + // + printf("HTTP Proxy installed on %s:%d/\n", proxyAddr, proxyPort); + printf("ICP Proxy installed on %s:%d/\n", icpAddr, icpPort); +#ifndef _WIN32 + { + pid_t pid = getpid(); + printf("PID=%d\n", (int)pid); + } +#endif + fflush(stdout); + fflush(stderr); + // + if (!proxytrack_start(index, soc, socICP)) { + fprintf(stderr, "Unable to create the server: %s\n", strerror(errno)); +#ifdef _WIN32 + closesocket(soc); +#else + close(soc); +#endif + printf("Done\n"); + returncode = 1; + } else { + returncode = 0; + } + } else { + fprintf(stderr, "Unable to initialize a temporary server : %s\n", strerror(errno)); + returncode = 1; + } + printf("EXITED\n"); + fflush(stdout); + fflush(stderr); + return returncode; +} + +static const char* GetHttpMessage(int statuscode) { + // Erreurs HTTP, selon RFC + switch( statuscode) { + case 100: return "Continue"; break; + case 101: return "Switching Protocols"; break; + case 200: return "OK"; break; + case 201: return "Created"; break; + case 202: return "Accepted"; break; + case 203: return "Non-Authoritative Information"; break; + case 204: return "No Content"; break; + case 205: return "Reset Content"; break; + case 206: return "Partial Content"; break; + case 207: return "Multi-Status"; break; + case 300: return "Multiple Choices"; break; + case 301: return "Moved Permanently"; break; + case 302: return "Moved Temporarily"; break; + case 303: return "See Other"; break; + case 304: return "Not Modified"; break; + case 305: return "Use Proxy"; break; + case 306: return "Undefined 306 error"; break; + case 307: return "Temporary Redirect"; break; + case 400: return "Bad Request"; break; + case 401: return "Unauthorized"; break; + case 402: return "Payment Required"; break; + case 403: return "Forbidden"; break; + case 404: return "Not Found"; break; + case 405: return "Method Not Allowed"; break; + case 406: return "Not Acceptable"; break; + case 407: return "Proxy Authentication Required"; break; + case 408: return "Request Time-out"; break; + case 409: return "Conflict"; break; + case 410: return "Gone"; break; + case 411: return "Length Required"; break; + case 412: return "Precondition Failed"; break; + case 413: return "Request Entity Too Large"; break; + case 414: return "Request-URI Too Large"; break; + case 415: return "Unsupported Media Type"; break; + case 416: return "Requested Range Not Satisfiable"; break; + case 417: return "Expectation Failed"; break; + case 500: return "Internal Server Error"; break; + case 501: return "Not Implemented"; break; + case 502: return "Bad Gateway"; break; + case 503: return "Service Unavailable"; break; + case 504: return "Gateway Time-out"; break; + case 505: return "HTTP Version Not Supported"; break; + default: return "Unknown HTTP Error"; break; + } +} + +#ifndef NO_WEBDAV +static void proxytrack_add_DAV_Item(String *item, String *buff, + const char* filename, + unsigned long int size, + time_t timestamp, + const char* mime, + int isDir, + int isRoot, + int isDefault) +{ + struct tm * timetm; + if (timestamp == (time_t) 0 || timestamp == (time_t) -1) { + timestamp = time(NULL); + } + if ((timetm = gmtime(×tamp)) != NULL) { + char tms[256 + 1]; + const char * name; + strftime(tms, 256, "%a, %d %b %Y %H:%M:%S GMT", timetm); /* Sun, 18 Sep 2005 11:45:45 GMT */ + + if (mime == NULL || *mime == 0) + mime = "application/octet-stream"; + + StringLength(*buff) = 0; + escapexml(filename, buff); + + name = strrchr(StringBuff(*buff), '/'); + if (name != NULL) + name++; + if (name == NULL || *name == 0) { + if (strcmp(mime, "text/html") == 0) + name = "Default Document for the Folder.html"; + else + name = "Default Document for the Folder"; + } + + StringRoom(*item, 1024); + sprintf(StringBuff(*item), + "<response xmlns=\"DAV:\">\r\n" + "<href>/webdav%s%s</href>\r\n" + "<propstat>\r\n" + "<prop>\r\n" + "<displayname>%s</displayname>\r\n" + "<iscollection>%d</iscollection>\r\n" + "<haschildren>%d</haschildren>\r\n" + "<isfolder>%d</isfolder>\r\n" + "<resourcetype>%s</resourcetype>\r\n" + "<creationdate>%d-%02d-%02dT%02d:%02d:%02dZ</creationdate>\r\n" + "<getlastmodified>%s</getlastmodified>\r\n" + "<supportedlock></supportedlock>\r\n" + "<lockdiscovery/>\r\n" + "<getcontenttype>%s</getcontenttype>\r\n" + "<getcontentlength>%d</getcontentlength>\r\n" + "<isroot>%d</isroot>\r\n" + "</prop>\r\n" + "<status>HTTP/1.1 200 OK</status>\r\n" + "</propstat>\r\n" + "</response>\r\n", + /* */ + ( StringBuff(*buff)[0] == '/' ) ? "" : "/", StringBuff(*buff), + name, + isDir ? 1 : 0, + isDir ? 1 : 0, + isDir ? 1 : 0, + isDir ? "<collection/>" : "", + timetm->tm_year + 1900, timetm->tm_mon + 1, timetm->tm_mday, timetm->tm_hour, timetm->tm_min, timetm->tm_sec, + tms, + isDir ? "httpd/unix-directory" : mime, + (int)size, + isRoot ? 1 : 0 + ); + StringLength(*item) = (int) strlen(StringBuff(*item)); + } +} + +/* Convert a RFC822 time to time_t */ +time_t get_time_rfc822(const char* s) { + struct tm result; + /* */ + char months[]="jan feb mar apr may jun jul aug sep oct nov dec"; + char str[256]; + char* a; + int i; + /* */ + int result_mm=-1; + int result_dd=-1; + int result_n1=-1; + int result_n2=-1; + int result_n3=-1; + int result_n4=-1; + /* */ + + if ((int) strlen(s) > 200) + return (time_t)0; + for(i = 0 ; s[i] != 0 ; i++) { + if (s[i] >= 'A' && s[i] <= 'Z') + str[i] = s[i] + ('a' - 'A'); + else + str[i] = s[i]; + } + str[i] = 0; + /* éliminer :,- */ + while( (a=strchr(str,'-')) ) *a=' '; + while( (a=strchr(str,':')) ) *a=' '; + while( (a=strchr(str,',')) ) *a=' '; + /* tokeniser */ + a=str; + while(*a) { + char *first,*last; + char tok[256]; + /* découper mot */ + while(*a==' ') a++; /* sauter espaces */ + first=a; + while((*a) && (*a!=' ')) a++; + last=a; + tok[0]='\0'; + if (first!=last) { + char* pos; + strncat(tok,first,(int) (last - first)); + /* analyser */ + if ( (pos=strstr(months,tok)) ) { /* month always in letters */ + result_mm=((int) (pos - months))/4; + } else { + int number; + if (sscanf(tok,"%d",&number) == 1) { /* number token */ + if (result_dd<0) /* day always first number */ + result_dd=number; + else if (result_n1<0) + result_n1=number; + else if (result_n2<0) + result_n2=number; + else if (result_n3<0) + result_n3=number; + else if (result_n4<0) + result_n4=number; + } /* sinon, bruit de fond(+1GMT for exampel) */ + } + } + } + if ((result_n1>=0) && (result_mm>=0) && (result_dd>=0) && (result_n2>=0) && (result_n3>=0) && (result_n4>=0)) { + if (result_n4>=1000) { /* Sun Nov 6 08:49:37 1994 */ + result.tm_year=result_n4-1900; + result.tm_hour=result_n1; + result.tm_min=result_n2; + result.tm_sec=max(result_n3,0); + } else { /* Sun, 06 Nov 1994 08:49:37 GMT or Sunday, 06-Nov-94 08:49:37 GMT */ + result.tm_hour=result_n2; + result.tm_min=result_n3; + result.tm_sec=max(result_n4,0); + if (result_n1<=50) /* 00 means 2000 */ + result.tm_year=result_n1+100; + else if (result_n1<1000) /* 99 means 1999 */ + result.tm_year=result_n1; + else /* 2000 */ + result.tm_year=result_n1-1900; + } + result.tm_isdst=0; /* assume GMT */ + result.tm_yday=-1; /* don't know */ + result.tm_wday=-1; /* don't know */ + result.tm_mon=result_mm; + result.tm_mday=result_dd; + return mktime(&result); + } + return (time_t) 0; +} + +static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes, const char * urlFull, int depth) { + const char * file = jump_protocol_and_auth(urlFull); + if ( (file = strchr(file, '/')) == NULL) + return NULL; + + if (strncmp(file, "/webdav", 7) != 0) { + PT_Element elt = PT_ElementNew(); + elt->statuscode = 405; + strcpy(elt->msg, "Method Not Allowed"); + return elt; + } + + /* Skip /webdav */ + file += 7; + + /* */ + { + PT_Element elt = PT_ElementNew(); + int i, isDir; + String url = STRING_EMPTY; + String response = STRING_EMPTY; + String item = STRING_EMPTY; + String itemUrl = STRING_EMPTY; + String buff = STRING_EMPTY; + StringClear(response); + StringClear(item); + StringClear(itemUrl); + StringClear(buff); + + /* Canonize URL */ + StringStrcpy(url, file + ((file[0] == '/') ? 1 : 0)); + if (StringLength(url) > 0) { + if (StringBuff(url)[StringLength(url) - 1] == '/') { + StringBuff(url)[StringLength(url) - 1] = '\0'; + StringLength(url)--; + } + } + + /* Form response */ + StringRoom(response, 1024); + sprintf(StringBuff(response), + "<?xml version=\"1.0\" encoding=\"utf-8\"?>\r\n" + "<multistatus xmlns=\"DAV:\">\r\n"); + StringLength(response) = (int) strlen(StringBuff(response)); + /* */ + + /* Root */ + StringLength(item) = 0; + proxytrack_add_DAV_Item(&item, &buff, + StringBuff(url), /*size*/0, /*timestamp*/(time_t) 0, /*mime*/NULL, /*isDir*/1, /*isRoot*/1, /*isDefault*/0); + StringMemcat(response, StringBuff(item), StringLength(item)); + + /* Childrens (Depth > 0) */ + if (depth > 0) { + time_t timestampRep = (time_t) -1; + const char * prefix = StringBuff(url); + unsigned int prefixLen = (unsigned int) strlen(prefix); + char ** list = PT_Enumerate(indexes, prefix, 0); + if (list != NULL) { + for(isDir = 1 ; isDir >= 0 ; isDir--) { + for(i = 0 ; list[i] != NULL ; i++) { + const char * thisUrl = list[i]; + const char * mimeType = "application/octet-stream"; + unsigned int thisUrlLen = (unsigned int) strlen(thisUrl); + int thisIsDir = (thisUrl[thisUrlLen - 1] == '/') ? 1 : 0; + + /* Item URL */ + StringRoom(itemUrl, thisUrlLen + prefixLen + sizeof("/webdav/") + 1); + StringClear(itemUrl); + sprintf(StringBuff(itemUrl), "/%s/%s", prefix, thisUrl); + if (!thisIsDir) + StringLength(itemUrl) = (int) strlen(StringBuff(itemUrl)); + else + StringLength(itemUrl) = (int) strlen(StringBuff(itemUrl)) - 1; + StringBuff(itemUrl)[StringLength(itemUrl)] = '\0'; + + if (thisIsDir == isDir) { + unsigned long size = 0; + time_t timestamp = (time_t) 0; + PT_Element file = NULL; + + /* Item stats */ + if (!isDir) { + file = PT_ReadIndex(indexes, StringBuff(itemUrl) + 1, FETCH_HEADERS); + if (file != NULL && file->statuscode == 200 ) { + size = file->size; + if (file->lastmodified) { + timestamp = get_time_rfc822(file->lastmodified); + } + if (timestamp == (time_t) 0) { + if (timestampRep == (time_t) -1) { + timestampRep = 0; + if (file->indexId != -1) { + timestampRep = PT_Index_Timestamp(PT_GetIndex(indexes, file->indexId)); + } + } + timestamp = timestampRep; + } + if (file->contenttype) { + mimeType = file->contenttype; + } + } + } + + /* Add item */ + StringLength(item) = 0; + proxytrack_add_DAV_Item(&item, &buff, + StringBuff(itemUrl), size, timestamp, mimeType, isDir, /*isRoot*/0, /*isDefault*/(thisUrlLen == 0)); + StringMemcat(response, StringBuff(item), StringLength(item)); + + /* Wipe element */ + if (file != NULL) + PT_Element_Delete(&file); + } + } + } + PT_Enumerate_Delete(&list); + } /* items != NULL */ + } /* Depth > 0 */ + + /* End of responses */ + StringStrcat(response, + "</multistatus>\r\n" + ); + + StringFree(item); + StringFree(itemUrl); + StringFree(url); + StringFree(buff); + + elt->size = StringLength(response); + elt->adr = StringAcquire(&response); + elt->statuscode = 207; /* Multi-Status */ + strcpy(elt->charset, "utf-8"); + strcpy(elt->contenttype, "text/xml"); + strcpy(elt->msg, "Multi-Status"); + StringFree(response); + + fprintf(stderr, "RESPONSE:\n%s\n", elt->adr); + + return elt; + } + return NULL; +} +#endif + +static PT_Element proxytrack_process_HTTP_List(PT_Indexes indexes, const char * url) { + char ** list = PT_Enumerate(indexes, url, 0); + if (list != NULL) { + PT_Element elt = PT_ElementNew(); + int i, isDir; + String html = STRING_EMPTY; + StringClear(html); + StringStrcat(html, + "<html>" + PROXYTRACK_COMMENT_HEADER + DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES + "<head>\r\n" + "<title>ProxyTrack " PROXYTRACK_VERSION " Catalog</title>" + "</head>\r\n" + "<body>\r\n" + "<h3>Directory index:</h3><br />" + "<br />" + "<hr>" + "<tt>[DIR] <a href=\"..\">..</a></tt><br />" + ); + for(isDir = 1 ; isDir >= 0 ; isDir--) { + for(i = 0 ; list[i] != NULL ; i++) { + char * thisUrl = list[i]; + unsigned int thisUrlLen = (unsigned int) strlen(thisUrl); + int thisIsDir = (thisUrl[thisUrlLen - 1] == '/') ? 1 : 0; + if (thisIsDir == isDir) { + if (isDir) + StringStrcat(html, "<tt>[DIR] "); + else + StringStrcat(html, "<tt> "); + StringStrcat(html, "<a href=\""); + if (isDir) { + StringStrcat(html, "http://proxytrack/"); + } + StringStrcat(html, url); + StringStrcat(html, list[i]); + StringStrcat(html, "\">"); + StringStrcat(html, list[i]); + StringStrcat(html, "</a></tt><br />"); + } + } + } + StringStrcat(html, + "</body>" + "</html>"); + PT_Enumerate_Delete(&list); + elt->size = StringLength(html); + elt->adr = StringAcquire(&html); + elt->statuscode = 200; + strcpy(elt->charset, "iso-8859-1"); + strcpy(elt->contenttype, "text/html"); + strcpy(elt->msg, "OK"); + StringFree(html); + return elt; + } + return NULL; +} + +static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) { + int timeout=30; + int retour=0; + int willexit=0; + int buffer_size = 32768; + char * buffer = (char*)malloc(buffer_size); + int line1Size = 1024; + char * line1 = (char*)malloc(line1Size); + int lineSize = 8192; + char * line = (char*)malloc(lineSize); + int length = 0; + int keepAlive = 1; + + String url = STRING_EMPTY; + String urlRedirect = STRING_EMPTY; + String headers = STRING_EMPTY; + String output = STRING_EMPTY; + String host = STRING_EMPTY; + String localhost = STRING_EMPTY; +#ifndef NO_WEBDAV + String davHeaders = STRING_EMPTY; + String davRequest = STRING_EMPTY; +#endif + + StringRoom(localhost, 256); + if (gethostname(StringBuff(localhost), StringCapacity(localhost) - 1) == 0) { + StringLength(localhost) = (int) strlen(StringBuff(localhost)); + } else { + StringStrcpy(localhost, "localhost"); + } + +#ifdef _DEBUG + Sleep(1000); +#endif + + if (buffer == NULL || line == NULL || line1 == NULL) { + CRITICAL("proxytrack_process_HTTP:memory exhausted"); +#if HTS_WIN + closesocket(soc_c); +#else + close(soc_c); +#endif + return ; + } + + do { + const char* msgError = NULL; + int msgCode = 0; + PT_Element element = NULL; + char* command; + char* proto; + char* surl; + int directHit = 0; + int headRequest = 0; + int listRequest = 0; +#ifndef NO_WEBDAV + int davDepth = 0; +#endif + + /* Clear context */ + line[0] = line1[0] = '\0'; + buffer[0] = '\0'; + command = line1; + StringClear(url); + StringClear(urlRedirect); + StringClear(headers); + StringClear(output); + StringClear(host); +#ifndef NO_WEBDAV + StringClear(davHeaders); + StringClear(davRequest); +#endif + + /* line1: "GET http://www.example.com/ HTTP/1.0" */ + if (linputsoc_t(soc_c, line1, line1Size - 2, timeout) > 0 + && ( surl = strchr(line1, ' ') ) + && !(*surl = '\0') + && ++surl + && (proto = strchr(surl, ' ')) && !(*proto = '\0') && ++proto) + { + /* Flush headers */ + while(linputsoc_t(soc_c, line, lineSize - 2, timeout) > 0 + && line[0] != 0) + { + int p; + if ((p = strfield(line, "Content-length:"))!=0) { + if (sscanf(line+p, "%d", &length) != 1) { + msgCode = 500; + msgError = "Bad HTTP Content-Length Field"; + keepAlive = 0; + length = 0; + } + } else if (strcasecmp(line, "Connection: close") == 0) { + keepAlive = 0; + } else if (strcasecmp(line, "Connection: keep-alive") == 0) { + keepAlive = 1; + } else if ((p = strfield(line, "Host:"))) { + char* chost = line + p; + if (*chost == ' ') + chost++; + StringStrcpy(host, chost); + } +#ifndef NO_WEBDAV + else if ((p = strfield(line, "Depth: "))) { + char* depth = line + p; + if (sscanf(depth, "%d", &davDepth) != 1) { + davDepth = 0; + } + } +#endif + } + + /* Flush body */ +#ifndef NO_WEBDAV + if (length > 0) { + if (length < 32768) { + StringRoom(davRequest, length + 1); + if (recv(soc_c, StringBuff(davRequest), length, 0) == length) { + StringBuff(davRequest)[length] = 0; + } else { + msgCode = 500; + msgError = "Posted Data Read Error"; + keepAlive = 0; + } + } else { + msgCode = 500; + msgError = "Posted Data Too Large"; + keepAlive = 0; + } + } +#endif + + /* Switch protocol ID */ + if (strcasecmp(command, "post") == 0) { +#ifndef NO_WEBDAV + msgCode = 404; +#else + msgCode = 501; + keepAlive = 0; +#endif + msgError = "Proxy Error (POST Request Forbidden)"; + } + else if (strcasecmp(command, "get") == 0) { + headRequest = 0; + } + else if (strcasecmp(command, "head") == 0) { + headRequest = 1; + } +#ifndef NO_WEBDAV + else if (strcasecmp(command, "options") == 0) { + const char * options = "GET, HEAD, OPTIONS, POST, PROPFIND, TRACE" + ", MKCOL, DELETE, PUT"; /* Not supported */ + msgCode = 200; + StringRoom(headers, 8192); + sprintf(StringBuff(headers), + "HTTP/1.1 %d %s\r\n" + "DAV: 1, 2\r\n" + "MS-Author-Via: DAV\r\n" + "Cache-Control: private\r\n" + "Allow: %s\r\n", + msgCode, GetHttpMessage(msgCode), options); + StringLength(headers) = (int) strlen(StringBuff(headers)); + } + else if (strcasecmp(command, "propfind") == 0) { + if (davDepth > 1) { + msgCode = 403; + msgError = "DAV Depth Limit Forbidden"; + } else { + fprintf(stderr, "DEBUG: DAV-DATA=<%s>\n", StringBuff(davRequest)); + listRequest = 2; /* propfind */ + } + } + else if (strcasecmp(command, "mkcol") == 0 + || strcasecmp(command, "delete") == 0 + || strcasecmp(command, "put") == 0 + || strcasecmp(command, "proppatch") == 0 + || strcasecmp(command, "lock") == 0 + || strcasecmp(command, "unlock") == 0 + || strcasecmp(command, "copy") == 0 + || strcasecmp(command, "trace") == 0) + { + msgCode = 403; + msgError = "Method Forbidden"; + } +#endif + else { + msgCode = 501; + msgError = "Proxy Error (Unsupported or Unknown HTTP Command Request)"; + keepAlive = 0; + } + if (strcasecmp(proto, "http/1.1") == 0) { + keepAlive = 1; + } else if (strcasecmp(proto, "http/1.0") == 0) { + keepAlive = 0; + } else { + msgCode = 505; + msgError = "Proxy Error (Unknown HTTP Version)"; + keepAlive = 0; + } + + /* Post-process request */ + if (link_has_authority(surl)) { + const unsigned int prefixLen = sizeof("http://proxytrack/") - 1; + if (strncasecmp(surl, "http://proxytrack/", prefixLen) == 0) { + directHit = 1; /* Another direct hit hack */ + } + StringStrcpy(url, surl); + } else { + if (StringLength(host) > 0) { + /* Direct hit */ + if ( +#ifndef NO_WEBDAV + listRequest != 2 + && +#endif + strncasecmp(StringBuff(host), StringBuff(localhost), StringLength(localhost)) == 0 + && + (StringBuff(host)[StringLength(localhost)] == '\0' + || StringBuff(host)[StringLength(localhost)] == ':') + && surl[0] == '/' + ) + { + const char * toHit = surl + 1; + if (strncmp(toHit, "webdav/", 7) == 0) { + toHit += 7; + } + /* Direct hit */ + directHit = 1; + StringStrcpy(url, ""); + if (!link_has_authority(toHit)) + StringStrcat(url, "http://"); + StringStrcat(url, toHit); + } else { + /* Transparent proxy */ + StringStrcpy(url, "http://"); + StringStrcat(url, StringBuff(host)); + StringStrcat(url, surl); + } + } else { + msgCode = 500; + msgError = "Transparent Proxy Error ('Host' HTTP Request Header Field Missing)"; + keepAlive = 0; + } + } + + /* Response */ + if (msgCode == 0) { + if (listRequest == 1) { + element = proxytrack_process_HTTP_List(indexes, StringBuff(url)); + } +#ifndef NO_WEBDAV + else if (listRequest == 2) { + if ((element = proxytrack_process_DAV_Request(indexes, StringBuff(url), davDepth)) != NULL) { + msgCode = element->statuscode; + StringRoom(davHeaders, 1024); + sprintf(StringBuff(davHeaders), + "DAV: 1, 2\r\n" + "MS-Author-Via: DAV\r\n" + "Cache-Control: private\r\n"); + StringLength(davHeaders) = (int) strlen(StringBuff(davHeaders)); + } + } +#endif + else { + element = PT_ReadIndex(indexes, StringBuff(url), FETCH_BODY); + } + if (element == NULL +#ifndef NO_WEBDAV + && listRequest == 2 +#endif + && StringLength(url) > 0 + && StringBuff(url)[StringLength(url) - 1] == '/' + ) + { + element = PT_Index_HTML_BuildRootInfo(indexes); + if (element != NULL) { + element->statuscode = 404; /* HTML page, but in error */ + } + } + if (element != NULL) { + msgCode = element->statuscode; + StringRoom(headers, 8192); + sprintf(StringBuff(headers), + "HTTP/1.1 %d %s\r\n" +#ifndef NO_WEBDAV + "%s" +#endif + "Content-Type: %s%s%s%s\r\n" + "%s%s%s" + "%s%s%s" + "%s%s%s", + /* */ + msgCode, + element->msg, +#ifndef NO_WEBDAV + /* DAV */ + StringBuff(davHeaders), +#endif + /* Content-type: foo; [ charset=bar ] */ + element->contenttype, + ( ( element->charset[0]) ? "; charset=\"" : ""), + element->charset, + ( ( element->charset[0]) ? "\"" : ""), + /* location */ + ( ( element->location != NULL && element->location[0]) ? "Location: " : ""), + ( ( element->location != NULL && element->location[0]) ? element->location : ""), + ( ( element->location != NULL && element->location[0]) ? "\r\n" : ""), + /* last-modified */ + ( ( element->lastmodified[0]) ? "Last-Modified: " : ""), + ( ( element->lastmodified[0]) ? element->lastmodified : ""), + ( ( element->lastmodified[0]) ? "\r\n" : ""), + /* etag */ + ( ( element->etag[0]) ? "ETag: " : ""), + ( ( element->etag[0]) ? element->etag : ""), + ( ( element->etag[0]) ? "\r\n" : "") + ); + StringLength(headers) = (int) strlen(StringBuff(headers)); + } else { + /* No query string, no ending / : check the the <url>/ page */ + if (StringLength(url) > 0 && StringBuff(url)[StringLength(url) - 1] != '/' && strchr(StringBuff(url), '?') == NULL) { + StringStrcpy(urlRedirect, StringBuff(url)); + StringStrcat(urlRedirect, "/"); + if (PT_LookupIndex(indexes, StringBuff(urlRedirect))) { + msgCode = 301; /* Moved Permanently */ + StringRoom(headers, 8192); + sprintf(StringBuff(headers), + "HTTP/1.1 %d %s\r\n" + "Content-Type: text/html\r\n" + "Location: %s\r\n", + /* */ + msgCode, + GetHttpMessage(msgCode), + StringBuff(urlRedirect) + ); + StringLength(headers) = (int) strlen(StringBuff(headers)); + /* */ + StringRoom(output, 1024 + sizeof(PROXYTRACK_COMMENT_HEADER) + sizeof(DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES)); + sprintf(StringBuff(output), + "<html>" + PROXYTRACK_COMMENT_HEADER + DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES + "<head>" + "<title>ProxyTrack - Page has moved</title>" + "</head>\r\n" + "<body>" + "<h3>The correct location is:</h3><br />" + "<b><a href=\"%s\">%s</a></b><br />" + "<br />" + "<br />\r\n" + "<i>Generated by ProxyTrack " PROXYTRACK_VERSION ", (C) Xavier Roche and other contributors</i>" + "\r\n" + "</body>" + "</header>", + StringBuff(urlRedirect), + StringBuff(urlRedirect)); + StringLength(output) = (int) strlen(StringBuff(output)); + } + } + if (msgCode == 0) { + msgCode = 404; + msgError = "Not Found in this cache"; + } + } + } + } else { + msgCode = 500; + msgError = "Server Error"; + keepAlive = 0; + } + if (StringLength(headers) == 0) { + if (msgCode == 0) { + msgCode = 500; + msgError = "Internal Proxy Error"; + } else if (msgError == NULL) { + msgError = GetHttpMessage(msgCode); + } + StringRoom(headers, 256); + sprintf(StringBuff(headers), + "HTTP/1.1 %d %s\r\n" + "Content-type: text/html\r\n", + msgCode, + msgError); + StringLength(headers) = (int) strlen(StringBuff(headers)); + StringRoom(output, 1024 + sizeof(PROXYTRACK_COMMENT_HEADER) + sizeof(DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES)); + sprintf(StringBuff(output), + "<html>" + PROXYTRACK_COMMENT_HEADER + DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES + "<head>" + "<title>ProxyTrack - HTTP Proxy Error %d</title>" + "</head>\r\n" + "<body>" + "<h3>A proxy error has occured while processing the request.</h3><br />" + "<b>Error HTTP %d: <i>%s</i></b><br />" + "<br />" + "<br />\r\n" + "<i>Generated by ProxyTrack " PROXYTRACK_VERSION ", (C) Xavier Roche and other contributors</i>" + "\r\n" + "</body>" + "</html>", + msgCode, + msgCode, + msgError); + StringLength(output) = (int) strlen(StringBuff(output)); + } + { + char tmp[20 + 1]; /* 2^64 = 18446744073709551616 */ + unsigned int dataSize = 0; + if (!headRequest) { + dataSize = StringLength(output); + if (dataSize == 0 && element != NULL) { + dataSize = element->size; + } + } + sprintf(tmp, "%d", (int) dataSize); + StringStrcat(headers, "Content-length: "); + StringStrcat(headers, tmp); + StringStrcat(headers, "\r\n"); + } + if (keepAlive) { + StringStrcat(headers, + "Connection: Keep-Alive\r\n" + "Proxy-Connection: Keep-Alive\r\n"); + } else { + StringStrcat(headers, + "Connection: Close\r\n" + "Proxy-Connection: Close\r\n"); + } + if (msgCode != 500) + StringStrcat(headers, "X-Cache: HIT from "); + else + StringStrcat(headers, "X-Cache: MISS from "); + StringStrcat(headers, StringBuff(localhost)); + StringStrcat(headers, "\r\n"); + + /* Logging */ + { + const char * contentType = "text/html"; + unsigned long int size = StringLength(output) ? StringLength(output) : ( element ? element->size : 0 ); + /* */ + String ip = STRING_EMPTY; + SOCaddr serverClient; + int lenServerClient = (int) sizeof(serverClient); + memset(&serverClient, 0, sizeof(serverClient)); + if (getsockname(soc_c, (struct sockaddr*) &serverClient, &lenServerClient) == 0) { + ip = getip(&serverClient, lenServerClient); + } else { + StringStrcpy(ip, "unknown"); + } + if (element != NULL && element->contenttype[0] != '\0') { + contentType = element->contenttype; + } + LOG("HTTP %s %d %d %s %s %s" _ StringBuff(ip) _ msgCode _ (int)size _ command _ StringBuff(url) _ contentType); + StringFree(ip); + } + + /* Send reply */ + StringStrcat(headers, "Server: ProxyTrack " PROXYTRACK_VERSION " (HTTrack " HTTRACK_VERSIONID ")\r\n"); + StringStrcat(headers, "\r\n"); /* Headers separator */ + if (send(soc_c, StringBuff(headers), StringLength(headers), 0) != StringLength(headers) + || ( !headRequest && StringLength(output) > 0 && send(soc_c, StringBuff(output), StringLength(output), 0) != StringLength(output)) + || ( !headRequest && StringLength(output) == 0 && element != NULL && element->adr != NULL && send(soc_c, element->adr, element->size, 0) != element->size) + ) + { + keepAlive = 0; /* Error, abort connection */ + } + PT_Element_Delete(&element); + + /* Shutdown (FIN) and wait until confirmed */ + if (!keepAlive) { + char c; +#ifdef _WIN32 + shutdown(soc_c, SD_SEND); +#else + shutdown(soc_c, 1); +#endif + while(recv(soc_c, ((char*)&c), 1, 0) > 0); + } + } while(keepAlive); + +#if HTS_WIN + closesocket(soc_c); +#else + close(soc_c); +#endif + + StringFree(url); + StringFree(urlRedirect); + StringFree(headers); + StringFree(output); + StringFree(host); + + if (buffer) + free(buffer); +} + +#ifdef _WIN32 +#define PTHREAD_RETURN +#define PTHREAD_TYPE void +#define PTHREAD_TYPE_FNC __cdecl +#else +#define PTHREAD_RETURN NULL +#define PTHREAD_TYPE void* +#define PTHREAD_TYPE_FNC +#endif + +/* Generic threaded function start */ +static int startThread(PTHREAD_TYPE (PTHREAD_TYPE_FNC * funct)(void* ), + void* param) +{ + if (param != NULL) { +#ifdef _WIN32 + if (_beginthread(funct, 0, param) == -1) { + free(param); + return 0; + } + return 1; +#else + pthread_t handle = 0; + int retcode; + retcode = pthread_create(&handle, NULL, funct, param); + if (retcode != 0) { /* error */ + free(param); + return 0; + } else { + /* detach the thread from the main process so that is can be independent */ + pthread_detach(handle); + return 1; + } +#endif + } else { + return 0; + } +} + +/* Generic socket/index structure */ +typedef struct proxytrack_process_th_p { + T_SOC soc_c; + PT_Indexes indexes; + void (*process)(PT_Indexes indexes, T_SOC soc_c); +} proxytrack_process_th_p; + +/* Generic socket/index function stub */ +static PTHREAD_TYPE PTHREAD_TYPE_FNC proxytrack_process_th(void* param_) { + proxytrack_process_th_p *param = (proxytrack_process_th_p *) param_; + T_SOC soc_c = param->soc_c; + PT_Indexes indexes = param->indexes; + void (*process)(PT_Indexes indexes, T_SOC soc_c) = param->process; + free(param); + process(indexes, soc_c); + return PTHREAD_RETURN ; +} + +/* Process generic socket/index operation */ +static int proxytrack_process_generic(void (*process)(PT_Indexes indexes, T_SOC soc_c), + PT_Indexes indexes, T_SOC soc_c) +{ + proxytrack_process_th_p *param = calloc(sizeof(proxytrack_process_th_p), 1); + if (param != NULL) { + param->soc_c = soc_c; + param->indexes = indexes; + param->process = process; + return startThread(proxytrack_process_th, param); + } else { + CRITICAL("proxytrack_process_generic:Memory exhausted"); + return 0; + } + return 0; +} + +/* Process HTTP proxy requests */ +static int proxytrack_process_HTTP_threaded(PT_Indexes indexes, T_SOC soc) { + return proxytrack_process_generic(proxytrack_process_HTTP, indexes, soc); +} + +/* HTTP Server */ +static int proxytrack_start_HTTP(PT_Indexes indexes, T_SOC soc) { + while(soc != INVALID_SOCKET) { + T_SOC soc_c; + struct sockaddr clientAddr; + int clientAddrLen = sizeof(struct sockaddr); + memset(&clientAddr, 0, sizeof(clientAddr)); + if ( (soc_c = accept(soc, &clientAddr, &clientAddrLen)) != INVALID_SOCKET) { + if (!proxytrack_process_HTTP_threaded(indexes, soc_c)) { + CRITICAL("proxytrack_start_HTTP::Can not fork a thread"); + } + } + } + if (soc != INVALID_SOCKET) { +#ifdef _WIN32 + closesocket(soc); +#else + close(soc); +#endif + } + return 1; +} + +/* Network order is big endian */ +#define READ_NET16(buffer) ( ( ((unsigned char*)buffer)[0] << 8 ) + ((unsigned char*)buffer)[1] ) +#define READ_NET32(buffer) ( ( READ_NET16(buffer) << 16 ) + READ_NET16(((unsigned char*)buffer) + 2) ) +#define WRITE_NET8(buffer, value) do { \ + ((unsigned char*)buffer)[0] = (unsigned char)(value); \ +} while(0) +#define WRITE_NET16(buffer, value) do { \ + ((unsigned char*)buffer)[0] = (((unsigned short)(value)) >> 8) & 0xff; \ + ((unsigned char*)buffer)[1] = ((unsigned short)(value)) & 0xff; \ +} while(0) +#define WRITE_NET32(buffer, value) do { \ + WRITE_NET16(buffer, ( ((unsigned int)(value)) >> 16 ) & 0xffff); \ + WRITE_NET16(((unsigned char*)buffer) + 2, ( ((unsigned int)(value)) ) & 0xffff); \ +} while(0) + +static int ICP_reply(struct sockaddr * clientAddr, + int clientAddrLen, + T_SOC soc, + /* */ + unsigned char Opcode, + unsigned char Version, + unsigned short Message_Length, + unsigned int Request_Number, + unsigned int Options, + unsigned int Option_Data, + unsigned int Sender_Host_Address, + unsigned char *Message + ) +{ + int ret = 0; + unsigned long int BufferSize; + unsigned char * buffer; + if (Message_Length == 0 && Message != NULL) /* We have to get the message size */ + Message_Length = (unsigned int) strlen(Message) + 1; /* NULL terminated */ + BufferSize = 20 + Message_Length; + buffer = malloc(BufferSize); + if (buffer != NULL) { + WRITE_NET8(&buffer[0], Opcode); + WRITE_NET8(&buffer[1], Version); + WRITE_NET16(&buffer[2], Message_Length); + WRITE_NET32(&buffer[4], Request_Number); + WRITE_NET32(&buffer[8], Options); + WRITE_NET32(&buffer[12], Option_Data); + WRITE_NET32(&buffer[16], Sender_Host_Address); + if (Message != NULL && Message_Length > 0) { + memcpy(buffer + 20, Message, Message_Length); + } + if (sendto(soc, buffer, BufferSize, 0, clientAddr, clientAddrLen) == BufferSize) { + ret = 1; + } + free(buffer); + } + return ret; +} + +/* ICP Server */ +static int proxytrack_start_ICP(PT_Indexes indexes, T_SOC soc) { + /* "ICP messages MUST not exceed 16,384 octets in length." (RFC2186) */ + int bufferSize = 16384; + unsigned char * buffer = (unsigned char*) malloc(bufferSize + 1); + if (buffer == NULL) { + CRITICAL("proxytrack_start_ICP:memory exhausted"); +#ifdef _WIN32 + closesocket(soc); +#else + close(soc); +#endif + return -1; + } + while(soc != INVALID_SOCKET) { + struct sockaddr clientAddr; + int clientAddrLen = sizeof(struct sockaddr); + int n; + memset(&clientAddr, 0, sizeof(clientAddr)); + n = recvfrom(soc, (char*)buffer, bufferSize, 0, &clientAddr, &clientAddrLen); + if (n != -1) { + const char * LogRequest = "ERROR"; + const char * LogReply = "ERROR"; + unsigned char * UrlRequest = NULL; + if (n >= 20) { + enum { + ICP_OP_MIN = 0, + ICP_OP_INVALID = 0, + ICP_OP_QUERY = 1, + ICP_OP_HIT = 2, + ICP_OP_MISS = 3, + ICP_OP_ERR = 4, + ICP_OP_SECHO = 10, + ICP_OP_DECHO = 11, + ICP_OP_MISS_NOFETCH = 21, + ICP_OP_DENIED = 22, + ICP_OP_HIT_OBJ = 23, + ICP_OP_MAX = ICP_OP_HIT_OBJ + }; + unsigned char Opcode = buffer[0]; + unsigned char Version = buffer[1]; + unsigned short Message_Length = READ_NET16(&buffer[2]); + unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */ + unsigned int Options = READ_NET32(&buffer[8]); + unsigned int Option_Data = READ_NET32(&buffer[12]); /* ICP_FLAG_SRC_RTT */ + unsigned int Sender_Host_Address = READ_NET32(&buffer[16]); /* ignored */ + unsigned char* Payload = &buffer[20]; + buffer[bufferSize] = '\0'; /* Ensure payload is NULL terminated */ + if (Message_Length <= bufferSize - 20) { + if (Opcode >= ICP_OP_MIN && Opcode <= ICP_OP_MAX) { + if (Version == 2) { + switch(Opcode) { + case ICP_OP_QUERY: + { + unsigned int UrlRequestSize; + UrlRequest = &Payload[4]; + UrlRequestSize = (unsigned int)strlen((char*)UrlRequest); + LogRequest = "ICP_OP_QUERY"; + if (indexes == NULL) { + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_DENIED, Version, 0, Request_Number, 0, 0, 0, UrlRequest); + LogReply = "ICP_OP_DENIED"; + } else if (PT_LookupIndex(indexes, UrlRequest)) { + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_HIT, Version, 0, Request_Number, 0, 0, 0, UrlRequest); + LogReply = "ICP_OP_HIT"; + } else { + if (UrlRequestSize > 0 && UrlRequest[UrlRequestSize - 1] != '/' && strchr(UrlRequest, '?') == NULL) { + char * UrlRedirect = malloc(UrlRequestSize + 1 + 1); + if (UrlRedirect != NULL) { + sprintf(UrlRedirect, "%s/", UrlRequest); + if (PT_LookupIndex(indexes, UrlRedirect)) { /* We'll generate a redirect */ + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_HIT, Version, 0, Request_Number, 0, 0, 0, UrlRequest); + LogReply = "ICP_OP_HIT"; + free(UrlRedirect); + break; + } + free(UrlRedirect); + } + } + /* We won't retrive the cache MISS online, no way! */ + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_MISS_NOFETCH, Version, 0, Request_Number, 0, 0, 0, UrlRequest); + LogReply = "ICP_OP_MISS_NOFETCH"; + } + } + break; + case ICP_OP_SECHO: + { + UrlRequest = &Payload[4]; + LogRequest = "ICP_OP_QUERY"; + LogReply = "ICP_OP_QUERY"; + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_SECHO, Version, 0, Request_Number, 0, 0, 0, UrlRequest); + } + break; + default: + LogRequest = "NOTIMPLEMENTED"; + LogReply = "ICP_OP_ERR"; + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_ERR, Version, 0, Request_Number, 0, 0, 0, NULL); + break; + } + } else { + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_ERR, 2, 0, Request_Number, 0, 0, 0, NULL); + } + } /* Ignored (RFC2186) */ + } else { + ICP_reply(&clientAddr, clientAddrLen, soc, ICP_OP_ERR, Version, 0, Request_Number, 0, 0, 0, NULL); + } + } + + /* Logging */ + { + String ip = STRING_EMPTY; + SOCaddr serverClient; + int lenServerClient = (int) sizeof(serverClient); + SOCaddr_copyaddr(serverClient, lenServerClient, &clientAddr, clientAddrLen); + if (lenServerClient > 0) { + ip = getip(&serverClient, lenServerClient); + } else { + StringStrcpy(ip, "unknown"); + } + LOG("ICP %s %s/%s %s" _ StringBuff(ip) _ LogRequest _ LogReply _ (UrlRequest ? UrlRequest : "-") ); + StringFree(ip); + } + + } + } + if (soc != INVALID_SOCKET) { +#ifdef _WIN32 + closesocket(soc); +#else + close(soc); +#endif + } + free(buffer); + return 1; +} + +static int proxytrack_start(PT_Indexes indexes, T_SOC soc, T_SOC socICP) { + int ret = 1; + if (proxytrack_process_generic(proxytrack_start_ICP, indexes, socICP)) { + //if (!proxytrack_process_generic(proxytrack_start_HTTP, indexes, soc)) + if (!proxytrack_start_HTTP(indexes, soc)) { + ret = 0; + } + } else { + ret = 0; + } + return ret; +} + diff --git a/src/proxy/proxytrack.h b/src/proxy/proxytrack.h new file mode 100644 index 0000000..498f4d8 --- /dev/null +++ b/src/proxy/proxytrack.h @@ -0,0 +1,288 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) Xavier Roche and other contributors + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +Please visit our Website: http://www.httrack.com +*/ + +/* ------------------------------------------------------------ */ +/* File: ProxyTrack, httrack cache-based proxy */ +/* Author: Xavier Roche */ +/* ------------------------------------------------------------ */ + +#ifndef WEBHTTRACK_PROXYTRACK +#define WEBHTTRACK_PROXYTRACK + +/* Version */ +#define PROXYTRACK_VERSION "0.4" + +/* Store manager */ +#include "../minizip/mztools.h" +#include "store.h" + +#include <sys/stat.h> + +/* generic */ + +int proxytrack_main(char* proxyAddr, int proxyPort, + char* icpAddr, int icpPort, + PT_Indexes index); + +/* Spaces: CR,LF,TAB,FF */ +#define is_space(c) ( ((c)==' ') || ((c)=='\"') || ((c)==10) || ((c)==13) || ((c)==9) || ((c)==12) || ((c)==11) || ((c)=='\'') ) +#define is_realspace(c) ( ((c)==' ') || ((c)==10) || ((c)==13) || ((c)==9) || ((c)==12) || ((c)==11) ) +#define is_taborspace(c) ( ((c)==' ') || ((c)==9) ) +#define is_quote(c) ( ((c)=='\"') || ((c)=='\'') ) +#define is_retorsep(c) ( ((c)==10) || ((c)==13) || ((c)==9) ) + +/* Static definitions */ + +#define _ , +#define CRITICAL_(msg, file, line) do { \ + fprintf(stderr, "* critical: "); \ + fprintf(stderr, msg); \ + fprintf(stderr, " at %s:%d\n", file, line); \ + fflush(stderr); \ +} while(0) +#define CRITICAL(msg) do { \ + fprintf(stderr, "* critical: "); \ + fprintf(stderr, msg); \ + fprintf(stderr, " at %s:%d\n", __FILE__, __LINE__); \ + fflush(stderr); \ +} while(0) + +#define WARNING(msg) do { \ + fprintf(stderr, "* warning: "); \ + fprintf(stderr, msg); \ + fprintf(stderr, "\n"); \ + fflush(stderr); \ +} while(0) + +#define LOG(msg) do { \ + fprintf(stderr, "* log: "); \ + fprintf(stderr, msg); \ + fprintf(stderr, "\n"); \ + fflush(stderr); \ +} while(0) + +#if defined(_DEBUG) || defined(DEBUG) +#define DEBUG(msg) do { \ + fprintf(stderr, "* debug: "); \ + fprintf(stderr, msg); \ + fprintf(stderr, "\n"); \ + fflush(stderr); \ +} while(0) +#else +#define DEBUG_(msg, file, line) do { } while(0) +#define DEBUG(msg) do { } while(0) +#endif + +/* Header for generated pages */ +#define PROXYTRACK_COMMENT_HEADER \ + "<!-- Generated by ProxyTrack " PROXYTRACK_VERSION " build " __DATE__ " -->\r\n" \ + "<!-- This is an add-on for HTTrack " HTTRACK_VERSIONID " -->\r\n" + +/* See IE "feature" (MSKB Q294807) */ +#define DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES \ + "<!-- Start Disable IE Friendly HTTP Error Messages -->\r\n" \ + "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ + "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ + "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ + "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ + "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ + "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ + "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ + "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ + "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ + "<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \ + "<!-- End Disable IE Friendly HTTP Error Messages -->\r\n" + +static char* gethomedir(void) { + char* home = getenv( "HOME" ); + if (home) + return home; + else + return "."; +} + +static int linput(FILE* fp,char* s,int max) { + int c; + int j=0; + do { + c=fgetc(fp); + if (c!=EOF) { + switch(c) { + case 13: break; // sauter CR + case 10: c=-1; break; + case 0: case 9: case 12: break; // sauter ces caractères + default: s[j++]=(char) c; break; + } + } + } while((c!=-1) && (c!=EOF) && (j<(max-1))); + s[j]='\0'; + return j; +} + +static int link_has_authority(const char* lien) { + const char* a=lien; + if (isalpha((const unsigned char)*a)) { + // Skip scheme? + while (isalpha((const unsigned char)*a)) + a++; + if (*a == ':') + a++; + else + return 0; + } + if (strncmp(a,"//",2) == 0) + return 1; + return 0; +} + +static const char* jump_protocol(const char* source) { + int p; + // scheme + // "Comparisons of scheme names MUST be case-insensitive" (RFC2616) + if ((p = strfield(source,"http:"))) + source+=p; + else if ((p = strfield(source,"ftp:"))) + source+=p; + else if ((p = strfield(source,"https:"))) + source+=p; + else if ((p = strfield(source,"file:"))) + source+=p; + else if ((p = strfield(source,"mms:"))) + source+=p; + // net_path + if (strncmp(source,"//",2)==0) + source+=2; + return source; +} + +static const char* strrchr_limit(const char* s, char c, const char* limit) { + if (limit == NULL) { + char* p = strrchr(s, c); + return p?(p+1):NULL; + } else { + char *a=NULL, *p; + for(;;) { + p=strchr((a)?a:s, c); + if ((p >= limit) || (p == NULL)) + return a; + a=p+1; + } + } +} + +static const char* jump_protocol_and_auth(const char* source) { + const char *a,*trytofind; + if (strcmp(source, "file://") == 0) + return source; + a = jump_protocol(source); + trytofind = strrchr_limit(a, '@', strchr(a,'/')); + return (trytofind != NULL)?trytofind:a; +} + +#ifndef min +#define min(a,b) ((a)>(b)?(b):(a)) +#endif +#ifndef max +#define max(a,b) ((a)>(b)?(a):(b)) +#endif +static int linput_trim(FILE* fp,char* s,int max) { + int rlen=0; + char* ls=(char*) malloc(max+2); + s[0]='\0'; + if (ls) { + char* a; + // lire ligne + rlen=linput(fp,ls,max); + if (rlen) { + // sauter espaces et tabs en fin + while( (rlen>0) && is_realspace(ls[max(rlen-1,0)]) ) + ls[--rlen]='\0'; + // sauter espaces en début + a=ls; + while((rlen>0) && ((*a==' ') || (*a=='\t'))) { + a++; + rlen--; + } + if (rlen>0) { + memcpy(s,a,rlen); // can copy \0 chars + s[rlen]='\0'; + } + } + // + free(ls); + } + return rlen; +} + +// copy of concat +#define HTS_URLMAXSIZE 1024 +typedef struct concat_strc { + char buff[16][HTS_URLMAXSIZE*2*2]; + int rol; +} concat_strc; +static char* concat(const char* a,const char* b) { + static concat_strc* strc = NULL; + if (strc == NULL) { + strc = (concat_strc*) calloc(16, sizeof(concat_strc)); + } + strc->rol=((strc->rol+1)%16); // roving pointer + strcpy(strc->buff[strc->rol],a); + if (b) strcat(strc->buff[strc->rol],b); + return strc->buff[strc->rol]; +} + +#ifndef S_ISREG +#define S_ISREG(m) ((m) & _S_IFREG) +#endif +static int fexist(char* s) { + struct stat st; + memset(&st, 0, sizeof(st)); + if (stat(s, &st) == 0) { + if (S_ISREG(st.st_mode)) { + return 1; + } + } + return 0; +} + +#ifndef _WIN32 +#define fconv(a) (a) +#define fconcat(a,b) concat(a,b) +#endif + +#ifdef _WIN32 +static char* __fconv(char* a) { + int i; + for(i=0;i<(int) strlen(a);i++) + if (a[i]=='/') // convertir + a[i]='\\'; + return a; +} +static char* fconcat(char* a,char* b) { + return __fconv(concat(a,b)); +} +static char* fconv(char* a) { + return __fconv(concat(a,"")); +} +#endif + +#endif diff --git a/src/proxy/store.c b/src/proxy/store.c new file mode 100644 index 0000000..1d17574 --- /dev/null +++ b/src/proxy/store.c @@ -0,0 +1,1505 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) Xavier Roche and other contributors + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +Please visit our Website: http://www.httrack.com +*/ + +/* ------------------------------------------------------------ */ +/* File: Cache manager for ProxyTrack */ +/* Author: Xavier Roche */ +/* ------------------------------------------------------------ */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/* Locking */ +#ifdef _WIN32 +#include <process.h> /* _beginthread, _endthread */ +#else +#include <pthread.h> +#endif + +#include "htsglobal.h" + +#define HTS_INTERNAL_BYTECODE +#include "htsinthash.h" +#undef HTS_INTERNAL_BYTECODE +#include "../minizip/mztools.h" + +#include "htscore.h" +#include "htsback.h" + +#include "store.h" +#include "proxystrings.h" +#include "proxytrack.h" + +/* Unlocked functions */ + +static int PT_LookupCache__New_u(PT_Index index, const char* url); +static PT_Element PT_ReadCache__New_u(PT_Index index, const char* url, int flags); + +static int PT_LookupCache__Old_u(PT_Index index, const char* url); +static PT_Element PT_ReadCache__Old_u(PT_Index index, const char* url, int flags); + + +/* Locking */ + +#ifdef _WIN32 +void MutexInit(PT_Mutex *pMutex) { + *pMutex = CreateMutex(NULL,FALSE,NULL); +} + +void MutexLock(PT_Mutex *pMutex) { + WaitForSingleObject(*pMutex, INFINITE); +} + +void MutexUnlock(PT_Mutex *pMutex) { + ReleaseMutex(*pMutex); +} + +void MutexFree(PT_Mutex *pMutex) { + CloseHandle(*pMutex); + *pMutex = NULL; +} +#else +void MutexInit(PT_Mutex *pMutex) { + (void) pthread_mutex_init(pMutex, 0); +} + +void MutexLock(PT_Mutex *pMutex) { + pthread_mutex_lock(pMutex); +} + +void MutexUnlock(PT_Mutex *pMutex) { + pthread_mutex_unlock(pMutex); +} + +void MutexFree(PT_Mutex *pMutex) { + pthread_mutex_destroy(pMutex); +} +#endif + +/* Indexes */ + +typedef struct _PT_Index__New _PT_Index__New; +typedef struct _PT_Index__Old _PT_Index__Old; +typedef struct _PT_Index_Functions _PT_Index_Functions; + +typedef struct _PT_Index__New *PT_Index__New; +typedef struct _PT_Index__Old *PT_Index__Old; +typedef struct _PT_Index_Functions *PT_Index_Functions; + +enum { + PT_CACHE_UNDEFINED = -1, + PT_CACHE_MIN = 0, + PT_CACHE__NEW = PT_CACHE_MIN, + PT_CACHE__OLD, + PT_CACHE_MAX = PT_CACHE__OLD +}; + +static int PT_LoadCache__New(PT_Index index, const char *filename); +static void PT_Index_Delete__New(PT_Index *pindex); +static PT_Element PT_ReadCache__New(PT_Index index, const char* url, int flags); +static int PT_LookupCache__New(PT_Index index, const char* url); +/**/ +static int PT_LoadCache__Old(PT_Index index, const char *filename); +static void PT_Index_Delete__Old(PT_Index *pindex); +static PT_Element PT_ReadCache__Old(PT_Index index, const char* url, int flags); +static int PT_LookupCache__Old(PT_Index index, const char* url); + +struct _PT_Index_Functions { + int (*PT_LoadCache)(PT_Index index, const char *filename); + void (*PT_Index_Delete)(PT_Index *pindex); + PT_Element (*PT_ReadCache)(PT_Index index, const char* url, int flags); + int (*PT_LookupCache)(PT_Index index, const char* url); +}; + +static _PT_Index_Functions _IndexFuncts[] = { + { PT_LoadCache__New, PT_Index_Delete__New, PT_ReadCache__New, PT_LookupCache__New }, + { PT_LoadCache__Old, PT_Index_Delete__Old, PT_ReadCache__Old, PT_LookupCache__Old }, + { NULL, NULL, NULL, NULL } +}; + +#define PT_INDEX_COMMON_STRUCTURE \ + time_t timestamp; \ + inthash hash; \ + char startUrl[1024] + +struct _PT_Index__New { + PT_INDEX_COMMON_STRUCTURE; + char path[1024]; /* either empty, or must include ending / */ + int fixedPath; + int safeCache; + unzFile zFile; + PT_Mutex zFileLock; +}; + +struct _PT_Index__Old { + PT_INDEX_COMMON_STRUCTURE; + char filenameDat[1024]; + char filenameNdx[1024]; + FILE *dat,*ndx; + PT_Mutex fileLock; + int version; + char lastmodified[1024]; + char path[1024]; /* either empty, or must include ending / */ + int fixedPath; + int safeCache; +}; + +struct _PT_Index { + int type; + union { + _PT_Index__New formatNew; + _PT_Index__Old formatOld; + struct { + PT_INDEX_COMMON_STRUCTURE; + } common; + } slots; +}; + +struct _PT_Indexes { + inthash cil; + struct _PT_Index **index; + int index_size; +}; + +struct _PT_CacheItem { + time_t lastUsed; + size_t size; + void* data; +}; + +struct _PT_Cache { + inthash index; + size_t maxSize; + size_t totalSize; + int count; +}; + +PT_Indexes PT_New() { + PT_Indexes index = (PT_Indexes) calloc(sizeof(_PT_Indexes), 1); + index->cil = inthash_new(127); + index->index_size = 0; + index->index = NULL; + return index; +} + +void PT_Delete(PT_Indexes index) { + if (index != NULL) { + inthash_delete(&index->cil); + free(index); + } +} + +int PT_RemoveIndex(PT_Indexes index, int indexId) { + return 0; +} + +#define assertf(exp) + +static int binput(char* buff,char* s,int max) { + int count = 0; + int destCount = 0; + + // Note: \0 will return 1 + while(destCount < max && buff[count] != '\0' && buff[count] != '\n') { + if (buff[count] != '\r') { + s[destCount++] = buff[count]; + } + count++; + } + s[destCount] = '\0'; + + // then return the supplemental jump offset + return count + 1; +} + +static time_t file_timestamp(const char* file) { + struct stat buf; + if (stat(file, &buf) == 0) { + time_t tt = buf.st_mtime; + if (tt != (time_t) 0 && tt != (time_t) -1) { + return tt; + } + } + return (time_t) 0; +} + +static int PT_Index_Check__(PT_Index index, const char* file, int line) { + if (index == NULL) + return 0; + if (index->type >= PT_CACHE_MIN && index->type <= PT_CACHE_MAX) + return 1; + CRITICAL_("index corrupted in memory", file, line); + return 0; +} +#define SAFE_INDEX(index) PT_Index_Check__(index, __FILE__, __LINE__) + + +/* ------------------------------------------------------------ */ +/* Generic cache dispatch */ +/* ------------------------------------------------------------ */ + +void PT_Index_Delete(PT_Index *pindex) { + if (pindex != NULL && (*pindex) != NULL) { + PT_Index index = *pindex; + if (SAFE_INDEX(index)) { + _IndexFuncts[index->type].PT_Index_Delete(pindex); + } + free(index); + *pindex = NULL; + } +} + +static void PT_Index_Delete__New(PT_Index *pindex) { + if (pindex != NULL && (*pindex) != NULL) { + PT_Index__New index = &(*pindex)->slots.formatNew; + if (index->zFile != NULL) { + unzClose(index->zFile); + index->zFile = NULL; + } + if (index->hash != NULL) { + inthash_delete(&index->hash); + index->hash = NULL; + } + MutexFree(&index->zFileLock); + } +} + +static void PT_Index_Delete__Old(PT_Index *pindex) { + if (pindex != NULL && (*pindex) != NULL) { + PT_Index__Old index = &(*pindex)->slots.formatOld; + if (index->dat != NULL) { + fclose(index->dat); + } + if (index->ndx != NULL) { + fclose(index->ndx); + } + if (index->hash != NULL) { + inthash_delete(&index->hash); + index->hash = NULL; + } + MutexFree(&index->fileLock); + } +} + +int PT_AddIndex(PT_Indexes indexes, const char *path) { + PT_Index index = PT_LoadCache(path); + if (index != NULL) { + int ret = PT_IndexMerge(indexes, &index); + if (index != NULL) { + PT_Index_Delete(&index); + } + return ret; + } + return -1; +} + +PT_Element PT_Index_HTML_BuildRootInfo(PT_Indexes indexes) { + if (indexes != NULL) { + PT_Element elt = PT_ElementNew(); + int i; + String html = STRING_EMPTY; + StringClear(html); + StringStrcat(html, + "<html>" + PROXYTRACK_COMMENT_HEADER + DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES + "<head>\r\n" + "<title>ProxyTrack " PROXYTRACK_VERSION " Catalog</title>" + "</head>\r\n" + "<body>\r\n" + "<h3>Available sites in this cache:</h3><br />" + "<br />" + ); + StringStrcat(html, "<ul>\r\n"); + for(i = 0 ; i < indexes->index_size ; i++) { + if (indexes->index[i] != NULL + && indexes->index[i]->slots.common.startUrl[0] != '\0') + { + const char * url = indexes->index[i]->slots.common.startUrl; + StringStrcat(html, "<li>\r\n"); + StringStrcat(html, "<a href=\""); + StringStrcat(html, url); + StringStrcat(html, "\">"); + StringStrcat(html, url); + StringStrcat(html, "</a>\r\n"); + StringStrcat(html, "</li>\r\n"); + } + } + StringStrcat(html, "</ul>\r\n"); + StringStrcat(html, "</body></html>\r\n"); + elt->size = StringLength(html); + elt->adr = StringAcquire(&html); + elt->statuscode = 200; + strcpy(elt->charset, "iso-8859-1"); + strcpy(elt->contenttype, "text/html"); + strcpy(elt->msg, "OK"); + StringFree(html); + return elt; + } + return NULL; +} + +static char* strchr_stop(char* str, char c, char stop) { + for( ; *str != 0 && *str != stop && *str != c ; str++); + if (*str == c) + return str; + return NULL; +} + +char ** PT_Enumerate(PT_Indexes indexes, const char *url, int subtree) { + // should be cached! + if (indexes != NULL && indexes->cil != NULL) { + unsigned int urlSize; + String list = STRING_EMPTY; + String listindexes = STRING_EMPTY; + String subitem = STRING_EMPTY; + unsigned int listCount = 0; + struct_inthash_enum en = inthash_enum_new(indexes->cil); + inthash_chain* chain; + inthash hdupes = NULL; + if (!subtree) + hdupes= inthash_new(127); + StringClear(list); + StringClear(listindexes); + StringClear(subitem); + if (strncmp(url, "http://", 7) == 0) + url += 7; + urlSize = (unsigned int) strlen(url); + while((chain = inthash_enum_next(&en))) { + long int index = (long int)chain->value.intg; + if (urlSize == 0 || strncmp(chain->name, url, urlSize) == 0) { + if (index >= 0 && index < indexes->index_size) { + char * item = chain->name + urlSize; + if (*item == '/') + item++; + { + char * pos = subtree ? 0 : strchr_stop(item, '/', '?'); + unsigned int len = pos ? (unsigned int)( pos - item ) : (unsigned int)strlen(item); + if (len > 0 /* default document */ || *item == 0) { + int isFolder = ( item[len] == '/' ); + StringClear(subitem); + if (len > 0) + StringMemcat(subitem, item, len); + if (len == 0 || !inthash_exists(hdupes, StringBuff(subitem))) { + char* ptr = NULL; + ptr += StringLength(list); + if (len > 0) + StringStrcat(list, StringBuff(subitem)); + if (isFolder) + StringStrcat(list, "/"); + StringMemcat(list, "\0", 1); /* NULL terminated strings */ + StringMemcat(listindexes, &ptr, sizeof(ptr)); + listCount++; + inthash_write(hdupes, StringBuff(subitem), 0); + } + } + } + } else { + CRITICAL("PT_Enumerate:Corrupted central index locator"); + } + } + } + StringFree(subitem); + inthash_delete(&hdupes); + if (listCount > 0) { + unsigned int i; + void* blk; + char *nullPointer = NULL; + char* startStrings; + /* NULL terminated index */ + StringMemcat(listindexes, &nullPointer, sizeof(nullPointer)); + /* start of all strings (index) */ + startStrings = nullPointer + StringLength(listindexes); + /* copy list of URLs after indexes */ + StringMemcat(listindexes, StringBuff(list), StringLength(list)); + /* ---- no reallocation beyond this point (fixed addresses) ---- */ + /* start of all strings (pointer) */ + startStrings = (startStrings - nullPointer) + StringBuff(listindexes); + /* transform indexes into references */ + for(i = 0 ; i < listCount ; i++) { + char *ptr = NULL; + unsigned int ndx; + memcpy(&ptr, &StringBuff(listindexes)[i*sizeof(char*)], sizeof(char*)); + ndx = (unsigned int) (ptr - nullPointer); + ptr = startStrings + ndx; + memcpy(&StringBuff(listindexes)[i*sizeof(char*)], &ptr, sizeof(char*)); + } + blk = StringAcquire(&listindexes); + StringFree(list); + StringFree(listindexes); + return (char **)blk; + } + } + return NULL; +} + +void PT_Enumerate_Delete(char ***plist) { + if (plist != NULL && *plist != NULL) { + free(*plist); + *plist = NULL; + } +} + +PT_Index PT_LoadCache(const char *filename) { + int type = PT_CACHE_UNDEFINED; + char * dot = strrchr(filename, '.'); + if (dot != NULL) { + if (strcasecmp(dot, ".zip") == 0) { + type = PT_CACHE__NEW; + } else if (strcasecmp(dot, ".ndx") == 0 || strcasecmp(dot, ".dat") == 0) { + type = PT_CACHE__OLD; + } + } + if (type != PT_CACHE_UNDEFINED) { + PT_Index index = calloc(sizeof(_PT_Index), 1); + if (index != NULL) { + index->type = type; + index->slots.common.timestamp = (time_t) time(NULL); + index->slots.common.startUrl[0] = '\0'; + index->slots.common.hash = inthash_new(8191); + if (!_IndexFuncts[type].PT_LoadCache(index, filename)) { + DEBUG("reading httrack cache (format #%d) %s : error" _ type _ filename ); + free(index); + index = NULL; + return NULL; + } else { + DEBUG("reading httrack cache (format #%d) %s : success" _ type _ filename ); + } + /* default starting URL is the first hash entry */ + if (index->slots.common.startUrl[0] == '\0') { + struct_inthash_enum en = inthash_enum_new(index->slots.common.hash); + inthash_chain* chain; + chain = inthash_enum_next(&en); + if (chain != NULL + && strstr(chain->name, "/robots.txt") != NULL) + { + chain = inthash_enum_next(&en); + } + if (chain != NULL) { + if (!link_has_authority(chain->name)) + strcat(index->slots.common.startUrl, "http://"); + strcat(index->slots.common.startUrl, chain->name); + } + } + } + return index; + } + return NULL; +} + + +static long int filesize(const char* filename) { + struct stat st; + memset(&st, 0, sizeof(st)); + if (stat(filename, &st) == 0) { + return (long int)st.st_size; + } + return -1; +} + +int PT_LookupCache(PT_Index index, const char* url) { + if (index != NULL && SAFE_INDEX(index)) { + return _IndexFuncts[index->type].PT_LookupCache(index, url); + } + return 0; +} + +time_t PT_Index_Timestamp(PT_Index index) { + return index->slots.common.timestamp; +} + +static int PT_LookupCache__New(PT_Index index, const char* url) { + int retCode; + MutexLock(&index->slots.formatNew.zFileLock); + { + retCode = PT_LookupCache__New_u(index, url); + } + MutexUnlock(&index->slots.formatNew.zFileLock); + return retCode; +} + +static int PT_LookupCache__New_u(PT_Index index_, const char* url) { + if (index_ != NULL) { + PT_Index__New index = &index_->slots.formatNew; + if (index->hash != NULL && index->zFile != NULL && url != NULL && *url != 0) { + int hash_pos_return; + if (strncmp(url, "http://", 7) == 0) + url += 7; + hash_pos_return = inthash_read(index->hash, url, NULL); + if (hash_pos_return) + return 1; + } + } + return 0; +} + +int PT_IndexMerge(PT_Indexes indexes, PT_Index *pindex) +{ + if (pindex != NULL && *pindex != NULL && (*pindex)->slots.common.hash != NULL + && indexes != NULL) + { + PT_Index index = *pindex; + struct_inthash_enum en = inthash_enum_new(index->slots.common.hash); + inthash_chain* chain; + int index_id = indexes->index_size++; + int nMerged = 0; + if ((indexes->index = realloc(indexes->index, sizeof(struct _PT_Index)*indexes->index_size)) != NULL) { + indexes->index[index_id] = index; + *pindex = NULL; + while((chain = inthash_enum_next(&en)) != NULL) { + const char * url = chain->name; + if (url != NULL && url[0] != '\0') { + long int previous_index_id = 0; + if (inthash_read(indexes->cil, url, (long int*)&previous_index_id)) { + if (previous_index_id >= 0 && previous_index_id < indexes->index_size) { + if (indexes->index[previous_index_id]->slots.common.timestamp > index->slots.common.timestamp) // existing entry is newer + break; + } else { + CRITICAL("PT_IndexMerge:Corrupted central index locator"); + } + } + inthash_write(indexes->cil, chain->name, index_id); + nMerged++; + } + } + } else { + CRITICAL("PT_IndexMerge:Memory exhausted"); + } + return nMerged; + } + return -1; +} + +void PT_Element_Delete(PT_Element *pentry) { + if (pentry != NULL) { + PT_Element entry = *pentry; + if (entry != NULL) { + if (entry->adr != NULL) { + free(entry->adr); + entry->adr = NULL; + } + if (entry->headers != NULL) { + free(entry->headers); + entry->headers = NULL; + } + if (entry->location != NULL) { + free(entry->location); + entry->location = NULL; + } + free(entry); + } + *pentry = NULL; + } +} + +PT_Element PT_ReadIndex(PT_Indexes indexes, const char* url, int flags) +{ + if (indexes != NULL) + { + long int index_id; + if (strncmp(url, "http://", 7) == 0) + url += 7; + if (inthash_read(indexes->cil, url, &index_id)) { + if (index_id >= 0 && index_id <= indexes->index_size) { + PT_Element item = PT_ReadCache(indexes->index[index_id], url, flags); + if (item != NULL) { + item->indexId = index_id; + return item; + } + } else { + CRITICAL("PT_ReadCache:Corrupted central index locator"); + } + } + } + return NULL; +} + +int PT_LookupIndex(PT_Indexes indexes, const char* url) { + if (indexes != NULL) + { + long int index_id; + if (strncmp(url, "http://", 7) == 0) + url += 7; + if (inthash_read(indexes->cil, url, &index_id)) { + if (index_id >= 0 && index_id <= indexes->index_size) { + return 1; + } else { + CRITICAL("PT_ReadCache:Corrupted central index locator"); + } + } + } + return 0; +} + +PT_Index PT_GetIndex(PT_Indexes indexes, int indexId) { + if (indexes != NULL && indexId >= 0 && indexId < indexes->index_size) + { + return indexes->index[indexId]; + } + return NULL; +} + +PT_Element PT_ElementNew() { + PT_Element r = NULL; + if ((r = calloc(sizeof(_PT_Element), 1)) == NULL) + return NULL; + r->statuscode=STATUSCODE_INVALID; + r->indexId = -1; + return r; +} + +PT_Element PT_ReadCache(PT_Index index, const char* url, int flags) { + if (index != NULL && SAFE_INDEX(index)) { + return _IndexFuncts[index->type].PT_ReadCache(index, url, flags); + } + return NULL; +} + +static PT_Element PT_ReadCache__New(PT_Index index, const char* url, int flags) { + PT_Element retCode; + MutexLock(&index->slots.formatNew.zFileLock); + { + retCode = PT_ReadCache__New_u(index, url, flags); + } + MutexUnlock(&index->slots.formatNew.zFileLock); + return retCode; +} + + +/* ------------------------------------------------------------ */ +/* New HTTrack cache (new.zip) format */ +/* ------------------------------------------------------------ */ + +#define ZIP_READFIELD_STRING(line, value, refline, refvalue) do { \ + if (line[0] != '\0' && strfield2(line, refline)) { \ + strcpy(refvalue, value); \ + line[0] = '\0'; \ + } \ +} while(0) +#define ZIP_READFIELD_INT(line, value, refline, refvalue) do { \ + if (line[0] != '\0' && strfield2(line, refline)) { \ + int intval = 0; \ + sscanf(value, "%d", &intval); \ + (refvalue) = intval; \ + line[0] = '\0'; \ + } \ +} while(0) + +int PT_LoadCache__New(PT_Index index_, const char *filename) { + if (index_ != NULL && filename != NULL) { + PT_Index__New index = &index_->slots.formatNew; + unzFile zFile = index->zFile = unzOpen(filename); + index->timestamp = file_timestamp(filename); + MutexInit(&index->zFileLock); + + // Opened ? + if (zFile!=NULL) { + const char * abpath; + int slashes; + inthash hashtable = index->hash; + + /* Compute base path for this index - the filename MUST be absolute! */ + for(slashes = 2, abpath = filename + (int)strlen(filename) - 1 + ; abpath > filename && ( ( *abpath != '/'&& *abpath != '\\' ) || --slashes > 0) + ; abpath--); + index->path[0] = '\0'; + if (slashes == 0 && *abpath != 0) { + int i; + strncat(index->path, filename, (int) ( abpath - filename ) + 1 ); + for(i = 0 ; index->path[i] != 0 ; i++) { + if (index->path[i] == '\\') { + index->path[i] = '/'; + } + } + } + + /* Ready directory entries */ + if (unzGoToFirstFile(zFile) == Z_OK) { + char comment[128]; + char filename[HTS_URLMAXSIZE * 4]; + int entries = 0; + int firstSeen = 0; + memset(comment, 0, sizeof(comment)); // for truncated reads + do { + int readSizeHeader = 0; + filename[0] = '\0'; + comment[0] = '\0'; + if (unzOpenCurrentFile(zFile) == Z_OK) { + if ( + (readSizeHeader = unzGetLocalExtrafield(zFile, comment, sizeof(comment) - 2)) > 0 + && + unzGetCurrentFileInfo(zFile, NULL, filename, sizeof(filename) - 2, NULL, 0, NULL, 0) == Z_OK + ) + { + long int pos = (long int) unzGetOffset(zFile); + assertf(readSizeHeader < sizeof(comment)); + comment[readSizeHeader] = '\0'; + entries++; + if (pos > 0) { + int dataincache = 0; // data in cache ? + char* filenameIndex = filename; + if (strncmp(filenameIndex, "http://", 7) == 0) { + filenameIndex += 7; + } + if (comment[0] != '\0') { + int maxLine = 2; + char* a = comment; + while(*a && maxLine-- > 0) { // parse only few first lines + char line[1024]; + line[0] = '\0'; + a+=binput(a, line, sizeof(line) - 2); + if (strncmp(line, "X-In-Cache:", 11) == 0) { + if (strcmp(line, "X-In-Cache: 1") == 0) { + dataincache = 1; + } else { + dataincache = 0; + } + break; + } + } + } + if (dataincache) + inthash_add(hashtable, filenameIndex, pos); + else + inthash_add(hashtable, filenameIndex, -pos); + + /* First link as starting URL */ + if (!firstSeen) { + if (strstr(filenameIndex, "/robots.txt") == NULL) { + firstSeen = 1; + if (!link_has_authority(filenameIndex)) + strcat(index->startUrl, "http://"); + strcat(index->startUrl, filenameIndex); + } + } + } else { + fprintf(stderr, "Corrupted cache meta entry #%d"LF, (int)entries); + } + } else { + fprintf(stderr, "Corrupted cache entry #%d"LF, (int)entries); + } + unzCloseCurrentFile(zFile); + } else { + fprintf(stderr, "Corrupted cache entry #%d"LF, (int)entries); + } + } while( unzGoToNextFile(zFile) == Z_OK ); + return 1; + } else { + inthash_delete(&index->hash); + index = NULL; + } + } else { + index = NULL; + } + } + return 0; +} + +static PT_Element PT_ReadCache__New_u(PT_Index index_, const char* url, int flags) +{ + PT_Index__New index = (PT_Index__New) &index_->slots.formatNew; + char location_default[HTS_URLMAXSIZE*2]; + char previous_save[HTS_URLMAXSIZE*2]; + char previous_save_[HTS_URLMAXSIZE*2]; + long int hash_pos; + int hash_pos_return; + PT_Element r = NULL; + if (index == NULL || index->hash == NULL || index->zFile == NULL || url == NULL || *url == 0) + return NULL; + if ((r = PT_ElementNew()) == NULL) + return NULL; + location_default[0] = '\0'; + previous_save[0] = previous_save_[0] = '\0'; + memset(r, 0, sizeof(_PT_Element)); + r->location = location_default; + strcpy(r->location, ""); + if (strncmp(url, "http://", 7) == 0) + url += 7; + hash_pos_return = inthash_read(index->hash, url, (long int*)&hash_pos); + + if (hash_pos_return) { + uLong posInZip; + if (hash_pos > 0) { + posInZip = (uLong) hash_pos; + } else { + posInZip = (uLong) -hash_pos; + } + if (unzSetOffset(index->zFile, posInZip) == Z_OK) { + /* Read header (Max 8KiB) */ + if (unzOpenCurrentFile(index->zFile) == Z_OK) { + char headerBuff[8192 + 2]; + int readSizeHeader; + int totalHeader = 0; + int dataincache = 0; + + /* For BIG comments */ + headerBuff[0] + = headerBuff[sizeof(headerBuff) - 1] + = headerBuff[sizeof(headerBuff) - 2] + = headerBuff[sizeof(headerBuff) - 3] = '\0'; + + if ( (readSizeHeader = unzGetLocalExtrafield(index->zFile, headerBuff, sizeof(headerBuff) - 2)) > 0) + { + int offset = 0; + char line[HTS_URLMAXSIZE + 2]; + int lineEof = 0; + headerBuff[readSizeHeader] = '\0'; + do { + char* value; + line[0] = '\0'; + offset += binput(headerBuff + offset, line, sizeof(line) - 2); + if (line[0] == '\0') { + lineEof = 1; + } + value = strchr(line, ':'); + if (value != NULL) { + *value++ = '\0'; + if (*value == ' ' || *value == '\t') value++; + ZIP_READFIELD_INT(line, value, "X-In-Cache", dataincache); + ZIP_READFIELD_INT(line, value, "X-Statuscode", r->statuscode); + ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r->msg); // msg + ZIP_READFIELD_INT(line, value, "X-Size", r->size); // size + ZIP_READFIELD_STRING(line, value, "Content-Type", r->contenttype); // contenttype + ZIP_READFIELD_STRING(line, value, "X-Charset", r->charset); // contenttype + ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified + ZIP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag + ZIP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved + ZIP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo); // Content-disposition + //ZIP_READFIELD_STRING(line, value, "X-Addr", ..); // Original address + //ZIP_READFIELD_STRING(line, value, "X-Fil", ..); // Original URI filename + ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename + } + } while(offset < readSizeHeader && !lineEof); + totalHeader = offset; + + /* Previous entry */ + if (previous_save_[0] != '\0') { + int pathLen = (int) strlen(index->path); + if (pathLen > 0 && strncmp(previous_save_, index->path, pathLen) == 0) { // old (<3.40) buggy format + strcpy(previous_save, previous_save_); + } + // relative ? (hack) + else if (index->safeCache + || (previous_save_[0] != '/' // /home/foo/bar.gif + && ( !isalpha(previous_save_[0]) || previous_save_[1] != ':' ) ) // c:/home/foo/bar.gif + ) + { + index->safeCache = 1; + sprintf(previous_save, "%s%s", index->path, previous_save_); + } + // bogus format (includes buggy absolute path) + else { + /* guess previous path */ + if (index->fixedPath == 0) { + const char * start = jump_protocol_and_auth(url); + const char * end = start ? strchr(start, '/') : NULL; + int len = (int) (end - start); + if (start != NULL && end != NULL && len > 0 && len < 128) { + char piece[128 + 2]; + const char * where; + piece[0] = '\0'; + strncat(piece, start, len); + if ((where = strstr(previous_save_, piece)) != NULL) { + index->fixedPath = (int) (where - previous_save_); // offset to relative path + } + } + } + if (index->fixedPath > 0) { + int saveLen = (int) strlen(previous_save_); + if (index->fixedPath < saveLen) { + sprintf(previous_save, "%s%s", index->path, previous_save_ + index->fixedPath); + } else { + sprintf(r->msg, "Bogus fixePath prefix for %s (prefixLen=%d)", previous_save_, (int)index->fixedPath); + r->statuscode = STATUSCODE_INVALID; + } + } else { + sprintf(previous_save, "%s%s", index->path, previous_save_); + } + } + } + + /* Complete fields */ + r->adr=NULL; + if (r->statuscode != STATUSCODE_INVALID) { /* Can continue */ + int ok = 0; + + // Court-circuit: + // Peut-on stocker le fichier directement sur disque? + if (ok) { + if (r->msg[0] == '\0') { + strcpy(r->msg,"Cache Read Error : Unexpected error"); + } + } else { // lire en mémoire + + if (!dataincache) { + /* Read in memory from cache */ + if (flags & FETCH_BODY) { + if (strnotempty(previous_save)) { + FILE* fp = fopen(fconv(previous_save), "rb"); + if (fp != NULL) { + r->adr = (char*) malloc(r->size + 4); + if (r->adr != NULL) { + if (r->size > 0 && fread(r->adr, 1, r->size, fp) != r->size) { + r->statuscode=STATUSCODE_INVALID; + sprintf(r->msg,"Read error in cache disk data: %s", strerror(errno)); + } + } else { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Read error (memory exhausted) from cache"); + } + fclose(fp); + } else { + r->statuscode=STATUSCODE_INVALID; + sprintf(r->msg, "Read error (can't open '%s') from cache", fconv(previous_save)); + } + } else { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cached file name is invalid"); + } + } + } else { + // lire fichier (d'un coup) + if (flags & FETCH_BODY) { + r->adr=(char*) malloc(r->size+1); + if (r->adr!=NULL) { + if (unzReadCurrentFile(index->zFile, r->adr, r->size) != r->size) { // erreur + free(r->adr); + r->adr=NULL; + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cache Read Error : Read Data"); + } else + *(r->adr+r->size)='\0'; + //printf(">%s status %d\n",back[p].r->contenttype,back[p].r->statuscode); + } else { // erreur + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cache Memory Error"); + } + } + } + } + } // si save==null, ne rien charger (juste en tête) + } else { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cache Read Error : Read Header Data"); + } + unzCloseCurrentFile(index->zFile); + } else { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cache Read Error : Open File"); + } + + } else { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cache Read Error : Bad Offset"); + } + } else { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"File Cache Entry Not Found"); + } + if (r->location[0] != '\0') { + r->location = strdup(r->location); + } else { + r->location = NULL; + } + return r; +} + + +/* ------------------------------------------------------------ */ +/* Old HTTrack cache (dat/ndx) format */ +/* ------------------------------------------------------------ */ + +static int cache_brstr(char* adr,char* s) { + int i; + int off; + char buff[256 + 1]; + off=binput(adr,buff,256); + adr+=off; + sscanf(buff,"%d",&i); + if (i>0) + strncpy(s,adr,i); + *(s+i)='\0'; + off+=i; + return off; +} + +static void cache_rstr(FILE* fp,char* s) { + INTsys i; + char buff[256+4]; + linput(fp,buff,256); + sscanf(buff,INTsysP,&i); + if (i < 0 || i > 32768) /* error, something nasty happened */ + i=0; + if (i>0) { + if ((int) fread(s,1,i,fp) != i) { + int fread_cache_failed = 0; + assertf(fread_cache_failed); + } + } + *(s+i)='\0'; +} + +static char* cache_rstr_addr(FILE* fp) { + INTsys i; + char* addr = NULL; + char buff[256+4]; + linput(fp,buff,256); + sscanf(buff,"%d",&i); + if (i < 0 || i > 32768) /* error, something nasty happened */ + i=0; + if (i > 0) { + addr = malloc(i + 1); + if (addr != NULL) { + if ((int) fread(addr,1,i,fp) != i) { + int fread_cache_failed = 0; + assertf(fread_cache_failed); + } + *(addr+i)='\0'; + } + } + return addr; +} + +static void cache_rint(FILE* fp,int* i) { + char s[256]; + cache_rstr(fp,s); + sscanf(s,"%d",i); +} + +static void cache_rLLint(FILE* fp,unsigned long* i) { + int l; + char s[256]; + cache_rstr(fp,s); + sscanf(s,"%d",&l); + *i = (unsigned long)l; +} + +static int PT_LoadCache__Old(PT_Index index_, const char *filename) { + if (index_ != NULL && filename != NULL) { + char * pos = strrchr(filename, '.'); + PT_Index__Old cache = &index_->slots.formatOld; + long int ndxSize; + cache->filenameDat[0] = '\0'; + cache->filenameNdx[0] = '\0'; + cache->path[0] = '\0'; + + { + PT_Index__Old index = cache; + const char * abpath; + int slashes; + /* -------------------- COPY OF THE __New() CODE -------------------- */ + /* Compute base path for this index - the filename MUST be absolute! */ + for(slashes = 2, abpath = filename + (int)strlen(filename) - 1 + ; abpath > filename && ( ( *abpath != '/'&& *abpath != '\\' ) || --slashes > 0) + ; abpath--); + index->path[0] = '\0'; + if (slashes == 0 && *abpath != 0) { + int i; + strncat(index->path, filename, (int) ( abpath - filename ) + 1 ); + for(i = 0 ; index->path[i] != 0 ; i++) { + if (index->path[i] == '\\') { + index->path[i] = '/'; + } + } + } + /* -------------------- END OF COPY OF THE __New() CODE -------------------- */ + } + + /* Index/data filenames */ + if (pos != NULL) { + int nLen = (int) (pos - filename); + strncat(cache->filenameDat, filename, nLen); + strncat(cache->filenameNdx, filename, nLen); + strcat(cache->filenameDat, ".dat"); + strcat(cache->filenameNdx, ".ndx"); + } + ndxSize = filesize(cache->filenameNdx); + cache->timestamp = file_timestamp(cache->filenameDat); + cache->dat = fopen(cache->filenameDat, "rb"); + cache->ndx = fopen(cache->filenameNdx, "rb"); + if (cache->dat != NULL && cache->ndx != NULL && ndxSize > 0) { + char * use = malloc(ndxSize + 1); + if (fread(use, 1, ndxSize, cache->ndx) == ndxSize) { + char firstline[256]; + char* a=use; + use[ndxSize] = '\0'; + a += cache_brstr(a, firstline); + if (strncmp(firstline,"CACHE-",6)==0) { // Nouvelle version du cache + if (strncmp(firstline,"CACHE-1.",8)==0) { // Version 1.1x + cache->version=(int)(firstline[8]-'0'); // cache 1.x + if (cache->version <= 5) { + a+=cache_brstr(a,firstline); + strcpy(cache->lastmodified,firstline); + } else { + // fprintf(opt->errlog,"Cache: version 1.%d not supported, ignoring current cache"LF,cache->version); + fclose(cache->dat); + cache->dat=NULL; + free(use); + use=NULL; + } + } else { // non supporté + // fspc(opt->errlog,"error"); fprintf(opt->errlog,"Cache: %s not supported, ignoring current cache"LF,firstline); + fclose(cache->dat); + cache->dat=NULL; + free(use); + use=NULL; + } + /* */ + } else { // Vieille version du cache + /* */ + // fspc(opt->log,"warning"); fprintf(opt->log,"Cache: importing old cache format"LF); + cache->version=0; // cache 1.0 + strcpy(cache->lastmodified,firstline); + } + + /* Create hash table for the cache (MUCH FASTER!) */ + if (use) { + char line[HTS_URLMAXSIZE*2]; + char linepos[256]; + int pos; + int firstSeen = 0; + while ( (a!=NULL) && (a < (use + ndxSize) ) ) { + a=strchr(a+1,'\n'); /* start of line */ + if (a) { + a++; + /* read "host/file" */ + a+=binput(a,line,HTS_URLMAXSIZE); + a+=binput(a,line+strlen(line),HTS_URLMAXSIZE); + /* read position */ + a+=binput(a,linepos,200); + sscanf(linepos,"%d",&pos); + + /* Add entry */ + inthash_add(cache->hash,line,pos); + + /* First link as starting URL */ + if (!firstSeen) { + if (strstr(line, "/robots.txt") == NULL) { + PT_Index__Old index = cache; + firstSeen = 1; + if (!link_has_authority(line)) + strcat(index->startUrl, "http://"); + strcat(index->startUrl, line); + } + } + + } + } + /* Not needed anymore! */ + free(use); + use=NULL; + return 1; + } + } + } + } + return 0; +} + +static String DecodeUrl(const char * url) { + int i; + String s = STRING_EMPTY; + StringClear(s); + for(i = 0 ; url[i] != '\0' ; i++) { + if (url[i] == '+') { + StringAddchar(s, ' '); + } else if (url[i] == '%') { + if (url[i + 1] == '%') { + StringAddchar(s, '%'); + i++; + } else if (url[i + 1] != 0 && url[i + 2] != 0) { + char tmp[3]; + int codepoint = 0; + tmp[0] = url[i + 1]; + tmp[1] = url[i + 2]; + tmp[2] = 0; + if (sscanf(tmp, "%x", &codepoint) == 1) { + StringAddchar(s, (char)codepoint); + } + i += 2; + } + } else { + StringAddchar(s, url[i]); + } + } + return s; +} + +static PT_Element PT_ReadCache__Old(PT_Index index, const char* url, int flags) { + PT_Element retCode; + MutexLock(&index->slots.formatOld.fileLock); + { + retCode = PT_ReadCache__Old_u(index, url, flags); + } + MutexUnlock(&index->slots.formatOld.fileLock); + return retCode; +} + +static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char* url, int flags) { + PT_Index__Old cache = (PT_Index__Old) &index_->slots.formatOld; + long int hash_pos; + int hash_pos_return; + char location_default[HTS_URLMAXSIZE*2]; + char previous_save[HTS_URLMAXSIZE*2]; + char previous_save_[HTS_URLMAXSIZE*2]; + PT_Element r; + int ok=0; + + if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0) + return NULL; + if ((r = PT_ElementNew()) == NULL) + return NULL; + location_default[0] = '\0'; + previous_save[0] = previous_save_[0] = '\0'; + memset(r, 0, sizeof(_PT_Element)); + r->location = location_default; + strcpy(r->location, ""); + if (strncmp(url, "http://", 7) == 0) + url += 7; + hash_pos_return=inthash_read(cache->hash, url, (long int*)&hash_pos); + + if (hash_pos_return) { + int pos = (int) hash_pos; /* simply */ + + if (fseek(cache->dat, (pos>0) ? pos : (-pos), SEEK_SET) == 0) { + /* Importer cache1.0 */ + if (cache->version==0) { + OLD_htsblk old_r; + if (fread((char*) &old_r,1,sizeof(old_r),cache->dat) == sizeof(old_r)) { // lire tout (y compris statuscode etc) + int i; + String urlDecoded; + r->statuscode = old_r.statuscode; + r->size = old_r.size; // taille fichier + strcpy(r->msg, old_r.msg); + strcpy(r->contenttype, old_r.contenttype); + + /* Guess the destination filename.. this sucks, because this method is not reliable. + Yes, the old 1.0 cache format was *that* bogus. /rx */ +#define FORBIDDEN_CHAR(c) (c == '~' \ + || c == '\\' \ + || c == ':' \ + || c == '*' \ + || c == '?' \ + || c == '\"' \ + || c == '<' \ + || c == '>' \ + || c == '|' \ + || c == '@' \ + || ((unsigned char) c ) <= 31 \ + || ((unsigned char) c ) == 127 \ + ) + urlDecoded = DecodeUrl(jump_protocol_and_auth(url)); + strcpy(previous_save_, StringBuff(urlDecoded)); + StringFree(urlDecoded); + for(i = 0 ; previous_save_[i] != '\0' && previous_save_[i] != '?' ; i++) { + if (FORBIDDEN_CHAR(previous_save_[i])) { + previous_save_[i] = '_'; + } + } + previous_save_[i] = '\0'; +#undef FORBIDDEN_CHAR + ok = 1; /* import ok */ + } + /* */ + /* Cache 1.1 */ + } else { + char check[256]; + unsigned long size_read; + check[0]='\0'; + // + cache_rint(cache->dat,&r->statuscode); + cache_rLLint(cache->dat,&r->size); + cache_rstr(cache->dat,r->msg); + cache_rstr(cache->dat,r->contenttype); + if (cache->version >= 3) + cache_rstr(cache->dat,r->charset); + cache_rstr(cache->dat,r->lastmodified); + cache_rstr(cache->dat,r->etag); + cache_rstr(cache->dat,r->location); + if (cache->version >= 2) + cache_rstr(cache->dat,r->cdispo); + if (cache->version >= 4) { + cache_rstr(cache->dat, previous_save_); // adr + cache_rstr(cache->dat, previous_save_); // fil + previous_save[0] = '\0'; + cache_rstr(cache->dat, previous_save_); // save + } + if (cache->version >= 5) { + r->headers = cache_rstr_addr(cache->dat); + } + // + cache_rstr(cache->dat,check); + if (strcmp(check,"HTS")==0) { /* intégrité OK */ + ok=1; + } + cache_rLLint(cache->dat, &size_read); /* lire size pour être sûr de la taille déclarée (réécrire) */ + if (size_read > 0) { /* si inscrite ici */ + r->size = size_read; + } else { /* pas de données directement dans le cache, fichier présent? */ + r->size = 0; + } + } + + /* Check destination filename */ + + { + PT_Index__Old index = cache; + /* -------------------- COPY OF THE __New() CODE -------------------- */ + if (previous_save_[0] != '\0') { + int pathLen = (int) strlen(index->path); + if (pathLen > 0 && strncmp(previous_save_, index->path, pathLen) == 0) { // old (<3.40) buggy format + strcpy(previous_save, previous_save_); + } + // relative ? (hack) + else if (index->safeCache + || (previous_save_[0] != '/' // /home/foo/bar.gif + && ( !isalpha(previous_save_[0]) || previous_save_[1] != ':' ) ) // c:/home/foo/bar.gif + ) + { + index->safeCache = 1; + sprintf(previous_save, "%s%s", index->path, previous_save_); + } + // bogus format (includes buggy absolute path) + else { + /* guess previous path */ + if (index->fixedPath == 0) { + const char * start = jump_protocol_and_auth(url); + const char * end = start ? strchr(start, '/') : NULL; + int len = (int) (end - start); + if (start != NULL && end != NULL && len > 0 && len < 128) { + char piece[128 + 2]; + const char * where; + piece[0] = '\0'; + strncat(piece, start, len); + if ((where = strstr(previous_save_, piece)) != NULL) { + index->fixedPath = (int) (where - previous_save_); // offset to relative path + } + } + } + if (index->fixedPath > 0) { + int saveLen = (int) strlen(previous_save_); + if (index->fixedPath < saveLen) { + sprintf(previous_save, "%s%s", index->path, previous_save_ + index->fixedPath); + } else { + sprintf(r->msg, "Bogus fixePath prefix for %s (prefixLen=%d)", previous_save_, (int)index->fixedPath); + r->statuscode = STATUSCODE_INVALID; + } + } else { + sprintf(previous_save, "%s%s", index->path, previous_save_); + } + } + } + /* -------------------- END OF COPY OF THE __New() CODE -------------------- */ + } + + /* Read data */ + if (ok) { + r->adr = NULL; + if ( (r->statuscode>=0) && (r->statuscode<=999)) { + r->adr = NULL; + if (pos<0) { + if (flags & FETCH_BODY) { + FILE* fp = fopen(previous_save, "rb"); + if (fp != NULL) { + r->adr = (char*) malloc(r->size + 1); + if (r->adr != NULL) { + if (r->size > 0 && fread(r->adr, 1, r->size, fp) != r->size) { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Read error in cache disk data"); + } + r->adr[r->size] = '\0'; + } else { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Read error (memory exhausted) from cache"); + } + fclose(fp); + } else { + r->statuscode = STATUSCODE_INVALID; + strcpy(r->msg, "Previous cache file not found (2)"); + } + } + } else { + // lire fichier (d'un coup) + if (flags & FETCH_BODY) { + r->adr=(char*) malloc(r->size + 1); + if (r->adr!=NULL) { + if (fread(r->adr, 1, r->size,cache->dat) != r->size) { // erreur + free(r->adr); + r->adr=NULL; + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cache Read Error : Read Data"); + } else + r->adr[r->size] = '\0'; + } else { // erreur + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cache Memory Error"); + } + } + } + } else { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cache Read Error : Bad Data"); + } + } else { // erreur + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cache Read Error : Read Header"); + } + } else { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"Cache Read Error : Seek Failed"); + } + } else { + r->statuscode=STATUSCODE_INVALID; + strcpy(r->msg,"File Cache Entry Not Found"); + } + if (r->location[0] != '\0') { + r->location = strdup(r->location); + } else { + r->location = NULL; + } + return r; +} + +static int PT_LookupCache__Old(PT_Index index, const char* url) { + int retCode; + MutexLock(&index->slots.formatOld.fileLock); + { + retCode = PT_LookupCache__Old_u(index, url); + } + MutexUnlock(&index->slots.formatOld.fileLock); + return retCode; +} + +static int PT_LookupCache__Old_u(PT_Index index_, const char* url) { + if (index_ != NULL) { + PT_Index__New cache = (PT_Index__New) &index_->slots.formatNew; + if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0) + return 0; + if (strncmp(url, "http://", 7) == 0) + url += 7; + if (inthash_read(cache->hash, url, NULL)) + return 1; + } + return 0; +} + diff --git a/src/proxy/store.h b/src/proxy/store.h new file mode 100644 index 0000000..805bc20 --- /dev/null +++ b/src/proxy/store.h @@ -0,0 +1,105 @@ +/* ------------------------------------------------------------ */ +/* +HTTrack Website Copier, Offline Browser for Windows and Unix +Copyright (C) Xavier Roche and other contributors + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +Please visit our Website: http://www.httrack.com +*/ + +/* ------------------------------------------------------------ */ +/* File: Cache manager for ProxyTrack */ +/* Author: Xavier Roche */ +/* ------------------------------------------------------------ */ + +#ifndef WEBHTTRACK_PROXYTRACK_STORE +#define WEBHTTRACK_PROXYTRACK_STORE + +/* Proxy */ + +typedef struct _PT_Index _PT_Index; +typedef struct _PT_Indexes _PT_Indexes; + +typedef struct _PT_Index *PT_Index; +typedef struct _PT_Indexes *PT_Indexes; + +typedef struct _PT_Cache _PT_Cache; +typedef struct _PT_Cache *PT_Cache; + +typedef struct _PT_CacheItem _PT_CacheItem; +typedef struct _PT_CacheItem *PT_CacheItem; + +typedef struct _PT_Element { + int indexId; // index identifier, if suitable (!= -1) + // + int statuscode; // status-code, -1=erreur, 200=OK,201=..etc (cf RFC1945) + char* adr; // adresse du bloc de mémoire, NULL=vide + char* headers; // adresse des en têtes si présents + unsigned long int size; // taille fichier + char msg[1024]; // error message ("\0"=undefined) + char contenttype[64]; // content-type ("text/html" par exemple) + char charset[64]; // charset ("iso-8859-1" par exemple) + char* location; // on copie dedans éventuellement la véritable 'location' + char lastmodified[64]; // Last-Modified + char etag[64]; // Etag + char cdispo[256]; // Content-Disposition coupé +} _PT_Element; +typedef struct _PT_Element *PT_Element; + +typedef enum PT_Fetch_Flags { + FETCH_HEADERS, // fetch headers + FETCH_BODY // fetch body +} PT_Fetch_Flags; + +/* Locking */ +#ifdef _WIN32 +typedef void* PT_Mutex; +#else +typedef pthread_mutex_t PT_Mutex; +#endif + +void MutexInit(PT_Mutex *pMutex); +void MutexLock(PT_Mutex *pMutex); +void MutexUnlock(PT_Mutex *pMutex); +void MutexFree(PT_Mutex *pMutex); + +/* Indexes */ +PT_Indexes PT_New(void); +void PT_Delete(PT_Indexes index); +PT_Element PT_ReadIndex(PT_Indexes indexes, const char* url, int flags); +int PT_LookupIndex(PT_Indexes indexes, const char* url); +int PT_AddIndex(PT_Indexes index, const char *path); +int PT_RemoveIndex(PT_Indexes index, int indexId); +int PT_IndexMerge(PT_Indexes indexes, PT_Index *pindex); +PT_Index PT_GetIndex(PT_Indexes indexes, int indexId); + +/* Indexes list */ +PT_Element PT_Index_HTML_BuildRootInfo(PT_Indexes indexes); +char ** PT_Enumerate(PT_Indexes indexes, const char *url, int subtree); +void PT_Enumerate_Delete(char ***plist); + +/* Index */ +PT_Index PT_LoadCache(const char *filename); +void PT_Index_Delete(PT_Index *pindex); +PT_Element PT_ReadCache(PT_Index index, const char* url, int flags); +int PT_LookupCache(PT_Index index, const char* url); +time_t PT_Index_Timestamp(PT_Index index); + +/* Elements*/ +PT_Element PT_ElementNew(void); +void PT_Element_Delete(PT_Element *pentry); + +#endif diff --git a/src/webhttrack b/src/webhttrack index ca3f512..aa0edca 100755 --- a/src/webhttrack +++ b/src/webhttrack @@ -59,8 +59,9 @@ fi # -remote if needed. Change the URL into openURL($url) too. # (thanks to Torsten Werner for the patch) # see http://www.mozilla.org/unix/remote.html +# 04/2006: openurl() fix from Samuel Suther if mozillabrowser ${browser}; then - if ! ${browser} -remote "${url}"; then + if ! ${browser} -remote "openurl(${url})"; then log "spawning browser.." ${browser} "${url}" fi @@ -129,7 +130,18 @@ test -n "$BROWSEREXE" || ! log "cound not find any suitable browser" || exit 1 # "browse" command if test "$1" = "browse"; then -launch_browser "${BROWSEREXE}" "file://${HOME}/websites/index.html" +if test -f "${HOME}/.httrack.ini"; then +INDEXF=`cat ${HOME}/.httrack.ini | tr '\r' '\n' | grep -E "^path=" | cut -f2- -d'='` +if test -n "${INDEXF}" -a -d "${INDEXF}" -a -f "${INDEXF}/index.html"; then +INDEXF="${INDEXF}/index.html" +else +INDEXF="" +fi +fi +if ! test -n "$INDEXF"; then +INDEXF="${HOME}/websites/index.html" +fi +launch_browser "${BROWSEREXE}" "file://${INDEXF}" exit $? fi |