1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
/*
HTTrack external callbacks example : crawling html pages depending on content
Example of <wrappername>_init and <wrappername>_exit call (httrack >> 3.31)
.c file
How to build: (callback.so or callback.dll)
With GNU-GCC:
gcc -O -g3 -Wall -D_REENTRANT -shared -o mycallback.so callbacks-example.c -lhttrack2
With MS-Visual C++:
cl -LD -nologo -W3 -Zi -Zp4 -DWIN32 -Fe"mycallback.dll" callbacks-example.c libhttrack.lib
Note: the httrack library linker option is only necessary when using libhttrack's functions inside the callback
How to use:
httrack --wrapper mycallback,stringtofind,stringtofind.. ..
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* Standard httrack module includes */
#include "httrack-library.h"
#include "htsopt.h"
#include "htsdefines.h"
/* Local function definitions */
static int process(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, const char* address, const char* filename);
static int end(t_hts_callbackarg *carg, httrackp *opt);
/* external functions */
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv);
/* TOLOWER */
#define TOLOWER_(a) (a >= 'A' && a <= 'Z') ? (a + ('a' - 'A')) : a
#define TOLOWER(a) ( TOLOWER_( (a) ) )
/*
This sample just crawls pages that contains certain keywords, and skips the other ones
*/
typedef struct t_my_userdef {
char stringfilter[8192];
char* stringfilters[128];
} t_my_userdef;
/*
module entry point
*/
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
const char *arg = strchr(argv, ',');
if (arg != NULL)
arg++;
/* Check args */
if (arg == NULL || *arg == '\0') {
fprintf(stderr, "** callback error: arguments expected or bad arguments\n");
fprintf(stderr, "usage: httrack --wrapper callback,stringtofind,stringtofind..\n");
fprintf(stderr, "example: httrack --wrapper callback,apple,orange,lemon\n");
return 0;
} else {
t_my_userdef *userdef = (t_my_userdef*) malloc(sizeof(t_my_userdef)); /* userdef */
char * const stringfilter = userdef->stringfilter;
char** const stringfilters = userdef->stringfilters;
/* */
char* a = stringfilter;
int i = 0;
fprintf(stderr, "** info: wrapper_init(%s) called!\n", arg);
fprintf(stderr, "** callback example: crawling pages only if specific keywords are found\n");
/* stringfilters = split(arg, ','); */
strcpy(stringfilter, arg);
while(a != NULL) {
stringfilters[i] = a;
a = strchr(a, ',');
if (a != NULL) {
*a = '\0';
a ++;
}
fprintf(stderr, "** callback info: will crawl pages with '%s' in them\n", stringfilters[i]);
i++;
}
stringfilters[i++] = NULL;
/* Plug callback functions */
CHAIN_FUNCTION(opt, check_html, process, userdef);
CHAIN_FUNCTION(opt, end, end, userdef);
}
return 1; /* success */
}
static int process(t_hts_callbackarg *carg, httrackp *opt, char* html, int len, const char* address, const char* filename) {
t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg);
/*char * const stringfilter = userdef->stringfilter;*/
char** const stringfilters = userdef->stringfilters;
/* */
int i = 0;
int getIt = 0;
char* pos;
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt, html, len, address, filename)) {
return 0; /* Abort */
}
}
/* Process */
if (strcmp(address, "primary") == 0 && strcmp(filename, "/primary") == 0) /* primary page (list of links) */
return 1;
while(stringfilters[i] != NULL && ! getIt) {
if ( ( pos = strstr(html, stringfilters[i]) ) != NULL) {
int j;
getIt = 1;
fprintf(stderr, "** callback info: found '%s' keyword in '%s%s', crawling this page!\n", stringfilters[i], address, filename);
fprintf(stderr, "** details:\n(..)");
for(j = 0; j < 72 && pos[j] ; j++) {
if (pos[j] > 32)
fprintf(stderr, "%c", pos[j]);
else
fprintf(stderr, "?");
}
fprintf(stderr, "(..)\n");
}
i++;
}
if (getIt) {
return 1; /* success */
} else {
fprintf(stderr, "** callback info: won't parse '%s%s' (no specified keywords found)\n", address, filename);
return 0; /* this page sucks, don't parse it */
}
}
static int end(t_hts_callbackarg *carg, httrackp *opt) {
t_my_userdef *userdef = (t_my_userdef*) CALLBACKARG_USERDEF(carg);
fprintf(stderr, "** info: wrapper_exit() called!\n");
if (userdef != NULL) {
free(userdef);
userdef = NULL;
}
/* Call parent functions if multiple callbacks are chained. */
if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
}
return 1; /* success */
}
|
