diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/list.h | 26 | ||||
| -rw-r--r-- | src/mscp.c | 190 | ||||
| -rw-r--r-- | src/mscp.h | 43 | ||||
| -rw-r--r-- | src/path.c | 250 | ||||
| -rw-r--r-- | src/path.h | 192 | ||||
| -rw-r--r-- | src/ssh.c | 4 | ||||
| -rw-r--r-- | src/ssh.h | 2 | ||||
| -rw-r--r-- | src/test.c | 68 |
8 files changed, 772 insertions, 3 deletions
@@ -208,6 +208,32 @@ static inline void list_splice(struct list_head *list, struct list_head *head) __list_splice(list, head); } +static inline void __list_splice_tail(struct list_head *list, + struct list_head *head) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + struct list_head *at = head->prev; + + first->prev = at; + at->next = first; + + last->next = head; + at->prev = last; +} + +/** + * list_splice_tail - join two lists + * @list: the new list to add. + * @head: the place to add it in the first list. + */ +static inline void list_splice_tail(struct list_head *list, struct list_head *head) +{ + if (!list_empty(list)) + __list_splice_tail(list, head); +} + + /** * list_splice_init - join two lists and reinitialise the emptied list. * @list: the new list to add. diff --git a/src/mscp.c b/src/mscp.c new file mode 100644 index 0000000..d1d8fa8 --- /dev/null +++ b/src/mscp.c @@ -0,0 +1,190 @@ +#include <stdbool.h> +#include <unistd.h> +#include <signal.h> +#include <sys/time.h> +#include <sys/ioctl.h> +#include <math.h> +#include <pthread.h> + +#include <list.h> +#include <util.h> +#include <ssh.h> +#include <path.h> +#include <pprint.h> +#include <atomic.h> +#include <platform.h> +#include <mscp.h> + +struct mscp { + const char *remote; /* remote host (and uername) */ + struct mscp_opts *opts; + struct ssh_opts ssh_opts; + + sftp_session first; /* first sftp session */ + + char dst_path[PATH_MAX]; + struct list_head src_list; + struct list_head path_list; + struct list_head chunk_list; + lock chunk_lock; + + struct mscp_thread *threads; +}; + +struct src { + struct list_head list; + char *path; +}; + +#define DEFAULT_MIN_CHUNK_SZ (64 << 20) /* 64MB */ +#define DEFAULT_NR_AHEAD 32 +#define DEFAULT_BUF_SZ 16384 + +struct mscp *mscp_init(const char *remote_host, struct mscp_opts *opts) +{ + struct mscp *m; + + m = malloc(sizeof(*m)); + if (!m) { + pr_err("failed to allocate memory: %s\n", strerrno()); + return NULL; + } + + memset(m, 0, sizeof(*m)); + INIT_LIST_HEAD(&m->src_list); + INIT_LIST_HEAD(&m->path_list); + INIT_LIST_HEAD(&m->chunk_list); + lock_init(&m->chunk_lock); + m->remote = strdup(remote_host); + if (!m->remote) { + pr_err("failed to allocate memory: %s\n", strerrno()); + free(m); + return NULL; + } + + m->opts = opts; + m->ssh_opts.login_name = opts->ssh_login_name; + m->ssh_opts.port = opts->ssh_port; + m->ssh_opts.identity = opts->ssh_identity; + m->ssh_opts.cipher = opts->ssh_cipher_spec; + m->ssh_opts.hmac = opts->ssh_hmac_spec; + m->ssh_opts.compress = opts->ssh_compress_level; + m->ssh_opts.debuglevel = opts->ssh_debug_level; + m->ssh_opts.no_hostkey_check = opts->ssh_no_hostkey_check; + m->ssh_opts.nodelay = opts->ssh_disable_tcp_nodely; + + m->first = ssh_init_sftp_session(m->remote, &m->ssh_opts); + if (!m->first) { + free(m); + return NULL; + } + + return m; +} + +int mscp_add_src_path(struct mscp *m, const char *src_path) +{ + struct src *s; + + s = malloc(sizeof(*s)); + if (!s) { + pr_err("failed to allocate memory: %s\n", strerrno()); + return -1; + } + + memset(s, 0, sizeof(*s)); + s->path = strdup(src_path); + if (!s->path) { + pr_err("failed to allocate memory: %s\n", strerrno()); + free(s); + return -1; + } + + list_add_tail(&s->list, &m->src_list); + return 0; +} + +static void mscp_free_src_list(struct mscp *m) +{ + struct src *s, *n; + + list_for_each_entry_safe(s, n, &m->src_list, list) { + free(s->path); + list_del(&s->list); + free(s); + } +} + +int mscp_set_dst_path(struct mscp *m, const char *dst_path) +{ + if (strlen(dst_path) + 1 >= PATH_MAX) { + pr_err("too long dst path: %s\n", dst_path); + return -1; + } + + strncpy(m->dst_path, dst_path, PATH_MAX); + return 0; +} + +int mscp_prepare(struct mscp *m) +{ + sftp_session src_sftp = NULL, dst_sftp = NULL; + bool src_path_is_dir, dst_path_is_dir; + struct list_head tmp; + struct src *s; + mstat ss, ds; + + switch (m->opts->direct) { + case MSCP_DIRECT_L2R: + src_sftp = NULL; + dst_sftp = m->first; + break; + case MSCP_DIRECT_R2L: + src_sftp = m->first; + dst_sftp = NULL; + break; + default: + pr_err("invalid mscp direction: %d\n", m->opts->direct); + return -1; + } + + if (mscp_stat(m->dst_path, &ds, dst_sftp) == 0) { + if (mstat_is_dir(ds)) + dst_path_is_dir = true; + mscp_stat_free(ds); + } else + dst_path_is_dir = false; + + /* walk a src_path recusively, and resolve path->dst_path for each src */ + list_for_each_entry(s, &m->src_list, list) { + if (mscp_stat(s->path, &ss, src_sftp) < 0) { + pr_err("stat: %s\n", mscp_stat_strerror(src_sftp)); + return -1; + } + src_path_is_dir = mstat_is_dir(ss); + mscp_stat_free(ss); + + INIT_LIST_HEAD(&tmp); + if (walk_src_path(src_sftp, s->path, &tmp) < 0) + return -1; + + if (resolve_dst_path(src_sftp, s->path, m->dst_path, &tmp, + src_path_is_dir, dst_path_is_dir) < 0) + return -1; + + list_splice_tail(&tmp, m->path_list.prev); + } + + if (prepare_chunk(&m->path_list, &m->chunk_list, m->opts->nr_threads, + m->opts->max_chunk_sz, m->opts->min_chunk_sz) < 0) + return -1; + + mscp_free_src_list(m); + + return 0; +} + +int mscp_start(struct mscp *m) +{ + return 0; +} diff --git a/src/mscp.h b/src/mscp.h new file mode 100644 index 0000000..beb48dd --- /dev/null +++ b/src/mscp.h @@ -0,0 +1,43 @@ +#ifndef _MSCP_H_ +#define _MSCP_H_ + +#include <stdbool.h> + +#define MSCP_DIRECT_L2R 1 +#define MSCP_DIRECT_R2L 2 + +struct mscp_opts { + /* mscp options */ + int direct; /* MSCP_DIRECT_ */ + + int nr_threads; + int nr_ahead; + size_t min_chunk_sz; + size_t max_chunk_sz; + size_t buf_sz; + + int verbose_level; + bool quiet; + bool dryrun; + + /* ssh options */ + char ssh_login_name[64]; + char ssh_port[32]; + char ssh_identity[PATH_MAX]; + char ssh_cipher_spec[64]; + char ssh_hmac_spec[32]; + int ssh_debug_level; + int ssh_compress_level; + bool ssh_no_hostkey_check; + bool ssh_disable_tcp_nodely; +}; + +struct mscp; + +struct mscp *mscp_init(const char *remote_host, struct mscp_opts *opts); +int mscp_add_src_path(struct mscp *m, const char *src_path); +int mscp_set_dst_path(struct mscp *m, const char *dst_path); +int mscp_prepare(struct mscp *m); +int mscp_start(struct mscp *m); + +#endif /* _MSCP_H_ */ diff --git a/src/path.c b/src/path.c new file mode 100644 index 0000000..2951dfd --- /dev/null +++ b/src/path.c @@ -0,0 +1,250 @@ +#include <string.h> +#include <unistd.h> +#include <dirent.h> +#include <sys/stat.h> +#include <libgen.h> + +#include <ssh.h> +#include <util.h> +#include <list.h> +#include <atomic.h> +#include <path.h> + + + +static int append_path(sftp_session sftp, const char *path, mstat s, + struct list_head *path_list) +{ + struct path *p; + + if (!(p = malloc(sizeof(*p)))) { + pr_err("failed to allocate memory: %s\n", strerrno()); + return -1; + } + + memset(p, 0, sizeof(*p)); + INIT_LIST_HEAD(&p->list); + strncpy(p->path, path, PATH_MAX - 1); + p->size = mstat_size(s); + p->mode = mstat_mode(s); + p->state = FILE_STATE_INIT; + lock_init(&p->lock); + list_add_tail(&p->list, path_list); + + return 0; +} + +static bool check_path_should_skip(const char *path) +{ + int len = strlen(path); + if ((len == 1 && strncmp(path, ".", 1) == 0) || + (len == 2 && strncmp(path, "..", 2) == 0)) { + return true; + } + return false; +} + +static int walk_path_recursive(sftp_session sftp, const char *path, + struct list_head *path_list) +{ + char next_path[PATH_MAX]; + mdirent *e; + mdir *d; + mstat s; + int ret; + + if (mscp_stat(path, &s, sftp) < 0) + return -1; + + if (mstat_is_regular(s)) { + /* this path is regular file. it is to be copied */ + ret = append_path(sftp, path, s, path_list); + mscp_stat_free(s); + return ret; + } + + if (!mstat_is_dir(s)) { + /* not regular file and not directory, skip it. */ + mscp_stat_free(s); + return 0; + } + + mscp_stat_free(s); + + + /* ok, this path is directory. walk it. */ + if (!(d = mscp_opendir(path, sftp))) + return -1; + + for (e = mscp_readdir(d); !mdirent_is_null(e); e = mscp_readdir(d)) { + if (check_path_should_skip(mdirent_name(e))) + continue; + + if (strlen(path) + 1 + strlen(mdirent_name(e)) > PATH_MAX) { + pr_err("too long path: %s/%s\n", path, mdirent_name(e)); + return -1; + } + snprintf(next_path, sizeof(next_path), "%s/%s", path, mdirent_name(e)); + ret = walk_path_recursive(sftp, next_path, path_list); + if (ret < 0) + return ret; + } + + mscp_closedir(d); + + return 0; +} + +int walk_src_path(sftp_session sftp, const char *src_path, struct list_head *path_list) +{ + return walk_path_recursive(sftp, src_path, path_list); +} + +static int src2dst_path(const char *src_path, const char *src_file_path, + const char *dst_path, char *dst_file_path, size_t len, + bool src_path_is_dir, bool dst_path_is_dir) +{ + char copy[PATH_MAX]; + char *prefix; + int offset; + + strncpy(copy, src_path, PATH_MAX - 1); + prefix = dirname(copy); + if (!prefix) { + pr_err("dirname: %s\n", strerrno()); + return -1; + } + if (strlen(prefix) == 1 && prefix[0] == '.') + offset = 0; + else + offset = strlen(prefix) + 1; + + + /* both are file */ + if (!src_path_is_dir && !dst_path_is_dir) + strncpy(dst_file_path, dst_path, len); + + /* src is file, and dst is dir */ + if (!src_path_is_dir && dst_path_is_dir) + snprintf(dst_file_path, len, "%s/%s", dst_path, src_path + offset); + + /* both are directory */ + if (src_path_is_dir && dst_path_is_dir) + snprintf(dst_file_path, len, "%s/%s", dst_path, src_file_path + offset); + + /* dst path does not exist. change dir name to dst_path */ + if (src_path_is_dir && !dst_path_is_dir) + snprintf(dst_file_path, len, "%s/%s", + dst_path, src_file_path + strlen(src_path) + 1); + + return 0; +} + +int resolve_dst_path(sftp_session sftp, const char *src_path, const char *dst_path, + struct list_head *path_list, bool src_is_dir, bool dst_is_dir) +{ + struct path *p; + + list_for_each_entry(p, path_list, list) { + if (src2dst_path(src_path, p->path, dst_path, p->dst_path, PATH_MAX, + src_is_dir, dst_is_dir) < 0) + return -1; + } + + return 0; +} + +void path_dump(struct list_head *path_list) +{ + struct path *p; + + list_for_each_entry(p, path_list, list) { + printf("src: %s %lu-byte\n", p->path, p->size); + printf("dst: %s\n", p->dst_path); + } +} + +/* chunk preparation */ + +static struct chunk *alloc_chunk(struct path *p) +{ + struct chunk *c; + + if (!(c = malloc(sizeof(*c)))) { + pr_err("%s\n", strerrno()); + return NULL; + } + memset(c, 0, sizeof(*c)); + + c->p = p; + c->off = 0; + c->len = 0; + refcnt_inc(&p->refcnt); + return c; +} + +static int get_page_mask(void) +{ + long page_sz = sysconf(_SC_PAGESIZE); + size_t page_mask = 0; + int n; + + for (n = 0; page_sz > 0; page_sz >>= 1, n++) { + page_mask <<= 1; + page_mask |= 1; + } + + return page_mask >> 1; +} + +int prepare_chunk(struct list_head *path_list, struct list_head *chunk_list, + int nr_conn, int min_chunk_sz, int max_chunk_sz) +{ + struct chunk *c; + struct path *p; + size_t page_mask; + size_t chunk_sz; + size_t size; + + page_mask = get_page_mask(); + + list_for_each_entry(p, path_list, list) { + if (p->size <= min_chunk_sz) + chunk_sz = p->size; + else if (max_chunk_sz) + chunk_sz = max_chunk_sz; + else { + chunk_sz = (p->size - (p->size % nr_conn)) / nr_conn; + chunk_sz &= ~page_mask; /* align with page_sz */ + if (chunk_sz <= min_chunk_sz) + chunk_sz = min_chunk_sz; + } + + /* for (size = f->size; size > 0;) does not create a + * file (chunk) when file size is 0. This do {} while + * (size > 0) creates just open/close a 0-byte file. + */ + size = p->size; + do { + c = alloc_chunk(p); + if (!c) + return -1; + c->off = p->size - size; + c->len = size < chunk_sz ? size : chunk_sz; + size -= c->len; + list_add_tail(&c->list, chunk_list); + } while (size > 0); + } + + return 0; +} + +void chunk_dump(struct list_head *chunk_list) +{ + struct chunk *c; + + list_for_each_entry(c, chunk_list, list) { + printf("chunk: %s 0x%lx-%lx bytes\n", + c->p->path, c->off, c->off + c->len); + } +} diff --git a/src/path.h b/src/path.h new file mode 100644 index 0000000..3cc5327 --- /dev/null +++ b/src/path.h @@ -0,0 +1,192 @@ +#ifndef _PATH_H_ +#define _PATH_H_ + +#include <limits.h> +#include <fcntl.h> +#include <dirent.h> +#include <sys/stat.h> + +#include <list.h> +#include <atomic.h> +#include <ssh.h> + +struct path { + struct list_head list; /* mscp->path_list */ + + char path[PATH_MAX]; /* file path */ + size_t size; /* size of file on this path */ + mode_t mode; /* permission */ + + char dst_path[PATH_MAX]; /* copy dst path */ + + int state; + lock lock; + refcnt refcnt; +}; +#define FILE_STATE_INIT 0 +#define FILE_STATE_OPENED 1 +#define FILE_STATE_DONE 2 + +struct chunk { + struct list_head list; /* mscp->chunk_list */ + + struct path *p; + size_t off; /* offset of this chunk on the file on path p */ + size_t len; /* length of this chunk */ + size_t done; /* copied bytes for this chunk by a thread */ +}; + + + +/* recursivly walk through src_path and fill path_list for each file */ +int walk_src_path(sftp_session sftp, const char *src_path, struct list_head *path_list); + +/* fill path->dst_path for all files */ +int resolve_dst_path(sftp_session sftp, const char *src_path, const char *dst_path, + struct list_head *path_list, + bool src_path_is_dir, bool dst_path_is_dir); + +/* prepare chunk_list for files in the path_list */ +int prepare_chunk(struct list_head *path_list, struct list_head *chunk_list, + int nr_conn, int min_chunk_sz, int max_chunk_sz); + +/* just print contents. just for debugging */ +void path_dump(struct list_head *path_list); +void chunk_dump(struct list_head *chunk_list); + + + + +/* wrap DIR/dirent and sftp_dir/sftp_attribute. not thread safe */ +struct mscp_dir { + DIR *l; + sftp_dir r; + sftp_session sftp; +}; +typedef struct mscp_dir mdir; + +struct mscp_dirent { + struct dirent *l; + sftp_attributes r; +}; +typedef struct mscp_dirent mdirent; + +#define mdirent_name(e) ((e->l) ? e->l->d_name : e->r->name) +#define mdirent_is_dir(e) ((e->l) ? \ + (e->l->d_type == DT_DIR) : \ + (e->r->type == SSH_FILEXFER_TYPE_DIRECTORY)) +#define mdirent_is_null(e) (e->l == NULL && e->r == NULL) + +static mdir *mscp_opendir(const char *path, sftp_session sftp) +{ + mdir *d; + + if (!(d = malloc(sizeof(*d)))) + return NULL; + memset(d, 0, sizeof(*d)); + + d->sftp = sftp; + + if (sftp) { + d->r = sftp_opendir(sftp, path); + if (!d->r) { + pr_err("sftp_opendir: %s: %s\n", path, sftp_get_ssh_error(sftp)); + free(d); + return NULL; + } + } else { + d->l = opendir(path); + if (!d->l) { + pr_err("opendir: %s: %s\n", path, strerrno()); + free(d); + return NULL; + } + } + return d; +} + +static int mscp_closedir(mdir *d) +{ + int ret; + if (d->r) + ret = sftp_closedir(d->r); + else + ret = closedir(d->l); + free(d); + return ret; +} + +static mdirent *mscp_readdir(mdir *d) +{ + static mdirent e; + + memset(&e, 0, sizeof(e)); + if (d->r) + e.r = sftp_readdir(d->sftp, d->r); + else + e.l = readdir(d->l); + return &e; +} + +/* warp stat/sftp_stat */ +struct mscp_stat { + struct stat l; + sftp_attributes r; +}; +typedef struct mscp_stat mstat; + +static int mscp_stat(const char *path, mstat *s, sftp_session sftp) +{ + memset(s, 0, sizeof(*s)); + + if (sftp) { + s->r = sftp_stat(sftp, path); + if (!s->r) + return -1; + } else { + if (stat(path, &s->l) < 0) + return -1; + } + + return 0; +} + +static const char *mscp_stat_strerror(sftp_session sftp) +{ + if (sftp) + return sftp_get_ssh_error(sftp); + return strerrno(); +} + +static int mscp_stat_check_err_noent(sftp_session sftp) +{ + if (sftp) { + if (sftp_get_error(sftp) == SSH_FX_NO_SUCH_PATH || + sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE) + return 0; + } else { + if (errno == ENOENT) + return 0; + } + return -1; +} + +static void mscp_stat_free(mstat s) { + if (s.r) + sftp_attributes_free(s.r); +} + +#define mstat_size(s) ((s.r) ? s.r->size : s.l.st_size) +#define mstat_mode(s) ((s.r) ? \ + s.r->permissions : \ + s.l.st_mode & (S_IRWXU|S_IRWXG|S_IRWXO)) +#define mstat_is_regular(s) ((s.r) ? \ + (s.r->type == SSH_FILEXFER_TYPE_REGULAR) : \ + S_ISREG(s.l.st_mode)) +#define mstat_is_dir(s) ((s.r) ? \ + (s.r->type == SSH_FILEXFER_TYPE_DIRECTORY) : \ + S_ISDIR(s.l.st_mode)) + + + +#endif /* _PATH_H_ */ @@ -148,7 +148,7 @@ static struct ssh_callbacks_struct cb = { .userdata = NULL, }; -static ssh_session ssh_init_session(char *sshdst, struct ssh_opts *opts) +static ssh_session ssh_init_session(const char *sshdst, struct ssh_opts *opts) { ssh_session ssh = ssh_new(); @@ -187,7 +187,7 @@ free_out: return NULL; } -sftp_session ssh_init_sftp_session(char *sshdst, struct ssh_opts *opts) +sftp_session ssh_init_sftp_session(const char *sshdst, struct ssh_opts *opts) { sftp_session sftp; ssh_session ssh = ssh_init_session(sshdst, opts); @@ -25,7 +25,7 @@ struct ssh_opts { /* ssh_init_sftp_session() creates sftp_session. sshdst accpets * user@hostname and hostname notations (by libssh). */ -sftp_session ssh_init_sftp_session(char *sshdst, struct ssh_opts *opts); +sftp_session ssh_init_sftp_session(const char *sshdst, struct ssh_opts *opts); void ssh_sftp_close(sftp_session sftp); #define sftp_ssh(sftp) (sftp)->session diff --git a/src/test.c b/src/test.c new file mode 100644 index 0000000..2452b6e --- /dev/null +++ b/src/test.c @@ -0,0 +1,68 @@ +#include <util.h> +#include <path.h> + +int path_walk_test(int argc, char **argv) +{ + struct list_head path_list, chunk_list, tmp; + mstat src, dst; + bool dst_is_dir = false, src_is_dir = false; + int ret, n; + + + if (mscp_stat(argv[argc - 1], &dst, NULL) == 0) { + if (mstat_is_dir(dst)) + dst_is_dir = true; + } + + INIT_LIST_HEAD(&path_list); + INIT_LIST_HEAD(&chunk_list); + + for (n = 1; n < argc - 1; n++) { + if (mscp_stat(argv[n], &src, NULL) < 0) { + pr_err("%s not found: %s\n", argv[n], strerrno()); + return -1; + } + src_is_dir = mstat_is_dir(src); + + INIT_LIST_HEAD(&tmp); + ret = walk_src_path(NULL, argv[n], &tmp); + if (ret < 0) + return ret; + + ret = resolve_dst_path(NULL, argv[n], argv[argc - 1], &tmp, + mstat_is_dir(src), dst_is_dir); + if (ret < 0) + return ret; + + list_splice_tail(&tmp, &path_list); + } + + path_dump(&path_list); + + ret = prepare_chunk(&path_list, &chunk_list, 4, 1024 * 1024, 0); + if (ret < 0) + return ret; + + //chunk_dump(&chunk_list); + + + return 0; +} + +void usage() +{ + printf("test [SRC_PATH] ... [DST_PATH]\n"); +} + +int main(int argc, char **argv) +{ + if (argc < 3) { + usage(); + return 1; + } + + if (path_walk_test(argc, argv) < 0) + return 1; + + return 0; +} |