diff options
| -rw-r--r-- | .clang-format | 2 | ||||
| -rw-r--r-- | CMakeLists.txt | 2 | ||||
| -rw-r--r-- | src/mscp.c | 59 | ||||
| -rw-r--r-- | src/pool.c | 91 | ||||
| -rw-r--r-- | src/pool.h | 74 |
5 files changed, 190 insertions, 38 deletions
diff --git a/.clang-format b/.clang-format index 0b172be..7779e21 100644 --- a/.clang-format +++ b/.clang-format @@ -689,6 +689,8 @@ ForEachMacros: - 'xbc_node_for_each_key_value' - 'xbc_node_for_each_subkey' - 'zorro_for_each_dev' + - 'pool_iter_for_each' + - 'pool_for_each' IncludeBlocks: Preserve IncludeCategories: diff --git a/CMakeLists.txt b/CMakeLists.txt index 610c2cc..8bb8525 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,7 +99,7 @@ list(APPEND MSCP_BUILD_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/include) # libmscp.a set(LIBMSCP_SRC src/mscp.c src/ssh.c src/fileops.c src/path.c src/platform.c - src/print.c src/strerrno.c + src/print.c src/pool.c src/strerrno.c ${OPENBSD_COMPAT_SRC}) add_library(mscp-static STATIC ${LIBMSCP_SRC}) target_include_directories(mscp-static @@ -7,6 +7,7 @@ #include <sys/time.h> #include <list.h> +#include <pool.h> #include <minmax.h> #include <ssh.h> #include <path.h> @@ -34,6 +35,7 @@ struct mscp { sftp_session first; /* first sftp session */ char dst_path[PATH_MAX]; + pool *src_pool; struct list_head src_list; struct list_head path_list; struct chunk_pool cp; @@ -60,11 +62,6 @@ struct mscp_thread { int ret; }; -struct src { - struct list_head list; /* mscp->src_list */ - char *path; -}; - #define DEFAULT_MIN_CHUNK_SZ (64 << 20) /* 64MB */ #define DEFAULT_NR_AHEAD 32 #define DEFAULT_BUF_SZ 16384 @@ -235,9 +232,14 @@ struct mscp *mscp_init(const char *remote_host, int direction, struct mscp_opts priv_set_errv("malloc: %s", strerrno()); return NULL; } - memset(m, 0, sizeof(*m)); - INIT_LIST_HEAD(&m->src_list); + + m->src_pool = pool_new(); + if (!m->src_pool) { + priv_set_errv("pool_new: %s", strerrno()); + goto free_out; + } + INIT_LIST_HEAD(&m->path_list); chunk_pool_init(&m->cp); @@ -275,6 +277,8 @@ struct mscp *mscp_init(const char *remote_host, int direction, struct mscp_opts return m; free_out: + if (m->src_pool) + pool_free(m->src_pool); free(m); return NULL; } @@ -290,23 +294,15 @@ int mscp_connect(struct mscp *m) int mscp_add_src_path(struct mscp *m, const char *src_path) { - struct src *s; - - s = malloc(sizeof(*s)); + char *s = strdup(src_path); if (!s) { - priv_set_errv("malloc: %s", strerrno()); + priv_set_errv("strdup: %s", strerrno()); return -1; } - - memset(s, 0, sizeof(*s)); - s->path = strdup(src_path); - if (!s->path) { - priv_set_errv("malloc: %s", strerrno()); - free(s); + if (pool_push(m->src_pool, s) < 0) { + priv_set_errv("pool_push: %s", strerrno()); return -1; } - - list_add_tail(&s->list, &m->src_list); return 0; } @@ -370,8 +366,8 @@ void *mscp_scan_thread(void *arg) struct path_resolve_args a; struct list_head tmp; struct path *p; - struct src *s; struct stat ss, ds; + char *src_path; glob_t pglob; int n; @@ -395,7 +391,7 @@ void *mscp_scan_thread(void *arg) memset(&a, 0, sizeof(a)); a.total_bytes = &m->total_bytes; - if (list_count(&m->src_list) > 1) + if (pool_size(m->src_pool) > 1) a.dst_path_should_dir = true; if (mscp_stat(m->dst_path, &ds, dst_sftp) == 0) { @@ -411,22 +407,22 @@ void *mscp_scan_thread(void *arg) pr_info("start to walk source path(s)"); - /* walk a src_path recusively, and resolve path->dst_path for each src */ - list_for_each_entry(s, &m->src_list, list) { + /* walk each src_path recusively, and resolve path->dst_path for each src */ + pool_iter_for_each(m->src_pool, src_path) { memset(&pglob, 0, sizeof(pglob)); - if (mscp_glob(s->path, GLOB_NOCHECK, &pglob, src_sftp) < 0) { + if (mscp_glob(src_path, GLOB_NOCHECK, &pglob, src_sftp) < 0) { pr_err("mscp_glob: %s", strerrno()); goto err_out; } for (n = 0; n < pglob.gl_pathc; n++) { if (mscp_stat(pglob.gl_pathv[n], &ss, src_sftp) < 0) { - pr_err("stat: %s %s", s->path, strerrno()); + pr_err("stat: %s %s", src_path, strerrno()); goto err_out; } if (!a.dst_path_should_dir && pglob.gl_pathc > 1) - a.dst_path_should_dir = true; /* we have over 1 src */ + a.dst_path_should_dir = true; /* we have over 1 srces */ /* set path specific args */ a.src_path = pglob.gl_pathv[n]; @@ -705,14 +701,6 @@ out: /* cleanup-related functions */ -static void list_free_src(struct list_head *list) -{ - struct src *s; - s = list_entry(list, typeof(*s), list); - free(s->path); - free(s); -} - static void list_free_path(struct list_head *list) { struct path *p; @@ -734,9 +722,6 @@ void mscp_cleanup(struct mscp *m) m->first = NULL; } - list_free_f(&m->src_list, list_free_src); - INIT_LIST_HEAD(&m->src_list); - list_free_f(&m->path_list, list_free_path); INIT_LIST_HEAD(&m->path_list); diff --git a/src/pool.c b/src/pool.c new file mode 100644 index 0000000..c7e7a88 --- /dev/null +++ b/src/pool.c @@ -0,0 +1,91 @@ + +#include <string.h> +#include <stdlib.h> +#include "pool.h" + +#define DEFAULT_START_SIZE 16 + +pool *pool_new(void) +{ + pool *p; + p = malloc(sizeof(*p)); + if (!p) + return NULL; + memset(p, 0, sizeof(*p)); + + p->array = calloc(DEFAULT_START_SIZE, sizeof(void *)); + if (!p->array) { + free(p); + return NULL; + } + + p->len = DEFAULT_START_SIZE; + p->num = 0; + lock_init(&p->lock); + return p; +} + +void pool_free(pool *p) +{ + if (p->array) + free(p->array); + free(p); +} + +int pool_push(pool *p, void *v) +{ + if (p->num == p->len) { + /* expand array */ + size_t newlen = p->len * 2 * sizeof(void *); + void **new = realloc(p->array, newlen); + if (new == NULL) + return -1; + p->len = newlen; + p->array = new; + } + p->array[p->num] = v; + p->num++; + return 0; +} + +int pool_push_lock(pool *p, void *v) +{ + int ret = -1; + pool_lock(p); + ret = pool_push(p, v); + pool_unlock(p); + return ret; +} + +void *pool_pop(pool *p) +{ + return p->num == 0 ? NULL : p->array[--p->num]; +} + +void *pool_pop_lock(pool *p) +{ + void *v; + pool_lock(p); + v = pool_pop(p); + pool_unlock(p); + return v; +} + +void *pool_iter_next(pool *p) +{ + if (p->num <= p->idx) + return NULL; + + void *v = p->array[p->idx]; + p->idx++; + return v; +} + +void *pool_iter_next_lock(pool *p) +{ + void *v = NULL; + pool_lock(p); + v = pool_iter_next(p); + pool_unlock(p); + return v; +} diff --git a/src/pool.h b/src/pool.h new file mode 100644 index 0000000..675c184 --- /dev/null +++ b/src/pool.h @@ -0,0 +1,74 @@ +#ifndef _POOL_H_ +#define _POOL_H_ + +#include <stdbool.h> +#include <stddef.h> + +#include <atomic.h> + +/* A pool like a stack with an iterator walking from the bottom to the + * top. The memory foot print for a pool never shrinks. Thus this is + * not suitable for long-term uses. */ + +struct pool_struct { + void **array; + size_t len; /* length of array */ + size_t num; /* number of items in the array */ + size_t idx; /* index used dy iter */ + int state; + lock lock; +}; + +typedef struct pool_struct pool; + +pool *pool_new(void); +void pool_free(pool *p); + +#define pool_lock(p) LOCK_ACQUIRE(&(p->lock)) +#define pool_unlock(p) LOCK_RELEASE() + +/* + * pool_push() pushes *v to pool *p. pool_push_lock() does this while + * locking *p. + */ +int pool_push(pool *p, void *v); +int pool_push_lock(pool *p, void *v); + +/* + * pool_pop() pops the last *v pushed to *p. pool_pop_lock() does this + * while locking *p. + */ +void *pool_pop(pool *p); +void *pool_pop_lock(pool *p); + +#define pool_size(p) ((p)->num) +#define pool_get(p, idx) ((p->num <= idx) ? NULL : p->array[idx]) + +/* + * pool->idx indicates next *v in an iteration. This has two + * use-cases. + * + * (1) A simple list: just a single thread has a pool, and the thread + * can call pool_iter_for_each() for the pool (not thread safe). + * + * (2) A thread-safe queue: one thread initializes the iterator for a + * pool by pool_iter_init(). Then, multiple threads get a next *v + * concurrently by pool_iter_next_lock(), which means dequeuing. At + * this time, other thread can add new *v by pool_push_lock(), which + * means enqueuing. During this, other threads must not intercept the + * pool by pool_iter_* functions. + */ + +#define pool_iter_init(p) (p->idx = 0) +void *pool_iter_next(pool *p); +void *pool_iter_next_lock(pool *p); + +#define pool_iter_for_each(p, v) \ + pool_iter_init(p); \ + for (v = pool_iter_next(p); v != NULL; v = pool_iter_next(p)) + +#define pool_for_each(p, v, idx) \ + idx = 0; \ + for (v = pool_get(p, idx); v != NULL; v = pool_get(p, ++idx)) + +#endif /* _POOL_H_ */ |