diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | cmd/main.go | 73 | ||||
| -rw-r--r-- | go.mod | 3 | ||||
| -rw-r--r-- | internal/config/config.go | 50 | ||||
| -rw-r--r-- | internal/local/files.go | 30 | ||||
| -rw-r--r-- | internal/remote/files.go | 48 | ||||
| -rw-r--r-- | internal/transfer/transfer.go | 126 |
7 files changed, 334 insertions, 0 deletions
@@ -1 +1,5 @@ # gscp +```bash +cd cmd && go build -o gscp main.go +./gscp +``` diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..887b506 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "log" + "os" + "time" + + "gscp/internal/config" + "gscp/internal/local" + "gscp/internal/remote" + "gscp/internal/transfer" +) + +func main() { + // Parse command-line arguments + cfg := config.ParseArgs() + + if cfg.Verbose { + log.Printf("Starting parallel SCP with configuration:") + log.Printf(" Destination: %s", cfg.Dest) + log.Printf(" Remote User: %s", cfg.RemoteUser) + log.Printf(" Remote Host: %s", cfg.RemoteHost) + log.Printf(" Remote Path: %s", cfg.RemotePath) + log.Printf(" Parallelism: %d", cfg.Parallelism) + log.Printf(" Lock Dir: %s", cfg.LockDir) + log.Printf(" Cipher: %s", cfg.CipherOption) + } + + // Create lock directory if it doesn't exist + err := os.MkdirAll(cfg.LockDir, 0755) + if err != nil { + log.Fatalf("Failed to create lock directory: %v", err) + } + + // Get remote file list + remoteFiles, err := remote.GetFiles(cfg) + if err != nil { + log.Fatalf("Failed to get remote files: %v", err) + } + if cfg.Verbose { + log.Printf("Found %d remote files", len(remoteFiles)) + } + + // Get local file list + localFiles, err := local.GetFiles(cfg.Dest) + if err != nil { + log.Fatalf("Failed to get local files: %v", err) + } + if cfg.Verbose { + log.Printf("Found %d local files", len(localFiles)) + } + + // Compute diff (files that exist on remote but not on local) + diffFiles := transfer.GetDiffFiles(remoteFiles, localFiles) + if cfg.Verbose { + log.Printf("Found %d files to copy", len(diffFiles)) + } + + // If only listing files, print them and exit + if cfg.OnlyListFiles { + for _, file := range diffFiles { + fmt.Println(file) + } + return + } + + // Copy files in parallel + start := time.Now() + copiedFiles := transfer.CopyFilesInParallel(diffFiles, cfg) + elapsed := time.Since(start) + log.Printf("Done! Copied %d/%d files in %s", copiedFiles, len(diffFiles), elapsed) +} @@ -0,0 +1,3 @@ +module gscp + +go 1.24.0 diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..3cfe5af --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,50 @@ +package config + +import ( + "flag" + "fmt" + "os" +) + +// Configuration holds all the parameters for the SCP operation +type Configuration struct { + Dest string + RemoteUser string + RemoteHost string + RemotePath string + Parallelism int + LockDir string + CipherOption string + Verbose bool + OnlyListFiles bool +} + +// ParseArgs parses command-line arguments and returns a Configuration +func ParseArgs() Configuration { + config := Configuration{ + LockDir: "/tmp/scp_lock", + Parallelism: 10, + CipherOption: "aes128-gcm@openssh.com", + } + + flag.StringVar(&config.Dest, "d", "", "Destination directory (local)") + flag.StringVar(&config.RemoteUser, "u", "", "Remote username") + flag.StringVar(&config.RemoteHost, "h", "", "Remote hostname or IP") + flag.StringVar(&config.RemotePath, "r", "", "Remote path") + flag.IntVar(&config.Parallelism, "P", 10, "Number of parallel SCP processes") + flag.StringVar(&config.LockDir, "L", "/tmp/scp_lock", "Lock directory") + flag.StringVar(&config.CipherOption, "c", "aes128-gcm@openssh.com", "SSH cipher option") + flag.BoolVar(&config.Verbose, "v", false, "Verbose output") + flag.BoolVar(&config.OnlyListFiles, "l", false, "Only list files to be copied, don't copy") + + flag.Parse() + + // Check required parameters + if config.Dest == "" || config.RemoteUser == "" || config.RemoteHost == "" || config.RemotePath == "" { + flag.Usage() + fmt.Println("\nRequired flags: -d, -u, -h, -r") + os.Exit(1) + } + + return config +} diff --git a/internal/local/files.go b/internal/local/files.go new file mode 100644 index 0000000..e4b4f67 --- /dev/null +++ b/internal/local/files.go @@ -0,0 +1,30 @@ +package local + +import ( + "os" + "path/filepath" +) + +// GetFiles retrieves a map of files in the local destination directory +func GetFiles(destDir string) (map[string]bool, error) { + files := make(map[string]bool) + + err := filepath.Walk(destDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() { + // Get relative path + relPath, err := filepath.Rel(destDir, path) + if err != nil { + return err + } + files[relPath] = true + } + + return nil + }) + + return files, err +} diff --git a/internal/remote/files.go b/internal/remote/files.go new file mode 100644 index 0000000..5b89811 --- /dev/null +++ b/internal/remote/files.go @@ -0,0 +1,48 @@ +package remote + +import ( + "bufio" + "fmt" + "log" + "os/exec" + + "gscp/internal/config" +) + +// GetFiles retrieves the list of files from the remote server +func GetFiles(config config.Configuration) ([]string, error) { + if config.Verbose { + log.Println("Getting remote file list...") + } + + cmd := exec.Command("ssh", fmt.Sprintf("%s@%s", config.RemoteUser, config.RemoteHost), + fmt.Sprintf("find '%s' -type f | sed 's|^%s/||'", config.RemotePath, config.RemotePath)) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("error creating stdout pipe: %v", err) + } + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("error starting ssh command: %v", err) + } + + var files []string + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + file := scanner.Text() + if file != "" { + files = append(files, file) + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading ssh output: %v", err) + } + + if err := cmd.Wait(); err != nil { + return nil, fmt.Errorf("ssh command failed: %v", err) + } + + return files, nil +} diff --git a/internal/transfer/transfer.go b/internal/transfer/transfer.go new file mode 100644 index 0000000..a57b073 --- /dev/null +++ b/internal/transfer/transfer.go @@ -0,0 +1,126 @@ +package transfer + +import ( + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "gscp/internal/config" +) + +// GetDiffFiles computes which files exist on remote but not on local +func GetDiffFiles(remoteFiles []string, localFiles map[string]bool) []string { + var diffFiles []string + + for _, remoteFile := range remoteFiles { + if !localFiles[remoteFile] { + diffFiles = append(diffFiles, remoteFile) + } + } + + return diffFiles +} + +// CopyFilesInParallel copies multiple files in parallel from remote to local +func CopyFilesInParallel(files []string, config config.Configuration) int { + if len(files) == 0 { + log.Println("No files to copy") + return 0 + } + + var wg sync.WaitGroup + sem := make(chan struct{}, config.Parallelism) + + totalFiles := len(files) + copiedFiles := 0 + var mu sync.Mutex + + for _, file := range files { + wg.Add(1) + go func(file string) { + defer wg.Done() + + // Acquire semaphore + sem <- struct{}{} + defer func() { <-sem }() + + if err := CopyFile(file, config); err != nil { + log.Printf("Error copying file %s: %v", file, err) + } else { + mu.Lock() + copiedFiles++ + if config.Verbose && copiedFiles%10 == 0 { + log.Printf("Progress: %d/%d files copied (%.2f%%)", + copiedFiles, totalFiles, float64(copiedFiles)/float64(totalFiles)*100) + } + mu.Unlock() + } + }(file) + } + + wg.Wait() + return copiedFiles +} + +// CopyFile copies a single file from remote to local +func CopyFile(file string, config config.Configuration) error { + // Create lock file path by replacing slashes with underscores + lockFileName := strings.ReplaceAll(file, "/", "_") + ".lock" + lockFilePath := filepath.Join(config.LockDir, lockFileName) + + // Try to create and lock the file + lockFile, err := os.OpenFile(lockFilePath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("cannot create lock file: %v", err) + } + defer lockFile.Close() + + // Create destination directory + destDir := filepath.Join(config.Dest, filepath.Dir(file)) + if err := os.MkdirAll(destDir, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %v", destDir, err) + } + + // Check if file already exists in destination + destFile := filepath.Join(config.Dest, file) + if _, err := os.Stat(destFile); err == nil { + // File already exists, skip + return nil + } + + // Construct scp command + src := fmt.Sprintf("%s@%s:%s/%s", config.RemoteUser, config.RemoteHost, config.RemotePath, file) + args := []string{ + "-q", // Quiet mode + "-c", config.CipherOption, + src, + destFile, + } + + if config.Verbose { + log.Printf("Copying file: %s", file) + } + + cmd := exec.Command("scp", args...) + + // Capture stdout and stderr + var stdoutBuf, stderrBuf strings.Builder + cmd.Stdout = io.MultiWriter(os.Stdout, &stdoutBuf) + cmd.Stderr = io.MultiWriter(os.Stderr, &stderrBuf) + + // Execute command + err = cmd.Run() + if err != nil { + return fmt.Errorf("scp failed: %v, stderr: %s", err, stderrBuf.String()) + } + + // Clean up lock file on success + os.Remove(lockFilePath) + + return nil +} |