From 51f6c3decb5510440acf8bd78aff8899c3e6562c Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Mon, 26 Aug 2024 01:09:56 +0530 Subject: [PATCH 01/26] Initial Commit --- ftpcli.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 10 ++++++++++ go.sum | 14 ++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 ftpcli.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/ftpcli.go b/ftpcli.go new file mode 100644 index 0000000..b33a35d --- /dev/null +++ b/ftpcli.go @@ -0,0 +1,55 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "log" + "time" + + "github.com/jlaffaye/ftp" +) + +func main() { + c1, err := ftp.Dial("ftp_server_1:21", ftp.DialWithTimeout(5*time.Second)) + if err != nil { + log.Fatal(err) + } + c2, err := ftp.Dial("ftp_server_2:21", ftp.DialWithTimeout(5*time.Second)) + if err != nil { + log.Fatal(err) + } + fmt.Println("Connection Made") + + err = c1.Login("user1", "pass1") + if err != nil { + log.Fatal(err) + } + err = c2.Login("user2", "pass2") + if err != nil { + log.Fatal(err) + } + fmt.Println("logged in") + + reader_ftp, err := c1.Retr("dumpdata.none") + if err != nil { + log.Fatal(err) + } + defer reader_ftp.Close() + + p := make([]byte, 4) + + for { + n, err := reader_ftp.Read(p) + + if err == io.EOF { + break + } + wdata := bytes.NewBufferString(string(p[:n])) + err = c2.Append("dumpdata.none", wdata) + if err != nil { + log.Fatal(err) + } + fmt.Println(string(p[:n])) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..657bbf3 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/abhijeetmohanan/ftpcli + +go 1.22.1 + +require github.com/jlaffaye/ftp v0.2.0 + +require ( + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e77a547 --- /dev/null +++ b/go.sum @@ -0,0 +1,14 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= +github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From f1f42032a2ad4bafd42cf1eb8789f6942062cb93 Mon Sep 17 00:00:00 2001 From: bhijeetmohanan Date: Wed, 28 Aug 2024 01:57:11 +0530 Subject: [PATCH 02/26] refactor: add structure to data add: argumnet based support --- ftpcli.go | 55 -------------------------------------- main.go | 37 ++++++++++++++++++++++++++ utils/ftphandler.go | 62 +++++++++++++++++++++++++++++++++++++++++++ utils/parsers.go | 64 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 55 deletions(-) delete mode 100644 ftpcli.go create mode 100644 main.go create mode 100644 utils/ftphandler.go create mode 100644 utils/parsers.go diff --git a/ftpcli.go b/ftpcli.go deleted file mode 100644 index b33a35d..0000000 --- a/ftpcli.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "io" - "log" - "time" - - "github.com/jlaffaye/ftp" -) - -func main() { - c1, err := ftp.Dial("ftp_server_1:21", ftp.DialWithTimeout(5*time.Second)) - if err != nil { - log.Fatal(err) - } - c2, err := ftp.Dial("ftp_server_2:21", ftp.DialWithTimeout(5*time.Second)) - if err != nil { - log.Fatal(err) - } - fmt.Println("Connection Made") - - err = c1.Login("user1", "pass1") - if err != nil { - log.Fatal(err) - } - err = c2.Login("user2", "pass2") - if err != nil { - log.Fatal(err) - } - fmt.Println("logged in") - - reader_ftp, err := c1.Retr("dumpdata.none") - if err != nil { - log.Fatal(err) - } - defer reader_ftp.Close() - - p := make([]byte, 4) - - for { - n, err := reader_ftp.Read(p) - - if err == io.EOF { - break - } - wdata := bytes.NewBufferString(string(p[:n])) - err = c2.Append("dumpdata.none", wdata) - if err != nil { - log.Fatal(err) - } - fmt.Println(string(p[:n])) - } -} diff --git a/main.go b/main.go new file mode 100644 index 0000000..15219da --- /dev/null +++ b/main.go @@ -0,0 +1,37 @@ +package main + +import ( + "flag" + "gufcli/utils" + "log" +) + +func main() { + log.Println("Starting Connection") + // Define flags + source_ftp := flag.String("src", "", "Source ftp endpoint Example: ftp://user:pass@host/filepath") + destination_ftp := flag.String("dest", "", "Destination ftp endpoint Example: ftp://user:pass@host/filepath") + + flag.Parse() + + utils.NullChecker("source", *source_ftp) + utils.NullChecker("destination", *destination_ftp) + + source_map_kv := utils.ParseInput("source", *source_ftp) + destination_map_kv := utils.ParseInput("destination", *destination_ftp) + + if utils.SchemeValidator(source_map_kv["scheme"]) && utils.SchemeValidator(destination_map_kv["scheme"]) { + log.Println("Good Boy ") + + // validate ftp parameters : panic on failure + utils.FtpParamsValidator(source_map_kv) + utils.FtpParamsValidator(destination_map_kv) + + // Create Coonection and start streaming + + utils.FtpClientHandler(source_map_kv, destination_map_kv) + + } else { + log.Panicln("Maradchod :: ? ftp hai dono") + } +} diff --git a/utils/ftphandler.go b/utils/ftphandler.go new file mode 100644 index 0000000..4038b8a --- /dev/null +++ b/utils/ftphandler.go @@ -0,0 +1,62 @@ +package utils + +import ( + "bytes" + "io" + "log" + "time" + + "github.com/jlaffaye/ftp" +) + +func FtpClientHandler(source map[string]string, destination map[string]string) { + // Make connections + source_ftp, err := ftp.Dial(source["host"], ftp.DialWithTimeout(5*time.Second)) + if err != nil { + log.Fatal(err) + } else { + log.Println("Source Connected Successfully") + } + destination_ftp, err := ftp.Dial(destination["host"], ftp.DialWithTimeout(5*time.Second)) + if err != nil { + log.Fatal(err) + } else { + log.Println("Destination Connected Successfully") + } + + // Login in using creds + err = source_ftp.Login(source["username"], source["password"]) + if err != nil { + log.Fatal(err) + } + err = destination_ftp.Login(destination["username"], destination["password"]) + if err != nil { + log.Fatal(err) + } + + // start Reading from Source + reader_ftp, err := source_ftp.Retr(source["path"]) + if err != nil { + log.Fatal(err) + } + + // initalize a buffer of size 64Kb + p := make([]byte, 64*1024) + + for { + n, err := reader_ftp.Read(p) + + if err == io.EOF { + break + } + + wdata := bytes.NewBufferString(string(p[:n])) + err = destination_ftp.Append(destination["path"], wdata) + if err != nil { + log.Fatal(err) + } + + } + // Close Source FTP + defer reader_ftp.Close() +} diff --git a/utils/parsers.go b/utils/parsers.go new file mode 100644 index 0000000..787fddf --- /dev/null +++ b/utils/parsers.go @@ -0,0 +1,64 @@ +package utils + +import ( + "log" + "net/url" +) + +func ParseInput(key string, furl string) map[string]string { + // Define return variable + + params := make(map[string]string) + // + + // Parse the URLs + sourceURL, err := url.Parse(furl) + if err != nil { + log.Panicf("Error parsing source URL: %v\n", err) + } + + // Output the parsed URLs + + // Extract and print the protocol (scheme) + params["scheme"] = sourceURL.Scheme + params["username"] = sourceURL.User.Username() + sourcePassword, _ := sourceURL.User.Password() + params["password"] = sourcePassword + params["host"] = sourceURL.Host + params["path"] = sourceURL.Path + + // log the paramters used for connection + // ToDo: sorted map implementation + parameters := "" + + for k, v := range params { + parameters = parameters + k + ": " + v + ", " + } + + log.Println(key + ":: " + parameters) + + return params +} + +func NullChecker(key string, ftpurl string) { + // Check if string is null + if ftpurl == "" { + log.Panicf("Error: Empty URL for %s", key) + } +} + +func SchemeValidator(scheme string) bool { + log.Println("scheme: %s", scheme) + if scheme == "ftp" { + return true + } + return false +} + +func FtpParamsValidator(params map[string]string) { + for k, v := range params { + if v == "" { + log.Panicf("Null Values passed for %v", k) + } + } +} From 3ace68c3d5e753d77e7e96f2ffd7386c4da70ff2 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Wed, 28 Aug 2024 02:00:51 +0530 Subject: [PATCH 03/26] fix: ref --- main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 15219da..9953d5a 100644 --- a/main.go +++ b/main.go @@ -2,8 +2,9 @@ package main import ( "flag" - "gufcli/utils" "log" + + "github.com/abhijeetmohanan/ftpcli/utils" ) func main() { From d7c7bde2ef6b406b12f94436c8f02d21b894a146 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Wed, 28 Aug 2024 02:06:34 +0530 Subject: [PATCH 04/26] fix: logs --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 9953d5a..8237df9 100644 --- a/main.go +++ b/main.go @@ -22,7 +22,7 @@ func main() { destination_map_kv := utils.ParseInput("destination", *destination_ftp) if utils.SchemeValidator(source_map_kv["scheme"]) && utils.SchemeValidator(destination_map_kv["scheme"]) { - log.Println("Good Boy ") + log.Println("Scheme Validated Both source and Destination are ftp endpoints") // validate ftp parameters : panic on failure utils.FtpParamsValidator(source_map_kv) @@ -33,6 +33,6 @@ func main() { utils.FtpClientHandler(source_map_kv, destination_map_kv) } else { - log.Panicln("Maradchod :: ? ftp hai dono") + log.Panicln("Only FTP endooints are supported as of now") } } From bf26be0c5477f579de1d92a0ad6eb7c34e40ee76 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Wed, 28 Aug 2024 22:25:05 +0530 Subject: [PATCH 05/26] add: customiziable byte size --- main.go | 20 +++++++++++++++++++- utils/ftphandler.go | 8 ++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 8237df9..aef7bed 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "log" + "strconv" "github.com/abhijeetmohanan/ftpcli/utils" ) @@ -12,12 +13,29 @@ func main() { // Define flags source_ftp := flag.String("src", "", "Source ftp endpoint Example: ftp://user:pass@host/filepath") destination_ftp := flag.String("dest", "", "Destination ftp endpoint Example: ftp://user:pass@host/filepath") + bytesize := flag.String("bs", "", "Defines the chuck byte size") flag.Parse() utils.NullChecker("source", *source_ftp) utils.NullChecker("destination", *destination_ftp) + bsvalue := 0 + var err error + + // validate byte size + if *bytesize == "" { + bsvalue, err = strconv.Atoi("64") + if err != nil { + log.Panicln("Invalid byte size ", *bytesize) + } + } else { + bsvalue, err = strconv.Atoi(*bytesize) + if err != nil { + log.Panicln("Failed :: Invalid default byte size", *bytesize) + } + } + source_map_kv := utils.ParseInput("source", *source_ftp) destination_map_kv := utils.ParseInput("destination", *destination_ftp) @@ -30,7 +48,7 @@ func main() { // Create Coonection and start streaming - utils.FtpClientHandler(source_map_kv, destination_map_kv) + utils.FtpClientHandler(source_map_kv, destination_map_kv, bsvalue) } else { log.Panicln("Only FTP endooints are supported as of now") diff --git a/utils/ftphandler.go b/utils/ftphandler.go index 4038b8a..f45cebf 100644 --- a/utils/ftphandler.go +++ b/utils/ftphandler.go @@ -9,7 +9,7 @@ import ( "github.com/jlaffaye/ftp" ) -func FtpClientHandler(source map[string]string, destination map[string]string) { +func FtpClientHandler(source map[string]string, destination map[string]string, bytesize int) { // Make connections source_ftp, err := ftp.Dial(source["host"], ftp.DialWithTimeout(5*time.Second)) if err != nil { @@ -41,16 +41,16 @@ func FtpClientHandler(source map[string]string, destination map[string]string) { } // initalize a buffer of size 64Kb - p := make([]byte, 64*1024) + chuckbuffer := make([]byte, bytesize*1024) for { - n, err := reader_ftp.Read(p) + n, err := reader_ftp.Read(chuckbuffer) if err == io.EOF { break } - wdata := bytes.NewBufferString(string(p[:n])) + wdata := bytes.NewBufferString(string(chuckbuffer[:n])) err = destination_ftp.Append(destination["path"], wdata) if err != nil { log.Fatal(err) From 1f06fba2372d9d685ff4e79d125ca9f035bfe16b Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Thu, 29 Aug 2024 11:06:58 +0530 Subject: [PATCH 06/26] fix: module reference --- go.mod | 2 +- main.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 657bbf3..d3259bf 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/abhijeetmohanan/ftpcli +module github.com/abhijeetmohanan/bridgeftp go 1.22.1 diff --git a/main.go b/main.go index aef7bed..b9de9ba 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,7 @@ import ( "log" "strconv" - "github.com/abhijeetmohanan/ftpcli/utils" + "github.com/abhijeetmohanan/bridgeftp/utils" ) func main() { @@ -20,7 +20,7 @@ func main() { utils.NullChecker("source", *source_ftp) utils.NullChecker("destination", *destination_ftp) - bsvalue := 0 + bsvalue := 64 var err error // validate byte size @@ -36,6 +36,8 @@ func main() { } } + log.Printf("Streaming Byte Size is %d Kb", bsvalue) + source_map_kv := utils.ParseInput("source", *source_ftp) destination_map_kv := utils.ParseInput("destination", *destination_ftp) From f97aba1d4cb9e95ef6ccd84a674396c592845f6d Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sat, 31 Aug 2024 03:12:19 +0530 Subject: [PATCH 07/26] add: support for sftp clients --- main.go | 16 ++++++++--- utils/ftphandler.go | 70 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index b9de9ba..5984d55 100644 --- a/main.go +++ b/main.go @@ -41,8 +41,8 @@ func main() { source_map_kv := utils.ParseInput("source", *source_ftp) destination_map_kv := utils.ParseInput("destination", *destination_ftp) - if utils.SchemeValidator(source_map_kv["scheme"]) && utils.SchemeValidator(destination_map_kv["scheme"]) { - log.Println("Scheme Validated Both source and Destination are ftp endpoints") + if source_map_kv["scheme"] == "ftp" && destination_map_kv["scheme"] == "ftp" { + log.Println("Source and Destination are ftp endpoints") // validate ftp parameters : panic on failure utils.FtpParamsValidator(source_map_kv) @@ -51,8 +51,16 @@ func main() { // Create Coonection and start streaming utils.FtpClientHandler(source_map_kv, destination_map_kv, bsvalue) + } + if source_map_kv["scheme"] == "sftp" && destination_map_kv["scheme"] == "sftp" { + log.Println("Source and Destination are ftp endpoints") - } else { - log.Panicln("Only FTP endooints are supported as of now") + // validate ftp parameters : panic on failure + utils.FtpParamsValidator(source_map_kv) + utils.FtpParamsValidator(destination_map_kv) + + // Create Coonection and start streaming + + utils.SftpClientHandler(source_map_kv, destination_map_kv, bsvalue) } } diff --git a/utils/ftphandler.go b/utils/ftphandler.go index f45cebf..6e21c98 100644 --- a/utils/ftphandler.go +++ b/utils/ftphandler.go @@ -7,6 +7,8 @@ import ( "time" "github.com/jlaffaye/ftp" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" ) func FtpClientHandler(source map[string]string, destination map[string]string, bytesize int) { @@ -60,3 +62,71 @@ func FtpClientHandler(source map[string]string, destination map[string]string, b // Close Source FTP defer reader_ftp.Close() } + +func connectToSFTP(user, password, host string) (*sftp.Client, error) { + log.Printf("starting clinet connection for %s", host) + config := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ + ssh.Password(password), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + conn, err := ssh.Dial("tcp", host, config) + if err != nil { + log.Fatalf("failed to dial: %w", err) + return nil, err + } + + client, err := sftp.NewClient(conn) + if err != nil { + log.Fatalf("failed to create SFTP client: %w", err) + return nil, err + } + log.Printf("SFTP Client initalized %s", host) + + return client, nil +} + +func SftpClientHandler(source map[string]string, destination map[string]string, bytesize int) { + // Establish connections to both SFTP servers + + srcClient, err := connectToSFTP(source["username"], source["password"], source["host"]) + if err != nil { + log.Fatalf("Failed to connect to source SFTP: %v", err) + } + defer srcClient.Close() + + dstClient, err := connectToSFTP(destination["username"], destination["password"], destination["host"]) + if err != nil { + log.Fatalf("Failed to connect to destination SFTP: %v", err) + } + defer dstClient.Close() + + // Open the source file + srcFile, err := srcClient.Open(source["path"]) + if err != nil { + log.Fatalf("Failed to open source file: %v", err) + } + defer srcFile.Close() + + // Create the destination file + dstFile, err := dstClient.Create(destination["path"]) + if err != nil { + log.Fatalf("Failed to create destination file: %v", err) + } + defer dstFile.Close() + + log.Println("starting stream") + + buffer := make([]byte, bytesize*1024) + + // Stream the file from source to destination + bytesCopied, err := io.CopyBuffer(dstFile, srcFile, buffer) + if err != nil { + log.Fatalf("Failed to copy file: %v", err) + } + + log.Printf("Successfully copied %d bytes ", bytesCopied) +} From 316d76c15d7a7fef5723fbc9d05954f86214dd59 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sat, 31 Aug 2024 03:24:14 +0530 Subject: [PATCH 08/26] add: Lint checker --- .github/workflows/lint.yml | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..5bb91f3 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,41 @@ +name: Go Lint + +on: + push: + branches: + - "**" # Run on push to any branch + pull_request: + branches: + - "**" # Run on any pull request, regardless of the source branch + +jobs: + lint: + name: Lint Go code + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" # Specify your Go version here + + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/.go/pkg/mod + key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-mod- + + - name: Install golangci-lint + run: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin + + - name: Run golangci-lint + run: | + golangci-lint run From 4c5860ab3b0d0691ca6d386c49da2b872d70bcf6 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sat, 31 Aug 2024 03:28:38 +0530 Subject: [PATCH 09/26] fix: golang version --- .github/workflows/lint.yml | 5 +---- go.mod | 8 ++++++- go.sum | 45 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5bb91f3..511adb6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,9 +1,6 @@ name: Go Lint on: - push: - branches: - - "**" # Run on push to any branch pull_request: branches: - "**" # Run on any pull request, regardless of the source branch @@ -20,7 +17,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: "1.20" # Specify your Go version here + go-version: "1.23" # Specify your Go version here - name: Cache Go modules uses: actions/cache@v3 diff --git a/go.mod b/go.mod index d3259bf..3d2f521 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,15 @@ module github.com/abhijeetmohanan/bridgeftp go 1.22.1 -require github.com/jlaffaye/ftp v0.2.0 +require ( + github.com/jlaffaye/ftp v0.2.0 + github.com/pkg/sftp v1.13.6 + golang.org/x/crypto v0.26.0 +) require ( github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/kr/fs v0.1.0 // indirect + golang.org/x/sys v0.23.0 // indirect ) diff --git a/go.sum b/go.sum index e77a547..51184ed 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -6,9 +7,53 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= +github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 3bbfb1c216bc1f332f26ed6095c8d21d95f21413 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sun, 1 Sep 2024 19:40:52 +0530 Subject: [PATCH 10/26] add: release configs --- .github/workflows/release.yml | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..50c900e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,49 @@ +name: Create Alpha Release + +on: + push: + tags: + - "alpha-*" # Trigger on tags that start with 'alpha-' + +jobs: + build: + name: Build and Release Binaries + runs-on: ubuntu-latest + + strategy: + matrix: + goos: [linux, darwin, windows] + goarch: [amd64, arm64, 386] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: "1.23" # Specify your Go version here + + - name: Build binary + run: | + GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o myapp-${{ matrix.goos }}-${{ matrix.goarch }} + + - name: Archive binary + run: | + zip myapp-${{ matrix.goos }}-${{ matrix.goarch }}.zip myapp-${{ matrix.goos }}-${{ matrix.goarch }} + + - name: Create GitHub Release + id: create_release + uses: softprops/action-gh-release@v1 + with: + files: myapp-${{ matrix.goos }}-${{ matrix.goarch }}.zip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload release assets + uses: actions/upload-release-asset@v1 + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./myapp-${{ matrix.goos }}-${{ matrix.goarch }}.zip + asset_name: myapp-${{ matrix.goos }}-${{ matrix.goarch }}.zip + asset_content_type: application/zip From dfc6c1aa64afbd279e3c8f7136963b955937a2c0 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sun, 1 Sep 2024 19:47:16 +0530 Subject: [PATCH 11/26] remove: multiple os + arch --- .github/workflows/release.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 50c900e..6169210 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,8 +12,8 @@ jobs: strategy: matrix: - goos: [linux, darwin, windows] - goarch: [amd64, arm64, 386] + goos: [linux, windows, darwin] + goarch: [amd64, arm64] steps: - name: Checkout code @@ -26,24 +26,26 @@ jobs: - name: Build binary run: | - GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o myapp-${{ matrix.goos }}-${{ matrix.goarch }} + GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }} - name: Archive binary run: | - zip myapp-${{ matrix.goos }}-${{ matrix.goarch }}.zip myapp-${{ matrix.goos }}-${{ matrix.goarch }} + zip bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }}.zip bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }} - name: Create GitHub Release id: create_release uses: softprops/action-gh-release@v1 with: - files: myapp-${{ matrix.goos }}-${{ matrix.goarch }}.zip + files: bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }}.zip env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.RELEASE_CREATION_TOKEN }} - name: Upload release assets uses: actions/upload-release-asset@v1 with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./myapp-${{ matrix.goos }}-${{ matrix.goarch }}.zip - asset_name: myapp-${{ matrix.goos }}-${{ matrix.goarch }}.zip + asset_path: ./bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }}.zip + asset_name: bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }}.zip asset_content_type: application/zip + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_CREATION_TOKEN }} From 973abfa22bca32377c059684b1867ba4e07a800c Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sun, 1 Sep 2024 20:25:43 +0530 Subject: [PATCH 12/26] refactor: let flag handle input --- main.go | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/main.go b/main.go index 5984d55..d354be3 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main import ( "flag" "log" - "strconv" "github.com/abhijeetmohanan/bridgeftp/utils" ) @@ -13,28 +12,14 @@ func main() { // Define flags source_ftp := flag.String("src", "", "Source ftp endpoint Example: ftp://user:pass@host/filepath") destination_ftp := flag.String("dest", "", "Destination ftp endpoint Example: ftp://user:pass@host/filepath") - bytesize := flag.String("bs", "", "Defines the chuck byte size") + bytesize := flag.Int("bs", 64, "Defines the chuck byte size") flag.Parse() utils.NullChecker("source", *source_ftp) utils.NullChecker("destination", *destination_ftp) - bsvalue := 64 - var err error - - // validate byte size - if *bytesize == "" { - bsvalue, err = strconv.Atoi("64") - if err != nil { - log.Panicln("Invalid byte size ", *bytesize) - } - } else { - bsvalue, err = strconv.Atoi(*bytesize) - if err != nil { - log.Panicln("Failed :: Invalid default byte size", *bytesize) - } - } + bsvalue := *bytesize log.Printf("Streaming Byte Size is %d Kb", bsvalue) From 358e0829ca261dac7b3929908da6e8c2c6b240d7 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sun, 1 Sep 2024 20:48:29 +0530 Subject: [PATCH 13/26] add: usage --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e7cd362..770d10e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,16 @@ -# ftpcli -Remote to Remote FTP Stream +# Bridgeftp + +A tool that can copy files from Remote to Remote FTP Servers + +## Supports + +- ftp +- sftp + +## Usage + +```bash + bridgeftp --src ftp://username:password@server:port/filepath --dest ftp://username@password@server:port/filepath +``` + +**change the protocol to switch between sftp and ftp From 4c22b6ab6d21847c976eebcb50026d898003f9f9 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sun, 1 Sep 2024 20:58:42 +0530 Subject: [PATCH 14/26] add: support for version logging --- .github/workflows/release.yml | 9 ++++++++- main.go | 4 +++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6169210..73421c3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,9 +24,16 @@ jobs: with: go-version: "1.23" # Specify your Go version here + - name: Get the tag value + id: get_tag + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Print the tag + run: echo "Tag value is ${{ env.TAG_NAME }}" + - name: Build binary run: | - GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }} + GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -ldflags "-X main.version=${{ env.TAG_NAME }}" -o bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }} - name: Archive binary run: | diff --git a/main.go b/main.go index d354be3..53ceacb 100644 --- a/main.go +++ b/main.go @@ -7,8 +7,10 @@ import ( "github.com/abhijeetmohanan/bridgeftp/utils" ) +var version string = "testing" + func main() { - log.Println("Starting Connection") + log.Printf("BridgeFTp Version = %s", version) // Define flags source_ftp := flag.String("src", "", "Source ftp endpoint Example: ftp://user:pass@host/filepath") destination_ftp := flag.String("dest", "", "Destination ftp endpoint Example: ftp://user:pass@host/filepath") From 34650a2b856fd51849abd405877f2be8f1651499 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Mon, 2 Sep 2024 01:04:25 +0530 Subject: [PATCH 15/26] add: run releases on all tags --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73421c3..284f3c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Create Alpha Release on: push: tags: - - "alpha-*" # Trigger on tags that start with 'alpha-' + - "*" # Trigger on tags that start with 'alpha-' jobs: build: From 880e0b1bf2a648c961e29ae5dced678f89e3ab9b Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Mon, 2 Sep 2024 01:11:10 +0530 Subject: [PATCH 16/26] add: pre commit hook for secret scans --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..c9528f4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/gitleaks/gitleaks + rev: v8.16.1 + hooks: + - id: gitleaks From d2f7d883bbc460af1fc20a33f98784ca58299f28 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Mon, 2 Sep 2024 21:43:45 +0530 Subject: [PATCH 17/26] add: GoReleaser --- .github/workflows/release.yml | 52 +++++++++-------------------------- .gitignore | 1 + .goreleaser.yaml | 10 +++++++ 3 files changed, 24 insertions(+), 39 deletions(-) create mode 100644 .gitignore create mode 100644 .goreleaser.yaml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 284f3c8..d8c75ae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,53 +6,27 @@ on: - "*" # Trigger on tags that start with 'alpha-' jobs: - build: - name: Build and Release Binaries + goreleaser: runs-on: ubuntu-latest - - strategy: - matrix: - goos: [linux, windows, darwin] - goarch: [amd64, arm64] - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v3 + - name: Checkout + uses: actions/checkout@v4 with: - go-version: "1.23" # Specify your Go version here - + fetch-depth: 0 - name: Get the tag value id: get_tag run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - name: Print the tag run: echo "Tag value is ${{ env.TAG_NAME }}" - - - name: Build binary - run: | - GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -ldflags "-X main.version=${{ env.TAG_NAME }}" -o bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }} - - - name: Archive binary - run: | - zip bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }}.zip bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }} - - - name: Create GitHub Release - id: create_release - uses: softprops/action-gh-release@v1 - with: - files: bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }}.zip - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_CREATION_TOKEN }} - - - name: Upload release assets - uses: actions/upload-release-asset@v1 + - name: Set up Go + uses: actions/setup-go@v5 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }}.zip - asset_name: bridgeftp-${{ matrix.goos }}-${{ matrix.goarch }}.zip - asset_content_type: application/zip + # either 'goreleaser' (default) or 'goreleaser-pro' + distribution: goreleaser + # 'latest', 'nightly', or a semver + version: "~> v2" + args: release --clean env: GITHUB_TOKEN: ${{ secrets.RELEASE_CREATION_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1521c8b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +dist diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..c2964af --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,10 @@ +builds: + - main: ./main.go + id: "bridgeftp" + binary: bridgeftp + goos: + - linux + - darwin + - windows + ldflags: + - -X "main.version=${{ .Env.TAG_NAME }}" From 14b2f12833d9142cccfbd1577e47bff02e44cde8 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sun, 8 Sep 2024 15:24:06 +0530 Subject: [PATCH 18/26] add: tests --- main.go | 12 +++++++++--- utils/ftphandler.go | 4 ++-- utils/parsers.go | 8 ++++---- utils/parsers_test.go | 17 +++++++++++++++++ 4 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 utils/parsers_test.go diff --git a/main.go b/main.go index 53ceacb..8cdb4f4 100644 --- a/main.go +++ b/main.go @@ -25,8 +25,14 @@ func main() { log.Printf("Streaming Byte Size is %d Kb", bsvalue) - source_map_kv := utils.ParseInput("source", *source_ftp) - destination_map_kv := utils.ParseInput("destination", *destination_ftp) + source_map_kv, err := utils.ParseInput("source", *source_ftp) + if err != nil { + log.Fatalf("Failed to parse input %v", err) + } + destination_map_kv, err := utils.ParseInput("destination", *destination_ftp) + if err != nil { + log.Fatalf("Failed to parse input %v", err) + } if source_map_kv["scheme"] == "ftp" && destination_map_kv["scheme"] == "ftp" { log.Println("Source and Destination are ftp endpoints") @@ -35,7 +41,7 @@ func main() { utils.FtpParamsValidator(source_map_kv) utils.FtpParamsValidator(destination_map_kv) - // Create Coonection and start streaming + // Create Connection and start streaming utils.FtpClientHandler(source_map_kv, destination_map_kv, bsvalue) } diff --git a/utils/ftphandler.go b/utils/ftphandler.go index 6e21c98..18caa91 100644 --- a/utils/ftphandler.go +++ b/utils/ftphandler.go @@ -75,13 +75,13 @@ func connectToSFTP(user, password, host string) (*sftp.Client, error) { conn, err := ssh.Dial("tcp", host, config) if err != nil { - log.Fatalf("failed to dial: %w", err) + log.Fatalf("failed to dial: %v", err) return nil, err } client, err := sftp.NewClient(conn) if err != nil { - log.Fatalf("failed to create SFTP client: %w", err) + log.Fatalf("failed to create SFTP client: %v", err) return nil, err } log.Printf("SFTP Client initalized %s", host) diff --git a/utils/parsers.go b/utils/parsers.go index 787fddf..def5004 100644 --- a/utils/parsers.go +++ b/utils/parsers.go @@ -5,7 +5,7 @@ import ( "net/url" ) -func ParseInput(key string, furl string) map[string]string { +func ParseInput(key string, furl string) (map[string]string, error) { // Define return variable params := make(map[string]string) @@ -14,7 +14,7 @@ func ParseInput(key string, furl string) map[string]string { // Parse the URLs sourceURL, err := url.Parse(furl) if err != nil { - log.Panicf("Error parsing source URL: %v\n", err) + return params, err } // Output the parsed URLs @@ -37,7 +37,7 @@ func ParseInput(key string, furl string) map[string]string { log.Println(key + ":: " + parameters) - return params + return params, nil } func NullChecker(key string, ftpurl string) { @@ -48,7 +48,7 @@ func NullChecker(key string, ftpurl string) { } func SchemeValidator(scheme string) bool { - log.Println("scheme: %s", scheme) + log.Printf("scheme: %s", scheme) if scheme == "ftp" { return true } diff --git a/utils/parsers_test.go b/utils/parsers_test.go new file mode 100644 index 0000000..da385d9 --- /dev/null +++ b/utils/parsers_test.go @@ -0,0 +1,17 @@ +package utils + +import "testing" + +func TestParseInput(t *testing.T) { + out, err := ParseInput("source", "sftp://user:password@host/myfile/out.txt") + if err != nil { + t.Error("failed parsing URL") + } + + got := len(out) + want := 5 + + if got != want { + t.Errorf("Wanted %d parameters git %d parameters ", want, got) + } +} From 1940ac4e3c904ad534f9c1264ef476f46aed0c6c Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Wed, 11 Sep 2024 01:56:41 +0530 Subject: [PATCH 19/26] fix: function response --- main.go | 4 ++-- utils/parsers.go | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index 8cdb4f4..a7481ab 100644 --- a/main.go +++ b/main.go @@ -34,7 +34,7 @@ func main() { log.Fatalf("Failed to parse input %v", err) } - if source_map_kv["scheme"] == "ftp" && destination_map_kv["scheme"] == "ftp" { + if utils.SchemeValidator(source_map_kv["scheme"], destination_map_kv["scheme"], "ftp") { log.Println("Source and Destination are ftp endpoints") // validate ftp parameters : panic on failure @@ -45,7 +45,7 @@ func main() { utils.FtpClientHandler(source_map_kv, destination_map_kv, bsvalue) } - if source_map_kv["scheme"] == "sftp" && destination_map_kv["scheme"] == "sftp" { + if utils.SchemeValidator(source_map_kv["scheme"], destination_map_kv["scheme"], "sftp") { log.Println("Source and Destination are ftp endpoints") // validate ftp parameters : panic on failure diff --git a/utils/parsers.go b/utils/parsers.go index def5004..c981207 100644 --- a/utils/parsers.go +++ b/utils/parsers.go @@ -47,10 +47,11 @@ func NullChecker(key string, ftpurl string) { } } -func SchemeValidator(scheme string) bool { - log.Printf("scheme: %s", scheme) - if scheme == "ftp" { - return true +func SchemeValidator(source, dest, key string) bool { + if source == dest { + if source == key { + return true + } } return false } From 0199f674bc422cddaf5f88e9f5d4feaa377b30b4 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Fri, 13 Sep 2024 22:53:08 +0530 Subject: [PATCH 20/26] add: tests --- .github/workflows/test.yml | 25 +++++++++++++++++++++++++ main.go | 8 ++++++-- utils/parsers.go | 24 ------------------------ utils/validators.go | 27 +++++++++++++++++++++++++++ utils/validators_test.go | 24 ++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 utils/validators.go create mode 100644 utils/validators_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a23f67e --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,25 @@ +name: Go Test + +on: + pull_request: + branches: + - "**" # Run on any pull request, regardless of the source branch + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: "1.23" # Specify your Go version here + + - name: Install dependencies + run: go mod download + + - name: Run tests + run: go test ./... -v diff --git a/main.go b/main.go index a7481ab..2cb159b 100644 --- a/main.go +++ b/main.go @@ -18,8 +18,12 @@ func main() { flag.Parse() - utils.NullChecker("source", *source_ftp) - utils.NullChecker("destination", *destination_ftp) + if utils.NullChecker(*source_ftp) { + log.Fatal("source_ftp Empty") + } + if utils.NullChecker(*destination_ftp) { + log.Fatal("destination_ftp Empty") + } bsvalue := *bytesize diff --git a/utils/parsers.go b/utils/parsers.go index c981207..69a92b1 100644 --- a/utils/parsers.go +++ b/utils/parsers.go @@ -39,27 +39,3 @@ func ParseInput(key string, furl string) (map[string]string, error) { return params, nil } - -func NullChecker(key string, ftpurl string) { - // Check if string is null - if ftpurl == "" { - log.Panicf("Error: Empty URL for %s", key) - } -} - -func SchemeValidator(source, dest, key string) bool { - if source == dest { - if source == key { - return true - } - } - return false -} - -func FtpParamsValidator(params map[string]string) { - for k, v := range params { - if v == "" { - log.Panicf("Null Values passed for %v", k) - } - } -} diff --git a/utils/validators.go b/utils/validators.go new file mode 100644 index 0000000..ee162ad --- /dev/null +++ b/utils/validators.go @@ -0,0 +1,27 @@ +package utils + +import ( + "log" +) + +func NullChecker(ftpurl string) bool { + // Check if string is null + return ftpurl == "" +} + +func SchemeValidator(source, dest, key string) bool { + if source == dest { + if source == key { + return true + } + } + return false +} + +func FtpParamsValidator(params map[string]string) { + for k, v := range params { + if v == "" { + log.Panicf("Null Values passed for %v", k) + } + } +} diff --git a/utils/validators_test.go b/utils/validators_test.go new file mode 100644 index 0000000..fc77650 --- /dev/null +++ b/utils/validators_test.go @@ -0,0 +1,24 @@ +package utils + +import ( + "testing" +) + +func TestNullChecker(t *testing.T) { + test := []struct { + input string + expected bool + }{ + {"", true}, + {"ftp://user:password@host/path", false}, + } + + for _, tt := range test { + t.Run(tt.input, func(t *testing.T) { + result := NullChecker(tt.input) + if result != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, result) + } + }) + } +} From 75446445d849f7a2bac508f3a19981b016ad8633 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Fri, 13 Sep 2024 22:59:03 +0530 Subject: [PATCH 21/26] refactor: Positions of the file --- main.go => bridgeftp/main.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename main.go => bridgeftp/main.go (100%) diff --git a/main.go b/bridgeftp/main.go similarity index 100% rename from main.go rename to bridgeftp/main.go From 31e5db3f395784354b8138e81498a92f6ab96605 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Fri, 13 Sep 2024 23:18:00 +0530 Subject: [PATCH 22/26] Fix: ReadME --- .github/workflows/test.yml | 2 +- README.md | 51 +++++++++++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a23f67e..07b0088 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Go Test +name: Go on: pull_request: diff --git a/README.md b/README.md index 770d10e..bb4d781 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,55 @@ + + +
+ # Bridgeftp +[![Tests](https://github.com/abhijeetmohanan/bridgeftp/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/abhijeetmohanan/bridgeftp/actions/workflows/test.yml/badge.svg?query=branch%3Amain) +![GitHub Release](https://img.shields.io/github/v/release/abhijeetmohanan/bridgeftp) + +
+ +
+ +## About + A tool that can copy files from Remote to Remote FTP Servers -## Supports +* It streams data from source to destination [ no need to save files locally just to transfer ] + +* Supported Protocols + * ftp + * sftp -- ftp -- sftp +## Installation + +**1: Binary Installation** + +download one of [releases](https://github.com/abhijeetmohanan/bridgeftp/releases) + +## Getting Started + + FTP + +```bash +bridgeftp --src ftp://username:password@server:port/filepath \ + --dest ftp://username@password@server:port/filepath \ + --bs 512 +``` -## Usage + SFTP ```bash - bridgeftp --src ftp://username:password@server:port/filepath --dest ftp://username@password@server:port/filepath +bridgeftp --src sftp://username:password@server:port/filepath \ + --dest sftp://username@password@server:port/filepath \ + --bs 512 ``` -**change the protocol to switch between sftp and ftp +## From 2f5023affc1840434d6cb51647cc83a8daff6138 Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sat, 14 Sep 2024 00:55:02 +0530 Subject: [PATCH 23/26] add: test for validators --- bridgeftp/main.go | 18 +++++++++++++----- utils/validators.go | 13 +++++-------- utils/validators_test.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/bridgeftp/main.go b/bridgeftp/main.go index 2cb159b..793bff8 100644 --- a/bridgeftp/main.go +++ b/bridgeftp/main.go @@ -42,8 +42,12 @@ func main() { log.Println("Source and Destination are ftp endpoints") // validate ftp parameters : panic on failure - utils.FtpParamsValidator(source_map_kv) - utils.FtpParamsValidator(destination_map_kv) + if utils.FtpParamsValidator(source_map_kv) { + log.Fatal("Null parameters passed in source") + } + if utils.FtpParamsValidator(destination_map_kv) { + log.Fatal("Null parameters passed in Destination") + } // Create Connection and start streaming @@ -53,10 +57,14 @@ func main() { log.Println("Source and Destination are ftp endpoints") // validate ftp parameters : panic on failure - utils.FtpParamsValidator(source_map_kv) - utils.FtpParamsValidator(destination_map_kv) + if utils.FtpParamsValidator(source_map_kv) { + log.Fatal("Null parameters passed in source") + } + if utils.FtpParamsValidator(destination_map_kv) { + log.Fatal("Null parameters passed in Destination") + } - // Create Coonection and start streaming + // Create Connection and start streaming utils.SftpClientHandler(source_map_kv, destination_map_kv, bsvalue) } diff --git a/utils/validators.go b/utils/validators.go index ee162ad..17c4f18 100644 --- a/utils/validators.go +++ b/utils/validators.go @@ -1,9 +1,5 @@ package utils -import ( - "log" -) - func NullChecker(ftpurl string) bool { // Check if string is null return ftpurl == "" @@ -18,10 +14,11 @@ func SchemeValidator(source, dest, key string) bool { return false } -func FtpParamsValidator(params map[string]string) { - for k, v := range params { - if v == "" { - log.Panicf("Null Values passed for %v", k) +func FtpParamsValidator(params map[string]string) bool { + for _, v := range params { + if NullChecker(v) { + return true } } + return false } diff --git a/utils/validators_test.go b/utils/validators_test.go index fc77650..19ab67a 100644 --- a/utils/validators_test.go +++ b/utils/validators_test.go @@ -22,3 +22,35 @@ func TestNullChecker(t *testing.T) { }) } } + +func TestFtpParamsValidator(t *testing.T) { + test := []struct { + input map[string]string + expected bool + }{ + {map[string]string{ + "host": "Myhost", + "scheme": "sftp", + "user": "myuser", + }, false}, + {map[string]string{ + "host": "", + "scheme": "sftp", + "user": "myuser", + }, true}, + {map[string]string{ + "host": "", + "scheme": "", + "user": "myuser", + }, true}, + } + + for _, tt := range test { + t.Run("FTP Parameters Validator", func(t *testing.T) { + result := FtpParamsValidator(tt.input) + if result != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, result) + } + }) + } +} From 3b1683257705eb7c53ef2145ebfb5428a2aee7ff Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sat, 14 Sep 2024 01:07:30 +0530 Subject: [PATCH 24/26] refactor: logic and add test --- utils/validators.go | 7 +------ utils/validators_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/utils/validators.go b/utils/validators.go index 17c4f18..dd89129 100644 --- a/utils/validators.go +++ b/utils/validators.go @@ -6,12 +6,7 @@ func NullChecker(ftpurl string) bool { } func SchemeValidator(source, dest, key string) bool { - if source == dest { - if source == key { - return true - } - } - return false + return (source == dest && dest == key) } func FtpParamsValidator(params map[string]string) bool { diff --git a/utils/validators_test.go b/utils/validators_test.go index 19ab67a..758c7c3 100644 --- a/utils/validators_test.go +++ b/utils/validators_test.go @@ -23,6 +23,27 @@ func TestNullChecker(t *testing.T) { } } +func TestSchemeValidator(t *testing.T) { + test := []struct { + source string + dest string + key string + expected bool + }{ + {"ftp", "ftp", "ftp", true}, + {"sftp", "ftp", "ftp", false}, + {"sftp", "sftp", "ftp", false}, + } + for _, tt := range test { + t.Run(tt.source, func(t *testing.T) { + result := SchemeValidator(tt.source, tt.dest, tt.key) + if result != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, result) + } + }) + } +} + func TestFtpParamsValidator(t *testing.T) { test := []struct { input map[string]string From aa2f1ef2b87e822bfbb1955f0ea41d95d8ac662a Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sat, 14 Sep 2024 01:15:19 +0530 Subject: [PATCH 25/26] add: ignore coverage reports --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1521c8b..2f24110 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ dist +coverage.* From 4e91d1fd91481d8f7b87df8b1b5183864c795f5e Mon Sep 17 00:00:00 2001 From: abhijeetmohanan Date: Sat, 14 Sep 2024 01:17:57 +0530 Subject: [PATCH 26/26] fix: naming convention and add coverage report --- .github/workflows/lint.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 511adb6..bacb0f4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: Go Lint +name: Lint on: pull_request: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d8c75ae..f6244e3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Create Alpha Release +name: Release on: push: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 07b0088..48b4764 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Go +name: Test on: pull_request: @@ -22,4 +22,4 @@ jobs: run: go mod download - name: Run tests - run: go test ./... -v + run: go test -cover -v ./...