First Release for BridgeFTP

.github/workflows/lint.yml

@@ -0,0 +1,38 @@
+name: Go Lint
+
+on:
+  pull_request:
+    branches:
+      - "**" # Run on any pull request, regardless of the source branch
+
+jobs:
+  lint:
+    name: Lint Go code
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.23" # Specify your Go version here
+
+      - name: Cache Go modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/.go/pkg/mod
+          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-mod-
+
+      - name: Install golangci-lint
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin
+
+      - name: Run golangci-lint
+        run: |
+          golangci-lint run

go.mod

@@ -0,0 +1,16 @@
+module github.com/abhijeetmohanan/bridgeftp
+
+go 1.22.1
+
+require (
+	github.com/jlaffaye/ftp v0.2.0
+	github.com/pkg/sftp v1.13.6
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	golang.org/x/sys v0.23.0 // indirect
+)

go.sum

@@ -0,0 +1,59 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
+github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
+github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
+golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
+golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"strconv"
+
+	"github.com/abhijeetmohanan/bridgeftp/utils"
+)
+
+func main() {
+	log.Println("Starting Connection")
+	// Define flags
+	source_ftp := flag.String("src", "", "Source ftp endpoint Example: ftp://user:pass@host/filepath")
+	destination_ftp := flag.String("dest", "", "Destination ftp endpoint Example: ftp://user:pass@host/filepath")
+	bytesize := flag.String("bs", "", "Defines the chuck byte size")
+
+	flag.Parse()
+
+	utils.NullChecker("source", *source_ftp)
+	utils.NullChecker("destination", *destination_ftp)
+
+	bsvalue := 64
+	var err error
+
+	// validate byte size
+	if *bytesize == "" {
+		bsvalue, err = strconv.Atoi("64")
+		if err != nil {
+			log.Panicln("Invalid byte size ", *bytesize)
+		}
+	} else {
+		bsvalue, err = strconv.Atoi(*bytesize)
+		if err != nil {
+			log.Panicln("Failed :: Invalid default byte size", *bytesize)
+		}
+	}
+
+	log.Printf("Streaming Byte Size is %d Kb", bsvalue)
+
+	source_map_kv := utils.ParseInput("source", *source_ftp)
+	destination_map_kv := utils.ParseInput("destination", *destination_ftp)
+
+	if source_map_kv["scheme"] == "ftp" && destination_map_kv["scheme"] == "ftp" {
+		log.Println("Source and Destination are ftp endpoints")
+
+		// validate ftp parameters : panic on failure
+		utils.FtpParamsValidator(source_map_kv)
+		utils.FtpParamsValidator(destination_map_kv)
+
+		// Create Coonection and start streaming
+
+		utils.FtpClientHandler(source_map_kv, destination_map_kv, bsvalue)
+	}
+	if source_map_kv["scheme"] == "sftp" && destination_map_kv["scheme"] == "sftp" {
+		log.Println("Source and Destination are ftp endpoints")
+
+		// validate ftp parameters : panic on failure
+		utils.FtpParamsValidator(source_map_kv)
+		utils.FtpParamsValidator(destination_map_kv)
+
+		// Create Coonection and start streaming
+
+		utils.SftpClientHandler(source_map_kv, destination_map_kv, bsvalue)
+	}
+}

utils/ftphandler.go

@@ -0,0 +1,132 @@
+package utils
+
+import (
+	"bytes"
+	"io"
+	"log"
+	"time"
+
+	"github.com/jlaffaye/ftp"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+func FtpClientHandler(source map[string]string, destination map[string]string, bytesize int) {
+	// Make connections
+	source_ftp, err := ftp.Dial(source["host"], ftp.DialWithTimeout(5*time.Second))
+	if err != nil {
+		log.Fatal(err)
+	} else {
+		log.Println("Source Connected Successfully")
+	}
+	destination_ftp, err := ftp.Dial(destination["host"], ftp.DialWithTimeout(5*time.Second))
+	if err != nil {
+		log.Fatal(err)
+	} else {
+		log.Println("Destination Connected Successfully")
+	}
+
+	// Login in using creds
+	err = source_ftp.Login(source["username"], source["password"])
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = destination_ftp.Login(destination["username"], destination["password"])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// start Reading from Source
+	reader_ftp, err := source_ftp.Retr(source["path"])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// initalize a buffer of size 64Kb
+	chuckbuffer := make([]byte, bytesize*1024)
+
+	for {
+		n, err := reader_ftp.Read(chuckbuffer)
+
+		if err == io.EOF {
+			break
+		}
+
+		wdata := bytes.NewBufferString(string(chuckbuffer[:n]))
+		err = destination_ftp.Append(destination["path"], wdata)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+	}
+	// Close Source FTP
+	defer reader_ftp.Close()
+}
+
+func connectToSFTP(user, password, host string) (*sftp.Client, error) {
+	log.Printf("starting clinet connection for  %s", host)
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			ssh.Password(password),
+		},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+
+	conn, err := ssh.Dial("tcp", host, config)
+	if err != nil {
+		log.Fatalf("failed to dial: %w", err)
+		return nil, err
+	}
+
+	client, err := sftp.NewClient(conn)
+	if err != nil {
+		log.Fatalf("failed to create SFTP client: %w", err)
+		return nil, err
+	}
+	log.Printf("SFTP Client initalized %s", host)
+
+	return client, nil
+}
+
+func SftpClientHandler(source map[string]string, destination map[string]string, bytesize int) {
+	// Establish connections to both SFTP servers
+
+	srcClient, err := connectToSFTP(source["username"], source["password"], source["host"])
+	if err != nil {
+		log.Fatalf("Failed to connect to source SFTP: %v", err)
+	}
+	defer srcClient.Close()
+
+	dstClient, err := connectToSFTP(destination["username"], destination["password"], destination["host"])
+	if err != nil {
+		log.Fatalf("Failed to connect to destination SFTP: %v", err)
+	}
+	defer dstClient.Close()
+
+	// Open the source file
+	srcFile, err := srcClient.Open(source["path"])
+	if err != nil {
+		log.Fatalf("Failed to open source file: %v", err)
+	}
+	defer srcFile.Close()
+
+	// Create the destination file
+	dstFile, err := dstClient.Create(destination["path"])
+	if err != nil {
+		log.Fatalf("Failed to create destination file: %v", err)
+	}
+	defer dstFile.Close()
+
+	log.Println("starting stream")
+
+	buffer := make([]byte, bytesize*1024)
+
+	// Stream the file from source to destination
+	bytesCopied, err := io.CopyBuffer(dstFile, srcFile, buffer)
+	if err != nil {
+		log.Fatalf("Failed to copy file: %v", err)
+	}
+
+	log.Printf("Successfully copied %d bytes ", bytesCopied)
+}

utils/parsers.go

@@ -0,0 +1,64 @@
+package utils
+
+import (
+	"log"
+	"net/url"
+)
+
+func ParseInput(key string, furl string) map[string]string {
+	// Define return variable
+
+	params := make(map[string]string)
+	//
+
+	// Parse the URLs
+	sourceURL, err := url.Parse(furl)
+	if err != nil {
+		log.Panicf("Error parsing source URL: %v\n", err)
+	}
+
+	// Output the parsed URLs
+
+	// Extract and print the protocol (scheme)
+	params["scheme"] = sourceURL.Scheme
+	params["username"] = sourceURL.User.Username()
+	sourcePassword, _ := sourceURL.User.Password()
+	params["password"] = sourcePassword
+	params["host"] = sourceURL.Host
+	params["path"] = sourceURL.Path
+
+	// log the paramters used for connection
+	// ToDo: sorted map implementation
+	parameters := ""
+
+	for k, v := range params {
+		parameters = parameters + k + ": " + v + ", "
+	}
+
+	log.Println(key + ":: " + parameters)
+
+	return params
+}
+
+func NullChecker(key string, ftpurl string) {
+	// Check if string is null
+	if ftpurl == "" {
+		log.Panicf("Error: Empty URL for %s", key)
+	}
+}
+
+func SchemeValidator(scheme string) bool {
+	log.Println("scheme: %s", scheme)
+	if scheme == "ftp" {
+		return true
+	}
+	return false
+}
+
+func FtpParamsValidator(params map[string]string) {
+	for k, v := range params {
+		if v == "" {
+			log.Panicf("Null Values passed for %v", k)
+		}
+	}
+}