From ce12f7275349487ad1d4dd16375ea37255b4509a Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Sat, 25 Jan 2025 11:36:48 +0100 Subject: [PATCH 1/3] feat(jobs-snapshot): various improvements and fixes --- jobs/instances-snapshot/Dockerfile | 2 +- jobs/instances-snapshot/README.md | 22 ++++++++---- jobs/instances-snapshot/go.mod | 4 +-- jobs/instances-snapshot/go.sum | 4 +-- jobs/instances-snapshot/main.go | 54 +++++++++++++++++------------- jobs/instances-snapshot/string.go | 21 ------------ 6 files changed, 51 insertions(+), 56 deletions(-) delete mode 100644 jobs/instances-snapshot/string.go diff --git a/jobs/instances-snapshot/Dockerfile b/jobs/instances-snapshot/Dockerfile index 4df584d..bc9c84e 100644 --- a/jobs/instances-snapshot/Dockerfile +++ b/jobs/instances-snapshot/Dockerfile @@ -1,5 +1,5 @@ # Using apline/golang image -FROM golang:1.22-alpine +FROM golang:1.23-alpine # Set destination for COPY WORKDIR /app diff --git a/jobs/instances-snapshot/README.md b/jobs/instances-snapshot/README.md index cc645a3..c485e39 100644 --- a/jobs/instances-snapshot/README.md +++ b/jobs/instances-snapshot/README.md @@ -15,7 +15,7 @@ This example is very simple, it generates snapshots of your desired Instance. ## Step 1 : Build and push to Container registry Serverless Jobs, like Serverless Containers (which are suited for HTTP applications), works -with Containers. So first, using your terminal reach this folder and run the following commands: +with containers. So first, use your terminal reach this folder and run the following commands: ```shell # First command is to login to container registry, you can find it in Scaleway console @@ -24,14 +24,16 @@ docker login rg.fr-par.scw.cloud/jobs-snapshot -u nologin --password-stdin <<< " # Here we build the image to push docker build -t rg.fr-par.scw.cloud/jobs-snapshot/jobs-snapshot:v1 . +## TIP: for Apple Silicon or other ARM processors, please use the following command as Serverless Jobs supports amd64 architecture +# docker buildx build --platform linux/amd64 -t rg.fr-par.scw.cloud/jobs-snapshot/jobs-snapshot:v1 . + # Push the image online to be used on Serverless Jobs docker push rg.fr-par.scw.cloud/jobs-snapshot/jobs-snapshot:v1 ``` +> [!TIP] +> As we do not expose a web server and we do not require features such as auto-scaling, Serverless Jobs are perfect for this use case. -Note about Serverless Containers versus Serverless Jobs. As we do not expose a web server and we do not -require features such as auto-scaling, Serverless Jobs are perfect for this use case. - -To check if everyting is ok, on the Console you can verify if your tag is present in Container Registry. +To check if everyting is ok, on the Scaleway Console you can verify if your tag is present in Container Registry. ## Step 2: Creating the Job Definition @@ -44,18 +46,25 @@ On Scaleway Console on the following link you can create a new Job Definition: h 1. To schedule your job for example every night at 2am, you can set the cron to `0 2 * * *`. 1. Important: advanced option, you need to set the following environment variables: +> [!TIP] +> For sensitive data like `SCW_ACCESS_KEY` and `SCW_SECRET_KEY` we recommend to inject them via Secret Manager, [more info here](https://www.scaleway.com/en/docs/serverless/jobs/how-to/reference-secret-in-job/). + - `INSTANCE_ID`: grab the instance ID you want to create snapshots from - `INSTANCE_ZONE`: you need to give the ZONE of you instance, like `fr-par-2` - `SCW_ACCESS_KEY`: your access key - `SCW_SECRET_KEY`: your secret key - `SCW_DEFAULT_ORGANIZATION_ID`: your organzation ID -1. Click "create job" +* Then click "create job" ## Step 3: Run the job On your created Job Definition, just click the button "Run Job" and within seconds it should be successful. +## Troubleshooting + +If your Job Run state goes in error, you can use the "Logs" tab in Scaleway Console to get more informations about the error. + # Possible improvements You can exercice by adding the following features: @@ -63,6 +72,7 @@ You can exercice by adding the following features: - Instead of managing a single instance, make it account wide - Add disk backups - Add alerts if something goes wrong +- Use secret manager # Additional content diff --git a/jobs/instances-snapshot/go.mod b/jobs/instances-snapshot/go.mod index fcee6c0..348a782 100644 --- a/jobs/instances-snapshot/go.mod +++ b/jobs/instances-snapshot/go.mod @@ -1,7 +1,7 @@ module github.com/scaleway/serverless-examples/jobs/instances-snapshot -go 1.22.2 +go 1.23 -require github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 +require github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 require gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/jobs/instances-snapshot/go.sum b/jobs/instances-snapshot/go.sum index b5ff887..41a467f 100644 --- a/jobs/instances-snapshot/go.sum +++ b/jobs/instances-snapshot/go.sum @@ -1,7 +1,7 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 h1:yoKAVkEVwAqbGbR8n87rHQ1dulL25rKloGadb3vm770= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30/go.mod h1:sH0u6fq6x4R5M7WxkoQFY/o7UaiItec0o1LinLCJNq8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/jobs/instances-snapshot/main.go b/jobs/instances-snapshot/main.go index 20bacbb..289f76e 100644 --- a/jobs/instances-snapshot/main.go +++ b/jobs/instances-snapshot/main.go @@ -3,23 +3,33 @@ package main import ( "fmt" "os" + "time" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" "github.com/scaleway/scaleway-sdk-go/scw" ) +const ( + envOrgID = "SCW_DEFAULT_ORGANIZATION_ID" + envAccessKey = "SCW_ACCESS_KEY" + envSecretKey = "SCW_SECRET_KEY" + envInstanceID = "INSTANCE_ID" + envInstanceZone = "INSTANCE_ZONE" +) + func main() { fmt.Println("creating snapshot of instance...") // Create a Scaleway client with credentials from environment variables. client, err := scw.NewClient( // Get your organization ID at https://console.scaleway.com/organization/settings - scw.WithDefaultOrganizationID(os.Getenv("SCW_DEFAULT_ORGANIZATION_ID")), + scw.WithDefaultOrganizationID(os.Getenv(envOrgID)), // Get your credentials at https://console.scaleway.com/iam/api-keys - scw.WithAuth(os.Getenv("SCW_ACCESS_KEY"), os.Getenv("SCW_SECRET_KEY")), + scw.WithAuth(os.Getenv(envAccessKey), os.Getenv(envSecretKey)), - // Get more about our availability zones at https://www.scaleway.com/en/docs/console/my-account/reference-content/products-availability/ + // Get more about our availability + // zones at https://www.scaleway.com/en/docs/console/my-account/reference-content/products-availability/ scw.WithDefaultRegion(scw.RegionFrPar), ) if err != nil { @@ -43,16 +53,24 @@ func createSnapshots(instanceAPI *instance.API) error { return fmt.Errorf("error while getting instance %w", err) } + now := time.Now().Format(time.DateOnly) + for _, volume := range gotInstance.Server.Volumes { + snapshotName := fmt.Sprintf("snap-vol-%s-%s-%s", + volume.VolumeType.String(), + now, + os.Getenv(envInstanceZone)) + snapshotResp, err := instanceAPI.CreateSnapshot(&instance.CreateSnapshotRequest{ - Name: volume.Name + RandomString(4), + Name: snapshotName, VolumeID: &volume.ID, - VolumeType: instance.SnapshotVolumeTypeBSSD, - Zone: scw.Zone(os.Getenv("INSTANCE_ZONE")), + VolumeType: instance.SnapshotVolumeType(volume.VolumeType), + Zone: scw.Zone(os.Getenv(envInstanceZone)), }) if err != nil { - return fmt.Errorf("error while creating snapshopt %w", err) + return fmt.Errorf("error while creating snapshot %w", err) } + fmt.Println("created snapshot ", snapshotResp.Snapshot.ID) } @@ -60,23 +78,11 @@ func createSnapshots(instanceAPI *instance.API) error { } func init() { - if os.Getenv("SCW_DEFAULT_ORGANIZATION_ID") == "" { - panic("missing SCW_DEFAULT_ORGANIZATION_ID") - } + mandatoryVariables := [...]string{envOrgID, envAccessKey, envSecretKey, envInstanceID, envInstanceZone} - if os.Getenv("SCW_ACCESS_KEY") == "" { - panic("missing SCW_ACCESS_KEY") - } - - if os.Getenv("SCW_SECRET_KEY") == "" { - panic("missing SCW_SECRET_KEY") - } - - if os.Getenv("INSTANCE_ID") == "" { - panic("missing INSTANCE_ID") - } - - if os.Getenv("INSTANCE_ZONE") == "" { - panic("missing INSTANCE_ZONE") + for idx := range mandatoryVariables { + if os.Getenv(mandatoryVariables[idx]) == "" { + panic("missing environment variable " + mandatoryVariables[idx]) + } } } diff --git a/jobs/instances-snapshot/string.go b/jobs/instances-snapshot/string.go deleted file mode 100644 index ec19790..0000000 --- a/jobs/instances-snapshot/string.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import "crypto/rand" - -// RandomString is used to generate a random string containing upper and lower characters -// + number, of size n. -func RandomString(n int) string { - const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - - bytes := make([]byte, n) - - if _, err := rand.Read(bytes); err != nil { - panic(err) - } - - for i, b := range bytes { - bytes[i] = letters[b%byte(len(letters))] - } - - return string(bytes) -} From ea307cd9def859e5754839012252435c8db6235c Mon Sep 17 00:00:00 2001 From: Thomas TACQUET Date: Mon, 3 Feb 2025 18:10:28 +0100 Subject: [PATCH 2/3] Update jobs/instances-snapshot/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Machavoine Rémy <52951124+RemyMach@users.noreply.github.com> --- jobs/instances-snapshot/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/instances-snapshot/README.md b/jobs/instances-snapshot/README.md index c485e39..bf6b62f 100644 --- a/jobs/instances-snapshot/README.md +++ b/jobs/instances-snapshot/README.md @@ -72,7 +72,7 @@ You can exercice by adding the following features: - Instead of managing a single instance, make it account wide - Add disk backups - Add alerts if something goes wrong -- Use secret manager +- Use secret manager instead of job environment variables # Additional content From 20eb12175a3522497c27fe633dfc460badb6661e Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Mon, 3 Feb 2025 18:11:34 +0100 Subject: [PATCH 3/3] fix sdk --- jobs/instances-snapshot/go.mod | 2 +- jobs/instances-snapshot/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/instances-snapshot/go.mod b/jobs/instances-snapshot/go.mod index 348a782..06ae8bf 100644 --- a/jobs/instances-snapshot/go.mod +++ b/jobs/instances-snapshot/go.mod @@ -2,6 +2,6 @@ module github.com/scaleway/serverless-examples/jobs/instances-snapshot go 1.23 -require github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 +require github.com/scaleway/scaleway-sdk-go v1.0.0-beta.32 require gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/jobs/instances-snapshot/go.sum b/jobs/instances-snapshot/go.sum index 41a467f..e287b70 100644 --- a/jobs/instances-snapshot/go.sum +++ b/jobs/instances-snapshot/go.sum @@ -1,7 +1,7 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 h1:yoKAVkEVwAqbGbR8n87rHQ1dulL25rKloGadb3vm770= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30/go.mod h1:sH0u6fq6x4R5M7WxkoQFY/o7UaiItec0o1LinLCJNq8= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.32 h1:4+LP7qmsLSGbmc66m1s5dKRMBwztRppfxFKlYqYte/c= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.32/go.mod h1:kzh+BSAvpoyHHdHBCDhmSWtBc1NbLMZ2lWHqnBoxFks= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=