Skip to content

Commit 4a7ae74

Browse files
motatoesbismuthdev[bot]
motatoes
and
bismuthdev[bot]
authored
iterative caching for terragrunt configs (#2051)
* add iterative caching support Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com> * Update backend/controllers/github.go Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com> * updated docs * Update backend/controllers/cache.go Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com> * fix log msg * fix io * Update backend/controllers/cache.go Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com> * Update libs/digger_config/digger_config.go Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com> --------- Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com>
1 parent 043926b commit 4a7ae74

File tree

139 files changed

+318
-198
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

139 files changed

+318
-198
lines changed

‎backend/controllers/cache.go

Lines changed: 84 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
package controllers
22

33
import (
4+
"bytes"
5+
"encoding/json"
46
"fmt"
7+
"github.com/diggerhq/digger/libs/digger_config/terragrunt/tac"
58
"github.com/diggerhq/digger/libs/git_utils"
9+
"io"
610
"log/slog"
711
"net/http"
12+
"net/url"
813
"os"
914
"path"
1015
"strings"
@@ -15,13 +20,14 @@ import (
1520
"github.com/gin-gonic/gin"
1621
)
1722

23+
type UpdateCacheRequest struct {
24+
RepoFullName string `json:"repo_full_name"`
25+
Branch string `json:"branch"`
26+
OrgId uint `json:"org_id"`
27+
InstallationId int64 `json:"installation_id"`
28+
}
29+
1830
func (d DiggerController) UpdateRepoCache(c *gin.Context) {
19-
type UpdateCacheRequest struct {
20-
RepoFullName string `json:"repo_full_name"`
21-
Branch string `json:"branch"`
22-
OrgId uint `json:"org_id"`
23-
InstallationId int64 `json:"installation_id"`
24-
}
2531

2632
var request UpdateCacheRequest
2733
err := c.BindJSON(&request)
@@ -65,13 +71,14 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
6571

6672
var diggerYmlStr string
6773
var config *dg_configuration.DiggerConfig
74+
var newAtlantisConfig *tac.AtlantisConfig
6875

6976
// update the cache here, do it async for immediate response
7077
go func() {
7178
err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
7279
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
7380
diggerYmlStr = string(diggerYmlBytes)
74-
config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil)
81+
config, _, _, newAtlantisConfig, err = dg_configuration.LoadDiggerConfig(dir, true, nil, nil)
7582
if err != nil {
7683
slog.Error("Error loading digger config", "error", err)
7784
return err
@@ -83,7 +90,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
8390
slog.Error("Could not load digger config", "error", err)
8491
return
8592
}
86-
_, err = models.DB.UpsertRepoCache(orgId, repoFullName, diggerYmlStr, *config)
93+
_, err = models.DB.UpsertRepoCache(orgId, repoFullName, diggerYmlStr, *config, newAtlantisConfig)
8794
if err != nil {
8895
slog.Error("Could not update repo cache", "error", err)
8996
return
@@ -93,3 +100,72 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
93100

94101
c.String(http.StatusOK, "successfully submitted cache for processing, check backend logs for progress")
95102
}
103+
104+
func sendProcessCacheRequest(repoFullName string, branch string, installationId int64) error {
105+
diggerHostname := os.Getenv("HOSTNAME")
106+
webhookSecret := os.Getenv("DIGGER_INTERNAL_SECRET")
107+
108+
installationLink, err := models.DB.GetGithubInstallationLinkForInstallationId(installationId)
109+
if err != nil {
110+
slog.Error("Error getting installation link", "installationId", installationId, "error", err)
111+
return err
112+
}
113+
114+
orgId := installationLink.OrganisationId
115+
116+
payload := UpdateCacheRequest{
117+
RepoFullName: repoFullName,
118+
Branch: branch,
119+
InstallationId: installationId,
120+
OrgId: orgId,
121+
}
122+
123+
cacheRefreshUrl, err := url.JoinPath(diggerHostname, "_internal/update_repo_cache")
124+
if err != nil {
125+
slog.Error("Error joining URL paths", "error", err)
126+
return err
127+
}
128+
129+
jsonPayload, err := json.Marshal(payload)
130+
if err != nil {
131+
slog.Error("Process Cache: error marshaling JSON", "error", err)
132+
return err
133+
}
134+
135+
req, err := http.NewRequest("POST", cacheRefreshUrl, bytes.NewBuffer(jsonPayload))
136+
if err != nil {
137+
slog.Error("Process Cache: Error creating request", "error", err)
138+
return err
139+
}
140+
141+
req.Header.Set("Content-Type", "application/json")
142+
req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", webhookSecret))
143+
144+
client := &http.Client{}
145+
resp, err := client.Do(req)
146+
if err != nil {
147+
fmt.Println("Error sending request:", err)
148+
return err
149+
}
150+
defer resp.Body.Close()
151+
152+
statusCode := resp.StatusCode
153+
if statusCode != 200 {
154+
// Read response body to get error details
155+
responseBody, err := io.ReadAll(resp.Body)
156+
if err != nil {
157+
slog.Error("Failed to read error response body", "error", err)
158+
}
159+
160+
slog.Error("got unexpected cache status",
161+
"statusCode", statusCode,
162+
"repoFullName", repoFullName,
163+
"orgId", orgId,
164+
"branch", branch,
165+
"installationId", installationId,
166+
"responseBody", string(responseBody))
167+
168+
return fmt.Errorf("cache update failed with status code %d: %s", statusCode, string(responseBody))
169+
}
170+
return nil
171+
}

‎backend/controllers/github.go

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"encoding/json"
77
"errors"
88
"fmt"
9+
"github.com/diggerhq/digger/libs/digger_config/terragrunt/tac"
910
"github.com/diggerhq/digger/libs/git_utils"
1011
"log/slog"
1112
"math/rand"
@@ -416,7 +417,6 @@ func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent, a
416417
loadProjectsOnPush := os.Getenv("DIGGER_LOAD_PROJECTS_ON_PUSH")
417418

418419
if loadProjectsOnPush == "true" {
419-
420420
if strings.HasSuffix(ref, defaultBranch) {
421421
slog.Debug("Loading projects from GitHub repo (push event)", "loadProjectsOnPush", loadProjectsOnPush, "ref", ref, "defaultBranch", defaultBranch)
422422
err := services.LoadProjectsFromGithubRepo(gh, strconv.FormatInt(installationId, 10), repoFullName, repoOwner, repoName, cloneURL, defaultBranch)
@@ -428,6 +428,15 @@ func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent, a
428428
slog.Debug("Skipping loading projects from GitHub repo", "loadProjectsOnPush", loadProjectsOnPush)
429429
}
430430

431+
repoCacheEnabled := os.Getenv("DIGGER_CONFIG_REPO_CACHE_ENABLED")
432+
if repoCacheEnabled == "1" && strings.HasSuffix(ref, defaultBranch) {
433+
go func() {
434+
if err := sendProcessCacheRequest(repoFullName, defaultBranch, installationId); err != nil {
435+
slog.Error("Failed to process cache request", "error", err, "repoFullName", repoFullName)
436+
}
437+
}()
438+
}
439+
431440
return nil
432441
}
433442

@@ -914,7 +923,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
914923
return nil
915924
}
916925

917-
func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, changedFiles []string) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) {
926+
func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, changedFiles []string, taConfig *tac.AtlantisConfig) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) {
918927
slog.Info("Getting Digger config for branch",
919928
slog.Group("repository",
920929
slog.String("fullName", repoFullName),
@@ -954,7 +963,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
954963

955964
slog.Debug("Successfully read digger.yml file", "configLength", len(diggerYmlStr))
956965

957-
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)
966+
config, _, dependencyGraph, _, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles, taConfig)
958967
if err != nil {
959968
slog.Error("Error loading and parsing Digger config",
960969
"directory", dir,
@@ -1046,6 +1055,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
10461055

10471056
// check if items should be loaded from cache
10481057
useCache := false
1058+
var taConfig *tac.AtlantisConfig = nil
10491059
if val, _ := os.LookupEnv("DIGGER_CONFIG_REPO_CACHE_ENABLED"); val == "1" && !slices.Contains(prLabels, "digger:no-cache") {
10501060
useCache = true
10511061
slog.Info("Attempting to load config from cache",
@@ -1054,7 +1064,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
10541064
"prNumber", prNumber,
10551065
)
10561066

1057-
diggerYmlStr, config, dependencyGraph, err := retrieveConfigFromCache(orgId, repoFullName)
1067+
_, _, _, taConfigTemp, err := retrieveConfigFromCache(orgId, repoFullName)
10581068
if err != nil {
10591069
slog.Info("Could not load from cache, falling back to live loading",
10601070
"orgId", orgId,
@@ -1065,9 +1075,8 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
10651075
slog.Info("Successfully loaded config from cache",
10661076
"orgId", orgId,
10671077
"repoFullName", repoFullName,
1068-
"projectCount", len(config.Projects),
10691078
)
1070-
return diggerYmlStr, ghService, config, *dependencyGraph, &prBranch, &prCommitSha, changedFiles, nil
1079+
taConfig = taConfigTemp
10711080
}
10721081
}
10731082

@@ -1084,7 +1093,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
10841093
"prNumber", prNumber,
10851094
)
10861095

1087-
diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, changedFiles)
1096+
diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, changedFiles, taConfig)
10881097
if err != nil {
10891098
slog.Error("Error loading Digger config from repository",
10901099
"prNumber", prNumber,
@@ -1098,7 +1107,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
10981107
return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, &prCommitSha, changedFiles, nil
10991108
}
11001109

1101-
func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_configuration.DiggerConfig, *graph.Graph[string, dg_configuration.Project], error) {
1110+
func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_configuration.DiggerConfig, *graph.Graph[string, dg_configuration.Project], *tac.AtlantisConfig, error) {
11021111
slog.Debug("Retrieving config from cache",
11031112
"orgId", orgId,
11041113
"repoFullName", repoFullName,
@@ -1111,7 +1120,7 @@ func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_confi
11111120
"repoFullName", repoFullName,
11121121
"error", err,
11131122
)
1114-
return "", nil, nil, fmt.Errorf("failed to load repo cache: %v", err)
1123+
return "", nil, nil, nil, fmt.Errorf("failed to load repo cache: %v", err)
11151124
}
11161125

11171126
var config dg_configuration.DiggerConfig
@@ -1122,7 +1131,18 @@ func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_confi
11221131
"repoFullName", repoFullName,
11231132
"error", err,
11241133
)
1125-
return "", nil, nil, fmt.Errorf("failed to unmarshal config from cache: %v", err)
1134+
return "", nil, nil, nil, fmt.Errorf("failed to unmarshal config from cache: %v", err)
1135+
}
1136+
1137+
var taConfig tac.AtlantisConfig
1138+
err = json.Unmarshal(repoCache.TerragruntAtlantisConfig, &taConfig)
1139+
if err != nil {
1140+
slog.Error("Failed to unmarshal config from cache",
1141+
"orgId", orgId,
1142+
"repoFullName", repoFullName,
1143+
"error", err,
1144+
)
1145+
return "", nil, nil, nil, fmt.Errorf("failed to unmarshal config from cache: %v", err)
11261146
}
11271147

11281148
slog.Debug("Creating project dependency graph from cached config",
@@ -1138,7 +1158,7 @@ func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_confi
11381158
"repoFullName", repoFullName,
11391159
"error", err,
11401160
)
1141-
return "", nil, nil, fmt.Errorf("error creating dependency graph from cached config: %v", err)
1161+
return "", nil, nil, nil, fmt.Errorf("error creating dependency graph from cached config: %v", err)
11421162
}
11431163

11441164
slog.Info("Successfully retrieved config from cache",
@@ -1147,7 +1167,7 @@ func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_confi
11471167
"projectCount", len(config.Projects),
11481168
)
11491169

1150-
return repoCache.DiggerYmlStr, &config, &projectsGraph, nil
1170+
return repoCache.DiggerYmlStr, &config, &projectsGraph, &taConfig, nil
11511171
}
11521172

11531173
func GetRepoByInstllationId(installationId int64, repoOwner string, repoName string) (*models.Repo, error) {

‎backend/migrations/20250725041417.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- Modify "repo_caches" table
2+
ALTER TABLE "public"."repo_caches" ADD COLUMN "terragrunt_atlantis_config" bytea NULL;

‎backend/migrations/atlas.sum

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
h1:pk1704W4CHtf2K+qFKIkjTh8pN2R8Lor32ecJPtQgoY=
1+
h1:DO/UG0tYCnZA69yRLMd2ZBKUORlJtZJLkVCiCogKp+k=
22
20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
33
20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
44
20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -60,3 +60,4 @@ h1:pk1704W4CHtf2K+qFKIkjTh8pN2R8Lor32ecJPtQgoY=
6060
20250716043004.sql h1:WFz35jhsJ1pswpt5mmewW7L8LP5taRcNMf+pqUi3Yxs=
6161
20250716222603.sql h1:a66teR/6BQwNwhJ/zgNA/NjUVTz5DcoQl7cDZvzVS8c=
6262
20250717032021.sql h1:HaIhNsz3C+c87CDmFjgFlc9zGqoE5BU4m0dofDpeDYk=
63+
20250725041417.sql h1:Dds6fqS415FD1jlsoVEahcEEm1px3EHV5435moL+Vp8=

‎backend/models/cache.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@ import (
77
// storing repo cache such as digger.yml configuration
88
type RepoCache struct {
99
gorm.Model
10-
OrgId uint
11-
RepoFullName string
12-
DiggerYmlStr string
13-
DiggerConfig []byte `gorm:"type:bytea"`
10+
OrgId uint
11+
RepoFullName string
12+
DiggerYmlStr string
13+
DiggerConfig []byte `gorm:"type:bytea"`
14+
TerragruntAtlantisConfig []byte `gorm:"type:bytea"`
1415
}

‎backend/models/storage.go

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"encoding/json"
55
"errors"
66
"fmt"
7+
"github.com/diggerhq/digger/libs/digger_config/terragrunt/tac"
78
"log/slog"
89
"math"
910
"net/http"
@@ -1746,7 +1747,7 @@ func (db *Database) GetDiggerLock(resource string) (*DiggerLock, error) {
17461747
return lock, nil
17471748
}
17481749

1749-
func (db *Database) UpsertRepoCache(orgId uint, repoFullName string, diggerYmlStr string, diggerConfig configuration.DiggerConfig) (*RepoCache, error) {
1750+
func (db *Database) UpsertRepoCache(orgId uint, repoFullName string, diggerYmlStr string, diggerConfig configuration.DiggerConfig, newAtlantisConfig *tac.AtlantisConfig) (*RepoCache, error) {
17501751
var repoCache RepoCache
17511752

17521753
configMarshalled, err := json.Marshal(diggerConfig)
@@ -1758,6 +1759,15 @@ func (db *Database) UpsertRepoCache(orgId uint, repoFullName string, diggerYmlSt
17581759
return nil, fmt.Errorf("could not marshal config: %v", err)
17591760
}
17601761

1762+
atlantisConfigMarshalled, err := json.Marshal(newAtlantisConfig)
1763+
if err != nil {
1764+
slog.Error("could not marshal terragrunt-atlantis-config",
1765+
"repoFullName", repoFullName,
1766+
"orgId", orgId,
1767+
"error", err)
1768+
return nil, fmt.Errorf("could not marshal config: %v", err)
1769+
}
1770+
17611771
// check if repo exist already, do nothing in this case
17621772
result := db.GormDB.Where("org_id = ? AND repo_full_name=?", orgId, repoFullName).Find(&repoCache)
17631773
if result.Error != nil {
@@ -1777,6 +1787,7 @@ func (db *Database) UpsertRepoCache(orgId uint, repoFullName string, diggerYmlSt
17771787

17781788
repoCache.DiggerConfig = configMarshalled
17791789
repoCache.DiggerYmlStr = diggerYmlStr
1790+
repoCache.TerragruntAtlantisConfig = atlantisConfigMarshalled
17801791
result = db.GormDB.Save(&repoCache)
17811792
} else {
17821793
// create record here
@@ -1785,10 +1796,11 @@ func (db *Database) UpsertRepoCache(orgId uint, repoFullName string, diggerYmlSt
17851796
"orgId", orgId)
17861797

17871798
repoCache = RepoCache{
1788-
OrgId: orgId,
1789-
RepoFullName: repoFullName,
1790-
DiggerYmlStr: diggerYmlStr,
1791-
DiggerConfig: configMarshalled,
1799+
OrgId: orgId,
1800+
RepoFullName: repoFullName,
1801+
DiggerYmlStr: diggerYmlStr,
1802+
DiggerConfig: configMarshalled,
1803+
TerragruntAtlantisConfig: atlantisConfigMarshalled,
17921804
}
17931805
result = db.GormDB.Save(&repoCache)
17941806
if result.Error != nil {

‎backend/utils/bitbucket.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ func GetDiggerConfigForBitbucketBranch(bb BitbucketProvider, token string, repoF
107107
}
108108

109109
diggerYmlStr = string(diggerYmlBytes)
110-
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)
110+
config, _, dependencyGraph, _, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles, nil)
111111
if err != nil {
112112
slog.Error("Error loading Digger config",
113113
"repoFullName", repoFullName,

‎backend/utils/gitlab.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ func GetDiggerConfigForBranchGitlab(gh GitlabProvider, projectId int, repoFullNa
131131
"configLength", len(diggerYmlStr),
132132
)
133133

134-
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)
134+
config, _, dependencyGraph, _, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles, nil)
135135
if err != nil {
136136
slog.Error("Failed to load Digger config",
137137
"projectId", projectId,

‎background/projects-refresh-service/projects_refesh_main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ func main() {
4646

4747
slog.Info("refreshing projects from repo", "repoFullName", repoFullName)
4848
err := utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error {
49-
config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil)
49+
config, _, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil, nil)
5050
if err != nil {
5151
slog.Error("failed to load digger.yml: %v", "error", err)
5252
return fmt.Errorf("error loading digger.yml %v", err)

‎cli/pkg/github/github.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
107107
usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get current dir. %s", err), 4)
108108
}
109109

110-
diggerConfig, diggerConfigYaml, dependencyGraph, err := digger_config.LoadDiggerConfig("./", true, nil)
110+
diggerConfig, diggerConfigYaml, dependencyGraph, _, err := digger_config.LoadDiggerConfig("./", true, nil, nil)
111111
if err != nil {
112112
usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to read Digger digger_config. %s", err), 4)
113113
}

0 commit comments

Comments
 (0)