feat: ArgoCD v2.13.3 support for private chart providers (#6766)

* refactor: update chart handling and dependency management in GitOps operations

* refactor: improve chart metadata handling and error logging in GitOps operations

* refactor: add support for setting APIVersion in chart metadata

* refactor: convert requirements.yaml to JSON before unmarshalling in GitOps operations

* refactor: convert chart.yaml to JSON before unmarshalling in GitOps operations

* refactor: enhance dependency validation by using unique keys for comparison in GitOps operations

* refactor: convert chart metadata from JSON to YAML format for GitOps operations

* refactor: update shouldMigrateProxyChartDependencies to accept expected chart metadata and remove requirements.yaml checks

* docs: add developer guide for creating API specs and generating Go beans

* wip

---------

Co-authored-by: Ash-exp <asutosh2000ad@gmail.com>
Co-authored-by: ayushmaheshwari <ayush@devtron.ai>
Co-authored-by: iamayushm <32041961+iamayushm@users.noreply.github.com>

Author: 3 people

cmd/external-app/wire_gen.go

@@ -0,0 +1,77 @@
+# Developer Guide: Creating API Specs and Generating Go Beans
+
+## 1. Write the OpenAPI Spec
+
+- Use OpenAPI 3.0+ YAML format.
+- Define:
+    - `info`, `servers`, `tags`
+    - `paths` for each endpoint, with HTTP methods, parameters, request bodies, and responses.
+    - `components.schemas` for all request/response objects.
+- Use `x-oapi-codegen-extra-tags` for Go struct tags (e.g., validation).
+- The default behavior is to generate the required fields as non-pointer types and optional fields as pointer types.
+  - Use `x-go-type-skip-optional-pointer: false` to ensure fields are generated as pointer type. 
+- Use `x-go-json-ignore: true` to add `json:"-"` tag to a field, preventing it from being serialized in JSON.
+- Use `$ref` to reuse schema definitions.
+
+**Reference:**  
+See `specs/bulkEdit/v1beta2/bulk_edit.yaml` for a complete example.
+---
+
+## 2. Create oapi-codegen Config
+
+- Create a YAML config file (e.g., `oapi-models-config.yaml`) at the same level as your OpenAPI spec file.
+- This file configures the code generation process.
+- Set:
+    - `package`: Go package name for generated code.
+    - `generate.models`: `true` to generate Go structs.
+    - `output`: Relative path for the generated file (e.g., `./bean/bean.go`).
+    - `output-options`: Customize struct naming, skip pruning, etc.
+    - `exclude-schemas`: Exclude error types if needed.
+
+**Reference:**  
+```yaml
+# yaml-language-server: ...
+package: api
+generate:
+  models: true
+output-options:
+  # to make sure that all types are generated
+  skip-prune: true
+  name-normalizer: ToCamelCaseWithDigits
+  prefer-skip-optional-pointer: true
+  prefer-skip-optional-pointer-on-container-types: true
+  exclude-schemas:
+    - "ErrorResponse"
+    - "ApiError"
+output: {{ // relative path to the generator.go file }}
+```
+---
+
+## 3. Add go:generate Directive
+- In your Go file (e.g., `generator.go`), add a `//go:generate` comment:
+    ```go
+        //go:generate go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen "--config=../../../../specs/bulkEdit/v1beta2/oapi-models-config.yaml" "../../../../specs/bulkEdit/v1beta2/bulk_edit.yaml"
+    ```
+- Here `--config` is the relative path (from the `generator.go` file) to your config file, i.e. `oapi-models-config.yaml` file.
+- And the last argument is the path to your OpenAPI spec file, i.e. `bulk_edit.yaml`.
+---
+
+## 4. Generate the Beans
+Run: (From the root dir)
+```bash
+    go generate ./pkg/...
+```
+> This will generate Go structs at the path specified in your config (e.g., bean/bean.go).
+---
+
+## 5. Best Practices
+Keep OpenAPI specs DRY: use $ref and shared schemas.
+Document every field and endpoint.
+Use validation tags for all struct fields.
+Exclude error response types from bean generation if not needed.
+Keep config and spec files versioned and close to your Go code.
+---
+
+## Summary:
+Write your OpenAPI YAML, create a codegen config, add a go:generate directive, and run go generate to produce Go beans at the desired path. 
+Follow the structure in bulk_edit.yaml and oapi-models-config.yaml for consistency.

developers-guide/StructGenerator.md

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"fmt"
 	dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry"
+	chartRefBean "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean"
 	dirCopy "github.com/otiai10/copy"
 	"go.opentelemetry.io/otel"
 	"go.uber.org/zap"
@@ -60,7 +61,7 @@ type ChartCreateRequest struct {
 
 type ChartCreateResponse struct {
 	BuiltChartPath string
-	valuesYaml     string
+	ChartMetaData  *chart.Metadata
 }
 
 type ChartTemplateService interface {
@@ -74,7 +75,7 @@ type ChartTemplateService interface {
 	LoadChartInBytes(ChartPath string, deleteChart bool) ([]byte, error)
 	LoadChartFromDir(dir string) (*chart.Chart, error)
 	CreateZipFileForChart(chart *chart.Chart, outputChartPathDir string) ([]byte, error)
-	PackageChart(tempReferenceTemplateDir string, chartMetaData *chart.Metadata) (*string, string, error)
+	PackageChart(tempReferenceTemplateDir string, chartMetaData *chart.Metadata) (*chart.Metadata, *string, string, error)
 }
 
 type ChartTemplateServiceImpl struct {
@@ -105,7 +106,7 @@ func (impl ChartTemplateServiceImpl) GetChartVersion(location string) (string, e
 		return "", fmt.Errorf("%q is not a directory", location)
 	}
 
-	chartYaml := filepath.Join(location, "Chart.yaml")
+	chartYaml := filepath.Join(location, chartRefBean.CHART_YAML_FILE)
 	if _, err := os.Stat(chartYaml); os.IsNotExist(err) {
 		return "", fmt.Errorf("Chart.yaml file not present in the directory %q", location)
 	}
@@ -135,7 +136,7 @@ func (impl ChartTemplateServiceImpl) FetchValuesFromReferenceChart(chartMetaData
 		impl.logger.Errorw("error in copying chart for app", "app", chartMetaData.Name, "error", err)
 		return nil, err
 	}
-	archivePath, valuesYaml, err := impl.PackageChart(chartDir, chartMetaData)
+	_, archivePath, valuesYaml, err := impl.PackageChart(chartDir, chartMetaData)
 	if err != nil {
 		impl.logger.Errorw("error in creating archive", "err", err)
 		return nil, err
@@ -175,7 +176,7 @@ func (impl ChartTemplateServiceImpl) BuildChart(ctx context.Context, chartMetaDa
 		return "", err
 	}
 	_, span := otel.Tracer("orchestrator").Start(ctx, "impl.PackageChart")
-	_, _, err = impl.PackageChart(tempReferenceTemplateDir, chartMetaData)
+	_, _, _, err = impl.PackageChart(tempReferenceTemplateDir, chartMetaData)
 	span.End()
 	if err != nil {
 		impl.logger.Errorw("error in creating archive", "err", err)
@@ -190,25 +191,24 @@ func (impl ChartTemplateServiceImpl) BuildChartProxyForHelmApps(chartCreateReque
 	chartMetaData.APIVersion = "v2" // ensure always v2
 	dir := impl.GetDir()
 	chartDir := filepath.Join(CHART_WORKING_DIR_PATH, dir)
-	impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", chartDir)
+	chartCreateResponse.BuiltChartPath = chartDir
+	impl.logger.Debugw("temp chart dir", "chart", chartMetaData.Name, "dir", chartDir)
 	err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling
 	if err != nil {
 		impl.logger.Errorw("err in creating dir", "dir", chartDir, "err", err)
 		return chartCreateResponse, err
 	}
 	err = dirCopy.Copy(chartCreateRequest.ChartPath, chartDir)
-
 	if err != nil {
 		impl.logger.Errorw("error in copying chart for app", "app", chartMetaData.Name, "error", err)
 		return chartCreateResponse, err
 	}
-	_, valuesYaml, err := impl.PackageChart(chartDir, chartMetaData)
+	chartMetaData, _, _, err = impl.PackageChart(chartDir, chartMetaData)
 	if err != nil {
 		impl.logger.Errorw("error in creating archive", "err", err)
 		return chartCreateResponse, err
 	}
-	chartCreateResponse.valuesYaml = valuesYaml
-	chartCreateResponse.BuiltChartPath = chartDir
+	chartCreateResponse.ChartMetaData = chartMetaData
 	return chartCreateResponse, nil
 }
 
@@ -292,58 +292,64 @@ func (impl ChartTemplateServiceImpl) overrideChartMetaDataInDir(chartDir string,
 		impl.logger.Errorw("error in loading template chart", "chartPath", chartDir, "err", err)
 		return nil, err
 	}
+	if len(chartMetaData.APIVersion) > 0 {
+		chart.Metadata.APIVersion = chartMetaData.APIVersion
+	}
 	if len(chartMetaData.Name) > 0 {
 		chart.Metadata.Name = chartMetaData.Name
 	}
 	if len(chartMetaData.Version) > 0 {
 		chart.Metadata.Version = chartMetaData.Version
 	}
+	if len(chartMetaData.Dependencies) > 0 {
+		chart.Metadata.Dependencies = append(chart.Metadata.Dependencies, chartMetaData.Dependencies...)
+	}
 	chartMetaDataBytes, err := yaml.Marshal(chart.Metadata)
 	if err != nil {
 		impl.logger.Errorw("error in marshaling chartMetadata", "err", err)
 		return chart, err
 	}
-	err = ioutil.WriteFile(filepath.Join(chartDir, "Chart.yaml"), chartMetaDataBytes, 0600)
+	err = os.WriteFile(filepath.Join(chartDir, chartRefBean.CHART_YAML_FILE), chartMetaDataBytes, 0600)
 	if err != nil {
 		impl.logger.Errorw("err in writing Chart.yaml", "err", err)
 		return chart, err
 	}
 	return chart, nil
 }
 
-func (impl ChartTemplateServiceImpl) PackageChart(tempReferenceTemplateDir string, chartMetaData *chart.Metadata) (*string, string, error) {
+func (impl ChartTemplateServiceImpl) PackageChart(tempReferenceTemplateDir string, chartMetaData *chart.Metadata) (*chart.Metadata, *string, string, error) {
 	valid, err := chartutil.IsChartDir(tempReferenceTemplateDir)
 	if err != nil {
 		impl.logger.Errorw("error in validating base chart", "dir", tempReferenceTemplateDir, "err", err)
-		return nil, "", err
+		return nil, nil, "", err
 	}
 	if !valid {
 		impl.logger.Errorw("invalid chart at ", "dir", tempReferenceTemplateDir)
-		return nil, "", fmt.Errorf("invalid base chart")
+		return nil, nil, "", fmt.Errorf("invalid base chart")
 	}
 	chart, err := impl.overrideChartMetaDataInDir(tempReferenceTemplateDir, chartMetaData)
 	if err != nil {
 		impl.logger.Errorw("error in overriding chart metadata", "chartPath", tempReferenceTemplateDir, "err", err)
-		return nil, "", err
+		return nil, nil, "", err
 	}
 	archivePath, err := chartutil.Save(chart, tempReferenceTemplateDir)
 	if err != nil {
 		impl.logger.Errorw("error in saving", "err", err, "dir", tempReferenceTemplateDir)
-		return nil, "", err
+		return nil, nil, "", err
 	}
 	impl.logger.Debugw("chart archive path", "path", archivePath)
 	var valuesYaml string
 	byteValues, err := json.Marshal(chart.Values)
 	if err != nil {
 		impl.logger.Errorw("error in json Marshal values", "values", chart.Values, "err", err)
-		return nil, "", err
+		return nil, nil, "", err
 	}
 	if chart.Values != nil {
 		valuesYaml = string(byteValues)
 	} else {
 		impl.logger.Warnw("values.yaml not found in helm chart", "dir", tempReferenceTemplateDir)
 	}
-	return &archivePath, valuesYaml, nil
+	return chart.Metadata, &archivePath, valuesYaml, nil
 }
 
 func (impl ChartTemplateServiceImpl) CleanDir(dir string) {
@@ -376,7 +382,7 @@ func (impl ChartTemplateServiceImpl) GetByteArrayRefChart(chartMetaData *chart.M
 		impl.logger.Errorw("error in copying chart for app", "app", chartMetaData.Name, "error", err)
 		return nil, err
 	}
-	activePath, _, err := impl.PackageChart(tempReferenceTemplateDir, chartMetaData)
+	_, activePath, _, err := impl.PackageChart(tempReferenceTemplateDir, chartMetaData)
 	if err != nil {
 		impl.logger.Errorw("error in creating archive", "err", err)
 		return nil, err

internal/util/ChartTemplateService.go

@@ -298,7 +298,6 @@ func (chart *InstallAppVersionDTO) GetFluxDeploymentConfig() *bean2.DeploymentCo
 	}
 }
 
-// /
 type RefChartProxyDir string
 
 const (
@@ -336,15 +335,6 @@ type AppNames struct {
 	SuggestedName string `json:"suggestedName,omitempty"`
 }
 
-type Dependencies struct {
-	Dependencies []Dependency `json:"dependencies"`
-}
-type Dependency struct {
-	Name       string `json:"name"`
-	Version    string `json:"version"`
-	Repository string `json:"repository"`
-}
-
 const REFERENCE_TYPE_DEFAULT string = "DEFAULT"
 const REFERENCE_TYPE_TEMPLATE string = "TEMPLATE"
 const REFERENCE_TYPE_DEPLOYED string = "DEPLOYED"

pkg/appStore/bean/bean.go

@@ -41,12 +41,13 @@ func ParseChartGitPushRequest(installAppRequestDTO *appStoreBean.InstallAppVersi
 	}
 }
 
-func ParseChartCreateRequest(appName string, includePackageChart bool) *util.ChartCreateRequest {
+func ParseChartCreateRequest(appName string, dependencies []*chart.Dependency, includePackageChart bool) *util.ChartCreateRequest {
 	chartPath := getRefProxyChartPath()
 	return &util.ChartCreateRequest{
 		ChartMetaData: &chart.Metadata{
-			Name:    appName,
-			Version: "1.0.1", // TODO Asutoh: Why not the actual version?
+			Name:         appName,
+			Version:      "1.0.1", // TODO Asutoh: Why not the actual version?
+			Dependencies: dependencies,
 		},
 		ChartPath:           chartPath,
 		IncludePackageChart: includePackageChart,