From b3cac1547935ccbc82588c46bb60d64827ec1223 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Sun, 10 Mar 2024 18:23:45 -0700 Subject: [PATCH 01/35] change admin to adhoc, also check why we dont exit early --- .../neptune/workflows/activities/terraform/job.go | 2 +- .../workflows/internal/terraform/job/runner.go | 4 +++- .../workflows/internal/terraform/root_fetcher.go | 2 +- .../workflows/internal/terraform/workflow.go | 13 ++++++++----- .../workflows/internal/terraform/workflow_test.go | 9 +++++---- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/server/neptune/workflows/activities/terraform/job.go b/server/neptune/workflows/activities/terraform/job.go index 64f6cf51f..2937dcae6 100644 --- a/server/neptune/workflows/activities/terraform/job.go +++ b/server/neptune/workflows/activities/terraform/job.go @@ -53,5 +53,5 @@ type WorkflowMode int const ( Deploy WorkflowMode = iota PR - Admin + Adhoc ) diff --git a/server/neptune/workflows/internal/terraform/job/runner.go b/server/neptune/workflows/internal/terraform/job/runner.go index a38d8237f..08b1035d7 100644 --- a/server/neptune/workflows/internal/terraform/job/runner.go +++ b/server/neptune/workflows/internal/terraform/job/runner.go @@ -2,6 +2,7 @@ package job import ( "context" + "github.com/runatlantis/atlantis/server/neptune/workflows/activities/command" key "github.com/runatlantis/atlantis/server/neptune/context" @@ -215,7 +216,8 @@ func (r *JobRunner) apply(executionCtx *ExecutionContext, planFile string, step } func (r *JobRunner) plan(ctx *ExecutionContext, mode *terraform.PlanMode, workflowMode terraform.WorkflowMode, extraArgs []string) (activities.TerraformPlanResponse, error) { - if workflowMode == terraform.Admin { + // TODO: Don't we already check this earlier? Why are we checking again, is there somewhere else this can get called? + if workflowMode == terraform.Adhoc { // Admin mode doesn't need to run a plan. return activities.TerraformPlanResponse{}, nil } diff --git a/server/neptune/workflows/internal/terraform/root_fetcher.go b/server/neptune/workflows/internal/terraform/root_fetcher.go index 66d6856bc..d714ab889 100644 --- a/server/neptune/workflows/internal/terraform/root_fetcher.go +++ b/server/neptune/workflows/internal/terraform/root_fetcher.go @@ -27,7 +27,7 @@ func (r *RootFetcher) Fetch(ctx workflow.Context) (*terraform.LocalRoot, func(wo return nil, func(_ workflow.Context) error { return nil }, err } - if r.Request.WorkflowMode == terraform.Admin { + if r.Request.WorkflowMode == terraform.Adhoc { return fetchRootResponse.LocalRoot, func(c workflow.Context) error { return nil }, nil } diff --git a/server/neptune/workflows/internal/terraform/workflow.go b/server/neptune/workflows/internal/terraform/workflow.go index 98e08aa4c..469b812cb 100644 --- a/server/neptune/workflows/internal/terraform/workflow.go +++ b/server/neptune/workflows/internal/terraform/workflow.go @@ -3,10 +3,11 @@ package terraform import ( "context" "fmt" - "github.com/runatlantis/atlantis/server/neptune/workflows/activities/conftest" "net/url" "time" + "github.com/runatlantis/atlantis/server/neptune/workflows/activities/conftest" + key "github.com/runatlantis/atlantis/server/neptune/context" "go.temporal.io/api/enums/v1" "go.temporal.io/sdk/client" @@ -341,6 +342,12 @@ func (r *Runner) run(ctx workflow.Context) (Response, error) { if err != nil { return Response{}, r.toExternalError(err, "fetching root") } + + // if we are in adhoc / terraform admin mode, we don't need to cleanup, plan, validate, or apply + if r.Request.WorkflowMode == terraform.Adhoc { + return Response{}, nil + } + defer func() { r.executeCleanup(ctx, cleanup) }() @@ -350,10 +357,6 @@ func (r *Runner) run(ctx workflow.Context) (Response, error) { return Response{}, r.toExternalError(err, "running plan job") } - if r.Request.WorkflowMode == terraform.Admin { - return Response{}, nil - } - if r.Request.WorkflowMode == terraform.PR { validationResults, err := r.Validate(ctx, root, response.ServerURL, planResponse.PlanJSONFile) if err != nil { diff --git a/server/neptune/workflows/internal/terraform/workflow_test.go b/server/neptune/workflows/internal/terraform/workflow_test.go index bb17b6dc0..1b9d1fad9 100644 --- a/server/neptune/workflows/internal/terraform/workflow_test.go +++ b/server/neptune/workflows/internal/terraform/workflow_test.go @@ -4,11 +4,12 @@ import ( "context" "errors" "fmt" - "github.com/runatlantis/atlantis/server/neptune/workflows/activities/conftest" "net/url" "testing" "time" + "github.com/runatlantis/atlantis/server/neptune/workflows/activities/conftest" + "go.temporal.io/sdk/activity" "go.temporal.io/sdk/client" @@ -176,7 +177,7 @@ func testTerraformWorkflow(ctx workflow.Context, req request) (*response, error) WorkflowMode: req.WorkflowMode, } - if req.WorkflowMode == terraformModel.Admin { + if req.WorkflowMode == terraformModel.Adhoc { tAct = nil } @@ -619,7 +620,7 @@ func TestSuccess_PRMode(t *testing.T) { }, resp.States) } -func TestSuccess_AdminMode(t *testing.T) { +func TestSuccess_AdhocMode(t *testing.T) { var suite testsuite.WorkflowTestSuite env := suite.NewTestWorkflowEnvironment() ga := &githubActivities{} @@ -649,7 +650,7 @@ func TestSuccess_AdminMode(t *testing.T) { // execute workflow env.ExecuteWorkflow(testTerraformWorkflow, request{ - WorkflowMode: terraformModel.Admin, + WorkflowMode: terraformModel.Adhoc, }) assert.True(t, env.IsWorkflowCompleted()) From d5654049598f73b46d98531fe6418f4bc125a4f9 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Sun, 10 Mar 2024 18:33:05 -0700 Subject: [PATCH 02/35] change admin to adhoc, also check why we dont exit early --- server/neptune/workflows/internal/terraform/job/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/neptune/workflows/internal/terraform/job/runner.go b/server/neptune/workflows/internal/terraform/job/runner.go index 08b1035d7..f594edd32 100644 --- a/server/neptune/workflows/internal/terraform/job/runner.go +++ b/server/neptune/workflows/internal/terraform/job/runner.go @@ -218,7 +218,7 @@ func (r *JobRunner) apply(executionCtx *ExecutionContext, planFile string, step func (r *JobRunner) plan(ctx *ExecutionContext, mode *terraform.PlanMode, workflowMode terraform.WorkflowMode, extraArgs []string) (activities.TerraformPlanResponse, error) { // TODO: Don't we already check this earlier? Why are we checking again, is there somewhere else this can get called? if workflowMode == terraform.Adhoc { - // Admin mode doesn't need to run a plan. + // Adhoc mode doesn't need to run a plan. return activities.TerraformPlanResponse{}, nil } var resp activities.TerraformPlanResponse From 708a5822c04593c258ab1569a4f60e8805725077 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 18 Mar 2024 18:42:31 -0700 Subject: [PATCH 03/35] first draft --- server/neptune/adhoc/server.go | 65 +++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 447b929f0..38b6ad2f9 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -17,6 +17,7 @@ import ( "github.com/runatlantis/atlantis/server/neptune/lyft/feature" "github.com/runatlantis/atlantis/server/neptune/sync/crons" ghClient "github.com/runatlantis/atlantis/server/neptune/workflows/activities/github" + "github.com/runatlantis/atlantis/server/neptune/workflows/activities/terraform" "github.com/runatlantis/atlantis/server/vcs/provider/github" assetfs "github.com/elazarl/go-bindata-assetfs" @@ -34,23 +35,24 @@ import ( "github.com/runatlantis/atlantis/server/static" "github.com/uber-go/tally/v4" "github.com/urfave/negroni" + "go.temporal.io/sdk/client" "go.temporal.io/sdk/interceptor" "go.temporal.io/sdk/worker" ) type Server struct { - Logger logging.Logger - CronScheduler *internalSync.CronScheduler - Crons []*internalSync.Cron - HTTPServerProxy *neptune_http.ServerProxy - Port int - StatsScope tally.Scope - StatsCloser io.Closer - TemporalClient *temporal.ClientWrapper - TerraformActivities *activities.Terraform - GithubActivities *activities.Github - - TerraformTaskQueue string + Logger logging.Logger + CronScheduler *internalSync.CronScheduler + Crons []*internalSync.Cron + HTTPServerProxy *neptune_http.ServerProxy + Port int + StatsScope tally.Scope + StatsCloser io.Closer + TemporalClient *temporal.ClientWrapper + TerraformActivities *activities.Terraform + GithubActivities *activities.Github + adhocExecutionParams AdhocTerraformWorkflowExecutionParams + TerraformTaskQueue string } func NewServer(config *adhocconfig.Config) (*Server, error) { @@ -172,6 +174,39 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { return &server, nil } +type AdhocTerraformWorkflowExecutionParams struct { + AtlantisRoot string + AtlantisRepo string + Revision string + DeploymentID string + // TODO: in separate PR, fill in github.Repo and terraform.Root (need helper funcs to get those) +} + +// This function constructs the request we want to send to the temporal client, +// then executes the Terraform workflow. Note normally this workflow is executed +// when a request is made to the server, but we are manually executing it here, +// since we don't care about requests in adhoc mode. +func (s Server) manuallyExecuteTerraformWorkflow(adhocExecutionParams AdhocTerraformWorkflowExecutionParams) (interface{}, error) { + request := workflows.TerraformRequest{ + DeploymentID: adhocExecutionParams.DeploymentID, + Revision: adhocExecutionParams.Revision, + WorkflowMode: terraform.Adhoc, + } + options := client.StartWorkflowOptions{ + TaskQueue: s.TerraformTaskQueue, + SearchAttributes: map[string]interface{}{ + "atlantis_repository": adhocExecutionParams.AtlantisRepo, + "atlantis_root": adhocExecutionParams.AtlantisRoot, + }, + } + + res, err := s.TemporalClient.ExecuteWorkflow(context.Background(), options, workflows.Terraform, request) + if err != nil { + s.Logger.Error(err.Error()) + } + return res, nil +} + func (s Server) Start() error { defer s.shutdown() @@ -191,6 +226,12 @@ func (s Server) Start() error { s.Logger.InfoContext(ctx, "Shutting down terraform worker, resource clean up may still be occurring in the background") }() + _, err := s.manuallyExecuteTerraformWorkflow(s.adhocExecutionParams) + if err != nil { + s.Logger.Error(err.Error()) + return err + } + // Ensure server gracefully drains connections when stopped. stop := make(chan os.Signal, 1) // Stop on SIGINTs and SIGTERMs. From f6743334cc88eeab132b227e9c3c0a04cfff0ba8 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 18 Mar 2024 18:44:01 -0700 Subject: [PATCH 04/35] first draft --- server/neptune/workflows/internal/terraform/job/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/neptune/workflows/internal/terraform/job/runner.go b/server/neptune/workflows/internal/terraform/job/runner.go index f594edd32..636f39b6a 100644 --- a/server/neptune/workflows/internal/terraform/job/runner.go +++ b/server/neptune/workflows/internal/terraform/job/runner.go @@ -216,7 +216,7 @@ func (r *JobRunner) apply(executionCtx *ExecutionContext, planFile string, step } func (r *JobRunner) plan(ctx *ExecutionContext, mode *terraform.PlanMode, workflowMode terraform.WorkflowMode, extraArgs []string) (activities.TerraformPlanResponse, error) { - // TODO: Don't we already check this earlier? Why are we checking again, is there somewhere else this can get called? + if workflowMode == terraform.Adhoc { // Adhoc mode doesn't need to run a plan. return activities.TerraformPlanResponse{}, nil From 6c5264945a32e7d7056bf4295fec2c8fd9020c83 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 18 Mar 2024 19:01:43 -0700 Subject: [PATCH 05/35] first draft --- server/neptune/adhoc/server.go | 1 + server/neptune/workflows/internal/terraform/job/runner.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 38b6ad2f9..fb72979a0 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -203,6 +203,7 @@ func (s Server) manuallyExecuteTerraformWorkflow(adhocExecutionParams AdhocTerra res, err := s.TemporalClient.ExecuteWorkflow(context.Background(), options, workflows.Terraform, request) if err != nil { s.Logger.Error(err.Error()) + return nil, err } return res, nil } diff --git a/server/neptune/workflows/internal/terraform/job/runner.go b/server/neptune/workflows/internal/terraform/job/runner.go index 636f39b6a..c55686c72 100644 --- a/server/neptune/workflows/internal/terraform/job/runner.go +++ b/server/neptune/workflows/internal/terraform/job/runner.go @@ -216,7 +216,6 @@ func (r *JobRunner) apply(executionCtx *ExecutionContext, planFile string, step } func (r *JobRunner) plan(ctx *ExecutionContext, mode *terraform.PlanMode, workflowMode terraform.WorkflowMode, extraArgs []string) (activities.TerraformPlanResponse, error) { - if workflowMode == terraform.Adhoc { // Adhoc mode doesn't need to run a plan. return activities.TerraformPlanResponse{}, nil From a570d5bcfa94d2021ea9ec43ba3dfe659ede37ca Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 18 Mar 2024 19:04:20 -0700 Subject: [PATCH 06/35] first draft --- server/neptune/adhoc/server.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index fb72979a0..20f1776ed 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -153,6 +153,8 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { cronScheduler := internalSync.NewCronScheduler(config.CtxLogger) + adhocExecutionParams := getAdhocExecutionParams(config) + server := Server{ Logger: config.CtxLogger, CronScheduler: cronScheduler, @@ -162,18 +164,24 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { Frequency: 1 * time.Minute, }, }, - HTTPServerProxy: httpServerProxy, - Port: config.ServerCfg.Port, - StatsScope: scope, - StatsCloser: statsCloser, - TemporalClient: temporalClient, - TerraformActivities: terraformActivities, - TerraformTaskQueue: config.TemporalCfg.TerraformTaskQueue, - GithubActivities: githubActivities, + HTTPServerProxy: httpServerProxy, + Port: config.ServerCfg.Port, + StatsScope: scope, + StatsCloser: statsCloser, + TemporalClient: temporalClient, + TerraformActivities: terraformActivities, + TerraformTaskQueue: config.TemporalCfg.TerraformTaskQueue, + GithubActivities: githubActivities, + adhocExecutionParams: adhocExecutionParams, } return &server, nil } +// TODO: complete this func +func getAdhocExecutionParams(config *adhocconfig.Config) AdhocTerraformWorkflowExecutionParams { + return AdhocTerraformWorkflowExecutionParams{} +} + type AdhocTerraformWorkflowExecutionParams struct { AtlantisRoot string AtlantisRepo string From 43dc515f57f4990e2cbbaeff0feacdaa9e5b9573 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 18 Mar 2024 19:33:18 -0700 Subject: [PATCH 07/35] first draft --- server/neptune/adhoc/server.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 20f1776ed..f89574949 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -179,7 +179,14 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { // TODO: complete this func func getAdhocExecutionParams(config *adhocconfig.Config) AdhocTerraformWorkflowExecutionParams { - return AdhocTerraformWorkflowExecutionParams{} + return AdhocTerraformWorkflowExecutionParams{ + AtlantisRoot: "", + AtlantisRepo: "", + Revision: "", + DeploymentID: "", + Root: terraform.Root{}, + Repo: ghClient.Repo{}, + } } type AdhocTerraformWorkflowExecutionParams struct { @@ -187,6 +194,8 @@ type AdhocTerraformWorkflowExecutionParams struct { AtlantisRepo string Revision string DeploymentID string + Root terraform.Root + Repo ghClient.Repo // TODO: in separate PR, fill in github.Repo and terraform.Root (need helper funcs to get those) } @@ -199,6 +208,8 @@ func (s Server) manuallyExecuteTerraformWorkflow(adhocExecutionParams AdhocTerra DeploymentID: adhocExecutionParams.DeploymentID, Revision: adhocExecutionParams.Revision, WorkflowMode: terraform.Adhoc, + Root: adhocExecutionParams.Root, + Repo: adhocExecutionParams.Repo, } options := client.StartWorkflowOptions{ TaskQueue: s.TerraformTaskQueue, From 1475bb85450b5f3d4e5fd691683d8807004b24a3 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 18 Mar 2024 20:04:03 -0700 Subject: [PATCH 08/35] first draft --- server/neptune/adhoc/server.go | 1 - server/neptune/workflows/internal/terraform/workflow.go | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index f89574949..578b374e6 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -196,7 +196,6 @@ type AdhocTerraformWorkflowExecutionParams struct { DeploymentID string Root terraform.Root Repo ghClient.Repo - // TODO: in separate PR, fill in github.Repo and terraform.Root (need helper funcs to get those) } // This function constructs the request we want to send to the temporal client, diff --git a/server/neptune/workflows/internal/terraform/workflow.go b/server/neptune/workflows/internal/terraform/workflow.go index 469b812cb..97e632a9d 100644 --- a/server/neptune/workflows/internal/terraform/workflow.go +++ b/server/neptune/workflows/internal/terraform/workflow.go @@ -343,11 +343,6 @@ func (r *Runner) run(ctx workflow.Context) (Response, error) { return Response{}, r.toExternalError(err, "fetching root") } - // if we are in adhoc / terraform admin mode, we don't need to cleanup, plan, validate, or apply - if r.Request.WorkflowMode == terraform.Adhoc { - return Response{}, nil - } - defer func() { r.executeCleanup(ctx, cleanup) }() @@ -357,6 +352,10 @@ func (r *Runner) run(ctx workflow.Context) (Response, error) { return Response{}, r.toExternalError(err, "running plan job") } + if r.Request.WorkflowMode == terraform.Adhoc { + return Response{}, nil + } + if r.Request.WorkflowMode == terraform.PR { validationResults, err := r.Validate(ctx, root, response.ServerURL, planResponse.PlanJSONFile) if err != nil { From cb6f933cdfaa46a05b1ba26700b69e838dd57594 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 18 Mar 2024 20:07:53 -0700 Subject: [PATCH 09/35] first draft --- server/neptune/adhoc/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 578b374e6..999209246 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -177,7 +177,7 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { return &server, nil } -// TODO: complete this func +// TODO: complete this func, filling in the values from global config or something func getAdhocExecutionParams(config *adhocconfig.Config) AdhocTerraformWorkflowExecutionParams { return AdhocTerraformWorkflowExecutionParams{ AtlantisRoot: "", From e3c65f268188493654813fec0ddf81cf3accd0ac Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Tue, 19 Mar 2024 13:10:27 -0700 Subject: [PATCH 10/35] merge --- server/neptune/adhoc/server.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 999209246..159f0733f 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -177,25 +177,23 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { return &server, nil } -// TODO: complete this func, filling in the values from global config or something func getAdhocExecutionParams(config *adhocconfig.Config) AdhocTerraformWorkflowExecutionParams { return AdhocTerraformWorkflowExecutionParams{ - AtlantisRoot: "", - AtlantisRepo: "", - Revision: "", - DeploymentID: "", - Root: terraform.Root{}, - Repo: ghClient.Repo{}, + AtlantisRoot: "", + AtlantisRepo: "", + Revision: "", + TerraformRoot: terraform.Root{}, + GithubRepo: ghClient.Repo{}, } } type AdhocTerraformWorkflowExecutionParams struct { - AtlantisRoot string - AtlantisRepo string - Revision string - DeploymentID string - Root terraform.Root - Repo ghClient.Repo + AtlantisRoot string + AtlantisRepo string + Revision string + TerraformRoot terraform.Root + GithubRepo ghClient.Repo + // Note that deploymentID is used in NewWorkflowStore(), but we don't care about that in adhoc mode so can leave it blank } // This function constructs the request we want to send to the temporal client, @@ -204,11 +202,10 @@ type AdhocTerraformWorkflowExecutionParams struct { // since we don't care about requests in adhoc mode. func (s Server) manuallyExecuteTerraformWorkflow(adhocExecutionParams AdhocTerraformWorkflowExecutionParams) (interface{}, error) { request := workflows.TerraformRequest{ - DeploymentID: adhocExecutionParams.DeploymentID, Revision: adhocExecutionParams.Revision, WorkflowMode: terraform.Adhoc, - Root: adhocExecutionParams.Root, - Repo: adhocExecutionParams.Repo, + Root: adhocExecutionParams.TerraformRoot, + Repo: adhocExecutionParams.GithubRepo, } options := client.StartWorkflowOptions{ TaskQueue: s.TerraformTaskQueue, From 317a808099a5b66146a730c206f327aad33d850a Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Wed, 20 Mar 2024 10:40:14 -0700 Subject: [PATCH 11/35] run in go routine --- server/neptune/adhoc/server.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 159f0733f..c91ce62b1 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -1,4 +1,4 @@ -package admin +package adhoc import ( "context" @@ -242,11 +242,14 @@ func (s Server) Start() error { s.Logger.InfoContext(ctx, "Shutting down terraform worker, resource clean up may still be occurring in the background") }() - _, err := s.manuallyExecuteTerraformWorkflow(s.adhocExecutionParams) - if err != nil { - s.Logger.Error(err.Error()) - return err - } + wg.Add(1) + go func() { + defer wg.Done() + _, err := s.manuallyExecuteTerraformWorkflow(s.adhocExecutionParams) + if err != nil { + s.Logger.Error(err.Error()) + } + }() // Ensure server gracefully drains connections when stopped. stop := make(chan os.Signal, 1) From c842b51b296b18b9c566ad5bf4c65f57ce1595b1 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Wed, 20 Mar 2024 11:45:13 -0700 Subject: [PATCH 12/35] ok --- server/neptune/adhoc/server.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index c91ce62b1..0d0fe9d64 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -177,16 +177,6 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { return &server, nil } -func getAdhocExecutionParams(config *adhocconfig.Config) AdhocTerraformWorkflowExecutionParams { - return AdhocTerraformWorkflowExecutionParams{ - AtlantisRoot: "", - AtlantisRepo: "", - Revision: "", - TerraformRoot: terraform.Root{}, - GithubRepo: ghClient.Repo{}, - } -} - type AdhocTerraformWorkflowExecutionParams struct { AtlantisRoot string AtlantisRepo string From 4ab6c43d6739d03b69df16b1ee63766dfa9ad093 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Wed, 10 Apr 2024 13:41:16 -0700 Subject: [PATCH 13/35] merge --- cmd/adhoc.go | 22 +++----- server/neptune/adhoc/config/config.go | 8 ++- server/neptune/adhoc/server.go | 74 +++++++++++++++------------ 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/cmd/adhoc.go b/cmd/adhoc.go index cd22bb6e7..197f645e1 100644 --- a/cmd/adhoc.go +++ b/cmd/adhoc.go @@ -7,11 +7,8 @@ import ( "github.com/runatlantis/atlantis/server/legacy" "github.com/runatlantis/atlantis/server/logging" adhoc "github.com/runatlantis/atlantis/server/neptune/adhoc" - adhocHelpers "github.com/runatlantis/atlantis/server/neptune/adhoc/adhocexecutionhelpers" adhocconfig "github.com/runatlantis/atlantis/server/neptune/adhoc/config" neptune "github.com/runatlantis/atlantis/server/neptune/temporalworker/config" - "github.com/runatlantis/atlantis/server/neptune/workflows/activities/github" - "github.com/runatlantis/atlantis/server/neptune/workflows/activities/terraform" ) type Adhoc struct{} @@ -58,18 +55,13 @@ func (a *Adhoc) NewServer(userConfig legacy.UserConfig, config legacy.Config) (S DownloadURL: userConfig.TFDownloadURL, LogFilters: globalCfg.TerraformLogFilter, }, - DataDir: userConfig.DataDir, - TemporalCfg: globalCfg.Temporal, - GithubCfg: globalCfg.Github, - App: appConfig, - CtxLogger: ctxLogger, - StatsNamespace: userConfig.StatsNamespace, - Metrics: globalCfg.Metrics, - AdhocExecutionParams: adhocHelpers.AdhocTerraformWorkflowExecutionParams{ - Revision: "", - TerraformRoots: []terraform.Root{}, - GithubRepo: github.Repo{}, - }, + DataDir: userConfig.DataDir, + TemporalCfg: globalCfg.Temporal, + GithubCfg: globalCfg.Github, + App: appConfig, + CtxLogger: ctxLogger, + StatsNamespace: userConfig.StatsNamespace, + Metrics: globalCfg.Metrics, GithubHostname: userConfig.GithubHostname, GithubAppID: userConfig.GithubAppID, GithubAppKeyFile: userConfig.GithubAppKeyFile, diff --git a/server/neptune/adhoc/config/config.go b/server/neptune/adhoc/config/config.go index c017e37ec..ac01acff5 100644 --- a/server/neptune/adhoc/config/config.go +++ b/server/neptune/adhoc/config/config.go @@ -4,7 +4,6 @@ import ( "github.com/palantir/go-githubapp/githubapp" "github.com/runatlantis/atlantis/server/config/valid" "github.com/runatlantis/atlantis/server/logging" - adhoc "github.com/runatlantis/atlantis/server/neptune/adhoc/adhocexecutionhelpers" neptune "github.com/runatlantis/atlantis/server/neptune/temporalworker/config" ) @@ -20,10 +19,9 @@ type Config struct { StatsNamespace string - DataDir string - CtxLogger logging.Logger - App githubapp.Config - AdhocExecutionParams adhoc.AdhocTerraformWorkflowExecutionParams + DataDir string + CtxLogger logging.Logger + App githubapp.Config GithubAppID int64 GithubAppKeyFile string diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 4a71ff558..483bcff4b 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -48,20 +48,22 @@ import ( ) type Server struct { - Logger logging.Logger - CronScheduler *internalSync.CronScheduler - Crons []*internalSync.Cron - HTTPServerProxy *neptune_http.ServerProxy - Port int - StatsScope tally.Scope - StatsCloser io.Closer - TemporalClient *temporal.ClientWrapper - TerraformActivities *activities.Terraform - GithubActivities *activities.Github - AdhocExecutionParams adhoc.AdhocTerraformWorkflowExecutionParams - TerraformTaskQueue string - RootConfigBuilder *root_config.Builder - GithubRetriever *adhocGithubHelpers.AdhocGithubRetriever + Logger logging.Logger + CronScheduler *internalSync.CronScheduler + Crons []*internalSync.Cron + HTTPServerProxy *neptune_http.ServerProxy + Port int + StatsScope tally.Scope + StatsCloser io.Closer + TemporalClient *temporal.ClientWrapper + TerraformActivities *activities.Terraform + GithubActivities *activities.Github + TerraformTaskQueue string + RootConfigBuilder *root_config.Builder + GithubRetriever *adhocGithubHelpers.AdhocGithubRetriever + Repo string + Root string + Revision string } func NewServer(config *adhocconfig.Config) (*Server, error) { @@ -222,17 +224,16 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { Frequency: 1 * time.Minute, }, }, - HTTPServerProxy: httpServerProxy, - Port: config.ServerCfg.Port, - StatsScope: scope, - StatsCloser: statsCloser, - TemporalClient: temporalClient, - TerraformActivities: terraformActivities, - TerraformTaskQueue: config.TemporalCfg.TerraformTaskQueue, - GithubActivities: githubActivities, - AdhocExecutionParams: config.AdhocExecutionParams, - RootConfigBuilder: rootConfigBuilder, - GithubRetriever: githubRetriever, + HTTPServerProxy: httpServerProxy, + Port: config.ServerCfg.Port, + StatsScope: scope, + StatsCloser: statsCloser, + TemporalClient: temporalClient, + TerraformActivities: terraformActivities, + TerraformTaskQueue: config.TemporalCfg.TerraformTaskQueue, + GithubActivities: githubActivities, + RootConfigBuilder: rootConfigBuilder, + GithubRetriever: githubRetriever, } return &server, nil } @@ -241,18 +242,18 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { // then executes the Terraform workflow. Note normally this workflow is executed // when a request is made to the server, but we are manually executing it here, // since we don't care about requests in adhoc mode. -func (s Server) manuallyExecuteTerraformWorkflow(adhocExecutionParams adhoc.AdhocTerraformWorkflowExecutionParams) (interface{}, error) { +func (s Server) manuallyExecuteTerraformWorkflow(repo ghClient.Repo, revision string, root terraform.Root) (interface{}, error) { request := workflows.TerraformRequest{ - Revision: adhocExecutionParams.Revision, + Revision: revision, WorkflowMode: terraform.Adhoc, - Root: adhocExecutionParams.TerraformRoot, - Repo: adhocExecutionParams.GithubRepo, + Root: root, + Repo: repo, } options := client.StartWorkflowOptions{ TaskQueue: s.TerraformTaskQueue, SearchAttributes: map[string]interface{}{ - "atlantis_repository": adhocExecutionParams.AtlantisRepo, - "atlantis_root": adhocExecutionParams.AtlantisRoot, + "atlantis_repository": repo.GetFullName(), + "atlantis_root": root.Name, }, } @@ -290,9 +291,18 @@ func (s Server) Start() error { wg.Add(1) go func() { defer wg.Done() - _, err := s.manuallyExecuteTerraformWorkflow(s.AdhocExecutionParams) + + adhocExecutionParams, err := adhoc.ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx, s.Repo, s.Revision, s.GithubRetriever, s.RootConfigBuilder) if err != nil { s.Logger.Error(err.Error()) + return + } + + for _, root := range adhocExecutionParams.TerraformRoots { + _, err := s.manuallyExecuteTerraformWorkflow(adhocExecutionParams.GithubRepo, adhocExecutionParams.Revision, root) + if err != nil { + s.Logger.Error(err.Error()) + } } }() From 6ed2c010fefce6065400e95766a63aabf8bc7355 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Fri, 12 Apr 2024 12:57:53 -0700 Subject: [PATCH 14/35] panic fixes --- cmd/adhoc.go | 4 ++++ server/neptune/adhoc/config/config.go | 2 ++ server/neptune/adhoc/server.go | 25 ++----------------------- 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/cmd/adhoc.go b/cmd/adhoc.go index 197f645e1..0603b3299 100644 --- a/cmd/adhoc.go +++ b/cmd/adhoc.go @@ -67,6 +67,10 @@ func (a *Adhoc) NewServer(userConfig legacy.UserConfig, config legacy.Config) (S GithubAppKeyFile: userConfig.GithubAppKeyFile, GithubAppSlug: userConfig.GithubAppSlug, GlobalCfg: globalCfg, + ValidationConfig: neptune.ValidationConfig{ + DefaultVersion: globalCfg.PolicySets.Version, + Policies: globalCfg.PolicySets, + }, } return adhoc.NewServer(cfg) } diff --git a/server/neptune/adhoc/config/config.go b/server/neptune/adhoc/config/config.go index ac01acff5..fd94d6b96 100644 --- a/server/neptune/adhoc/config/config.go +++ b/server/neptune/adhoc/config/config.go @@ -29,4 +29,6 @@ type Config struct { GithubHostname string GlobalCfg valid.GlobalCfg + + ValidationConfig neptune.ValidationConfig } diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 483bcff4b..f66f8b755 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -15,7 +15,6 @@ import ( "github.com/palantir/go-githubapp/githubapp" "github.com/runatlantis/atlantis/server/legacy/events/vcs" - "github.com/runatlantis/atlantis/server/neptune/lyft/feature" "github.com/runatlantis/atlantis/server/neptune/sync/crons" ghClient "github.com/runatlantis/atlantis/server/neptune/workflows/activities/github" "github.com/runatlantis/atlantis/server/neptune/workflows/activities/terraform" @@ -35,7 +34,6 @@ import ( neptune_http "github.com/runatlantis/atlantis/server/neptune/http" internalSync "github.com/runatlantis/atlantis/server/neptune/sync" "github.com/runatlantis/atlantis/server/neptune/temporal" - neptune "github.com/runatlantis/atlantis/server/neptune/temporalworker/config" "github.com/runatlantis/atlantis/server/neptune/workflows" "github.com/runatlantis/atlantis/server/neptune/workflows/activities" "github.com/runatlantis/atlantis/server/static" @@ -110,7 +108,7 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { terraformActivities, err := activities.NewTerraform( config.TerraformCfg, - neptune.ValidationConfig{}, + config.ValidationConfig, config.App, config.DataDir, config.ServerCfg.URL, @@ -130,33 +128,14 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { return nil, errors.Wrap(err, "client creator") } - repoConfig := feature.RepoConfig{ - Owner: config.FeatureConfig.FFOwner, - Repo: config.FeatureConfig.FFRepo, - Branch: config.FeatureConfig.FFBranch, - Path: config.FeatureConfig.FFPath, - } installationFetcher := &github.InstallationRetriever{ ClientCreator: clientCreator, } - fileFetcher := &github.SingleFileContentsFetcher{ - ClientCreator: clientCreator, - } - retriever := &feature.CustomGithubInstallationRetriever{ - InstallationFetcher: installationFetcher, - FileContentsFetcher: fileFetcher, - Cfg: repoConfig, - } - featureAllocator, err := feature.NewGHSourcedAllocator(retriever, config.CtxLogger) - if err != nil { - return nil, errors.Wrap(err, "initializing feature allocator") - } - githubActivities, err := activities.NewGithub( clientCreator, config.GithubCfg.TemporalAppInstallationID, config.DataDir, - featureAllocator, + nil, ) if err != nil { return nil, errors.Wrap(err, "initializing github activities") From cb78afdbea2de9a062f1c843d03df3b3180b5d68 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Fri, 12 Apr 2024 13:59:39 -0700 Subject: [PATCH 15/35] somehow forgot to add repo and revision woops --- server/neptune/adhoc/server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index f66f8b755..24d1a2c3d 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -213,6 +213,8 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { GithubActivities: githubActivities, RootConfigBuilder: rootConfigBuilder, GithubRetriever: githubRetriever, + Repo: config.GlobalCfg.AdhocMode.Repo, + Revision: config.GlobalCfg.AdhocMode.Revision, } return &server, nil } From d08d2d079ee6a0cdaa965342df3e3e1fdb8463b7 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Fri, 12 Apr 2024 14:50:47 -0700 Subject: [PATCH 16/35] add more logging --- server/neptune/exec/exec.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/neptune/exec/exec.go b/server/neptune/exec/exec.go index 2c640860e..fd078bf27 100644 --- a/server/neptune/exec/exec.go +++ b/server/neptune/exec/exec.go @@ -2,13 +2,14 @@ package exec import ( "context" - "github.com/pkg/errors" - "github.com/runatlantis/atlantis/server/logging" - key "github.com/runatlantis/atlantis/server/neptune/context" "os" "os/exec" "syscall" "time" + + "github.com/pkg/errors" + "github.com/runatlantis/atlantis/server/logging" + key "github.com/runatlantis/atlantis/server/neptune/context" ) type Cmd struct { @@ -48,7 +49,7 @@ func (c *Cmd) RunWithNewProcessGroup(ctx context.Context) error { err := c.Wait() if ctx.Err() != nil { - return errors.Wrap(ctx.Err(), "waiting for process") + return errors.Wrap(ctx.Err(), "context error, waiting for process") } if err != nil { return errors.Wrap(err, "waiting for process") From ec7f526d12c9a8c9444c70d2518ac55de39fb162 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Sun, 14 Apr 2024 09:26:49 -0700 Subject: [PATCH 17/35] add more logging --- server/vcs/provider/github/repo_fetcher.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/vcs/provider/github/repo_fetcher.go b/server/vcs/provider/github/repo_fetcher.go index 5b487ee65..3acd7cd12 100644 --- a/server/vcs/provider/github/repo_fetcher.go +++ b/server/vcs/provider/github/repo_fetcher.go @@ -91,7 +91,8 @@ func (g *RepoFetcher) clone(ctx context.Context, repo models.Repo, branch string } _, err := g.run(ctx, cloneCmd, destinationPath) if err != nil { - return "", nil, errors.Wrap(err, "failed to clone directory") + debugStr := fmt.Sprintf("clone cmd is %v, destination path is %s", cloneCmd, destinationPath) + return "", nil, errors.Wrap(err, "failed to clone directory, debug info: "+debugStr) } // Return immediately if commit at HEAD of clone matches request commit @@ -137,7 +138,7 @@ func (g *RepoFetcher) run(ctx context.Context, args []string, destinationPath st cmd.Stderr = &b err := cmd.RunWithNewProcessGroup(ctx) if err != nil { - return nil, errors.Wrap(err, "running command in separate process group") + return nil, errors.Wrap(err, "running command in separate process group, command is "+cmd.String()) } return b.Bytes(), nil } From 86843add35d505d79f414729bccd2536fe23790b Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Sun, 14 Apr 2024 10:06:53 -0700 Subject: [PATCH 18/35] add more logging --- server/vcs/provider/github/repo_fetcher.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/vcs/provider/github/repo_fetcher.go b/server/vcs/provider/github/repo_fetcher.go index 3acd7cd12..41590974b 100644 --- a/server/vcs/provider/github/repo_fetcher.go +++ b/server/vcs/provider/github/repo_fetcher.go @@ -72,6 +72,9 @@ func (g *RepoFetcher) Fetch(ctx context.Context, repo models.Repo, branch string } func (g *RepoFetcher) clone(ctx context.Context, repo models.Repo, branch string, sha string, options RepoFetcherOptions) (string, func(ctx context.Context, filePath string), error) { + // debug + options.SimplePath = false + destinationPath := g.generateDirPath(repo.Name) // If simple path is enabled, we don't need a prefix and UUID if options.SimplePath { @@ -91,7 +94,7 @@ func (g *RepoFetcher) clone(ctx context.Context, repo models.Repo, branch string } _, err := g.run(ctx, cloneCmd, destinationPath) if err != nil { - debugStr := fmt.Sprintf("clone cmd is %v, destination path is %s", cloneCmd, destinationPath) + debugStr := fmt.Sprintf("destination path is %s, repo is %v, sha is %v", destinationPath, repo, sha) return "", nil, errors.Wrap(err, "failed to clone directory, debug info: "+debugStr) } From 9352a3f6b0bfca9ff98875ccc310905fc0632e25 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Sun, 14 Apr 2024 11:34:55 -0700 Subject: [PATCH 19/35] fix the underlying issue --- .../adhoc_execution_params.go | 17 ++++++++++++++++- server/vcs/provider/github/repo_fetcher.go | 3 --- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go index ba22b0afa..013923a4d 100644 --- a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go +++ b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go @@ -4,6 +4,7 @@ import ( "context" "github.com/pkg/errors" + "github.com/runatlantis/atlantis/server/models" "github.com/runatlantis/atlantis/server/neptune/adhoc/adhocgithubhelpers" "github.com/runatlantis/atlantis/server/neptune/gateway/config" root_config "github.com/runatlantis/atlantis/server/neptune/gateway/config" @@ -33,7 +34,21 @@ func ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx context.Cont }, } - rootCfgs, err := rootCfgBuilder.Build(ctx, &root_config.RepoCommit{}, repo.Credentials.InstallationToken, opts) + rootCfgs, err := rootCfgBuilder.Build(ctx, &root_config.RepoCommit{ + Repo: models.Repo{ + FullName: repo.GetFullName(), + Owner: repo.Owner, + Name: repoName, + CloneURL: repo.URL, + VCSHost: models.VCSHost{ + Hostname: "github.com", + Type: models.Github, + }, + DefaultBranch: repo.DefaultBranch, + }, + Branch: repo.DefaultBranch, + Sha: revision, + }, repo.Credentials.InstallationToken, opts) if err != nil { return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "building root cfgs") } diff --git a/server/vcs/provider/github/repo_fetcher.go b/server/vcs/provider/github/repo_fetcher.go index 41590974b..aab5bd9ea 100644 --- a/server/vcs/provider/github/repo_fetcher.go +++ b/server/vcs/provider/github/repo_fetcher.go @@ -72,9 +72,6 @@ func (g *RepoFetcher) Fetch(ctx context.Context, repo models.Repo, branch string } func (g *RepoFetcher) clone(ctx context.Context, repo models.Repo, branch string, sha string, options RepoFetcherOptions) (string, func(ctx context.Context, filePath string), error) { - // debug - options.SimplePath = false - destinationPath := g.generateDirPath(repo.Name) // If simple path is enabled, we don't need a prefix and UUID if options.SimplePath { From 8e954a4e41d0bee5ab4d92c8663b11b5d0d75330 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Sun, 14 Apr 2024 18:04:58 -0700 Subject: [PATCH 20/35] moar logging --- server/neptune/gateway/config/root_config_builder.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/neptune/gateway/config/root_config_builder.go b/server/neptune/gateway/config/root_config_builder.go index 30108fc00..e18544a33 100644 --- a/server/neptune/gateway/config/root_config_builder.go +++ b/server/neptune/gateway/config/root_config_builder.go @@ -54,7 +54,8 @@ func (s *ModifiedRootsStrategy) FindMatches(ctx context.Context, config valid.Re Sha: repo.RepoCommit.Sha, }) if err != nil { - return nil, errors.Wrapf(err, "finding modified files: %s", modifiedFiles) + debugStr := fmt.Sprintf("sha: %s, prNum: %d, dir %s", repo.RepoCommit.Sha, repo.RepoCommit.OptionalPRNum, repo.Dir) + return nil, errors.Wrapf(err, "finding modified files: %s, debug str: %s", modifiedFiles, debugStr) } matchingRoots, err := s.RootFinder.FindRoots(ctx, config, repo.Dir, modifiedFiles) @@ -131,6 +132,7 @@ func (b *Builder) build(ctx context.Context, commit *RepoCommit, installationTok RepoCommit: commit, Dir: repoDir, } + b.Logger.Info(fmt.Sprintf("localRepo is: full repo name: %s, commit sha: %s, commit branch: %s, commit repo name: %s, repodir: %s", commit.Repo.FullName, commit.Sha, commit.Branch, commit.Repo.Name, repoDir)) // Run pre-workflow hooks err = b.HooksRunner.Run(ctx, localRepo.Repo, localRepo.Dir) From 34cc02f44ad9fd0d1a6e2af540c0eb2a39a46f13 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Sun, 14 Apr 2024 18:37:39 -0700 Subject: [PATCH 21/35] moar logging --- server/neptune/adhoc/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 24d1a2c3d..bd3566a46 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -193,6 +193,7 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { RepoRetriever: repoRetriever, InstallationRetriever: installationFetcher, } + config.CtxLogger.Info(fmt.Sprintf("Starting adhoc server, params are: repo: %s, root: %s, revision: %s", config.GlobalCfg.AdhocMode.Repo, config.GlobalCfg.AdhocMode.Root, config.GlobalCfg.AdhocMode.Revision)) server := Server{ Logger: config.CtxLogger, From ebf578433cb1c7297d38d7793eb2d0236c34ac49 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 15 Apr 2024 09:44:47 -0700 Subject: [PATCH 22/35] moar logging --- cmd/adhoc.go | 4 ---- server/neptune/adhoc/config/config.go | 2 -- server/neptune/adhoc/server.go | 3 ++- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/cmd/adhoc.go b/cmd/adhoc.go index 0603b3299..197f645e1 100644 --- a/cmd/adhoc.go +++ b/cmd/adhoc.go @@ -67,10 +67,6 @@ func (a *Adhoc) NewServer(userConfig legacy.UserConfig, config legacy.Config) (S GithubAppKeyFile: userConfig.GithubAppKeyFile, GithubAppSlug: userConfig.GithubAppSlug, GlobalCfg: globalCfg, - ValidationConfig: neptune.ValidationConfig{ - DefaultVersion: globalCfg.PolicySets.Version, - Policies: globalCfg.PolicySets, - }, } return adhoc.NewServer(cfg) } diff --git a/server/neptune/adhoc/config/config.go b/server/neptune/adhoc/config/config.go index fd94d6b96..ac01acff5 100644 --- a/server/neptune/adhoc/config/config.go +++ b/server/neptune/adhoc/config/config.go @@ -29,6 +29,4 @@ type Config struct { GithubHostname string GlobalCfg valid.GlobalCfg - - ValidationConfig neptune.ValidationConfig } diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index bd3566a46..f5cef06a8 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -34,6 +34,7 @@ import ( neptune_http "github.com/runatlantis/atlantis/server/neptune/http" internalSync "github.com/runatlantis/atlantis/server/neptune/sync" "github.com/runatlantis/atlantis/server/neptune/temporal" + neptune "github.com/runatlantis/atlantis/server/neptune/temporalworker/config" "github.com/runatlantis/atlantis/server/neptune/workflows" "github.com/runatlantis/atlantis/server/neptune/workflows/activities" "github.com/runatlantis/atlantis/server/static" @@ -108,7 +109,7 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { terraformActivities, err := activities.NewTerraform( config.TerraformCfg, - config.ValidationConfig, + neptune.ValidationConfig{}, config.App, config.DataDir, config.ServerCfg.URL, From 3d86fb5d947fd16a1fc9612468481146aa10a506 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 15 Apr 2024 14:26:07 -0700 Subject: [PATCH 23/35] moar logging --- server/neptune/gateway/config/root_config_builder.go | 4 ++++ server/vcs/provider/github/repo_fetcher.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/server/neptune/gateway/config/root_config_builder.go b/server/neptune/gateway/config/root_config_builder.go index e18544a33..48a601d77 100644 --- a/server/neptune/gateway/config/root_config_builder.go +++ b/server/neptune/gateway/config/root_config_builder.go @@ -135,6 +135,7 @@ func (b *Builder) build(ctx context.Context, commit *RepoCommit, installationTok b.Logger.Info(fmt.Sprintf("localRepo is: full repo name: %s, commit sha: %s, commit branch: %s, commit repo name: %s, repodir: %s", commit.Repo.FullName, commit.Sha, commit.Branch, commit.Repo.Name, repoDir)) // Run pre-workflow hooks + b.Logger.Info(fmt.Sprintf("running pre-workflow hooks")) err = b.HooksRunner.Run(ctx, localRepo.Repo, localRepo.Dir) if err != nil { return nil, errors.Wrap(err, "running pre-workflow hooks") @@ -144,16 +145,19 @@ func (b *Builder) build(ctx context.Context, commit *RepoCommit, installationTok // TODO: rename project to roots var mergedRootCfgs []*valid.MergedProjectCfg + b.Logger.Info(fmt.Sprintf("parsing repo config")) repoCfg, err := b.ParserValidator.ParseRepoCfg(localRepo.Dir, localRepo.Repo.ID()) if err != nil { return nil, errors.Wrapf(err, "parsing %s", config.AtlantisYAMLFilename) } + b.Logger.Info(fmt.Sprintf("getting matching roots")) matchingRoots, err := b.getMatchingRoots(ctx, repoCfg, localRepo, installationToken, rootNames) if err != nil { return nil, errors.Wrap(err, "getting matching roots") } + b.Logger.Info(fmt.Sprintf("merging roots for %s", localRepo.Repo.FullName)) for _, mr := range matchingRoots { mergedRootCfg := b.GlobalCfg.MergeProjectCfg(localRepo.Repo.ID(), mr, repoCfg) mergedRootCfgs = append(mergedRootCfgs, &mergedRootCfg) diff --git a/server/vcs/provider/github/repo_fetcher.go b/server/vcs/provider/github/repo_fetcher.go index aab5bd9ea..b16006bfb 100644 --- a/server/vcs/provider/github/repo_fetcher.go +++ b/server/vcs/provider/github/repo_fetcher.go @@ -62,12 +62,14 @@ func (g *RepoFetcher) Fetch(ctx context.Context, repo models.Repo, branch string authURL := fmt.Sprintf("://x-access-token:%s", ghToken) repo.CloneURL = strings.Replace(repo.CloneURL, "://:", authURL, 1) repo.SanitizedCloneURL = strings.Replace(repo.SanitizedCloneURL, "://:", "://x-access-token:", 1) + g.Logger.Info(fmt.Sprintf("about to clone inside RepoFetcher Fetch with params: repo: %v. branch: %s, sha: %s", repo, branch, sha)) path, cleanup, err := g.clone(ctx, repo, branch, sha, options) if err != nil { g.Scope.Counter(metrics.ExecutionErrorMetric).Inc(1) return path, cleanup, err } g.Scope.Counter(metrics.ExecutionSuccessMetric).Inc(1) + g.Logger.Info(fmt.Sprintf("cloned repo %s to path %s", repo.Name, path)) return path, cleanup, err } @@ -79,6 +81,7 @@ func (g *RepoFetcher) clone(ctx context.Context, repo models.Repo, branch string } // Create the directory and parents if necessary. + g.Logger.Info("creating new directory inside clone at path: " + destinationPath) if err := os.MkdirAll(destinationPath, 0700); err != nil { return "", nil, errors.Wrap(err, "creating new directory") } @@ -144,6 +147,7 @@ func (g *RepoFetcher) run(ctx context.Context, args []string, destinationPath st } func (g *RepoFetcher) Cleanup(ctx context.Context, filePath string) { + g.Logger.Info(fmt.Sprintf("cleaning up cloned repo at path %s", filePath)) if err := os.RemoveAll(filePath); err != nil { g.Logger.ErrorContext(ctx, "failed deleting cloned repo", map[string]interface{}{ "err": err, From 5c60e8a74515821ea1477f7576ea4ceea131cc6c Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 15 Apr 2024 15:10:05 -0700 Subject: [PATCH 24/35] moar logging --- .../adhoc/adhocexecutionhelpers/adhoc_execution_params.go | 2 ++ server/neptune/adhoc/server.go | 5 +++++ server/neptune/gateway/config/root_config_builder.go | 6 +++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go index 013923a4d..b9202879f 100644 --- a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go +++ b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go @@ -53,8 +53,10 @@ func ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx context.Cont return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "building root cfgs") } + rootCfgBuilder.Logger.Info("getting roots from merged project cfgs") roots := getRootsFromMergedProjectCfgs(rootCfgs) + rootCfgBuilder.Logger.Info("returning adhocexecution params") return AdhocTerraformWorkflowExecutionParams{ Revision: revision, GithubRepo: repo, diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index f5cef06a8..8e721206b 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -281,7 +281,12 @@ func (s Server) Start() error { return } + if len(adhocExecutionParams.TerraformRoots) == 0 { + s.Logger.Info("no roots found") + } + for _, root := range adhocExecutionParams.TerraformRoots { + s.Logger.Info("running terraform workflow for root " + root.Name) _, err := s.manuallyExecuteTerraformWorkflow(adhocExecutionParams.GithubRepo, adhocExecutionParams.Revision, root) if err != nil { s.Logger.Error(err.Error()) diff --git a/server/neptune/gateway/config/root_config_builder.go b/server/neptune/gateway/config/root_config_builder.go index 48a601d77..50fa1de93 100644 --- a/server/neptune/gateway/config/root_config_builder.go +++ b/server/neptune/gateway/config/root_config_builder.go @@ -135,7 +135,7 @@ func (b *Builder) build(ctx context.Context, commit *RepoCommit, installationTok b.Logger.Info(fmt.Sprintf("localRepo is: full repo name: %s, commit sha: %s, commit branch: %s, commit repo name: %s, repodir: %s", commit.Repo.FullName, commit.Sha, commit.Branch, commit.Repo.Name, repoDir)) // Run pre-workflow hooks - b.Logger.Info(fmt.Sprintf("running pre-workflow hooks")) + b.Logger.Info("running pre-workflow hooks") err = b.HooksRunner.Run(ctx, localRepo.Repo, localRepo.Dir) if err != nil { return nil, errors.Wrap(err, "running pre-workflow hooks") @@ -145,13 +145,13 @@ func (b *Builder) build(ctx context.Context, commit *RepoCommit, installationTok // TODO: rename project to roots var mergedRootCfgs []*valid.MergedProjectCfg - b.Logger.Info(fmt.Sprintf("parsing repo config")) + b.Logger.Info("parsing repo config") repoCfg, err := b.ParserValidator.ParseRepoCfg(localRepo.Dir, localRepo.Repo.ID()) if err != nil { return nil, errors.Wrapf(err, "parsing %s", config.AtlantisYAMLFilename) } - b.Logger.Info(fmt.Sprintf("getting matching roots")) + b.Logger.Info("getting matching roots") matchingRoots, err := b.getMatchingRoots(ctx, repoCfg, localRepo, installationToken, rootNames) if err != nil { return nil, errors.Wrap(err, "getting matching roots") From f4c4300778cff0ddf94108aecab660d571880716 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Thu, 18 Apr 2024 20:29:52 -0700 Subject: [PATCH 25/35] fix all issues and remove all unused code --- cmd/adhoc.go | 2 + .../adhoc_execution_params.go | 57 ++++++++----- .../adhoc_github_helpers.go | 50 ------------ server/neptune/adhoc/config/config.go | 2 + server/neptune/adhoc/server.go | 80 ++++++++++--------- 5 files changed, 82 insertions(+), 109 deletions(-) delete mode 100644 server/neptune/adhoc/adhocgithubhelpers/adhoc_github_helpers.go diff --git a/cmd/adhoc.go b/cmd/adhoc.go index 197f645e1..0a4d82cec 100644 --- a/cmd/adhoc.go +++ b/cmd/adhoc.go @@ -67,6 +67,8 @@ func (a *Adhoc) NewServer(userConfig legacy.UserConfig, config legacy.Config) (S GithubAppKeyFile: userConfig.GithubAppKeyFile, GithubAppSlug: userConfig.GithubAppSlug, GlobalCfg: globalCfg, + GithubUser: userConfig.GithubUser, + GithubToken: userConfig.GithubToken, } return adhoc.NewServer(cfg) } diff --git a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go index b9202879f..2b6b49141 100644 --- a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go +++ b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go @@ -4,13 +4,12 @@ import ( "context" "github.com/pkg/errors" - "github.com/runatlantis/atlantis/server/models" - "github.com/runatlantis/atlantis/server/neptune/adhoc/adhocgithubhelpers" "github.com/runatlantis/atlantis/server/neptune/gateway/config" root_config "github.com/runatlantis/atlantis/server/neptune/gateway/config" "github.com/runatlantis/atlantis/server/neptune/workflows/activities/github" "github.com/runatlantis/atlantis/server/neptune/workflows/activities/terraform" internal_gh "github.com/runatlantis/atlantis/server/vcs/provider/github" + "github.com/runatlantis/atlantis/server/vcs/provider/github/converter" ) type AdhocTerraformWorkflowExecutionParams struct { @@ -20,11 +19,30 @@ type AdhocTerraformWorkflowExecutionParams struct { // Note that deploymentID is used in NewWorkflowStore(), but we don't care about that in adhoc mode so can leave it blank } -func ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx context.Context, repoName string, revision string, githubRetriever *adhocgithubhelpers.AdhocGithubRetriever, rootCfgBuilder *root_config.Builder) (AdhocTerraformWorkflowExecutionParams, error) { +func ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever( + ctx context.Context, + repoName string, + PRNum int, + pullFetcher *internal_gh.PRFetcher, + pullConverter converter.PullConverter, + installationRetriever *internal_gh.InstallationRetriever, + rootCfgBuilder *root_config.Builder) (AdhocTerraformWorkflowExecutionParams, error) { + + orgName := "lyft" + installationToken, err := installationRetriever.FindOrganizationInstallation(ctx, orgName) + if err != nil { + return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "finding organization installation") + } + // TODO: in the future, could potentially pass in the owner instead of hardcoding lyft - repo, err := githubRetriever.GetRepository(ctx, "lyft", repoName) + ghCommit, err := pullFetcher.Fetch(ctx, installationToken.Token, orgName, repoName, PRNum) if err != nil { - return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "getting repo") + return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "fetching commit") + } + + actualCommit, err := pullConverter.Convert(ghCommit) + if err != nil { + return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "converting commit") } opts := config.BuilderOptions{ @@ -35,20 +53,11 @@ func ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx context.Cont } rootCfgs, err := rootCfgBuilder.Build(ctx, &root_config.RepoCommit{ - Repo: models.Repo{ - FullName: repo.GetFullName(), - Owner: repo.Owner, - Name: repoName, - CloneURL: repo.URL, - VCSHost: models.VCSHost{ - Hostname: "github.com", - Type: models.Github, - }, - DefaultBranch: repo.DefaultBranch, - }, - Branch: repo.DefaultBranch, - Sha: revision, - }, repo.Credentials.InstallationToken, opts) + Repo: actualCommit.HeadRepo, + Branch: actualCommit.HeadBranch, + Sha: actualCommit.HeadCommit, + OptionalPRNum: actualCommit.Num, + }, installationToken.Token, opts) if err != nil { return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "building root cfgs") } @@ -58,8 +67,14 @@ func ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx context.Cont rootCfgBuilder.Logger.Info("returning adhocexecution params") return AdhocTerraformWorkflowExecutionParams{ - Revision: revision, - GithubRepo: repo, + Revision: actualCommit.HeadCommit, + GithubRepo: github.Repo{ + Owner: orgName, + Name: repoName, + URL: actualCommit.HeadRepo.CloneURL, + DefaultBranch: actualCommit.HeadRepo.DefaultBranch, + Credentials: github.AppCredentials{InstallationToken: installationToken.Token}, + }, TerraformRoots: roots, }, nil } diff --git a/server/neptune/adhoc/adhocgithubhelpers/adhoc_github_helpers.go b/server/neptune/adhoc/adhocgithubhelpers/adhoc_github_helpers.go deleted file mode 100644 index fb0854c57..000000000 --- a/server/neptune/adhoc/adhocgithubhelpers/adhoc_github_helpers.go +++ /dev/null @@ -1,50 +0,0 @@ -package adhocgithubhelpers - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/runatlantis/atlantis/server/models" - "github.com/runatlantis/atlantis/server/neptune/workflows/activities/github" - internal "github.com/runatlantis/atlantis/server/vcs/provider/github" -) - -type repoRetriever interface { - Get(ctx context.Context, installationToken int64, owner, repo string) (models.Repo, error) -} - -type installationRetriever interface { - FindOrganizationInstallation(ctx context.Context, org string) (internal.Installation, error) -} - -type AdhocGithubRetriever struct { - RepoRetriever repoRetriever - InstallationRetriever installationRetriever -} - -func (r *AdhocGithubRetriever) GetRepository(ctx context.Context, owner string, repoName string) (github.Repo, error) { - installation, err := r.InstallationRetriever.FindOrganizationInstallation(ctx, owner) - if err != nil { - return github.Repo{}, errors.Wrap(err, "finding installation") - } - - repo, err := r.RepoRetriever.Get(ctx, installation.Token, owner, repoName) - if err != nil { - return github.Repo{}, errors.Wrap(err, "getting repo") - } - - if len(repo.DefaultBranch) == 0 { - return github.Repo{}, fmt.Errorf("default branch was nil, this is a bug on github's side") - } - - return github.Repo{ - Owner: repo.Owner, - Name: repo.Name, - URL: repo.CloneURL, - DefaultBranch: repo.DefaultBranch, - Credentials: github.AppCredentials{ - InstallationToken: installation.Token, - }, - }, nil -} diff --git a/server/neptune/adhoc/config/config.go b/server/neptune/adhoc/config/config.go index ac01acff5..eb52a8293 100644 --- a/server/neptune/adhoc/config/config.go +++ b/server/neptune/adhoc/config/config.go @@ -27,6 +27,8 @@ type Config struct { GithubAppKeyFile string GithubAppSlug string GithubHostname string + GithubUser string + GithubToken string GlobalCfg valid.GlobalCfg } diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 8e721206b..783eb9e6b 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -19,6 +19,7 @@ import ( ghClient "github.com/runatlantis/atlantis/server/neptune/workflows/activities/github" "github.com/runatlantis/atlantis/server/neptune/workflows/activities/terraform" "github.com/runatlantis/atlantis/server/vcs/provider/github" + "github.com/runatlantis/atlantis/server/vcs/provider/github/converter" assetfs "github.com/elazarl/go-bindata-assetfs" "github.com/gorilla/mux" @@ -26,7 +27,6 @@ import ( "github.com/runatlantis/atlantis/server/logging" "github.com/runatlantis/atlantis/server/metrics" adhoc "github.com/runatlantis/atlantis/server/neptune/adhoc/adhocexecutionhelpers" - adhocGithubHelpers "github.com/runatlantis/atlantis/server/neptune/adhoc/adhocgithubhelpers" adhocconfig "github.com/runatlantis/atlantis/server/neptune/adhoc/config" root_config "github.com/runatlantis/atlantis/server/neptune/gateway/config" "github.com/runatlantis/atlantis/server/neptune/gateway/deploy" @@ -38,7 +38,6 @@ import ( "github.com/runatlantis/atlantis/server/neptune/workflows" "github.com/runatlantis/atlantis/server/neptune/workflows/activities" "github.com/runatlantis/atlantis/server/static" - github_converter "github.com/runatlantis/atlantis/server/vcs/provider/github/converter" "github.com/uber-go/tally/v4" "github.com/urfave/negroni" "go.temporal.io/sdk/client" @@ -47,22 +46,23 @@ import ( ) type Server struct { - Logger logging.Logger - CronScheduler *internalSync.CronScheduler - Crons []*internalSync.Cron - HTTPServerProxy *neptune_http.ServerProxy - Port int - StatsScope tally.Scope - StatsCloser io.Closer - TemporalClient *temporal.ClientWrapper - TerraformActivities *activities.Terraform - GithubActivities *activities.Github - TerraformTaskQueue string - RootConfigBuilder *root_config.Builder - GithubRetriever *adhocGithubHelpers.AdhocGithubRetriever - Repo string - Root string - Revision string + Logger logging.Logger + CronScheduler *internalSync.CronScheduler + Crons []*internalSync.Cron + HTTPServerProxy *neptune_http.ServerProxy + Port int + StatsScope tally.Scope + StatsCloser io.Closer + TemporalClient *temporal.ClientWrapper + TerraformActivities *activities.Terraform + GithubActivities *activities.Github + TerraformTaskQueue string + RootConfigBuilder *root_config.Builder + Repo string + PRNum int + InstallationRetriever *github.InstallationRetriever + PullFetcher *github.PRFetcher + PullConverter converter.PullConverter } func NewServer(config *adhocconfig.Config) (*Server, error) { @@ -183,17 +183,19 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { Scope: scope.SubScope("event.filters.root"), } - repoConverter := github_converter.RepoConverter{} - repoRetriever := &github.RepoRetriever{ + pullFetcher := &github.PRFetcher{ ClientCreator: clientCreator, - RepoConverter: repoConverter, } - // This exists to convert a repo name to a repo object - githubRetriever := &adhocGithubHelpers.AdhocGithubRetriever{ - RepoRetriever: repoRetriever, - InstallationRetriever: installationFetcher, + repoConverter := converter.RepoConverter{ + GithubUser: config.GithubUser, + GithubToken: config.GithubToken, } + + pullConverter := converter.PullConverter{ + RepoConverter: repoConverter, + } + config.CtxLogger.Info(fmt.Sprintf("Starting adhoc server, params are: repo: %s, root: %s, revision: %s", config.GlobalCfg.AdhocMode.Repo, config.GlobalCfg.AdhocMode.Root, config.GlobalCfg.AdhocMode.Revision)) server := Server{ @@ -205,18 +207,20 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { Frequency: 1 * time.Minute, }, }, - HTTPServerProxy: httpServerProxy, - Port: config.ServerCfg.Port, - StatsScope: scope, - StatsCloser: statsCloser, - TemporalClient: temporalClient, - TerraformActivities: terraformActivities, - TerraformTaskQueue: config.TemporalCfg.TerraformTaskQueue, - GithubActivities: githubActivities, - RootConfigBuilder: rootConfigBuilder, - GithubRetriever: githubRetriever, - Repo: config.GlobalCfg.AdhocMode.Repo, - Revision: config.GlobalCfg.AdhocMode.Revision, + HTTPServerProxy: httpServerProxy, + Port: config.ServerCfg.Port, + StatsScope: scope, + StatsCloser: statsCloser, + TemporalClient: temporalClient, + TerraformActivities: terraformActivities, + TerraformTaskQueue: config.TemporalCfg.TerraformTaskQueue, + GithubActivities: githubActivities, + RootConfigBuilder: rootConfigBuilder, + Repo: config.GlobalCfg.AdhocMode.Repo, + PRNum: config.GlobalCfg.AdhocMode.PRNum, + InstallationRetriever: installationFetcher, + PullFetcher: pullFetcher, + PullConverter: pullConverter, } return &server, nil } @@ -275,7 +279,7 @@ func (s Server) Start() error { go func() { defer wg.Done() - adhocExecutionParams, err := adhoc.ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx, s.Repo, s.Revision, s.GithubRetriever, s.RootConfigBuilder) + adhocExecutionParams, err := adhoc.ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx, s.Repo, s.PRNum, s.PullFetcher, s.PullConverter, s.InstallationRetriever, s.RootConfigBuilder) if err != nil { s.Logger.Error(err.Error()) return From b5cbd0405caa69c3f97710fc79db4b3c40df0c2f Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Thu, 18 Apr 2024 20:32:23 -0700 Subject: [PATCH 26/35] fix all issues and remove all unused code --- .../adhoc/adhocexecutionhelpers/adhoc_execution_params.go | 2 +- server/neptune/adhoc/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go index 2b6b49141..5775a2bde 100644 --- a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go +++ b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go @@ -19,7 +19,7 @@ type AdhocTerraformWorkflowExecutionParams struct { // Note that deploymentID is used in NewWorkflowStore(), but we don't care about that in adhoc mode so can leave it blank } -func ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever( +func ConstructAdhocExecParams( ctx context.Context, repoName string, PRNum int, diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 783eb9e6b..a96afd879 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -279,7 +279,7 @@ func (s Server) Start() error { go func() { defer wg.Done() - adhocExecutionParams, err := adhoc.ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx, s.Repo, s.PRNum, s.PullFetcher, s.PullConverter, s.InstallationRetriever, s.RootConfigBuilder) + adhocExecutionParams, err := adhoc.ConstructAdhocExecParams(ctx, s.Repo, s.PRNum, s.PullFetcher, s.PullConverter, s.InstallationRetriever, s.RootConfigBuilder) if err != nil { s.Logger.Error(err.Error()) return From 43d4e1a64fc7f17a08b03b798ede01a011117593 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Fri, 19 Apr 2024 08:24:37 -0700 Subject: [PATCH 27/35] merge --- server/neptune/adhoc/server.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index a96afd879..33a581be1 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -196,8 +196,6 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { RepoConverter: repoConverter, } - config.CtxLogger.Info(fmt.Sprintf("Starting adhoc server, params are: repo: %s, root: %s, revision: %s", config.GlobalCfg.AdhocMode.Repo, config.GlobalCfg.AdhocMode.Root, config.GlobalCfg.AdhocMode.Revision)) - server := Server{ Logger: config.CtxLogger, CronScheduler: cronScheduler, From 55256eb8082c6291e3f1988b97ed3b3bb85b4114 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Fri, 19 Apr 2024 08:52:20 -0700 Subject: [PATCH 28/35] lint --- .../adhoc/adhocexecutionhelpers/adhoc_execution_params.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go index 5775a2bde..6b99f2707 100644 --- a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go +++ b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go @@ -27,7 +27,6 @@ func ConstructAdhocExecParams( pullConverter converter.PullConverter, installationRetriever *internal_gh.InstallationRetriever, rootCfgBuilder *root_config.Builder) (AdhocTerraformWorkflowExecutionParams, error) { - orgName := "lyft" installationToken, err := installationRetriever.FindOrganizationInstallation(ctx, orgName) if err != nil { From 6973870dd6a7bb21a8e909cee70d6ec26453d884 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Fri, 19 Apr 2024 12:44:31 -0700 Subject: [PATCH 29/35] does adding jobstore stuff make it work? --- cmd/adhoc.go | 1 + server/neptune/adhoc/config/config.go | 1 + server/neptune/adhoc/server.go | 11 ++++++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cmd/adhoc.go b/cmd/adhoc.go index 0a4d82cec..d84b5c6fd 100644 --- a/cmd/adhoc.go +++ b/cmd/adhoc.go @@ -69,6 +69,7 @@ func (a *Adhoc) NewServer(userConfig legacy.UserConfig, config legacy.Config) (S GlobalCfg: globalCfg, GithubUser: userConfig.GithubUser, GithubToken: userConfig.GithubToken, + JobConfig: globalCfg.PersistenceConfig.Jobs, } return adhoc.NewServer(cfg) } diff --git a/server/neptune/adhoc/config/config.go b/server/neptune/adhoc/config/config.go index eb52a8293..064495121 100644 --- a/server/neptune/adhoc/config/config.go +++ b/server/neptune/adhoc/config/config.go @@ -17,6 +17,7 @@ type Config struct { TerraformCfg neptune.TerraformConfig Metrics valid.Metrics + JobConfig valid.StoreConfig StatsNamespace string DataDir string diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 33a581be1..54c60549a 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -35,6 +35,7 @@ import ( internalSync "github.com/runatlantis/atlantis/server/neptune/sync" "github.com/runatlantis/atlantis/server/neptune/temporal" neptune "github.com/runatlantis/atlantis/server/neptune/temporalworker/config" + "github.com/runatlantis/atlantis/server/neptune/temporalworker/job" "github.com/runatlantis/atlantis/server/neptune/workflows" "github.com/runatlantis/atlantis/server/neptune/workflows/activities" "github.com/runatlantis/atlantis/server/static" @@ -81,6 +82,14 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { "mode": "adhoc", }) + jobStore, err := job.NewStorageBackendStore(config.JobConfig, scope.SubScope("job.store"), config.CtxLogger) + if err != nil { + return nil, errors.Wrapf(err, "initializing job store") + } + receiverRegistry := job.NewReceiverRegistry() + + jobStreamHandler := job.NewStreamHandler(jobStore, receiverRegistry, config.TerraformCfg.LogFilters, config.CtxLogger) + opts := &temporal.Options{ StatsReporter: statsReporter, } @@ -115,7 +124,7 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { config.ServerCfg.URL, config.TemporalCfg.TerraformTaskQueue, config.GithubCfg.TemporalAppInstallationID, - nil, + jobStreamHandler, ) if err != nil { return nil, errors.Wrap(err, "initializing terraform activities") From e472942f7f3f7187460e1454c65a3544ad1784f9 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Fri, 19 Apr 2024 13:51:49 -0700 Subject: [PATCH 30/35] does adding deploymentid make it work? --- server/neptune/adhoc/server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 54c60549a..47dcd42cb 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -237,11 +237,13 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { // when a request is made to the server, but we are manually executing it here, // since we don't care about requests in adhoc mode. func (s Server) manuallyExecuteTerraformWorkflow(repo ghClient.Repo, revision string, root terraform.Root) (interface{}, error) { + deploymentIDStr := revision + "-" + root.Name + "-" + repo.Name request := workflows.TerraformRequest{ Revision: revision, WorkflowMode: terraform.Adhoc, Root: root, Repo: repo, + DeploymentID: deploymentIDStr, } options := client.StartWorkflowOptions{ TaskQueue: s.TerraformTaskQueue, From 1bf16d5de291d40a87b4e8c0d5324d405858abd9 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Fri, 19 Apr 2024 18:25:48 -0700 Subject: [PATCH 31/35] dont signal parent --- server/neptune/workflows/internal/terraform/workflow.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/neptune/workflows/internal/terraform/workflow.go b/server/neptune/workflows/internal/terraform/workflow.go index 97e632a9d..0c547c4d1 100644 --- a/server/neptune/workflows/internal/terraform/workflow.go +++ b/server/neptune/workflows/internal/terraform/workflow.go @@ -99,6 +99,10 @@ func newRunner(ctx workflow.Context, request Request) *Runner { // be no situation where we are deploying while this is failing. store := state.NewWorkflowStore( func(s *state.Workflow) error { + // in adhoc mode we have no parent workflow to signal + if request.WorkflowMode == terraform.Adhoc { + return nil + } return workflow.SignalExternalWorkflow(ctx, parent.ID, parent.RunID, state.WorkflowStateChangeSignal, s).Get(ctx, nil) }, request.WorkflowMode, From 3776c554c08df668318c82251de166e5e591ac42 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Fri, 19 Apr 2024 19:32:07 -0700 Subject: [PATCH 32/35] clean up some of teh logs, going to leave some of them though for future --- .../adhoc/adhocexecutionhelpers/adhoc_execution_params.go | 2 -- server/neptune/gateway/config/root_config_builder.go | 3 --- 2 files changed, 5 deletions(-) diff --git a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go index 6b99f2707..cae5502de 100644 --- a/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go +++ b/server/neptune/adhoc/adhocexecutionhelpers/adhoc_execution_params.go @@ -61,10 +61,8 @@ func ConstructAdhocExecParams( return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "building root cfgs") } - rootCfgBuilder.Logger.Info("getting roots from merged project cfgs") roots := getRootsFromMergedProjectCfgs(rootCfgs) - rootCfgBuilder.Logger.Info("returning adhocexecution params") return AdhocTerraformWorkflowExecutionParams{ Revision: actualCommit.HeadCommit, GithubRepo: github.Repo{ diff --git a/server/neptune/gateway/config/root_config_builder.go b/server/neptune/gateway/config/root_config_builder.go index 50fa1de93..b63c1af91 100644 --- a/server/neptune/gateway/config/root_config_builder.go +++ b/server/neptune/gateway/config/root_config_builder.go @@ -135,7 +135,6 @@ func (b *Builder) build(ctx context.Context, commit *RepoCommit, installationTok b.Logger.Info(fmt.Sprintf("localRepo is: full repo name: %s, commit sha: %s, commit branch: %s, commit repo name: %s, repodir: %s", commit.Repo.FullName, commit.Sha, commit.Branch, commit.Repo.Name, repoDir)) // Run pre-workflow hooks - b.Logger.Info("running pre-workflow hooks") err = b.HooksRunner.Run(ctx, localRepo.Repo, localRepo.Dir) if err != nil { return nil, errors.Wrap(err, "running pre-workflow hooks") @@ -145,13 +144,11 @@ func (b *Builder) build(ctx context.Context, commit *RepoCommit, installationTok // TODO: rename project to roots var mergedRootCfgs []*valid.MergedProjectCfg - b.Logger.Info("parsing repo config") repoCfg, err := b.ParserValidator.ParseRepoCfg(localRepo.Dir, localRepo.Repo.ID()) if err != nil { return nil, errors.Wrapf(err, "parsing %s", config.AtlantisYAMLFilename) } - b.Logger.Info("getting matching roots") matchingRoots, err := b.getMatchingRoots(ctx, repoCfg, localRepo, installationToken, rootNames) if err != nil { return nil, errors.Wrap(err, "getting matching roots") From bb17db4874726c972d39195fca213cf95a7fa8e3 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 22 Apr 2024 08:14:46 -0700 Subject: [PATCH 33/35] merge and remove a log --- server/vcs/provider/github/repo_fetcher.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/vcs/provider/github/repo_fetcher.go b/server/vcs/provider/github/repo_fetcher.go index b16006bfb..514e59e4b 100644 --- a/server/vcs/provider/github/repo_fetcher.go +++ b/server/vcs/provider/github/repo_fetcher.go @@ -81,7 +81,6 @@ func (g *RepoFetcher) clone(ctx context.Context, repo models.Repo, branch string } // Create the directory and parents if necessary. - g.Logger.Info("creating new directory inside clone at path: " + destinationPath) if err := os.MkdirAll(destinationPath, 0700); err != nil { return "", nil, errors.Wrap(err, "creating new directory") } From 6babeda129ce8c29d29cb4db2899bd2c6c45665f Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 22 Apr 2024 15:19:26 -0700 Subject: [PATCH 34/35] remove more logs --- server/neptune/adhoc/server.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 47dcd42cb..7db4248d5 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -294,12 +294,7 @@ func (s Server) Start() error { return } - if len(adhocExecutionParams.TerraformRoots) == 0 { - s.Logger.Info("no roots found") - } - for _, root := range adhocExecutionParams.TerraformRoots { - s.Logger.Info("running terraform workflow for root " + root.Name) _, err := s.manuallyExecuteTerraformWorkflow(adhocExecutionParams.GithubRepo, adhocExecutionParams.Revision, root) if err != nil { s.Logger.Error(err.Error()) From 534e333a36698afdecf3c6d46fdda5a37f2b3991 Mon Sep 17 00:00:00 2001 From: Shawna Monero Date: Mon, 22 Apr 2024 15:20:00 -0700 Subject: [PATCH 35/35] remove jobstream handler --- server/neptune/adhoc/server.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/server/neptune/adhoc/server.go b/server/neptune/adhoc/server.go index 7db4248d5..171bc0ce5 100644 --- a/server/neptune/adhoc/server.go +++ b/server/neptune/adhoc/server.go @@ -35,7 +35,6 @@ import ( internalSync "github.com/runatlantis/atlantis/server/neptune/sync" "github.com/runatlantis/atlantis/server/neptune/temporal" neptune "github.com/runatlantis/atlantis/server/neptune/temporalworker/config" - "github.com/runatlantis/atlantis/server/neptune/temporalworker/job" "github.com/runatlantis/atlantis/server/neptune/workflows" "github.com/runatlantis/atlantis/server/neptune/workflows/activities" "github.com/runatlantis/atlantis/server/static" @@ -82,14 +81,6 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { "mode": "adhoc", }) - jobStore, err := job.NewStorageBackendStore(config.JobConfig, scope.SubScope("job.store"), config.CtxLogger) - if err != nil { - return nil, errors.Wrapf(err, "initializing job store") - } - receiverRegistry := job.NewReceiverRegistry() - - jobStreamHandler := job.NewStreamHandler(jobStore, receiverRegistry, config.TerraformCfg.LogFilters, config.CtxLogger) - opts := &temporal.Options{ StatsReporter: statsReporter, } @@ -124,7 +115,7 @@ func NewServer(config *adhocconfig.Config) (*Server, error) { config.ServerCfg.URL, config.TemporalCfg.TerraformTaskQueue, config.GithubCfg.TemporalAppInstallationID, - jobStreamHandler, + nil, ) if err != nil { return nil, errors.Wrap(err, "initializing terraform activities")