Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DEVCON-6862] Execute terraform workflow from adhoc server mode #732

Merged
merged 43 commits into from
Apr 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
b3cac15
change admin to adhoc, also check why we dont exit early
smonero Mar 11, 2024
d565404
change admin to adhoc, also check why we dont exit early
smonero Mar 11, 2024
708a582
first draft
smonero Mar 19, 2024
f674333
first draft
smonero Mar 19, 2024
6c52649
first draft
smonero Mar 19, 2024
a570d5b
first draft
smonero Mar 19, 2024
43dc515
first draft
smonero Mar 19, 2024
1475bb8
first draft
smonero Mar 19, 2024
cb6f933
first draft
smonero Mar 19, 2024
f12419e
merge
smonero Mar 19, 2024
e3c65f2
merge
smonero Mar 19, 2024
317a808
run in go routine
smonero Mar 20, 2024
e857330
Merge branch 'main' of github.com:lyft/atlantis
smonero Mar 20, 2024
c842b51
ok
smonero Mar 20, 2024
adccfce
merge
smonero Mar 21, 2024
61c1029
Merge branch 'main' of github.com:lyft/atlantis
smonero Mar 28, 2024
357bae4
Merge branch 'main' of github.com:lyft/atlantis
smonero Apr 10, 2024
4ab6c43
merge
smonero Apr 10, 2024
6ed2c01
panic fixes
smonero Apr 12, 2024
cb78afd
somehow forgot to add repo and revision woops
smonero Apr 12, 2024
d08d2d0
add more logging
smonero Apr 12, 2024
ec7f526
add more logging
smonero Apr 14, 2024
86843ad
add more logging
smonero Apr 14, 2024
9352a3f
fix the underlying issue
smonero Apr 14, 2024
8e954a4
moar logging
smonero Apr 15, 2024
34cc02f
moar logging
smonero Apr 15, 2024
81831e0
Merge branch 'main' of github.com:lyft/atlantis
smonero Apr 15, 2024
ebf5784
moar logging
smonero Apr 15, 2024
3d86fb5
moar logging
smonero Apr 15, 2024
5c60e8a
moar logging
smonero Apr 15, 2024
f4c4300
fix all issues and remove all unused code
smonero Apr 19, 2024
b5cbd04
fix all issues and remove all unused code
smonero Apr 19, 2024
1d41e09
Merge branch 'main' of github.com:lyft/atlantis
smonero Apr 19, 2024
43d4e1a
merge
smonero Apr 19, 2024
55256eb
lint
smonero Apr 19, 2024
6973870
does adding jobstore stuff make it work?
smonero Apr 19, 2024
e472942
does adding deploymentid make it work?
smonero Apr 19, 2024
1bf16d5
dont signal parent
smonero Apr 20, 2024
3776c55
clean up some of teh logs, going to leave some of them though for future
smonero Apr 20, 2024
53f4581
Merge branch 'main' of github.com:lyft/atlantis
smonero Apr 22, 2024
bb17db4
merge and remove a log
smonero Apr 22, 2024
6babeda
remove more logs
smonero Apr 22, 2024
534e333
remove jobstream handler
smonero Apr 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 10 additions & 15 deletions cmd/adhoc.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,8 @@ import (
"github.com/runatlantis/atlantis/server/legacy"
"github.com/runatlantis/atlantis/server/logging"
adhoc "github.com/runatlantis/atlantis/server/neptune/adhoc"
adhocHelpers "github.com/runatlantis/atlantis/server/neptune/adhoc/adhocexecutionhelpers"
adhocconfig "github.com/runatlantis/atlantis/server/neptune/adhoc/config"
neptune "github.com/runatlantis/atlantis/server/neptune/temporalworker/config"
"github.com/runatlantis/atlantis/server/neptune/workflows/activities/github"
"github.com/runatlantis/atlantis/server/neptune/workflows/activities/terraform"
)

type Adhoc struct{}
Expand Down Expand Up @@ -58,23 +55,21 @@ func (a *Adhoc) NewServer(userConfig legacy.UserConfig, config legacy.Config) (S
DownloadURL: userConfig.TFDownloadURL,
LogFilters: globalCfg.TerraformLogFilter,
},
DataDir: userConfig.DataDir,
TemporalCfg: globalCfg.Temporal,
GithubCfg: globalCfg.Github,
App: appConfig,
CtxLogger: ctxLogger,
StatsNamespace: userConfig.StatsNamespace,
Metrics: globalCfg.Metrics,
AdhocExecutionParams: adhocHelpers.AdhocTerraformWorkflowExecutionParams{
Revision: "",
TerraformRoots: []terraform.Root{},
GithubRepo: github.Repo{},
},
DataDir: userConfig.DataDir,
TemporalCfg: globalCfg.Temporal,
GithubCfg: globalCfg.Github,
App: appConfig,
CtxLogger: ctxLogger,
StatsNamespace: userConfig.StatsNamespace,
Metrics: globalCfg.Metrics,
GithubHostname: userConfig.GithubHostname,
GithubAppID: userConfig.GithubAppID,
GithubAppKeyFile: userConfig.GithubAppKeyFile,
GithubAppSlug: userConfig.GithubAppSlug,
GlobalCfg: globalCfg,
GithubUser: userConfig.GithubUser,
GithubToken: userConfig.GithubToken,
JobConfig: globalCfg.PersistenceConfig.Jobs,
}
return adhoc.NewServer(cfg)
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ import (
"context"

"github.com/pkg/errors"
"github.com/runatlantis/atlantis/server/neptune/adhoc/adhocgithubhelpers"
"github.com/runatlantis/atlantis/server/neptune/gateway/config"
root_config "github.com/runatlantis/atlantis/server/neptune/gateway/config"
"github.com/runatlantis/atlantis/server/neptune/workflows/activities/github"
"github.com/runatlantis/atlantis/server/neptune/workflows/activities/terraform"
internal_gh "github.com/runatlantis/atlantis/server/vcs/provider/github"
"github.com/runatlantis/atlantis/server/vcs/provider/github/converter"
)

type AdhocTerraformWorkflowExecutionParams struct {
Expand All @@ -19,11 +19,29 @@ type AdhocTerraformWorkflowExecutionParams struct {
// Note that deploymentID is used in NewWorkflowStore(), but we don't care about that in adhoc mode so can leave it blank
}

func ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx context.Context, repoName string, revision string, githubRetriever *adhocgithubhelpers.AdhocGithubRetriever, rootCfgBuilder *root_config.Builder) (AdhocTerraformWorkflowExecutionParams, error) {
func ConstructAdhocExecParams(
ctx context.Context,
repoName string,
PRNum int,
pullFetcher *internal_gh.PRFetcher,
pullConverter converter.PullConverter,
installationRetriever *internal_gh.InstallationRetriever,
rootCfgBuilder *root_config.Builder) (AdhocTerraformWorkflowExecutionParams, error) {
orgName := "lyft"
installationToken, err := installationRetriever.FindOrganizationInstallation(ctx, orgName)
if err != nil {
return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "finding organization installation")
}

// TODO: in the future, could potentially pass in the owner instead of hardcoding lyft
repo, err := githubRetriever.GetRepository(ctx, "lyft", repoName)
ghCommit, err := pullFetcher.Fetch(ctx, installationToken.Token, orgName, repoName, PRNum)
if err != nil {
return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "getting repo")
return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "fetching commit")
}

actualCommit, err := pullConverter.Convert(ghCommit)
if err != nil {
return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "converting commit")
}

opts := config.BuilderOptions{
Expand All @@ -33,16 +51,27 @@ func ConstructAdhocExecParamsWithRootCfgBuilderAndRepoRetriever(ctx context.Cont
},
}

rootCfgs, err := rootCfgBuilder.Build(ctx, &root_config.RepoCommit{}, repo.Credentials.InstallationToken, opts)
rootCfgs, err := rootCfgBuilder.Build(ctx, &root_config.RepoCommit{
Repo: actualCommit.HeadRepo,
Branch: actualCommit.HeadBranch,
Sha: actualCommit.HeadCommit,
OptionalPRNum: actualCommit.Num,
}, installationToken.Token, opts)
if err != nil {
return AdhocTerraformWorkflowExecutionParams{}, errors.Wrap(err, "building root cfgs")
}

roots := getRootsFromMergedProjectCfgs(rootCfgs)

return AdhocTerraformWorkflowExecutionParams{
Revision: revision,
GithubRepo: repo,
Revision: actualCommit.HeadCommit,
GithubRepo: github.Repo{
Owner: orgName,
Name: repoName,
URL: actualCommit.HeadRepo.CloneURL,
DefaultBranch: actualCommit.HeadRepo.DefaultBranch,
Credentials: github.AppCredentials{InstallationToken: installationToken.Token},
},
TerraformRoots: roots,
}, nil
}
50 changes: 0 additions & 50 deletions server/neptune/adhoc/adhocgithubhelpers/adhoc_github_helpers.go

This file was deleted.

11 changes: 6 additions & 5 deletions server/neptune/adhoc/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"github.com/palantir/go-githubapp/githubapp"
"github.com/runatlantis/atlantis/server/config/valid"
"github.com/runatlantis/atlantis/server/logging"
adhoc "github.com/runatlantis/atlantis/server/neptune/adhoc/adhocexecutionhelpers"
neptune "github.com/runatlantis/atlantis/server/neptune/temporalworker/config"
)

Expand All @@ -18,17 +17,19 @@ type Config struct {
TerraformCfg neptune.TerraformConfig
Metrics valid.Metrics

JobConfig valid.StoreConfig
StatsNamespace string

DataDir string
CtxLogger logging.Logger
App githubapp.Config
AdhocExecutionParams adhoc.AdhocTerraformWorkflowExecutionParams
DataDir string
CtxLogger logging.Logger
App githubapp.Config

GithubAppID int64
GithubAppKeyFile string
GithubAppSlug string
GithubHostname string
GithubUser string
GithubToken string

GlobalCfg valid.GlobalCfg
}
Loading
Loading