diff --git a/launcher_go/README.md b/launcher_go/README.md new file mode 100644 index 000000000..a3f7baa28 --- /dev/null +++ b/launcher_go/README.md @@ -0,0 +1,134 @@ +# Launcher2 + +Build and run discourse images. Drop in replacement for launcher the shell script. + +## Changes from launcher + +No software prerequisites are checked here. It assumes you have docker set up and whatever minimum requirements setup for Discourse: namely a recent enough version of docker, git. + +Some things are not implemented from launcher1. + +* `DOCKER_HOST_IP` - container can use `host.docker.internal` in most cases. Supported on mac and windows... can also be [added on linux via docker args](https://stackoverflow.com/questions/72827527/what-is-running-on-host-docker-internal-host). +* debug containers - not implemented. No debug containers saved on build. Under the hood, launcher2 uses docker build which does not allow images to be saved along the way. +* stable `mac-address` - not implemented. + +## New features + +In a nutshell: split bootstrap/rebuild process up into distinct parts to allow for greater flexibility in how we build and deploy Discourse containers. + +### Separates bootstrap process into distinct build, configure, and migrate steps. + +Separating the larger bootstrap process into separate steps allows us to break up the work. + +`bootstrap` becomes an alias for: `build`, `migrate`, `configure`. There are multiple benefits to this. + +#### Build: Easier creation for prebuilt docker images + +Share built docker images by only running a `build` step - this build step does not need to connect to a database. +It does not need postgres or redis running. This makes for a simple way to install custom plugins to your Discourse image. + +The resulting image is able to be used in Kubernetes and other docker environments. + +This is done by deferring finishing the build step, to a later configure step -- which boostraps the db, and precompiles assets. + +The `configure` and `migrate` steps can now be done on boot through use of env vars set in the `app.yml` config: `CREATE_DB_ON_BOOT`, `MIGRATE_ON_BOOT`, and `PRECOMPILE_ON_BOOT`, which allows for more portable containers able to drop in and bootstrap themselves and the database as they come into service. + +#### Build: Better environment management + +The resulting image from a build is a container with no environment (unless `--bake-env` is specified). Additionally, well-known secrets are excluded from the build environment, resulting in a clean history of the prebuilt image that may be more easily shared. + +Environment is only bound to a container either with `--bake-env` on build, or on a subsequent `configure` step. + +#### Migrate: Adds support to *when* migrations are run + +`Build` and `Configure` steps do not run migrations, allowing for external tooling to specify exactly when migrations are run. + +`Migrate`, (and`bootstrap`, and `rebuild`) steps are the only ones that run migrations. + +#### Migrate: Adds support for *how* migrations are run: `SKIP_POST_DEPLOYMENT_MIGRATIONS` support + +the `migrate` step exposes env vars that turn on separate post deploy migration steps. + +Allows the ability to turn on and skip post migration steps from launcher when running a stand-alone migrate step. + +#### Rebuild: Minimize downtime + +Both standalone and multi-container setups' downtime have been minimized for rebuilds + +##### Standalone +On standalone builds, only stop the running container after the base build is done. +Standalone sites will only need to be offline during migration and configure steps. + +For standalone, `rebuild` runs `build`, `stop`, `migrate`, `configure`, `destroy`, `start`. + +##### Multiple container, web only +On multi-container setups or setups with a configured external database using web only containers, rebuilds attempt to run migrations without stopping the container. +A multi-container stays up as migration (skipping post deployment migrations) and as any necessary configuration steps are run. After deploy, post deployment migrations are run to clean up any destructive migrations. + +For web-only, `rebuild` runs `build`, `migrate (skip post migrations)`, `configure`, `destroy`, `start`, `migrate`. + +#### Rebuild: Serve offline page during downtime + +Adds the ability to build and run an image that finishes a build on boot, allowing the server to display an offline page. +For standalone builds above, this allows for the accrued downtime from migration and configure steps to happen more gracefully. + +Additional container env vars get turned on by adding the `offline-page.template.yml` template: + * `CREATE_DB_ON_BOOT` + * `MIGRATE_ON_BOOT` + * `PRECOMPILE_ON_BOOT` + +These allow containers to boot cleanly from a cold state, and complete db creation, migration, and precompile steps on boot. + +During this time, nginx can be up which allows standalone builds to display an offline page. + +These variables may also be used for other applications where more flexible bootstrapping is desired. + +##### Standalone +On rebuild, a standalone site will skip migration if it detects the presence of `MIGRATE_ON_BOOT` in the app config, and will skip configure steps if it detects the presence of `PRECOMPILE_ON_BOOT` in the app config. + +For standalone, `rebuild` runs `build`, `destroy`, `start`, skipping `migrate` and `configure`. The started container then serves an offline page, and runs migrate and precompiles assets before fully entering service. + +##### Multiple container, web only +On rebuild, a web only container will act in the same way as a standalone container. This may result in the same downtime as standalone services, as the containers are swapped, and the new container is still responsible for migration and precompiling before serving traffic. + +For web-only containers, it may be desired to either ensure that `MIGRATE_ON_BOOT` and `PRECOMPILE_ON_BOOT` are false. Alternatively, you may run with `--full-build` which will ensure that migration and precompile steps are not deferred for the 'live' deploy. + +### Multiline env support + +Allows the use of multiline env vars so this is valid config, and is passed through to the container as expected: +``` +env: + SECRET_KEY: | + ---START OF SECRET KEY--- + 123456 + 78910 + ---END OF SECRET KEY--- +``` + +### More dependable SIGINT/SIGTERM handling. + +Launcher wraps docker run commands, which run as children in process trees. Launcher2 does the same, but attempts to kill or stop the underlying docker processes from interrupt signals. + +Tools that extend or depend on launcher should be able to send SIGINT/SIGTERM signals to tell launcher to shut down, and launcher should clean up child processes appropriately. + +### Docker compose generation. + +Allows easier exporting of configuration from discourse's pups configuration to a docker compose configuration. + +### Autocomplete support + +Run `source <(./launcher2 sh)` to activate completions for the current shell, or add the results of `./launcher2 sh` to your dotfiles + +Autocompletes commands, subcommands, and suggests `app` config files from your containers directory. Having a long site name should not feel like a pain to type. + +## Maintainability + +Golang is well suited as a drop in replacement as just like a shellscript, the deployed binary can still carry minimal assumptions about a particular platform to run. (IE, no dependency on ruby, python, etc) + +Golang allows us to use a fully fleshed out programming language to run native yaml parsing: Calling out to ruby through a docker container worked well enough, but got complicated shuffling results through stdout into shell variables. + +Launcher has outgrown being a simple wrapper script around Docker. Golang has good support for tests and breaking up code into separate modules to better support further growth around additional subcommands we may wish to add. + +## Roadmap + +Scaffolding out subcommands, possibly as a later rewrite for `discourse-setup` as having native YAML libraries should make config parsing and editing simpler to do. diff --git a/launcher_go/v2/Makefile b/launcher_go/v2/Makefile new file mode 100644 index 000000000..cd2e89246 --- /dev/null +++ b/launcher_go/v2/Makefile @@ -0,0 +1,10 @@ +.PHONY: default +default: build + +.PHONY: build +build: + go build -o bin/launcher2 + +.PHONY: test +test: + go test ./... diff --git a/launcher_go/v2/bin/.gitignore b/launcher_go/v2/bin/.gitignore new file mode 100644 index 000000000..89f3d385c --- /dev/null +++ b/launcher_go/v2/bin/.gitignore @@ -0,0 +1 @@ +launcher2* \ No newline at end of file diff --git a/launcher_go/v2/cli_build.go b/launcher_go/v2/cli_build.go new file mode 100644 index 000000000..0a5575789 --- /dev/null +++ b/launcher_go/v2/cli_build.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "errors" + "github.com/discourse/discourse_docker/launcher_go/v2/config" + "github.com/discourse/discourse_docker/launcher_go/v2/docker" + "os" + "strings" +) + +/* + * build + * migrate + * configure + * bootstrap + */ +type DockerBuildCmd struct { + BakeEnv bool `short:"e" help:"Bake in the configured environment to image after build."` + Tag string `default:"latest" help:"Resulting image tag."` + + Config string `arg:"" name:"config" help:"configuration" predictor:"config"` +} + +func (r *DockerBuildCmd) Run(cli *Cli, ctx *context.Context) error { + config, err := config.LoadConfig(cli.ConfDir, r.Config, true, cli.TemplatesDir) + if err != nil { + return errors.New("YAML syntax error. Please check your containers/*.yml config files.") + } + + dir := cli.BuildDir + "/" + r.Config + if err := os.MkdirAll(dir, 0755); err != nil && !os.IsExist(err) { + return err + } + if err := config.WriteYamlConfig(dir); err != nil { + return err + } + + pupsArgs := "--skip-tags=precompile,migrate,db" + builder := docker.DockerBuilder{ + Config: config, + Ctx: ctx, + Stdin: strings.NewReader(config.Dockerfile(pupsArgs, r.BakeEnv)), + Dir: dir, + ImageTag: r.Tag, + } + if err := builder.Run(); err != nil { + return err + } + cleaner := CleanCmd{Config: r.Config} + cleaner.Run(cli) + + return nil +} + +type CleanCmd struct { + Config string `arg:"" name:"config" help:"config to clean" predictor:"config"` +} + +func (r *CleanCmd) Run(cli *Cli) error { + dir := cli.BuildDir + "/" + r.Config + os.Remove(dir + "/config.yaml") + if err := os.Remove(dir); err != nil { + return err + } + return nil +} diff --git a/launcher_go/v2/cli_build_test.go b/launcher_go/v2/cli_build_test.go new file mode 100644 index 000000000..3aa8f7975 --- /dev/null +++ b/launcher_go/v2/cli_build_test.go @@ -0,0 +1,67 @@ +package main_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "bytes" + "context" + ddocker "github.com/discourse/discourse_docker/launcher_go/v2" + . "github.com/discourse/discourse_docker/launcher_go/v2/test_utils" + "github.com/discourse/discourse_docker/launcher_go/v2/utils" + "io" + "os" + "os/exec" + "strings" +) + +var _ = Describe("Build", func() { + var testDir string + var out *bytes.Buffer + var cli *ddocker.Cli + var ctx context.Context + + BeforeEach(func() { + utils.DockerPath = "docker" + out = &bytes.Buffer{} + utils.Out = out + testDir, _ = os.MkdirTemp("", "ddocker-test") + + ctx = context.Background() + + cli = &ddocker.Cli{ + ConfDir: "./test/containers", + TemplatesDir: "./test", + BuildDir: testDir, + } + utils.CmdRunner = CreateNewFakeCmdRunner() + }) + AfterEach(func() { + os.RemoveAll(testDir) + }) + + Context("When running build commands", func() { + var checkBuildCmd = func(cmd exec.Cmd) { + Expect(cmd.String()).To(ContainSubstring("docker build")) + Expect(cmd.String()).To(ContainSubstring("--build-arg DISCOURSE_DEVELOPER_EMAILS")) + Expect(cmd.Dir).To(Equal(testDir + "/test")) + + //db password is ignored + Expect(cmd.Env).ToNot(ContainElement("DISCOURSE_DB_PASSWORD=SOME_SECRET")) + Expect(cmd.Env).ToNot(ContainElement("DISCOURSEDB_SOCKET=")) + buf := new(strings.Builder) + io.Copy(buf, cmd.Stdin) + // docker build's stdin is a dockerfile + Expect(buf.String()).To(ContainSubstring("COPY config.yaml /temp-config.yaml")) + Expect(buf.String()).To(ContainSubstring("--skip-tags=precompile,migrate,db")) + Expect(buf.String()).ToNot(ContainSubstring("SKIP_EMBER_CLI_COMPILE=1")) + } + + It("Should run docker build with correct arguments", func() { + runner := ddocker.DockerBuildCmd{Config: "test"} + runner.Run(cli, &ctx) + Expect(len(RanCmds)).To(Equal(1)) + checkBuildCmd(RanCmds[0]) + }) + }) +}) diff --git a/launcher_go/v2/config/config.go b/launcher_go/v2/config/config.go new file mode 100644 index 000000000..4c9ab2281 --- /dev/null +++ b/launcher_go/v2/config/config.go @@ -0,0 +1,209 @@ +package config + +import ( + "dario.cat/mergo" + "errors" + "fmt" + "github.com/discourse/discourse_docker/launcher_go/v2/utils" + "os" + "regexp" + "runtime" + "slices" + "strings" + + "gopkg.in/yaml.v3" +) + +const defaultBootCommand = "/sbin/boot" + +func DefaultBaseImage() string { + if runtime.GOARCH == "arm64" { + return "discourse/base:aarch64" + } + return "discourse/base:2.0.20231121-0024" +} + +type Config struct { + Name string `yaml:-` + rawYaml []string + Base_Image string `yaml:,omitempty` + Update_Pups bool `yaml:,omitempty` + Run_Image string `yaml:,omitempty` + Boot_Command string `yaml:,omitempty` + No_Boot_Command bool `yaml:,omitempty` + Docker_Args string `yaml:,omitempty` + Templates []string `yaml:templates,omitempty` + Expose []string `yaml:expose,omitempty` + Params map[string]string `yaml:params,omitempty` + Env map[string]string `yaml:env,omitempty` + Labels map[string]string `yaml:labels,omitempty` + Volumes []struct { + Volume struct { + Host string `yaml:host` + Guest string `yaml:guest` + } `yaml:volume` + } `yaml:volumes,omitempty` + Links []struct { + Link struct { + Name string `yaml:name` + Alias string `yaml:alias` + } `yaml:link` + } `yaml:links,omitempty` +} + +func (config *Config) loadTemplate(templateDir string, template string) error { + template_filename := strings.TrimRight(templateDir, "/") + "/" + string(template) + content, err := os.ReadFile(template_filename) + if err != nil { + if os.IsNotExist(err) { + fmt.Println("template file does not exist: " + template_filename) + } + return err + } + templateConfig := &Config{} + if err := yaml.Unmarshal(content, templateConfig); err != nil { + return err + } + if err := mergo.Merge(config, templateConfig, mergo.WithOverride); err != nil { + return err + } + config.rawYaml = append(config.rawYaml, string(content[:])) + return nil +} + +func LoadConfig(dir string, configName string, includeTemplates bool, templatesDir string) (*Config, error) { + config := &Config{ + Name: configName, + Boot_Command: defaultBootCommand, + Base_Image: DefaultBaseImage(), + } + matched, _ := regexp.MatchString("[[:upper:]/ !@#$%^&*()+~`=]", configName) + if matched { + msg := "ERROR: Config name '" + configName + "' must not contain upper case characters, spaces or special characters. Correct config name and rerun." + fmt.Println(msg) + return nil, errors.New(msg) + } + + config_filename := string(strings.TrimRight(dir, "/") + "/" + config.Name + ".yml") + content, err := os.ReadFile(config_filename) + if err != nil { + if os.IsNotExist(err) { + fmt.Println("config file does not exist: " + config_filename) + } + return nil, err + } + baseConfig := &Config{} + + if err := yaml.Unmarshal(content, baseConfig); err != nil { + return nil, err + } + + if includeTemplates { + for _, t := range baseConfig.Templates { + if err := config.loadTemplate(templatesDir, t); err != nil { + return nil, err + } + } + } + if err := mergo.Merge(config, baseConfig, mergo.WithOverride); err != nil { + return nil, err + } + config.rawYaml = append(config.rawYaml, string(content[:])) + if err != nil { + return nil, err + } + + for k, v := range config.Labels { + val := strings.ReplaceAll(v, "{{config}}", config.Name) + config.Labels[k] = val + } + + for k, v := range config.Env { + val := strings.ReplaceAll(v, "{{config}}", config.Name) + config.Env[k] = val + } + + return config, nil +} + +func (config *Config) Yaml() string { + return strings.Join(config.rawYaml, "_FILE_SEPERATOR_") +} + +func (config *Config) Dockerfile(pupsArgs string, bakeEnv bool) string { + builder := strings.Builder{} + builder.WriteString("ARG dockerfile_from_image=" + config.Base_Image + "\n") + builder.WriteString("FROM ${dockerfile_from_image}\n") + builder.WriteString(config.dockerfileArgs() + "\n") + if bakeEnv { + builder.WriteString(config.dockerfileEnvs() + "\n") + } + builder.WriteString(config.dockerfileExpose() + "\n") + builder.WriteString("COPY config.yaml /temp-config.yaml\n") + builder.WriteString("RUN " + + "cat /temp-config.yaml | /usr/local/bin/pups " + pupsArgs + " --stdin " + + "&& rm /temp-config.yaml\n") + builder.WriteString("CMD [\"" + config.bootCommand() + "\"]") + return builder.String() +} + +func (config *Config) WriteYamlConfig(dir string) error { + file := strings.TrimRight(dir, "/") + "/config.yaml" + if err := os.WriteFile(file, []byte(config.Yaml()), 0660); err != nil { + return errors.New("error writing config file " + file) + } + return nil +} + +func (config *Config) bootCommand() string { + if len(config.Boot_Command) > 0 { + return config.Boot_Command + } else if config.No_Boot_Command { + return "" + } else { + return defaultBootCommand + } +} + +func (config *Config) EnvArray(includeKnownSecrets bool) []string { + envs := []string{} + for k, v := range config.Env { + if !includeKnownSecrets && slices.Contains(utils.KnownSecrets, k) { + continue + } + envs = append(envs, k+"="+v) + } + slices.Sort(envs) + return envs +} + +func (config *Config) dockerfileEnvs() string { + builder := []string{} + for k, _ := range config.Env { + builder = append(builder, "ENV "+k+"=${"+k+"}") + } + slices.Sort(builder) + return strings.Join(builder, "\n") +} + +func (config *Config) dockerfileArgs() string { + builder := []string{} + for k, _ := range config.Env { + builder = append(builder, "ARG "+k) + } + slices.Sort(builder) + return strings.Join(builder, "\n") +} + +func (config *Config) dockerfileExpose() string { + builder := []string{} + for _, p := range config.Expose { + port := p + if strings.Contains(p, ":") { + _, port, _ = strings.Cut(p, ":") + } + builder = append(builder, "EXPOSE "+port) + } + slices.Sort(builder) + return strings.Join(builder, "\n") +} diff --git a/launcher_go/v2/config/config_suite_test.go b/launcher_go/v2/config/config_suite_test.go new file mode 100644 index 000000000..c6e29ba71 --- /dev/null +++ b/launcher_go/v2/config/config_suite_test.go @@ -0,0 +1,13 @@ +package config_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Config Suite") +} diff --git a/launcher_go/v2/config/config_test.go b/launcher_go/v2/config/config_test.go new file mode 100644 index 000000000..24bd4b8ca --- /dev/null +++ b/launcher_go/v2/config/config_test.go @@ -0,0 +1,46 @@ +package config_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/discourse/discourse_docker/launcher_go/v2/config" + "os" + "strings" +) + +var _ = Describe("Config", func() { + var testDir string + var conf *config.Config + BeforeEach(func() { + testDir, _ = os.MkdirTemp("", "ddocker-test") + conf, _ = config.LoadConfig("../test/containers", "test", true, "../test") + }) + AfterEach(func() { + os.RemoveAll(testDir) + }) + It("should be able to run LoadConfig to load yaml configuration", func() { + conf, err := config.LoadConfig("../test/containers", "test", true, "../test") + Expect(err).To(BeNil()) + result := conf.Yaml() + Expect(result).To(ContainSubstring("DISCOURSE_DEVELOPER_EMAILS: 'me@example.com,you@example.com'")) + Expect(result).To(ContainSubstring("_FILE_SEPERATOR_")) + Expect(result).To(ContainSubstring("version: tests-passed")) + }) + + It("can write raw yaml config", func() { + err := conf.WriteYamlConfig(testDir) + Expect(err).To(BeNil()) + out, err := os.ReadFile(testDir + "/config.yaml") + Expect(err).To(BeNil()) + Expect(strings.Contains(string(out[:]), "")) + Expect(string(out[:])).To(ContainSubstring("DISCOURSE_DEVELOPER_EMAILS: 'me@example.com,you@example.com'")) + }) + + It("can convert pups config to dockerfile format", func() { + dockerfile := conf.Dockerfile("", false) + Expect(dockerfile).To(ContainSubstring("ARG DISCOURSE_DEVELOPER_EMAILS")) + Expect(dockerfile).To(ContainSubstring("RUN cat /temp-config.yaml")) + Expect(dockerfile).To(ContainSubstring("EXPOSE 80")) + }) +}) diff --git a/launcher_go/v2/docker/commands.go b/launcher_go/v2/docker/commands.go new file mode 100644 index 000000000..04c4879ee --- /dev/null +++ b/launcher_go/v2/docker/commands.go @@ -0,0 +1,54 @@ +package docker + +import ( + "context" + "github.com/discourse/discourse_docker/launcher_go/v2/config" + "github.com/discourse/discourse_docker/launcher_go/v2/utils" + "golang.org/x/sys/unix" + "io" + "os" + "os/exec" + "syscall" +) + +type DockerBuilder struct { + Config *config.Config + Ctx *context.Context + Stdin io.Reader + Dir string + ImageTag string +} + +func (r *DockerBuilder) Run() error { + if r.ImageTag == "" { + r.ImageTag = "latest" + } + cmd := exec.CommandContext(*r.Ctx, utils.DockerPath, "build") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + return unix.Kill(-cmd.Process.Pid, unix.SIGINT) + } + cmd.Dir = r.Dir + cmd.Env = r.Config.EnvArray(false) + cmd.Env = append(cmd.Env, "BUILDKIT_PROGRESS=plain") + for k, _ := range r.Config.Env { + cmd.Args = append(cmd.Args, "--build-arg") + cmd.Args = append(cmd.Args, k) + } + cmd.Args = append(cmd.Args, "--no-cache") + cmd.Args = append(cmd.Args, "--pull") + cmd.Args = append(cmd.Args, "--force-rm") + cmd.Args = append(cmd.Args, "-t") + cmd.Args = append(cmd.Args, utils.BaseImageName+r.Config.Name+":"+r.ImageTag) + cmd.Args = append(cmd.Args, "--shm-size=512m") + cmd.Args = append(cmd.Args, "-f") + cmd.Args = append(cmd.Args, "-") + cmd.Args = append(cmd.Args, ".") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = r.Stdin + if err := utils.CmdRunner(cmd).Run(); err != nil { + return err + } + return nil +} diff --git a/launcher_go/v2/docker/docker_suite_test.go b/launcher_go/v2/docker/docker_suite_test.go new file mode 100644 index 000000000..4001151b1 --- /dev/null +++ b/launcher_go/v2/docker/docker_suite_test.go @@ -0,0 +1,13 @@ +package docker_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDocker(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Docker Suite") +} diff --git a/launcher_go/v2/go.mod b/launcher_go/v2/go.mod new file mode 100644 index 000000000..30d730377 --- /dev/null +++ b/launcher_go/v2/go.mod @@ -0,0 +1,29 @@ +module github.com/discourse/discourse_docker/launcher_go/v2 + +go 1.21 + +require ( + dario.cat/mergo v1.0.0 + github.com/Wing924/shellwords v1.1.0 + github.com/alecthomas/kong v0.8.1 + github.com/google/uuid v1.3.1 + github.com/onsi/ginkgo/v2 v2.13.0 + github.com/onsi/gomega v1.29.0 + github.com/posener/complete v1.2.3 + github.com/willabides/kongplete v0.4.0 + golang.org/x/sys v0.13.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/tools v0.12.0 // indirect +) diff --git a/launcher_go/v2/go.sum b/launcher_go/v2/go.sum new file mode 100644 index 000000000..b311624b7 --- /dev/null +++ b/launcher_go/v2/go.sum @@ -0,0 +1,73 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Wing924/shellwords v1.1.0 h1:dSiaG54kIH5pP636vlQSnRFhnSrFBrDPokMUj1CwySU= +github.com/Wing924/shellwords v1.1.0/go.mod h1:VWXBb1GU2vKj0ts/tn+TkAIs/uTn60rYcclSv02wSQg= +github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= +github.com/alecthomas/assert/v2 v2.1.0/go.mod h1:b/+1DI2Q6NckYi+3mXyH3wFb8qG37K/DuK80n7WefXA= +github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY= +github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= +github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= +github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab h1:ZjX6I48eZSFetPb41dHudEyVr5v953N15TsNZXlkcWY= +github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab/go.mod h1:/PfPXh0EntGc3QAAyUaviy4S9tzy4Zp0e2ilq4voC6E= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/willabides/kongplete v0.4.0 h1:eivXxkp5ud5+4+NVN9e4goxC5mSh3n1RHov+gsblM2g= +github.com/willabides/kongplete v0.4.0/go.mod h1:0P0jtWD9aTsqPSUAl4de35DLghrr57XcayPyvqSi2X8= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/launcher_go/v2/go_suite_test.go b/launcher_go/v2/go_suite_test.go new file mode 100644 index 000000000..754999475 --- /dev/null +++ b/launcher_go/v2/go_suite_test.go @@ -0,0 +1,13 @@ +package main_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMain(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Main Suite") +} diff --git a/launcher_go/v2/main.go b/launcher_go/v2/main.go new file mode 100644 index 000000000..0c7258f3d --- /dev/null +++ b/launcher_go/v2/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "fmt" + "github.com/alecthomas/kong" + "github.com/discourse/discourse_docker/launcher_go/v2/utils" + "golang.org/x/sys/unix" + "os" + "os/exec" + "os/signal" +) + +type Cli struct { + Version kong.VersionFlag `help:"Show version."` + ConfDir string `default:"./containers" hidden:"" help:"Discourse pups config directory." predictor:"dir"` + TemplatesDir string `default:"." hidden:"" help:"Home project directory containing a templates/ directory which in turn contains pups yaml templates." predictor:"dir"` + BuildDir string `default:"./tmp" hidden:"" help:"Temporary build folder for building images." predictor:"dir"` + BuildCmd DockerBuildCmd `cmd:"" name:"build" help:"Build a base image. This command does not need a running database. Saves resulting container."` +} + +func main() { + cli := Cli{} + runCtx, cancel := context.WithCancel(context.Background()) + + parser := kong.Must(&cli, kong.UsageOnError(), kong.Bind(&runCtx), kong.Vars{"version": utils.Version}) + + ctx, err := parser.Parse(os.Args[1:]) + parser.FatalIfErrorf(err) + + defer cancel() + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, unix.SIGTERM) + signal.Notify(sigChan, unix.SIGINT) + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-sigChan: + fmt.Fprintln(utils.Out, "Command interrupted") + cancel() + case <-done: + } + }() + err = ctx.Run() + if err == nil { + return + } + if exiterr, ok := err.(*exec.ExitError); ok { + // Magic exit code that indicates a retry + if exiterr.ExitCode() == 77 { + os.Exit(77) + } else if runCtx.Err() != nil { + fmt.Fprintln(utils.Out, "Aborted with exit code", exiterr.ExitCode()) + } else { + ctx.Fatalf( + "run failed with exit code %v\n"+ + "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one.\n"+ + "./discourse-doctor may help diagnose the problem.", exiterr.ExitCode()) + } + } else { + ctx.FatalIfErrorf(err) + } +} diff --git a/launcher_go/v2/test/containers/standalone.yml b/launcher_go/v2/test/containers/standalone.yml new file mode 100644 index 000000000..aef14df4d --- /dev/null +++ b/launcher_go/v2/test/containers/standalone.yml @@ -0,0 +1,109 @@ +## this is the all-in-one, standalone Discourse Docker container template +## +## After making changes to this file, you MUST rebuild +## /var/discourse/launcher rebuild app +## +## BE *VERY* CAREFUL WHEN EDITING! +## YAML FILES ARE SUPER SUPER SENSITIVE TO MISTAKES IN WHITESPACE OR ALIGNMENT! +## visit http://www.yamllint.com/ to validate this file as needed + +templates: + #- "templates/postgres.template.yml" + #- "templates/redis.template.yml" + - "templates/web.template.yml" + ## Uncomment the next line to enable the IPv6 listener + #- "templates/web.ipv6.template.yml" + #- "templates/web.ratelimited.template.yml" + ## Uncomment these two lines if you wish to add Lets Encrypt (https) + #- "templates/web.ssl.template.yml" + #- "templates/web.letsencrypt.ssl.template.yml" + +## which TCP/IP ports should this container expose? +## If you want Discourse to share a port with another webserver like Apache or nginx, +## see https://meta.discourse.org/t/17247 for details +expose: + - "80:80" # http + - "443:443" # https + +params: + db_default_text_search_config: "pg_catalog.english" + + ## Set db_shared_buffers to a max of 25% of the total memory. + ## will be set automatically by bootstrap based on detected RAM, or you can override + #db_shared_buffers: "256MB" + + ## can improve sorting performance, but adds memory usage per-connection + #db_work_mem: "40MB" + + ## Which Git revision should this container use? (default: tests-passed) + #version: tests-passed + +env: + LC_ALL: en_US.UTF-8 + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + # DISCOURSE_DEFAULT_LOCALE: en + + ## How many concurrent web requests are supported? Depends on memory and CPU cores. + ## will be set automatically by bootstrap based on detected CPUs, or you can override + #UNICORN_WORKERS: 3 + + ## TODO: The domain name this Discourse instance will respond to + ## Required. Discourse will not work with a bare IP number. + DISCOURSE_HOSTNAME: 'discourse.example.com' + + ## Uncomment if you want the container to be started with the same + ## hostname (-h option) as specified above (default "$hostname-$config") + #DOCKER_USE_HOSTNAME: true + + ## TODO: List of comma delimited emails that will be made admin and developer + ## on initial signup example 'user1@example.com,user2@example.com' + DISCOURSE_DEVELOPER_EMAILS: 'me@example.com,you@example.com' + + ## TODO: The SMTP mail server used to validate new accounts and send notifications + # SMTP ADDRESS, username, and password are required + # WARNING the char '#' in SMTP password can cause problems! + DISCOURSE_SMTP_ADDRESS: smtp.example.com + #DISCOURSE_SMTP_PORT: 587 + DISCOURSE_SMTP_USER_NAME: user@example.com + DISCOURSE_SMTP_PASSWORD: pa$$word + #DISCOURSE_SMTP_ENABLE_START_TLS: true # (optional, default true) + #DISCOURSE_SMTP_DOMAIN: discourse.example.com # (required by some providers) + #DISCOURSE_NOTIFICATION_EMAIL: noreply@discourse.example.com # (address to send notifications from) + + ## If you added the Lets Encrypt template, uncomment below to get a free SSL certificate + #LETSENCRYPT_ACCOUNT_EMAIL: me@example.com + + ## The http or https CDN address for this Discourse instance (configured to pull) + ## see https://meta.discourse.org/t/14857 for details + #DISCOURSE_CDN_URL: https://discourse-cdn.example.com + + ## The maxmind geolocation IP address key for IP address lookup + ## see https://meta.discourse.org/t/-/137387/23 for details + #DISCOURSE_MAXMIND_LICENSE_KEY: 1234567890123456 + +## The Docker container is stateless; all data is stored in /shared +volumes: + - volume: + host: /var/discourse/shared/standalone + guest: /shared + - volume: + host: /var/discourse/shared/standalone/log/var-log + guest: /var/log + +## Plugins go here +## see https://meta.discourse.org/t/19157 for details +hooks: + after_code: + - exec: + cd: $home/plugins + cmd: + - git clone https://github.com/discourse/docker_manager.git + +## Any custom commands to run after building +run: + - exec: echo "Beginning of custom commands" + ## If you want to set the 'From' email address for your first registration, uncomment and change: + ## After getting the first signup email, re-comment the line. It only needs to run once. + #- exec: rails r "SiteSetting.notification_email='info@unconfigured.discourse.org'" + - exec: echo "End of custom commands" diff --git a/launcher_go/v2/test/containers/test.yml b/launcher_go/v2/test/containers/test.yml new file mode 100644 index 000000000..ef3b70636 --- /dev/null +++ b/launcher_go/v2/test/containers/test.yml @@ -0,0 +1,120 @@ +# IMPORTANT: SET A SECRET PASSWORD in Postgres for the Discourse User +# TODO: change SOME_SECRET in this template + +templates: + - "templates/web.template.yml" + ## Uncomment the next line to enable the IPv6 listener + #- "templates/web.ipv6.template.yml" + #- "templates/web.ratelimited.template.yml" + ## Uncomment these two lines if you wish to add Lets Encrypt (https) + #- "templates/web.ssl.template.yml" + #- "templates/web.letsencrypt.ssl.template.yml" + +## which TCP/IP ports should this container expose? +## If you want Discourse to share a port with another webserver like Apache or nginx, +## see https://meta.discourse.org/t/17247 for details +expose: + - "80:80" # http + - "443:443" # https + - 90 + +# Use 'links' key to link containers together, aka use Docker --link flag. +links: + - link: + name: data + alias: data + +# any extra arguments for Docker? +docker_args: "--expose 100" + +params: + ## Which Git revision should this container use? (default: tests-passed) + #version: tests-passed + +env: + LC_ALL: en_US.UTF-8 + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + # DISCOURSE_DEFAULT_LOCALE: en + REPLACED: "{{config}}/{{config}}/{{config}}" + MULTI: | + test + multiline with some spaces + var + + ## How many concurrent web requests are supported? Depends on memory and CPU cores. + ## will be set automatically by bootstrap based on detected CPUs, or you can override + #UNICORN_WORKERS: 3 + + ## TODO: The domain name this Discourse instance will respond to + DISCOURSE_HOSTNAME: 'discourse.example.com' + + ## Uncomment if you want the container to be started with the same + ## hostname (-h option) as specified above (default "$hostname-$config") + #DOCKER_USE_HOSTNAME: true + + ## TODO: List of comma delimited emails that will be made admin and developer + ## on initial signup example 'user1@example.com,user2@example.com' + DISCOURSE_DEVELOPER_EMAILS: 'me@example.com,you@example.com' + + ## TODO: The SMTP mail server used to validate new accounts and send notifications + # SMTP ADDRESS, username, and password are required + # WARNING the char '#' in SMTP password can cause problems! + DISCOURSE_SMTP_ADDRESS: smtp.example.com + #DISCOURSE_SMTP_PORT: 587 + DISCOURSE_SMTP_USER_NAME: user@example.com + DISCOURSE_SMTP_PASSWORD: pa$$word + #DISCOURSE_SMTP_ENABLE_START_TLS: true # (optional, default true) + #DISCOURSE_SMTP_DOMAIN: discourse.example.com # (required by some providers) + #DISCOURSE_NOTIFICATION_EMAIL: noreply@discourse.example.com # (address to send notifications from) + + ## If you added the Lets Encrypt template, uncomment below to get a free SSL certificate + #LETSENCRYPT_ACCOUNT_EMAIL: me@example.com + + ## TODO: configure connectivity to the databases + DISCOURSE_DB_SOCKET: '' + #DISCOURSE_DB_USERNAME: discourse + DISCOURSE_DB_PASSWORD: SOME_SECRET + DISCOURSE_DB_HOST: data + DISCOURSE_REDIS_HOST: data + + ## The http or https CDN address for this Discourse instance (configured to pull) + ## see https://meta.discourse.org/t/14857 for details + #DISCOURSE_CDN_URL: https://discourse-cdn.example.com + + ## The maxmind geolocation IP address key for IP address lookup + ## see https://meta.discourse.org/t/-/137387/23 for details + #DISCOURSE_MAXMIND_LICENSE_KEY: 1234567890123456 + +volumes: + - volume: + host: /var/discourse/shared/web-only + guest: /shared + - volume: + host: /var/discourse/shared/web-only/log/var-log + guest: /var/log + +## Plugins go here +## see https://meta.discourse.org/t/19157 for details +hooks: + after_code: + - exec: + cd: $home/plugins + cmd: + - git clone https://github.com/discourse/docker_manager.git + +## Remember, this is YAML syntax - you can only have one block with a name +run: + - exec: echo "Beginning of custom commands" + + ## If you want to configure password login for root, uncomment and change: + ## Use only one of the following lines: + #- exec: /usr/sbin/usermod -p 'PASSWORD_HASH' root + #- exec: /usr/sbin/usermod -p "$(mkpasswd -m sha-256 'RAW_PASSWORD')" root + + ## If you want to authorized additional users, uncomment and change: + #- exec: ssh-import-id username + #- exec: ssh-import-id anotherusername + + - exec: echo "End of custom commands" + - exec: awk -F\# '{print $1;}' ~/.ssh/authorized_keys | awk 'BEGIN { print "Authorized SSH keys for this container:"; } NF>=2 {print $NF;}' diff --git a/launcher_go/v2/test/containers/test2.yaml b/launcher_go/v2/test/containers/test2.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/launcher_go/v2/test/containers/test3.not-a-yaml b/launcher_go/v2/test/containers/test3.not-a-yaml new file mode 100644 index 000000000..e69de29bb diff --git a/launcher_go/v2/test/containers/web_only.yml b/launcher_go/v2/test/containers/web_only.yml new file mode 100644 index 000000000..5cf77ff42 --- /dev/null +++ b/launcher_go/v2/test/containers/web_only.yml @@ -0,0 +1,114 @@ +# IMPORTANT: SET A SECRET PASSWORD in Postgres for the Discourse User +# TODO: change SOME_SECRET in this template + +templates: + - "templates/web.template.yml" + ## Uncomment the next line to enable the IPv6 listener + #- "templates/web.ipv6.template.yml" + #- "templates/web.ratelimited.template.yml" + ## Uncomment these two lines if you wish to add Lets Encrypt (https) + #- "templates/web.ssl.template.yml" + #- "templates/web.letsencrypt.ssl.template.yml" + +## which TCP/IP ports should this container expose? +## If you want Discourse to share a port with another webserver like Apache or nginx, +## see https://meta.discourse.org/t/17247 for details +expose: + - "80:80" # http + - "443:443" # https + +# Use 'links' key to link containers together, aka use Docker --link flag. +links: + - link: + name: data + alias: data + +# any extra arguments for Docker? +# docker_args: + +params: + ## Which Git revision should this container use? (default: tests-passed) + #version: tests-passed + +env: + LC_ALL: en_US.UTF-8 + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + # DISCOURSE_DEFAULT_LOCALE: en + + ## How many concurrent web requests are supported? Depends on memory and CPU cores. + ## will be set automatically by bootstrap based on detected CPUs, or you can override + #UNICORN_WORKERS: 3 + + ## TODO: The domain name this Discourse instance will respond to + DISCOURSE_HOSTNAME: 'discourse.example.com' + + ## Uncomment if you want the container to be started with the same + ## hostname (-h option) as specified above (default "$hostname-$config") + #DOCKER_USE_HOSTNAME: true + + ## TODO: List of comma delimited emails that will be made admin and developer + ## on initial signup example 'user1@example.com,user2@example.com' + DISCOURSE_DEVELOPER_EMAILS: 'me@example.com,you@example.com' + + ## TODO: The SMTP mail server used to validate new accounts and send notifications + # SMTP ADDRESS, username, and password are required + # WARNING the char '#' in SMTP password can cause problems! + DISCOURSE_SMTP_ADDRESS: smtp.example.com + #DISCOURSE_SMTP_PORT: 587 + DISCOURSE_SMTP_USER_NAME: user@example.com + DISCOURSE_SMTP_PASSWORD: pa$$word + #DISCOURSE_SMTP_ENABLE_START_TLS: true # (optional, default true) + #DISCOURSE_SMTP_DOMAIN: discourse.example.com # (required by some providers) + #DISCOURSE_NOTIFICATION_EMAIL: noreply@discourse.example.com # (address to send notifications from) + + ## If you added the Lets Encrypt template, uncomment below to get a free SSL certificate + #LETSENCRYPT_ACCOUNT_EMAIL: me@example.com + + ## TODO: configure connectivity to the databases + DISCOURSE_DB_SOCKET: '' + #DISCOURSE_DB_USERNAME: discourse + DISCOURSE_DB_PASSWORD: SOME_SECRET + DISCOURSE_DB_HOST: data + DISCOURSE_REDIS_HOST: data + + ## The http or https CDN address for this Discourse instance (configured to pull) + ## see https://meta.discourse.org/t/14857 for details + #DISCOURSE_CDN_URL: https://discourse-cdn.example.com + + ## The maxmind geolocation IP address key for IP address lookup + ## see https://meta.discourse.org/t/-/137387/23 for details + #DISCOURSE_MAXMIND_LICENSE_KEY: 1234567890123456 + +volumes: + - volume: + host: /var/discourse/shared/web-only + guest: /shared + - volume: + host: /var/discourse/shared/web-only/log/var-log + guest: /var/log + +## Plugins go here +## see https://meta.discourse.org/t/19157 for details +hooks: + after_code: + - exec: + cd: $home/plugins + cmd: + - git clone https://github.com/discourse/docker_manager.git + +## Remember, this is YAML syntax - you can only have one block with a name +run: + - exec: echo "Beginning of custom commands" + + ## If you want to configure password login for root, uncomment and change: + ## Use only one of the following lines: + #- exec: /usr/sbin/usermod -p 'PASSWORD_HASH' root + #- exec: /usr/sbin/usermod -p "$(mkpasswd -m sha-256 'RAW_PASSWORD')" root + + ## If you want to authorized additional users, uncomment and change: + #- exec: ssh-import-id username + #- exec: ssh-import-id anotherusername + + - exec: echo "End of custom commands" + - exec: awk -F\# '{print $1;}' ~/.ssh/authorized_keys | awk 'BEGIN { print "Authorized SSH keys for this container:"; } NF>=2 {print $NF;}' diff --git a/launcher_go/v2/test/templates/web.template.yml b/launcher_go/v2/test/templates/web.template.yml new file mode 100644 index 000000000..d3d992b9b --- /dev/null +++ b/launcher_go/v2/test/templates/web.template.yml @@ -0,0 +1,438 @@ +env: + # You can have redis on a different box + RAILS_ENV: 'production' + UNICORN_WORKERS: 3 + UNICORN_SIDEKIQS: 1 + # stop heap doubling in size so aggressively, this conserves memory + RUBY_GC_HEAP_GROWTH_MAX_SLOTS: 40000 + RUBY_GC_HEAP_INIT_SLOTS: 400000 + RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR: 1.5 + + DISCOURSE_DB_SOCKET: /var/run/postgresql + DISCOURSE_DB_HOST: + DISCOURSE_DB_PORT: + + +params: + version: tests-passed + + home: /var/www/discourse + upload_size: 10m + nginx_worker_connections: 4000 + +run: + - exec: thpoff echo "thpoff is installed!" + - exec: + tag: precompile + cmd: + - /usr/local/bin/ruby -e 'if ENV["DISCOURSE_SMTP_ADDRESS"] == "smtp.example.com"; puts "Aborting! Mail is not configured!"; exit 1; end' + - /usr/local/bin/ruby -e 'if ENV["DISCOURSE_HOSTNAME"] == "discourse.example.com"; puts "Aborting! Domain is not configured!"; exit 1; end' + - /usr/local/bin/ruby -e 'if (ENV["DISCOURSE_CDN_URL"] || "")[0..1] == "//"; puts "Aborting! CDN must have a protocol specified. Once fixed you should rebake your posts now to correct all posts."; exit 1; end' + # TODO: move to base image (anacron can not be fired up using rc.d) + - exec: rm -f /etc/cron.d/anacron + - file: + path: /etc/cron.d/anacron + contents: | + SHELL=/bin/sh + PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + + 30 7 * * * root /usr/sbin/anacron -s >/dev/null + - file: + path: /etc/runit/1.d/copy-env + chmod: "+x" + contents: | + #!/bin/bash + env > ~/boot_env + conf=/var/www/discourse/config/discourse.conf + + # find DISCOURSE_ env vars, strip the leader, lowercase the key + /usr/local/bin/ruby -e 'ENV.each{|k,v| puts "#{$1.downcase} = '\''#{v}'\''" if k =~ /^DISCOURSE_(.*)/}' > $conf + + - file: + path: /etc/service/unicorn/run + chmod: "+x" + contents: | + #!/bin/bash + exec 2>&1 + # redis + # postgres + cd $home + chown -R discourse:www-data /shared/log/rails + if [[ -z "$PRECOMPILE_ON_BOOT" ]]; then + PRECOMPILE_ON_BOOT=1 + fi + if [ -f /usr/local/bin/create_db ] && [ "$CREATE_DB_ON_BOOT" = "1" ]; then /usr/local/bin/create_db; rm -f /usr/local/bin/create_db; fi; + if [ "$MIGRATE_ON_BOOT" = "1" ]; then su discourse -c 'bundle exec rake db:migrate'; fi + if [ "$PRECOMPILE_ON_BOOT" = "1" ]; then SKIP_EMBER_CLI_COMPILE=1 su discourse -c 'bundle exec rake assets:precompile'; fi + LD_PRELOAD=$RUBY_ALLOCATOR HOME=/home/discourse USER=discourse exec thpoff chpst -u discourse:www-data -U discourse:www-data bundle exec config/unicorn_launcher -E production -c config/unicorn.conf.rb + + - file: + path: /etc/service/nginx/run + chmod: "+x" + contents: | + #!/bin/sh + exec 2>&1 + exec /usr/sbin/nginx + + - file: + path: /etc/runit/3.d/01-nginx + chmod: "+x" + contents: | + #!/bin/bash + sv stop nginx + + - file: + path: /etc/runit/3.d/02-unicorn + chmod: "+x" + contents: | + #!/bin/bash + sv stop unicorn + + - exec: + cd: $home + hook: code + cmd: + - sudo -H -E -u discourse git reset --hard + - sudo -H -E -u discourse git clean -f + # TODO Remove the special handling of shallow clones when everyone uses images without that clone type + - |- + sudo -H -E -u discourse bash -c ' + set -o errexit + if [ $(git rev-parse --is-shallow-repository) == "true" ]; then + git remote set-branches --add origin main + git remote set-branches origin $version + git fetch --depth 1 origin $version + else + git fetch --tags --prune-tags --prune --force origin + fi + ' + - |- + sudo -H -E -u discourse bash -c ' + set -o errexit + if [[ $(git symbolic-ref --short HEAD) == $version ]] ; then + git pull + else + git -c advice.detachedHead=false checkout $version + fi + ' + - sudo -H -E -u discourse git config user.discourse-version $version + - mkdir -p tmp + - chown discourse:www-data tmp + - mkdir -p tmp/pids + - mkdir -p tmp/sockets + - touch tmp/.gitkeep + - mkdir -p /shared/log/rails + - bash -c "touch -a /shared/log/rails/{production,production_errors,unicorn.stdout,unicorn.stderr,sidekiq}.log" + - bash -c "ln -s /shared/log/rails/{production,production_errors,unicorn.stdout,unicorn.stderr,sidekiq}.log $home/log" + - bash -c "mkdir -p /shared/{uploads,backups}" + - bash -c "ln -s /shared/{uploads,backups} $home/public" + - bash -c "mkdir -p /shared/tmp/{backups,restores}" + - bash -c "ln -s /shared/tmp/{backups,restores} $home/tmp" + - chown -R discourse:www-data /shared/log/rails /shared/uploads /shared/backups /shared/tmp + # scrub broken symlinks from plugins that have been removed + - "[ ! -d public/plugins ] || find public/plugins/ -maxdepth 1 -xtype l -delete" + + - exec: + cmd: + - "cp $home/config/nginx.sample.conf /etc/nginx/conf.d/discourse.conf" + - "rm /etc/nginx/sites-enabled/default" + - "mkdir -p /var/nginx/cache" + + - replace: + filename: /etc/nginx/nginx.conf + from: pid /run/nginx.pid; + to: daemon off; + + - replace: + filename: "/etc/nginx/conf.d/discourse.conf" + from: /upstream[^\}]+\}/m + to: "upstream discourse { + server 127.0.0.1:3000; + }" + + - replace: + filename: "/etc/nginx/conf.d/discourse.conf" + from: /server_name.+$/ + to: server_name _ ; + + - replace: + filename: "/etc/nginx/conf.d/discourse.conf" + from: /client_max_body_size.+$/ + to: client_max_body_size $upload_size ; + + - replace: + filename: "/etc/nginx/nginx.conf" + from: /worker_connections.+$/ + to: worker_connections $nginx_worker_connections ; + + - exec: + cmd: echo "done configuring web" + hook: web_config + + - exec: + cd: $home + hook: web + cmd: + # install bundler version to match Gemfile.lock + - gem install bundler --conservative -v $(awk '/BUNDLED WITH/ { getline; gsub(/ /,""); print $0 }' Gemfile.lock) + - find $home ! -user discourse -exec chown discourse {} \+ + + - exec: + cd: $home + cmd: + - |- + if [ "$version" != "tests-passed" ]; then + rm -rf app/assets/javascripts/node_modules + fi + - su discourse -c 'yarn install --frozen-lockfile && yarn cache clean' + + - exec: + cd: $home + hook: bundle_exec + cmd: + - su discourse -c 'bundle config --local deployment true' + - su discourse -c 'bundle config --local without "development test"' + - su discourse -c 'bundle install --retry 3 --jobs 4' + + - exec: + cd: $home + cmd: + - su discourse -c 'LOAD_PLUGINS=0 bundle exec rake plugin:pull_compatible_all' + raise_on_fail: false + + - exec: + cd: $home + tag: migrate + hook: db_migrate + cmd: + - su discourse -c 'bundle exec rake db:migrate' + - exec: + cd: $home + tag: build + hook: assets_precompile_build + cmd: + - su discourse -c 'bundle exec rake assets:precompile:build' + - exec: + cd: $home + tag: precompile + hook: assets_precompile + cmd: + - su discourse -c 'SKIP_EMBER_CLI_COMPILE=1 bundle exec rake themes:update assets:precompile' + - replace: + tag: precompile + filename: /etc/service/unicorn/run + from: PRECOMPILE_ON_BOOT=1 + to: "PRECOMPILE_ON_BOOT=0" + + - file: + path: /usr/local/bin/discourse + chmod: +x + contents: | + #!/bin/bash + (cd /var/www/discourse && RAILS_ENV=production sudo -H -E -u discourse bundle exec script/discourse "$@") + + - file: + path: /usr/local/bin/rails + chmod: +x + contents: | + #!/bin/bash + # If they requested a console, load pry instead + if [ "$*" == "c" -o "$*" == "console" ] + then + (cd /var/www/discourse && RAILS_ENV=production sudo -H -E -u discourse bundle exec pry -r ./config/environment) + else + (cd /var/www/discourse && RAILS_ENV=production sudo -H -E -u discourse bundle exec script/rails "$@") + fi + + - file: + path: /usr/local/bin/rake + chmod: +x + contents: | + #!/bin/bash + (cd /var/www/discourse && RAILS_ENV=production sudo -H -E -u discourse bundle exec bin/rake "$@") + + - file: + path: /usr/local/bin/rbtrace + chmod: +x + contents: | + #!/bin/bash + (cd /var/www/discourse && RAILS_ENV=production sudo -H -E -u discourse bundle exec rbtrace "$@") + + - file: + path: /usr/local/bin/stackprof + chmod: +x + contents: | + #!/bin/bash + (cd /var/www/discourse && RAILS_ENV=production sudo -H -E -u discourse bundle exec stackprof "$@") + + - file: + path: /etc/update-motd.d/10-web + chmod: +x + contents: | + #!/bin/bash + echo + echo Use: rails, rake or discourse to execute commands in production + echo + + - file: + path: /etc/logrotate.d/rails + contents: | + /shared/log/rails/*.log + { + rotate 7 + dateext + daily + missingok + delaycompress + compress + postrotate + sv 1 unicorn + endscript + } + + - file: + path: /etc/logrotate.d/nginx + contents: | + /var/log/nginx/*.log { + daily + missingok + rotate 7 + compress + delaycompress + create 0644 www-data www-data + sharedscripts + postrotate + sv 1 nginx + endscript + } + + # move state out of the container this fancy is done to support rapid rebuilds of containers, + # we store anacron and logrotate state outside the container to ensure its maintained across builds + # later move this snipped into an initialization script + # we also ensure all the symlinks we need to /shared are in place in the correct structure + # this allows us to bootstrap on one machine and then run on another + - file: + path: /etc/runit/1.d/00-ensure-links + chmod: +x + contents: | + #!/bin/bash + if [[ ! -L /var/lib/logrotate ]]; then + rm -fr /var/lib/logrotate + mkdir -p /shared/state/logrotate + ln -s /shared/state/logrotate /var/lib/logrotate + fi + if [[ ! -L /var/spool/anacron ]]; then + rm -fr /var/spool/anacron + mkdir -p /shared/state/anacron-spool + ln -s /shared/state/anacron-spool /var/spool/anacron + fi + if [[ ! -d /shared/log/rails ]]; then + mkdir -p /shared/log/rails + chown -R discourse:www-data /shared/log/rails + fi + if [[ ! -d /shared/uploads ]]; then + mkdir -p /shared/uploads + chown -R discourse:www-data /shared/uploads + fi + if [[ ! -d /shared/backups ]]; then + mkdir -p /shared/backups + chown -R discourse:www-data /shared/backups + fi + + rm -rf /shared/tmp/{backups,restores} + mkdir -p /shared/tmp/{backups,restores} + chown -R discourse:www-data /shared/tmp/{backups,restores} + - file: + path: /etc/runit/1.d/01-cleanup-web-pids + chmod: +x + contents: | + #!/bin/bash + /bin/rm -f /var/www/discourse/tmp/pids/*.pid + # change login directory to Discourse home + - file: + path: /root/.bash_profile + chmod: 644 + contents: | + cd $home + + - file: + path: /usr/local/etc/ImageMagick-7/policy.xml + contents: | + + + + + + ]> + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/launcher_go/v2/test_utils/utils.go b/launcher_go/v2/test_utils/utils.go new file mode 100644 index 000000000..1c716e641 --- /dev/null +++ b/launcher_go/v2/test_utils/utils.go @@ -0,0 +1,36 @@ +package test_utils + +import ( + "github.com/discourse/discourse_docker/launcher_go/v2/utils" + "os/exec" +) + +var RanCmds []exec.Cmd +var CmdOutputResponse []byte +var CmdOutputError error + +type FakeCmdRunner struct { + Cmd *exec.Cmd +} + +func (r FakeCmdRunner) Run() error { + RanCmds = append(RanCmds, *r.Cmd) + return CmdOutputError +} + +func (r FakeCmdRunner) Output() ([]byte, error) { + RanCmds = append(RanCmds, *r.Cmd) + return CmdOutputResponse, CmdOutputError +} + +// Swap out CmdRunner with a fake instance that also returns created ICmdRunners on a channel +// so tests can inspect commands after they're run +func CreateNewFakeCmdRunner() func(cmd *exec.Cmd) utils.ICmdRunner { + RanCmds = []exec.Cmd{} + CmdOutputResponse = []byte{} + CmdOutputError = nil + return func(cmd *exec.Cmd) utils.ICmdRunner { + cmdRunner := &FakeCmdRunner{Cmd: cmd} + return cmdRunner + } +} diff --git a/launcher_go/v2/utils/cmd_runner.go b/launcher_go/v2/utils/cmd_runner.go new file mode 100644 index 000000000..765bee1a1 --- /dev/null +++ b/launcher_go/v2/utils/cmd_runner.go @@ -0,0 +1,28 @@ +package utils + +import ( + "os/exec" +) + +type ICmdRunner interface { + Run() error + Output() ([]byte, error) +} + +type ExecCmdRunner struct { + Cmd *exec.Cmd +} + +func (r *ExecCmdRunner) Run() error { + return r.Cmd.Run() +} + +func (r *ExecCmdRunner) Output() ([]byte, error) { + return r.Cmd.Output() +} + +func NewExecCmdRunner(cmd *exec.Cmd) ICmdRunner { + return &ExecCmdRunner{Cmd: cmd} +} + +var CmdRunner = NewExecCmdRunner diff --git a/launcher_go/v2/utils/consts.go b/launcher_go/v2/utils/consts.go new file mode 100644 index 000000000..21cc3a629 --- /dev/null +++ b/launcher_go/v2/utils/consts.go @@ -0,0 +1,46 @@ +package utils + +import ( + "io" + "os" + "os/exec" +) + +const Version = "v2.0.0" + +const BaseImageName = "local_discourse/" + +// Known secrets, or otherwise not public info from config so we can build public images +var KnownSecrets = []string{ + "DISCOURSE_DB_HOST", + "DISCOURSE_DB_PORT", + "DISCOURSE_DB_REPLICA_HOST", + "DISCOURSE_DB_REPLICA_PORT", + "DISCOURSE_DB_PASSWORD", + "DISCOURSE_REDIS_HOST", + "DISCOURSE_REDIS_REPLICA_HOST", + "DISCOURSE_REDIS_PASSWORD", + "DISCOURSE_SMTP_ADDRESS", + "DISCOURSE_SMTP_USER_NAME", + "DISCOURSE_SMTP_PASSWORD", + "DISCOURSE_DEVELOPER_EMAILS", + "DISCOURSE_SECRET_KEY_BASE", + "DISCOURSE_HOSTNAME", + "DISCOURSE_SAML_CERT", + "DISCOURSE_SAML_TITLE", + "DISCOURSE_SAML_TARGET_URL", + "DISCOURSE_SAML_NAME_IDENTIFIER_FORMAT", +} + +func findDockerPath() string { + location, err := exec.LookPath("docker.io") + if err != nil { + location, _ := exec.LookPath("docker") + return location + } + return location +} + +var DockerPath = findDockerPath() + +var Out io.Writer = os.Stdout diff --git a/launcher_go/v2/utils/utils_suite_test.go b/launcher_go/v2/utils/utils_suite_test.go new file mode 100644 index 000000000..9ca82ff0d --- /dev/null +++ b/launcher_go/v2/utils/utils_suite_test.go @@ -0,0 +1,13 @@ +package utils_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestUtils(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Utils Suite") +}