diff --git a/api/app.bytetrade.io/v1alpha1/application_types.go b/api/app.bytetrade.io/v1alpha1/application_types.go index 4f9dd80..47479c2 100644 --- a/api/app.bytetrade.io/v1alpha1/application_types.go +++ b/api/app.bytetrade.io/v1alpha1/application_types.go @@ -44,12 +44,20 @@ type ApplicationSpec struct { //Entrances []Entrance `json:"entrances,omitempty"` Entrances []Entrance `json:"entrances,omitempty"` - Ports []ServicePort `json:"ports,omitempty"` + Ports []ServicePort `json:"ports,omitempty"` + TailScaleACLs []ACL `json:"tailscaleAcls,omitempty"` // the extend settings of the application Settings map[string]string `json:"settings,omitempty"` } +type ACL struct { + Action string `json:"action,omitempty"` + Src []string `json:"src,omitempty"` + Proto string `json:"proto"` + Dst []string `json:"dst"` +} + type EntranceState string const ( diff --git a/api/app.bytetrade.io/v1alpha1/zz_generated.deepcopy.go b/api/app.bytetrade.io/v1alpha1/zz_generated.deepcopy.go index 8c9d8b5..c26cdf4 100644 --- a/api/app.bytetrade.io/v1alpha1/zz_generated.deepcopy.go +++ b/api/app.bytetrade.io/v1alpha1/zz_generated.deepcopy.go @@ -9,6 +9,31 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ACL) DeepCopyInto(out *ACL) { + *out = *in + if in.Src != nil { + in, out := &in.Src, &out.Src + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Dst != nil { + in, out := &in.Dst, &out.Dst + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ACL. +func (in *ACL) DeepCopy() *ACL { + if in == nil { + return nil + } + out := new(ACL) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Application) DeepCopyInto(out *Application) { *out = *in @@ -192,6 +217,13 @@ func (in *ApplicationSpec) DeepCopyInto(out *ApplicationSpec) { *out = make([]ServicePort, len(*in)) copy(*out, *in) } + if in.TailScaleACLs != nil { + in, out := &in.TailScaleACLs, &out.TailScaleACLs + *out = make([]ACL, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Settings != nil { in, out := &in.Settings, &out.Settings *out = make(map[string]string, len(*in)) diff --git a/cmd/app-service/main.go b/cmd/app-service/main.go index 425070e..54d5059 100644 --- a/cmd/app-service/main.go +++ b/cmd/app-service/main.go @@ -129,6 +129,13 @@ func main() { os.Exit(1) } + if err = (&controllers.TailScaleACLController{ + Client: mgr.GetClient(), + }).SetUpWithManager(mgr); err != nil { + setupLog.Error(err, "Unable to create controller", "controller", "tailScaleACLA manager") + os.Exit(1) + } + //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/crd/bases/app.bytetrade.io_applications.yaml b/config/crd/bases/app.bytetrade.io_applications.yaml index 1d7dc6d..fac8d43 100644 --- a/config/crd/bases/app.bytetrade.io_applications.yaml +++ b/config/crd/bases/app.bytetrade.io_applications.yaml @@ -127,8 +127,8 @@ spec: format: int32 type: integer protocol: - description: The protocol for this entrance. Supports "TCP" - and "UDP". Default is TCP. + description: The protocol for this entrance. Supports "tcp" + and "udp". Default is tcp. type: string required: - host @@ -141,6 +141,26 @@ spec: type: string description: the extend settings of the application type: object + tailscaleAcls: + items: + properties: + action: + type: string + dst: + items: + type: string + type: array + proto: + type: string + src: + items: + type: string + type: array + required: + - dst + - proto + type: object + type: array required: - appid - isSysApp diff --git a/controllers/application_controller.go b/controllers/application_controller.go index 2cf34db..40b1395 100644 --- a/controllers/application_controller.go +++ b/controllers/application_controller.go @@ -330,6 +330,10 @@ func (r *ApplicationReconciler) createApplication(ctx context.Context, req ctrl. if err != nil { klog.Errorf("failed to get app ports err=%v", err) } + tailScaleACLs, err := r.getAppACLs(deployment) + if err != nil { + klog.Errorf("failed to get app tailscale acls err=%v", err) + } var appid string var isSysApp bool @@ -355,6 +359,7 @@ func (r *ApplicationReconciler) createApplication(ctx context.Context, req ctrl. DeploymentName: deployment.GetName(), Entrances: entrancesMap[name], Ports: servicePortsMap[name], + TailScaleACLs: tailScaleACLs, Icon: icon[name], Settings: settings, }, @@ -415,6 +420,11 @@ func (r *ApplicationReconciler) updateApplication(ctx context.Context, req ctrl. deployment client.Object, app *appv1alpha1.Application, name string) error { appCopy := app.DeepCopy() + tailScaleACLs, err := r.getAppACLs(deployment) + if err != nil { + klog.Errorf("failed to get tailscale err=%v", err) + } + owner := deployment.GetLabels()[constants.ApplicationOwnerLabel] klog.Infof("in updateApplication ....") icons := getAppIcon(deployment) @@ -428,6 +438,8 @@ func (r *ApplicationReconciler) updateApplication(ctx context.Context, req ctrl. appCopy.Spec.DeploymentName = deployment.GetName() appCopy.Spec.Icon = icon + appCopy.Spec.TailScaleACLs = tailScaleACLs + actionConfig, _, err := helm.InitConfig(r.Kubeconfig, appCopy.Spec.Namespace) if err != nil { ctrl.Log.Error(err, "init helm config error") @@ -708,6 +720,16 @@ func (r *ApplicationReconciler) getAppPorts(ctx context.Context, deployment clie return portsMap, nil } +func (r *ApplicationReconciler) getAppACLs(deployment client.Object) ([]appv1alpha1.ACL, error) { + acls := make([]appv1alpha1.ACL, 0) + aclsString := deployment.GetAnnotations()[constants.ApplicationTailScaleACLKey] + err := json.Unmarshal([]byte(aclsString), &acls) + if err != nil { + return nil, err + } + return acls, nil +} + func checkPortOfService(s *corev1.Service, port int32) bool { for _, p := range s.Spec.Ports { if p.Port == port { diff --git a/controllers/appmgr_controller.go b/controllers/appmgr_controller.go index 5c98d01..8da0ce5 100644 --- a/controllers/appmgr_controller.go +++ b/controllers/appmgr_controller.go @@ -449,9 +449,6 @@ func (r *ApplicationManagerController) install(ctx context.Context, appMgr *appv klog.Errorf("Failed to update applicationmanagers status name=%s err=%v", appMgr.Name, err) } - err = r.Get(ctx, types.NamespacedName{Name: appMgr.Name}, appMgr) - var curAppMgr appv1alpha1.ApplicationManager - err = r.Get(ctx, types.NamespacedName{Name: appMgr.Name}, &curAppMgr) return err } diff --git a/controllers/tailscale_acl_controller.go b/controllers/tailscale_acl_controller.go new file mode 100644 index 0000000..eb24a00 --- /dev/null +++ b/controllers/tailscale_acl_controller.go @@ -0,0 +1,194 @@ +package controllers + +import ( + "context" + "encoding/json" + "fmt" + + "bytetrade.io/web3os/app-service/api/app.bytetrade.io/v1alpha1" + "bytetrade.io/web3os/app-service/pkg/utils" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const tailScaleACLPolicyMd5Key = "tailscale-acl-md5" + +var defaultHTTPSACL = v1alpha1.ACL{ + Action: "accept", + Src: []string{"*"}, + Proto: "", + Dst: []string{"*:443"}, +} + +type ACLPolicy struct { + ACLs []v1alpha1.ACL `json:"acls"` + AutoApprovers AutoApprovers `json:"autoApprovers"` +} + +type AutoApprovers struct { + Routes map[string][]string `json:"routes"` + ExitNode []string `json:"exitNode"` +} + +type TailScaleACLController struct { + client.Client +} + +func (r *TailScaleACLController) SetUpWithManager(mgr ctrl.Manager) error { + c, err := controller.New("app's tailscale acls manager controller", mgr, controller.Options{ + Reconciler: r, + }) + if err != nil { + return err + } + err = c.Watch( + &source.Kind{Type: &v1alpha1.Application{}}, + handler.EnqueueRequestsFromMapFunc( + func(obj client.Object) []reconcile.Request { + app, ok := obj.(*v1alpha1.Application) + if !ok { + return nil + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{ + Name: app.Name, + Namespace: app.Spec.Owner, + }}} + }), + predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return true + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return true + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return true + }, + }, + ) + if err != nil { + return err + } + return nil +} + +func (r *TailScaleACLController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + klog.Infof("reconcile tailscale acls request name=%v, owner=%v", req.Name, req.Namespace) + + // for this request req.Namespace is owner + // list all apps by owner and generate acls by owner + var apps v1alpha1.ApplicationList + err := r.List(ctx, &apps) + if err != nil { + return ctrl.Result{}, err + } + filteredApps := make([]v1alpha1.Application, 0) + for _, app := range apps.Items { + if app.Spec.Owner != req.Namespace { + continue + } + filteredApps = append(filteredApps, app) + } + + tailScaleACLConfig := "tailscale-acl" + headScaleNamespace := fmt.Sprintf("user-space-%s", req.Namespace) + + // calculate acls + acls := make([]v1alpha1.ACL, 0) + for _, app := range filteredApps { + acls = append(acls, app.Spec.TailScaleACLs...) + } + aclPolicyByte, err := makeACLPolicy(acls) + if err != nil { + return ctrl.Result{}, err + } + klog.Infof("aclPolicyByte:string: %s", string(aclPolicyByte)) + configMap := &corev1.ConfigMap{} + err = r.Get(ctx, types.NamespacedName{Name: tailScaleACLConfig, Namespace: headScaleNamespace}, configMap) + if err != nil { + return ctrl.Result{}, err + } + oldTailScaleACLPolicyMd5Sum := "" + if configMap.Annotations != nil { + oldTailScaleACLPolicyMd5Sum = configMap.Annotations[tailScaleACLPolicyMd5Key] + } + curTailScaleACLPolicyMd5Sum := utils.Md5String(string(aclPolicyByte)) + + if curTailScaleACLPolicyMd5Sum != oldTailScaleACLPolicyMd5Sum { + if configMap.Annotations == nil { + configMap.Annotations = make(map[string]string) + } + if configMap.Data == nil { + configMap.Data = make(map[string]string) + } + + configMap.Annotations[tailScaleACLPolicyMd5Key] = curTailScaleACLPolicyMd5Sum + configMap.Data["acl.json"] = string(aclPolicyByte) + err = r.Update(ctx, configMap) + if err != nil { + return ctrl.Result{}, err + } + } + + deploy := &appsv1.Deployment{} + err = r.Get(ctx, types.NamespacedName{Namespace: headScaleNamespace, Name: "headscale"}, deploy) + if err != nil { + return ctrl.Result{}, err + } + headScaleACLMd5 := "" + if deploy.Spec.Template.Annotations != nil { + headScaleACLMd5 = deploy.Spec.Template.Annotations[tailScaleACLPolicyMd5Key] + } + if headScaleACLMd5 != curTailScaleACLPolicyMd5Sum { + if deploy.Spec.Template.Annotations == nil { + deploy.Spec.Template.Annotations = make(map[string]string) + } + + // update headscale deploy template annotations for rolling update + deploy.Spec.Template.Annotations[tailScaleACLPolicyMd5Key] = curTailScaleACLPolicyMd5Sum + err = r.Update(ctx, deploy) + if err != nil { + return ctrl.Result{}, err + } + klog.Infof("rolling update headscale...") + } + + return ctrl.Result{}, nil +} + +func makeACLPolicy(acls []v1alpha1.ACL) ([]byte, error) { + acls = append(acls, defaultHTTPSACL) + for i := range acls { + acls[i].Action = "accept" + acls[i].Src = []string{"*"} + } + aclPolicy := ACLPolicy{ + ACLs: acls, + AutoApprovers: AutoApprovers{ + Routes: map[string][]string{ + "10.0.0.0/8": {"default"}, + "172.16.0.0/12": {"default"}, + "192.168.0.0/16": {"default"}, + }, + ExitNode: []string{}, + }, + } + aclPolicyByte, err := json.Marshal(aclPolicy) + if err != nil { + return nil, err + } + return aclPolicyByte, nil +} diff --git a/pkg/apiserver/handler_appupgrade.go b/pkg/apiserver/handler_appupgrade.go index afbdbbc..17d595c 100644 --- a/pkg/apiserver/handler_appupgrade.go +++ b/pkg/apiserver/handler_appupgrade.go @@ -80,6 +80,11 @@ func (h *Handler) appUpgrade(req *restful.Request, resp *restful.Response) { api.HandleError(resp, req, err) return } + err = utils.CheckTailScaleACLs(appConfig.TailScaleACLs) + if err != nil { + api.HandleError(resp, req, err) + return + } if !utils.MatchVersion(appConfig.CfgFileVersion, MinCfgFileVersion) { api.HandleBadRequest(resp, req, fmt.Errorf("olaresManifest.version must %s", MinCfgFileVersion)) diff --git a/pkg/apiserver/handler_installer.go b/pkg/apiserver/handler_installer.go index d3ce092..30f1437 100644 --- a/pkg/apiserver/handler_installer.go +++ b/pkg/apiserver/handler_installer.go @@ -60,6 +60,11 @@ func (h *Handler) install(req *restful.Request, resp *restful.Response) { api.HandleBadRequest(resp, req, err) return } + err = utils.CheckTailScaleACLs(appConfig.TailScaleACLs) + if err != nil { + api.HandleError(resp, req, err) + return + } if !utils.MatchVersion(appConfig.CfgFileVersion, MinCfgFileVersion) { api.HandleBadRequest(resp, req, fmt.Errorf("olaresManifest.version must %s", MinCfgFileVersion)) @@ -509,6 +514,7 @@ func toApplicationConfig(app, chart string, cfg *appinstaller.AppConfiguration) ChartsName: chart, Entrances: cfg.Entrances, Ports: cfg.Ports, + TailScaleACLs: cfg.TailScaleACLs, Icon: cfg.Metadata.Icon, Permission: permission, Requirement: appinstaller.AppRequirement{ diff --git a/pkg/appinstaller/appcfg_types.go b/pkg/appinstaller/appcfg_types.go index a7e3bfc..7394dc2 100644 --- a/pkg/appinstaller/appcfg_types.go +++ b/pkg/appinstaller/appcfg_types.go @@ -24,6 +24,7 @@ type AppConfiguration struct { Metadata AppMetaData `yaml:"metadata" json:"metadata"` Entrances []v1alpha1.Entrance `yaml:"entrances" json:"entrances"` Ports []v1alpha1.ServicePort `yaml:"ports" json:"ports"` + TailScaleACLs []v1alpha1.ACL `yaml:"tailscaleAcls" json:"tailscaleAcls"` Spec AppSpec `yaml:"spec" json:"spec"` Permission Permission `yaml:"permission" json:"permission" description:"app permission request"` Middleware *tapr.Middleware `yaml:"middleware" json:"middleware" description:"app middleware request"` diff --git a/pkg/appinstaller/appcfg_utils.go b/pkg/appinstaller/appcfg_utils.go index 4c231e9..77a1c30 100644 --- a/pkg/appinstaller/appcfg_utils.go +++ b/pkg/appinstaller/appcfg_utils.go @@ -149,6 +149,7 @@ func getAppConfigFromConfigurationFile(app, chart string) (*ApplicationConfig, e ChartsName: chart, Entrances: cfg.Entrances, Ports: cfg.Ports, + TailScaleACLs: cfg.TailScaleACLs, Icon: cfg.Metadata.Icon, Permission: permission, Requirement: AppRequirement{ @@ -213,3 +214,8 @@ func ToAppTCPUDPPorts(ports []v1alpha1.ServicePort) string { portsLabel, _ := json.Marshal(ports) return string(portsLabel) } + +func ToTailScaleACL(acls []v1alpha1.ACL) string { + aclLabel, _ := json.Marshal(acls) + return string(aclLabel) +} diff --git a/pkg/appinstaller/application.go b/pkg/appinstaller/application.go index 26e60f6..e256a38 100644 --- a/pkg/appinstaller/application.go +++ b/pkg/appinstaller/application.go @@ -61,6 +61,7 @@ type ApplicationConfig struct { OwnerName string // name of owner who installed application Entrances []v1alpha1.Entrance Ports []v1alpha1.ServicePort + TailScaleACLs []v1alpha1.ACL Icon string // base64 icon data Permission []AppPermission // app permission requests Requirement AppRequirement diff --git a/pkg/appinstaller/helm.go b/pkg/appinstaller/helm.go index a132e99..91c52af 100644 --- a/pkg/appinstaller/helm.go +++ b/pkg/appinstaller/helm.go @@ -200,6 +200,8 @@ func (h *HelmOps) addApplicationLabelsToDeployment() error { services := ToEntrancesLabel(h.app.Entrances) ports := ToAppTCPUDPPorts(h.app.Ports) + acls := ToTailScaleACL(h.app.TailScaleACLs) + patchData := map[string]interface{}{ "metadata": map[string]interface{}{ "labels": map[string]string{ @@ -209,12 +211,13 @@ func (h *HelmOps) addApplicationLabelsToDeployment() error { constants.ApplicationRunAsUserLabel: strconv.FormatBool(h.app.RunAsUser), }, "annotations": map[string]string{ - constants.ApplicationIconLabel: h.app.Icon, - constants.ApplicationTitleLabel: h.app.Title, - constants.ApplicationVersionLabel: h.app.Version, - constants.ApplicationEntrancesKey: services, - constants.ApplicationPortsKey: ports, - constants.ApplicationSourceLabel: h.options.Source, + constants.ApplicationIconLabel: h.app.Icon, + constants.ApplicationTitleLabel: h.app.Title, + constants.ApplicationVersionLabel: h.app.Version, + constants.ApplicationEntrancesKey: services, + constants.ApplicationPortsKey: ports, + constants.ApplicationSourceLabel: h.options.Source, + constants.ApplicationTailScaleACLKey: acls, }, }, } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index d6ff1de..6847936 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -22,6 +22,7 @@ const ( ApplicationRunAsUserLabel = "applications.apps.bytetrade.io/runasuser" ApplicationVersionLabel = "applications.app.bytetrade.io/version" ApplicationSourceLabel = "applications.app.bytetrade.io/source" + ApplicationTailScaleACLKey = "applications.app.bytetrade.io/tailscal-acls" ApplicationAnalytics = "applications.app.bytetrade.io/analytics" ApplicationPolicies = "applications.app.bytetrade.io/policies" ApplicationMobileSupported = "applications.app.bytetrade.io/mobile_supported" diff --git a/pkg/utils/app.go b/pkg/utils/app.go index 18fb6f8..12362d4 100644 --- a/pkg/utils/app.go +++ b/pkg/utils/app.go @@ -2,8 +2,10 @@ package utils import ( "context" + "errors" "fmt" "net" + "net/netip" "strings" "time" @@ -15,11 +17,18 @@ import ( "helm.sh/helm/v3/pkg/action" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" ) +const expectedTokenItems = 2 + +var ( + ErrInvalidAction = errors.New("invalid action") + ErrInvalidPortFormat = errors.New("invalid port format") +) var protectedNamespace = []string{ "default", "kube-node-lease", @@ -409,3 +418,87 @@ func IsForbidNamespace(namespace string) bool { } return false } + +// ACLProto If the ACL proto field is empty, it allows ICMPv4, ICMPv6, TCP, and UDP as per Tailscale behaviour +var ACLProto = sets.NewString("", "igmp", "ipv4", "ip-in-ip", "tcp", "egp", "igp", "udp", "gre", "esp", "ah", "sctp", "icmp") + +func CheckTailScaleACLs(acls []v1alpha1.ACL) error { + if len(acls) == 0 { + return nil + } + var err error + // fill default value fro ACL + for i := range acls { + acls[i].Action = "accept" + acls[i].Src = []string{"*"} + } + for _, acl := range acls { + err = parseProtocol(acl.Proto) + if err != nil { + return err + } + for _, dest := range acl.Dst { + _, _, err = parseDestination(dest) + if err != nil { + return err + } + } + } + return nil +} + +func parseProtocol(protocol string) error { + if ACLProto.Has(protocol) { + return nil + } + return fmt.Errorf("unsupported protocol: %v", protocol) +} + +// parseDestination from +// https://github.com/juanfont/headscale/blob/770f3dcb9334adac650276dcec90cd980af53c6e/hscontrol/policy/acls.go#L475 +func parseDestination(dest string) (string, string, error) { + var tokens []string + + // Check if there is a IPv4/6:Port combination, IPv6 has more than + // three ":". + tokens = strings.Split(dest, ":") + if len(tokens) < expectedTokenItems || len(tokens) > 3 { + port := tokens[len(tokens)-1] + + maybeIPv6Str := strings.TrimSuffix(dest, ":"+port) + + filteredMaybeIPv6Str := maybeIPv6Str + if strings.Contains(maybeIPv6Str, "/") { + networkParts := strings.Split(maybeIPv6Str, "/") + filteredMaybeIPv6Str = networkParts[0] + } + + if maybeIPv6, err := netip.ParseAddr(filteredMaybeIPv6Str); err != nil && !maybeIPv6.Is6() { + + return "", "", fmt.Errorf( + "failed to parse destination, tokens %v: %w", + tokens, + ErrInvalidPortFormat, + ) + } else { + tokens = []string{maybeIPv6Str, port} + } + } + + var alias string + // We can have here stuff like: + // git-server:* + // 192.168.1.0/24:22 + // fd7a:115c:a1e0::2:22 + // fd7a:115c:a1e0::2/128:22 + // tag:montreal-webserver:80,443 + // tag:api-server:443 + // example-host-1:* + if len(tokens) == expectedTokenItems { + alias = tokens[0] + } else { + alias = fmt.Sprintf("%s:%s", tokens[0], tokens[1]) + } + + return alias, tokens[len(tokens)-1], nil +}