CVE-2024-56513
Description
Karmada is a Kubernetes management system that allows users to run cloud-native applications across multiple Kubernetes clusters and clouds. Prior to version 1.12.0, the PULL mode clusters registered with the karmadactl register command have excessive privileges to access control plane resources. By abusing these permissions, an attacker able to authenticate as the karmada-agent to a karmada cluster would be able to obtain administrative privileges over the entire federation system including all registered member clusters. Since Karmada v1.12.0, command karmadactl register restricts the access permissions of pull mode member clusters to control plane resources. This way, an attacker able to authenticate as the karmada-agent cannot control other member clusters in Karmada. As a workaround, one may restrict the access permissions of pull mode member clusters to control plane resources according to Karmada Component Permissions Docs.
Affected packages
Versions sourced from the GitHub Security Advisory.
| Package | Affected versions | Patched versions |
|---|---|---|
github.com/karmada-io/karmadaGo | < 1.12.0 | 1.12.0 |
Patches
21bc955a23c652c82055c4c7fMerge pull request #5793 from zhzhuang-zju/register
11 files changed · +722 −496
artifacts/deploy/bootstrap-token-configuration.yaml+60 −256 modified@@ -13,6 +13,7 @@ data: kind: Config --- +# Define a role with permission to get the cluster-info configmap apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -31,6 +32,8 @@ rules: - get --- +# An anonymous user can get `cluster-info` configmap, which is used to obtain the control plane API server's server +# address and `certificate-authority-data` during the `karmadactl register` process. apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -48,6 +51,8 @@ subjects: name: system:anonymous --- +# Group `system:bootstrappers:karmada:default-cluster-token` is the user group of the bootstrap token +# used by `karmadactl register` when registering a new pull mode cluster. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -64,6 +69,26 @@ subjects: name: system:bootstrappers:karmada:default-cluster-token --- +# Define a ClusterRole with permissions to automatically approve the agent CSRs when the agentcsrapproving controller is enabled by karmada-controller-manager. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + karmada.io/bootstrapping: rbac-defaults + name: system:karmada:certificatesigningrequest:autoapprover +rules: + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/clusteragent + verbs: + - create + +--- +# Group `system:bootstrappers:karmada:default-cluster-token` is the user group of the bootstrap token +# used by `karmadactl register` when registering a new pull mode cluster. +# When the `agentcsrapproving` controller is enabled by the karmada-controller-manager, +# it can automatically approve the agent CSRs requested by the user group system:bootstrappers:karmada:default-cluster-token. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -73,13 +98,33 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:certificates.k8s.io:certificatesigningrequests:nodeclient + name: system:karmada:certificatesigningrequest:autoapprover subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:bootstrappers:karmada:default-cluster-token --- +# Define a ClusterRole with permissions to automatically approve the agent CSRs +# where the user name and group of requester match those in the CSRs when the agentcsrapproving controller is enabled by karmada-controller-manager. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + karmada.io/bootstrapping: rbac-defaults + name: system:karmada:certificatesigningrequest:selfautoapprover +rules: + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/selfclusteragent + verbs: + - create + +--- +# Group `system:karmada:agents` is the user group used by the karmada-agent to access the Karmada API server. +# When the agentcsrapproving controller is enabled by the karmada-controller-manager, it can automatically approve +# the agent CSRs(csr.Subject.CommonName = agent username) requested by the user group system:karmada:agents. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -89,280 +134,39 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + name: system:karmada:certificatesigningrequest:selfautoapprover subjects: - apiGroup: rbac.authorization.k8s.io kind: Group - name: system:nodes + name: system:karmada:agents --- +# ClusterRole `system:karmada:agent-rbac-generator` is not used for the connection between the karmada-agent and the control plane, +# but is used by karmadactl register to generate the RBAC resources required by the karmada-agent. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: karmada.io/bootstrapping: rbac-defaults - name: system:karmada:agent + name: system:karmada:agent-rbac-generator rules: -- apiGroups: - - cluster.karmada.io - resources: - - clusters - verbs: - - create - - get - - list - - watch - - delete -- apiGroups: - - cluster.karmada.io - resources: - - clusters/status - verbs: - - update -- apiGroups: - - work.karmada.io - resources: - - works - verbs: - - create - - get - - list - - watch - - update - - delete -- apiGroups: - - work.karmada.io - resources: - - works/status - verbs: - - patch - - update -- apiGroups: - - config.karmada.io - resources: - - resourceinterpreterwebhookconfigurations - - resourceinterpretercustomizations - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create - - patch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - update -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests - verbs: - - create - - get -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update + - apiGroups: ['*'] + resources: ['*'] + verbs: ['*'] --- +# User `system:karmada:agent:rbac-generator` is specifically used during the `karmadactl register` process to generate restricted RBAC resources for the `karmada-agent`. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: karmada.io/bootstrapping: rbac-defaults - name: system:karmada:agent + name: system:karmada:agent-rbac-generator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:karmada:agent + name: system:karmada:agent-rbac-generator subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes - -# To ensure the agent has the minimal RBAC permissions, the ideal approach is to -# use different RBAC configurations for different agents of member clusters with pull mode. -# Below is the minimal set of RBAC permissions required for a single pull mode member cluster. -# Here are the definitions of the variables used: -# -# - clustername: the name of the member cluster. -# - cluster_namespace: the namespace where the member cluster secrets are stored, default to karmada-cluster. -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRole -# metadata: -# name: system:karmada:agent -# rules: -# - apiGroups: -# - cluster.karmada.io -# resources: -# - clusters -# resourceNames: -# - {{clustername}} -# verbs: -# - create -# - get -# - delete -# - apiGroups: -# - cluster.karmada.io -# resources: -# - clusters -# verbs: -# - list -# - watch -# - apiGroups: -# - cluster.karmada.io -# resources: -# - clusters/status -# resourceNames: -# - {{clustername}} -# verbs: -# - update -# - apiGroups: -# - config.karmada.io -# resources: -# - resourceinterpreterwebhookconfigurations -# - resourceinterpretercustomizations -# verbs: -# - get -# - list -# - watch -# - apiGroups: -# - "" -# resources: -# - namespaces -# verbs: -# - get -# - apiGroups: -# - coordination.k8s.io -# resources: -# - leases -# verbs: -# - create -# - get -# - update -# - apiGroups: -# - certificates.k8s.io -# resources: -# - certificatesigningrequests -# verbs: -# - create -# - get -# - apiGroups: -# - "" -# resources: -# - events -# verbs: -# - create -# - patch -# - update -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRoleBinding -# metadata: -# name: system:karmada:agent -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: ClusterRole -# name: system:karmada:agent -# subjects: -# - apiGroup: rbac.authorization.k8s.io -# kind: Group -# name: system:nodes -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: Role -# metadata: -# name: system:karmada:agent-secret -# namespace: "{{cluster_namespace}}" -# rules: -# - apiGroups: -# - "" -# resources: -# - secrets -# resourceNames: -# - {{clustername}}-impersonator -# - {{clustername}} -# verbs: -# - get -# - create -# - patch -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: RoleBinding -# metadata: -# name: system:karmada:agent-secret -# namespace: "{{cluster_namespace}}" -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: Role -# name: system:karmada:agent-secret -# subjects: -# - apiGroup: rbac.authorization.k8s.io -# kind: Group -# name: system:nodes -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: Role -# metadata: -# name: system:karmada:agent-work -# namespace: "karmada-es-{{clustername}}" -# rules: -# - apiGroups: -# - work.karmada.io -# resources: -# - works -# verbs: -# - create -# - get -# - list -# - watch -# - update -# - delete -# - apiGroups: -# - work.karmada.io -# resources: -# - works/status -# verbs: -# - patch -# - update -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: RoleBinding -# metadata: -# name: system:karmada:agent-work -# namespace: "karmada-es-{{clustername}}" -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: Role -# name: system:karmada:agent-work -# subjects: -# - apiGroup: rbac.authorization.k8s.io -# kind: Group -# name: system:nodes + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:karmada:agent:rbac-generator
charts/karmada/templates/_karmada_bootstrap_token_configuration.tpl+43 −103 modified@@ -75,6 +75,22 @@ subjects: name: system:bootstrappers:karmada:default-cluster-token --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:karmada:certificatesigningrequest:autoapprover + {{- if "karmada.commonLabels" }} + labels: + {{- include "karmada.commonLabels" . | nindent 4 }} + {{- end }} +rules: +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/clusteragent + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:karmada:agent-autoapprove-bootstrap @@ -85,13 +101,29 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:certificates.k8s.io:certificatesigningrequests:nodeclient + name: system:karmada:certificatesigningrequest:autoapprover subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:bootstrappers:karmada:default-cluster-token --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:karmada:certificatesigningrequest:selfautoapprover + {{- if "karmada.commonLabels" }} + labels: + {{- include "karmada.commonLabels" . | nindent 4 }} + {{- end }} +rules: +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/selfclusteragent + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:karmada:agent-autoapprove-certificate-rotation @@ -102,134 +134,42 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + name: system:karmada:certificatesigningrequest:selfautoapprover subjects: - apiGroup: rbac.authorization.k8s.io kind: Group - name: system:nodes + name: system:karmada:agents --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: system:karmada:agent + name: system:karmada:agent-rbac-generator {{- if "karmada.commonLabels" }} labels: {{- include "karmada.commonLabels" . | nindent 4 }} {{- end }} rules: - apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - cluster.karmada.io - resources: - - clusters - verbs: - - create - - get - - list - - watch - - patch - - update - - delete -- apiGroups: - - cluster.karmada.io - resources: - - clusters/status - verbs: - - patch - - update -- apiGroups: - - work.karmada.io + - "*" resources: - - works + - "*" verbs: - - create - - get - - list - - watch - - update - - delete -- apiGroups: - - work.karmada.io - resources: - - works/status - verbs: - - patch - - update -- apiGroups: - - config.karmada.io - resources: - - resourceinterpreterwebhookconfigurations - - resourceinterpretercustomizations - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - watch - - create -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - create - - patch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - delete - - get - - patch - - update -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests - verbs: - - create - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update + - "*" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: system:karmada:agent + name: system:karmada:agent-rbac-generator {{- if "karmada.commonLabels" }} labels: {{- include "karmada.commonLabels" . | nindent 4 }} {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:karmada:agent + name: system:karmada:agent-rbac-generator subjects: - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes + kind: User + name: system:karmada:agent:rbac-generator {{- end -}}
pkg/karmadactl/cmdinit/bootstraptoken/agent/tlsbootstrap.go+26 −4 modified@@ -31,15 +31,15 @@ const ( // KarmadaAgentBootstrap defines the name of the ClusterRoleBinding that lets Karmada Agent post CSRs KarmadaAgentBootstrap = "system:karmada:agent-bootstrap" // KarmadaAgentGroup defines the group of Karmada Agent - KarmadaAgentGroup = "system:nodes" + KarmadaAgentGroup = "system:karmada:agents" // KarmadaAgentAutoApproveBootstrapClusterRoleBinding defines the name of the ClusterRoleBinding that makes the csrapprover approve agent CSRs KarmadaAgentAutoApproveBootstrapClusterRoleBinding = "system:karmada:agent-autoapprove-bootstrap" // KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding defines name of the ClusterRoleBinding that makes the csrapprover approve agent auto rotated CSRs KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding = "system:karmada:agent-autoapprove-certificate-rotation" // CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR - CSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient" + CSRAutoApprovalClusterRoleName = "system:karmada:certificatesigningrequest:autoapprover" // KarmadaAgentSelfCSRAutoApprovalClusterRoleName is a role for automatic CSR approvals for automatically rotated agent certificates - KarmadaAgentSelfCSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:selfnodeclient" + KarmadaAgentSelfCSRAutoApprovalClusterRoleName = "system:karmada:certificatesigningrequest:selfautoapprover" // KarmadaAgentBootstrapTokenAuthGroup specifies which group a Karmada Agent Bootstrap Token should be authenticated in KarmadaAgentBootstrapTokenAuthGroup = "system:bootstrappers:karmada:default-cluster-token" ) @@ -60,7 +60,18 @@ func AllowBootstrapTokensToPostCSRs(clientSet kubernetes.Interface) error { // AutoApproveKarmadaAgentBootstrapTokens creates RBAC rules in a way that makes Karmada Agent Bootstrap Tokens' CSR auto-approved by the csrapprover controller func AutoApproveKarmadaAgentBootstrapTokens(clientSet kubernetes.Interface) error { - klog.Infoln("[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Karmada Agent Bootstrap Token") + klog.Infoln("[bootstrap-token] configured RBAC rules to allow the agentcsrapproving controller automatically approve CSRs from a Karmada Agent Bootstrap Token") + csrAutoApprovalClusterRole := utils.ClusterRoleFromRules(CSRAutoApprovalClusterRoleName, []rbacv1.PolicyRule{ + { + APIGroups: []string{"certificates.k8s.io"}, + Resources: []string{"certificatesigningrequests/clusteragent"}, + Verbs: []string{"create"}, + }, + }, nil, nil) + err := cmdutil.CreateOrUpdateClusterRole(clientSet, csrAutoApprovalClusterRole) + if err != nil { + return err + } clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(KarmadaAgentAutoApproveBootstrapClusterRoleBinding, CSRAutoApprovalClusterRoleName, []rbacv1.Subject{ @@ -75,6 +86,17 @@ func AutoApproveKarmadaAgentBootstrapTokens(clientSet kubernetes.Interface) erro // AutoApproveAgentCertificateRotation creates RBAC rules in a way that makes Agent certificate rotation CSR auto-approved by the csrapprover controller func AutoApproveAgentCertificateRotation(clientSet kubernetes.Interface) error { klog.Infoln("[bootstrap-token] configured RBAC rules to allow certificate rotation for all agent client certificates in the member cluster") + karmadaAgentSelfCSRAutoApprovalClusterRole := utils.ClusterRoleFromRules(KarmadaAgentSelfCSRAutoApprovalClusterRoleName, []rbacv1.PolicyRule{ + { + APIGroups: []string{"certificates.k8s.io"}, + Resources: []string{"certificatesigningrequests/selfclusteragent"}, + Verbs: []string{"create"}, + }, + }, nil, nil) + err := cmdutil.CreateOrUpdateClusterRole(clientSet, karmadaAgentSelfCSRAutoApprovalClusterRole) + if err != nil { + return err + } clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding, KarmadaAgentSelfCSRAutoApprovalClusterRoleName, []rbacv1.Subject{
pkg/karmadactl/cmdinit/karmada/deploy.go+2 −2 modified@@ -185,8 +185,8 @@ func createExtraResources(clientSet *kubernetes.Clientset, dir string) error { return fmt.Errorf("error creating clusterinfo RBAC rules: %v", err) } - // grant limited access permission to 'karmada-agent' - if err := grantAccessPermissionToAgent(clientSet); err != nil { + // grant access permission to 'karmada-agent-rbac-generator' + if err := grantAccessPermissionToAgentRBACGenerator(clientSet); err != nil { return err }
pkg/karmadactl/cmdinit/karmada/rbac.go+15 −64 modified@@ -26,10 +26,11 @@ import ( ) const ( - karmadaViewClusterRole = "karmada-view" - karmadaEditClusterRole = "karmada-edit" - karmadaAgentAccessClusterRole = "system:karmada:agent" - karmadaAgentGroup = "system:nodes" + karmadaViewClusterRole = "karmada-view" + karmadaEditClusterRole = "karmada-edit" + karmadaAgentRBACGeneratorClusterRole = "system:karmada:agent-rbac-generator" + karmadaAgentRBACGeneratorClusterRoleBinding = "system:karmada:agent-rbac-generator" + agentRBACGenerator = "system:karmada:agent:rbac-generator" ) // grantProxyPermissionToAdmin grants the proxy permission to "system:admin" @@ -62,78 +63,28 @@ func grantProxyPermissionToAdmin(clientSet kubernetes.Interface) error { return nil } -// grantAccessPermissionToAgent grants the limited access permission to 'karmada-agent' -func grantAccessPermissionToAgent(clientSet kubernetes.Interface) error { - clusterRole := utils.ClusterRoleFromRules(karmadaAgentAccessClusterRole, []rbacv1.PolicyRule{ +// grantAccessPermissionToAgentRBACGenerator grants the access permission to 'karmada-agent-rbac-generator' +func grantAccessPermissionToAgentRBACGenerator(clientSet kubernetes.Interface) error { + clusterRole := utils.ClusterRoleFromRules(karmadaAgentRBACGeneratorClusterRole, []rbacv1.PolicyRule{ { - APIGroups: []string{"authentication.k8s.io"}, - Resources: []string{"tokenreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"cluster.karmada.io"}, - Resources: []string{"clusters"}, - Verbs: []string{"create", "get", "list", "watch", "patch", "update", "delete"}, - }, - { - APIGroups: []string{"cluster.karmada.io"}, - Resources: []string{"clusters/status"}, - Verbs: []string{"patch", "update"}, - }, - { - APIGroups: []string{"work.karmada.io"}, - Resources: []string{"works"}, - Verbs: []string{"create", "get", "list", "watch", "update", "delete"}, - }, - { - APIGroups: []string{"work.karmada.io"}, - Resources: []string{"works/status"}, - Verbs: []string{"patch", "update"}, - }, - { - APIGroups: []string{"config.karmada.io"}, - Resources: []string{"resourceinterpreterwebhookconfigurations", "resourceinterpretercustomizations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "list", "watch", "create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "watch", "create", "patch"}, - }, - { - APIGroups: []string{"coordination.k8s.io"}, - Resources: []string{"leases"}, - Verbs: []string{"create", "delete", "get", "patch", "update"}, - }, - { - APIGroups: []string{"certificates.k8s.io"}, - Resources: []string{"certificatesigningrequests"}, - Verbs: []string{"create", "get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch", "update"}, + APIGroups: []string{"*"}, + Resources: []string{"*"}, + Verbs: []string{"*"}, }, }, nil, nil) err := cmdutil.CreateOrUpdateClusterRole(clientSet, clusterRole) if err != nil { return err } - clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(karmadaAgentAccessClusterRole, karmadaAgentAccessClusterRole, + clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(karmadaAgentRBACGeneratorClusterRoleBinding, karmadaAgentRBACGeneratorClusterRole, []rbacv1.Subject{ { - Kind: rbacv1.GroupKind, - Name: karmadaAgentGroup, + Kind: rbacv1.UserKind, + Name: agentRBACGenerator, }}, nil) - klog.V(1).Info("Grant the limited access permission to 'karmada-agent'") + klog.V(1).Info("Grant the access permission to 'karmada-agent-rbac-generator'") err = cmdutil.CreateOrUpdateClusterRoleBinding(clientSet, clusterRoleBinding) if err != nil { return err
pkg/karmadactl/cmdinit/karmada/rbac_test.go+2 −2 modified@@ -31,8 +31,8 @@ func Test_grantProxyPermissionToAdmin(t *testing.T) { func Test_grantAccessPermissionToAgent(t *testing.T) { client := fake.NewSimpleClientset() - if err := grantAccessPermissionToAgent(client); err != nil { - t.Errorf("grantAccessPermissionToAgent() expected no error, but got err: %v", err) + if err := grantAccessPermissionToAgentRBACGenerator(client); err != nil { + t.Errorf("grantAccessPermissionToAgentRBACGenerator() expected no error, but got err: %v", err) } }
pkg/karmadactl/register/register.go+415 −33 modified@@ -51,7 +51,6 @@ import ( "github.com/karmada-io/karmada/pkg/apis/cluster/validation" karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" - addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" "github.com/karmada-io/karmada/pkg/karmadactl/options" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" @@ -82,9 +81,11 @@ const ( // CACertPath defines default location of CA certificate on Linux CACertPath = "/etc/karmada/pki/ca.crt" // ClusterPermissionPrefix defines the common name of karmada agent certificate - ClusterPermissionPrefix = "system:node:" + ClusterPermissionPrefix = "system:karmada:agent:" // ClusterPermissionGroups defines the organization of karmada agent certificate - ClusterPermissionGroups = "system:nodes" + ClusterPermissionGroups = "system:karmada:agents" + // AgentRBACGenerator defines the common name of karmada agent rbac generator certificate + AgentRBACGenerator = "system:karmada:agent:rbac-generator" // KarmadaAgentBootstrapKubeConfigFileName defines the file name for the kubeconfig that the karmada-agent will use to do // the TLS bootstrap to get itself an unique credential KarmadaAgentBootstrapKubeConfigFileName = "bootstrap-karmada-agent.conf" @@ -97,8 +98,8 @@ const ( KarmadaAgentName = "karmada-agent" // KarmadaAgentServiceAccountName is the name of karmada-agent serviceaccount KarmadaAgentServiceAccountName = "karmada-agent-sa" - // SignerName defines the signer name for csr, 'kubernetes.io/kube-apiserver-client-kubelet' can sign the csr automatically - SignerName = "kubernetes.io/kube-apiserver-client-kubelet" + // SignerName defines the signer name for csr, 'kubernetes.io/kube-apiserver-client' can sign the csr with `O=system:agents,CN=system:agent:` automatically if agentcsrapproving controller if enabled. + SignerName = "kubernetes.io/kube-apiserver-client" // BootstrapUserName defines bootstrap user name BootstrapUserName = "token-bootstrap-client" // DefaultClusterName defines the default cluster name @@ -260,6 +261,9 @@ type CommandRegisterOption struct { memberClusterEndpoint string memberClusterClient *kubeclient.Clientset + + // rbacResources contains RBAC resources that grant the necessary permissions for pull mode cluster to access to Karmada control plane. + rbacResources *RBACResources } // Complete ensures that options are valid and marshals them if necessary. @@ -288,6 +292,8 @@ func (o *CommandRegisterOption) Complete(args []string) error { o.ClusterName = config.Contexts[config.CurrentContext].Cluster } + o.rbacResources = GenerateRBACResources(o.ClusterName, o.ClusterNamespace) + o.memberClusterEndpoint = restConfig.Host o.memberClusterClient, err = apiclient.NewClientSet(restConfig) @@ -358,53 +364,107 @@ func (o *CommandRegisterOption) Run(parentCommand string) error { return err } - // construct the final kubeconfig file used by karmada agent to connect to karmada apiserver - fmt.Println("[karmada-agent-start] Waiting to construct karmada-agent kubeconfig") - karmadaAgentCfg, err := o.constructKarmadaAgentConfig(bootstrapClient, karmadaClusterInfo) + var rbacClient *kubeclient.Clientset + defer func() { + if err != nil && rbacClient != nil { + fmt.Println("karmadactl register failed and started deleting the created resources") + err = o.rbacResources.Delete(rbacClient) + if err != nil { + klog.Warningf("Failed to delete rbac resources: %v", err) + } + } + }() + + rbacClient, err = o.EnsureNecessaryResourcesExistInControlPlane(bootstrapClient, karmadaClusterInfo) if err != nil { return err } - fmt.Println("[karmada-agent-start] Waiting to check cluster exists") - karmadaClient, err := ToKarmadaClient(karmadaAgentCfg) + err = o.EnsureNecessaryResourcesExistInMemberCluster(bootstrapClient, karmadaClusterInfo) if err != nil { return err } + + fmt.Printf("\ncluster(%s) is joined successfully\n", o.ClusterName) + + return nil +} + +// EnsureNecessaryResourcesExistInControlPlane ensures that all necessary resources are exist in Karmada control plane. +func (o *CommandRegisterOption) EnsureNecessaryResourcesExistInControlPlane(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*kubeclient.Clientset, error) { + csrName := "agent-rbac-generator-" + o.ClusterName + k8srand.String(5) + rbacCfg, err := o.constructAgentRBACGeneratorConfig(bootstrapClient, karmadaClusterInfo, csrName) + if err != nil { + return nil, err + } + + kubelient, err := ToClientSet(rbacCfg) + if err != nil { + return nil, err + } + defer func() { + err = kubelient.CertificatesV1().CertificateSigningRequests().Delete(context.Background(), csrName, metav1.DeleteOptions{}) + if err != nil { + klog.Warningf("Failed to delete CertificateSigningRequests %s: %v", csrName, err) + } + }() + + fmt.Println("[karmada-agent-start] Waiting to check cluster exists") + karmadaClient, err := ToKarmadaClient(rbacCfg) + if err != nil { + return kubelient, err + } _, exist, err := karmadautil.GetClusterWithKarmadaClient(karmadaClient, o.ClusterName) if err != nil { - return err + return kubelient, err } else if exist { - return fmt.Errorf("failed to register as cluster with name %s already exists", o.ClusterName) + return kubelient, fmt.Errorf("failed to register as cluster with name %s already exists", o.ClusterName) + } + + fmt.Println("[karmada-agent-start] Assign the necessary RBAC permissions to the agent") + err = o.ensureAgentRBACResourcesExistInControlPlane(kubelient) + if err != nil { + return kubelient, err + } + + return kubelient, nil +} + +// EnsureNecessaryResourcesExistInMemberCluster ensures that all necessary resources are exist in the registering cluster. +func (o *CommandRegisterOption) EnsureNecessaryResourcesExistInMemberCluster(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) error { + // construct the final kubeconfig file used by karmada agent to connect to karmada apiserver + fmt.Println("[karmada-agent-start] Waiting to construct karmada-agent kubeconfig") + karmadaAgentCfg, err := o.constructKarmadaAgentConfig(bootstrapClient, karmadaClusterInfo) + if err != nil { + return err } // It's necessary to set the label of namespace to make sure that the namespace is created by Karmada. labels := map[string]string{ karmadautil.KarmadaSystemLabel: karmadautil.KarmadaSystemLabelValue, } // ensure namespace where the karmada-agent resources be deployed exists in the member cluster - if _, err := karmadautil.EnsureNamespaceExistWithLabels(o.memberClusterClient, o.Namespace, o.DryRun, labels); err != nil { + if _, err = karmadautil.EnsureNamespaceExistWithLabels(o.memberClusterClient, o.Namespace, o.DryRun, labels); err != nil { return err } // create the necessary secret and RBAC in the member cluster fmt.Println("[karmada-agent-start] Waiting the necessary secret and RBAC") - if err := o.createSecretAndRBACInMemberCluster(karmadaAgentCfg); err != nil { + if err = o.createSecretAndRBACInMemberCluster(karmadaAgentCfg); err != nil { return err } // create karmada-agent Deployment in the member cluster fmt.Println("[karmada-agent-start] Waiting karmada-agent Deployment") KarmadaAgentDeployment := o.makeKarmadaAgentDeployment() - if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil { + if _, err = o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil { return err } - if err := addonutils.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { + if err = cmdutil.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { return err } - fmt.Printf("\ncluster(%s) is joined successfully\n", o.ClusterName) - return nil } @@ -505,11 +565,316 @@ func (o *CommandRegisterOption) discoveryBootstrapConfigAndClusterInfo(bootstrap return bootstrapClient, clusterinfo, nil } -// constructKarmadaAgentConfig construct the final kubeconfig used by karmada-agent -func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*clientcmdapi.Config, error) { +// ensureAgentRBACResourcesExistInControlPlane ensures that necessary RBAC resources for karmada-agent are exist in control plane. +func (o *CommandRegisterOption) ensureAgentRBACResourcesExistInControlPlane(client kubeclient.Interface) error { + for i := range o.rbacResources.ClusterRoles { + _, err := karmadautil.CreateClusterRole(client, o.rbacResources.ClusterRoles[i]) + if err != nil { + return err + } + } + for i := range o.rbacResources.ClusterRoleBindings { + _, err := karmadautil.CreateClusterRoleBinding(client, o.rbacResources.ClusterRoleBindings[i]) + if err != nil { + return err + } + } + + for i := range o.rbacResources.Roles { + roleNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: o.rbacResources.Roles[i].GetNamespace(), + Labels: map[string]string{ + karmadautil.KarmadaSystemLabel: karmadautil.KarmadaSystemLabelValue, + }, + }, + } + _, err := karmadautil.CreateNamespace(client, roleNamespace) + if err != nil { + return err + } + _, err = karmadautil.CreateRole(client, o.rbacResources.Roles[i]) + if err != nil { + return err + } + } + + for i := range o.rbacResources.RoleBindings { + _, err := karmadautil.CreateRoleBinding(client, o.rbacResources.RoleBindings[i]) + if err != nil { + return err + } + } + + return nil +} + +// RBACResources defines the list of rbac resources. +type RBACResources struct { + ClusterRoles []*rbacv1.ClusterRole + ClusterRoleBindings []*rbacv1.ClusterRoleBinding + Roles []*rbacv1.Role + RoleBindings []*rbacv1.RoleBinding +} + +// GenerateRBACResources generates rbac resources. +func GenerateRBACResources(clusterName, clusterNamespace string) *RBACResources { + return &RBACResources{ + ClusterRoles: []*rbacv1.ClusterRole{GenerateClusterRole(clusterName)}, + ClusterRoleBindings: []*rbacv1.ClusterRoleBinding{GenerateClusterRoleBinding(clusterName)}, + Roles: []*rbacv1.Role{GenerateSecretAccessRole(clusterName, clusterNamespace), GenerateWorkAccessRole(clusterName)}, + RoleBindings: []*rbacv1.RoleBinding{GenerateSecretAccessRoleBinding(clusterName, clusterNamespace), GenerateWorkAccessRoleBinding(clusterName)}, + } +} + +// List return the list of rbac resources. +func (r *RBACResources) List() []Obj { + var obj []Obj + for i := range r.ClusterRoles { + obj = append(obj, Obj{Kind: "ClusterRole", Name: r.ClusterRoles[i].GetName()}) + } + for i := range r.ClusterRoleBindings { + obj = append(obj, Obj{Kind: "ClusterRoleBinding", Name: r.ClusterRoleBindings[i].GetName()}) + } + for i := range r.Roles { + obj = append(obj, Obj{Kind: "Role", Name: r.Roles[i].GetName(), Namespace: r.Roles[i].GetNamespace()}) + } + for i := range r.RoleBindings { + obj = append(obj, Obj{Kind: "RoleBinding", Name: r.RoleBindings[i].GetName(), Namespace: r.RoleBindings[i].GetNamespace()}) + } + return obj +} + +// ToString returns a list of RBAC resources in string format. +func (r *RBACResources) ToString() string { + var resources []string + for i := range r.List() { + resources = append(resources, r.List()[i].ToString()) + } + return strings.Join(resources, "\n") +} + +// Delete deletes RBAC resources. +func (r *RBACResources) Delete(client kubeclient.Interface) error { + var err error + for _, resource := range r.List() { + switch resource.Kind { + case "ClusterRole": + err = karmadautil.DeleteClusterRole(client, resource.Name) + case "ClusterRoleBinding": + err = karmadautil.DeleteClusterRoleBinding(client, resource.Name) + case "Role": + err = karmadautil.DeleteRole(client, resource.Namespace, resource.Name) + case "RoleBinding": + err = karmadautil.DeleteRoleBinding(client, resource.Namespace, resource.Name) + } + if err != nil { + return err + } + } + return nil +} + +// Obj defines the struct which contains the information of kind, name and namespace. +type Obj struct{ Kind, Name, Namespace string } + +// ToString returns a string that concatenates kind, name, and namespace using "/". +func (o *Obj) ToString() string { + if o.Namespace == "" { + return fmt.Sprintf("%s/%s", o.Kind, o.Name) + } + return fmt.Sprintf("%s/%s/%s", o.Kind, o.Namespace, o.Name) +} + +// GenerateClusterRole generates the clusterRole that karmada-agent needed. +func GenerateClusterRole(clusterName string) *rbacv1.ClusterRole { + clusterRoleName := fmt.Sprintf("system:karmada:%s:agent", clusterName) + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "ClusterRole", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters"}, + ResourceNames: []string{clusterName}, + Verbs: []string{"get", "delete"}, + }, + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters"}, + Verbs: []string{"create", "list", "watch"}, + }, + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters/status"}, + ResourceNames: []string{clusterName}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"config.karmada.io"}, + Resources: []string{"resourceinterpreterwebhookconfigurations", "resourceinterpretercustomizations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"get", "create", "update"}, + }, + { + APIGroups: []string{"certificates.k8s.io"}, + Resources: []string{"certificatesigningrequests"}, + Verbs: []string{"get", "create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"patch", "create", "update"}, + }, + }, + } +} + +// GenerateClusterRoleBinding generates the clusterRoleBinding that karmada-agent needed. +func GenerateClusterRoleBinding(clusterName string) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("system:karmada:%s:agent", clusterName)}, + Subjects: []rbacv1.Subject{ + { + APIGroup: "rbac.authorization.k8s.io", + Kind: "User", + Name: generateAgentUserName(clusterName), + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: fmt.Sprintf("system:karmada:%s:agent", clusterName), + }, + } +} + +// GenerateSecretAccessRole generates the secret-related Role that karmada-agent needed. +func GenerateSecretAccessRole(clusterName, clusterNamespace string) *rbacv1.Role { + secretAccessRoleName := fmt.Sprintf("system:karmada:%s:agent-secret", clusterName) + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretAccessRoleName, + Namespace: clusterNamespace, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "patch"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + ResourceNames: []string{clusterName, clusterName + "-impersonator"}, + }, + { + Verbs: []string{"create"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + }, + }, + } +} + +// GenerateSecretAccessRoleBinding generates the secret-related RoleBinding that karmada-agent needed. +func GenerateSecretAccessRoleBinding(clusterName, clusterNamespace string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("system:karmada:%s:agent-secret", clusterName), + Namespace: clusterNamespace, + }, + Subjects: []rbacv1.Subject{ + { + APIGroup: "rbac.authorization.k8s.io", + Kind: "User", + Name: generateAgentUserName(clusterName), + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: fmt.Sprintf("system:karmada:%s:agent-secret", clusterName), + }, + } +} + +// GenerateWorkAccessRole generates the work-related Role that karmada-agent needed. +func GenerateWorkAccessRole(clusterName string) *rbacv1.Role { + workAccessRoleName := fmt.Sprintf("system:karmada:%s:agent-work", clusterName) + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: workAccessRoleName, + Namespace: "karmada-es-" + clusterName, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "list", "watch", "update", "delete"}, + APIGroups: []string{"work.karmada.io"}, + Resources: []string{"works"}, + }, + { + Verbs: []string{"patch", "update"}, + APIGroups: []string{"work.karmada.io"}, + Resources: []string{"works/status"}, + }, + }, + } +} + +// GenerateWorkAccessRoleBinding generates the work-related RoleBinding that karmada-agent needed. +func GenerateWorkAccessRoleBinding(clusterName string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("system:karmada:%s:agent-work", clusterName), + Namespace: "karmada-es-" + clusterName, + }, + Subjects: []rbacv1.Subject{ + { + APIGroup: "rbac.authorization.k8s.io", + Kind: "User", + Name: generateAgentUserName(clusterName), + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: fmt.Sprintf("system:karmada:%s:agent-work", clusterName), + }, + } +} + +func (o *CommandRegisterOption) constructKubeConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster, csrName, commonName string, organization []string) (*clientcmdapi.Config, error) { var cert []byte - pk, csr, err := generateKeyAndCSR(o.ClusterName) + pk, csr, err := generateKeyAndCSR(commonName, organization) if err != nil { return nil, err } @@ -519,8 +884,6 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub return nil, err } - csrName := o.ClusterName + "-" + k8srand.String(5) - certificateSigningRequest := &certificatesv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ Name: csrName, @@ -547,35 +910,45 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub if err != nil { return nil, err } - - klog.V(1).Infof("Waiting for the client certificate to be issued") + klog.V(1).Infof(fmt.Sprintf("Waiting for the client certificate %s to be issued", csrName)) err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, o.Timeout, false, func(context.Context) (done bool, err error) { csrOK, err := bootstrapClient.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), csrName, metav1.GetOptions{}) if err != nil { - return false, fmt.Errorf("failed to get the cluster csr %s. err: %v", o.ClusterName, err) + return false, fmt.Errorf("failed to get the cluster csr %s. err: %v", csrName, err) } if csrOK.Status.Certificate != nil { - klog.V(1).Infof("Signing certificate successfully") + klog.V(1).Infof(fmt.Sprintf("Signing certificate of csr %s successfully", csrName)) cert = csrOK.Status.Certificate return true, nil } - klog.V(1).Infof("Waiting for the client certificate to be issued") + klog.V(1).Infof(fmt.Sprintf("Waiting for the client certificate of csr %s to be issued", csrName)) + klog.V(1).Infof("Approve the CSR %s manually by executing `kubectl certificate approve %s` on the control plane", csrName, csrName) return false, nil }) if err != nil { return nil, err } - karmadaAgentCfg := CreateWithCert( + return CreateWithCert( karmadaClusterInfo.Server, DefaultClusterName, o.ClusterName, karmadaClusterInfo.CertificateAuthorityData, cert, pkData, - ) + ), nil +} + +// constructKarmadaAgentConfig constructs the final kubeconfig used by karmada-agent +func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*clientcmdapi.Config, error) { + csrName := o.ClusterName + "-" + k8srand.String(5) + + karmadaAgentCfg, err := o.constructKubeConfig(bootstrapClient, karmadaClusterInfo, csrName, generateAgentUserName(o.ClusterName), []string{ClusterPermissionGroups}) + if err != nil { + return nil, err + } kubeConfigFile := filepath.Join(KarmadaDir, KarmadaAgentKubeConfigFileName) @@ -588,6 +961,11 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub return karmadaAgentCfg, nil } +// constructKarmadaAgentConfig constructs the kubeconfig to generate rbac config for karmada-agent. +func (o *CommandRegisterOption) constructAgentRBACGeneratorConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster, csrName string) (*clientcmdapi.Config, error) { + return o.constructKubeConfig(bootstrapClient, karmadaClusterInfo, csrName, AgentRBACGenerator, []string{ClusterPermissionGroups}) +} + // createSecretAndRBACInMemberCluster create required secrets and rbac in member cluster func (o *CommandRegisterOption) createSecretAndRBACInMemberCluster(karmadaAgentCfg *clientcmdapi.Config) error { configBytes, err := clientcmd.Write(*karmadaAgentCfg) @@ -781,16 +1159,16 @@ func (o *CommandRegisterOption) makeKarmadaAgentDeployment() *appsv1.Deployment } // generateKeyAndCSR generate private key and csr -func generateKeyAndCSR(clusterName string) (*rsa.PrivateKey, []byte, error) { +func generateKeyAndCSR(commonName string, organization []string) (*rsa.PrivateKey, []byte, error) { pk, err := rsa.GenerateKey(rand.Reader, 3072) if err != nil { return nil, nil, err } csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{ Subject: pkix.Name{ - CommonName: ClusterPermissionPrefix + clusterName, - Organization: []string{ClusterPermissionGroups}, + CommonName: commonName, + Organization: organization, }, }, pk) if err != nil { @@ -1070,3 +1448,7 @@ func ToKarmadaClient(config *clientcmdapi.Config) (*karmadaclientset.Clientset, return karmadaClient, nil } + +func generateAgentUserName(clusterName string) string { + return ClusterPermissionPrefix + clusterName +}
pkg/karmadactl/unregister/unregister.go+57 −32 modified@@ -135,6 +135,9 @@ type CommandUnregisterOption struct { // MemberClusterClient member cluster client set MemberClusterClient kubeclient.Interface + + // rbacResources contains RBAC resources that grant the necessary permissions for the unregistering cluster to access to Karmada control plane. + rbacResources *register.RBACResources } // AddFlags adds flags to the specified FlagSet. @@ -157,6 +160,8 @@ func (j *CommandUnregisterOption) Complete(args []string) error { if len(args) > 0 { j.ClusterName = args[0] } + + j.rbacResources = register.GenerateRBACResources(j.ClusterName, j.ClusterNamespace) return nil } @@ -303,30 +308,67 @@ func (j *CommandUnregisterOption) getKarmadaAgentConfig(agent *appsv1.Deployment return clientcmd.Load(agentConfigSecret.Data[fileName]) } -type obj struct{ Kind, Name, Namespace string } - -func (o *obj) ToString() string { - if o.Namespace == "" { - return fmt.Sprintf("%s/%s", o.Kind, o.Name) - } - return fmt.Sprintf("%s/%s/%s", o.Kind, o.Namespace, o.Name) -} - // RunUnregisterCluster unregister the pull mode cluster from karmada. func (j *CommandUnregisterOption) RunUnregisterCluster() error { if j.DryRun { return nil } - // 1. delete the cluster object from the Karmada control plane + start := time.Now() + // 1. delete the work object from the Karmada control plane + // When deleting a cluster, the deletion triggers the removal of executionSpace, which can lead to the deletion of RBAC roles related to work. + // Therefore, the deletion of work should be performed before deleting the cluster. + err := cmdutil.EnsureWorksDeleted(j.ControlPlaneClient, names.GenerateExecutionSpaceName(j.ClusterName), j.Wait) + if err != nil { + klog.Errorf("Failed to delete works object. cluster name: %s, error: %v", j.ClusterName, err) + return err + } + j.Wait = j.Wait - time.Since(start) + + // 2. delete the cluster object from the Karmada control plane //TODO: add flag --force to implement force deletion. - if err := cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil { + if err = cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil { klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) return err } klog.Infof("Successfully delete cluster object (%s) from control plane.", j.ClusterName) - // 2. delete resource created by karmada in member cluster + if j.KarmadaConfig != "" { + if err = j.rbacResources.Delete(j.ControlPlaneKubeClient); err != nil { + klog.Errorf("Failed to delete karmada-agent RBAC resources from control plane. cluster name: %s, error: %v", j.ClusterName, err) + return err + } + klog.Infof("Successfully delete karmada-agent RBAC resources from control plane. cluster name: %s", j.ClusterName) + } else { + klog.Warningf("The RBAC resources on the control plane need to be manually cleaned up, including the following resources:\n%s", j.rbacResources.ToString()) + } + + // 3. delete resource created by karmada in member cluster + if err = j.cleanupMemberClusterResources(); err != nil { + return err + } + + // 4. delete local obsolete files generated by karmadactl + localObsoleteFiles := []register.Obj{ + {Kind: "File", Name: filepath.Join(register.KarmadaDir, register.KarmadaAgentKubeConfigFileName)}, + {Kind: "File", Name: register.CACertPath}, + } + for _, obj := range localObsoleteFiles { + if err = os.Remove(obj.Name); err != nil { + if os.IsNotExist(err) { + continue + } + klog.Errorf("Failed to delete local file (%v) in current node: %+v.", obj.Name, err) + return err + } + klog.Infof("Successfully delete local file (%v) in current node.", obj.Name) + } + + return nil +} + +// cleanupMemberClusterResources clean up the resources which created by karmada in member cluster +func (j *CommandUnregisterOption) cleanupMemberClusterResources() error { var err error for _, resource := range j.listMemberClusterResources() { switch resource.Kind { @@ -350,29 +392,12 @@ func (j *CommandUnregisterOption) RunUnregisterCluster() error { } klog.Infof("Successfully delete resource (%v) from member cluster (%s).", resource, j.ClusterName) } - - // 3. delete local obsolete files generated by karmadactl - localObsoleteFiles := []obj{ - {Kind: "File", Name: filepath.Join(register.KarmadaDir, register.KarmadaAgentKubeConfigFileName)}, - {Kind: "File", Name: register.CACertPath}, - } - for _, obj := range localObsoleteFiles { - if err = os.Remove(obj.Name); err != nil { - if os.IsNotExist(err) { - continue - } - klog.Errorf("Failed to delete local file (%v) in current node: %+v.", obj.Name, err) - return err - } - klog.Infof("Successfully delete local file (%v) in current node.", obj.Name) - } - - return nil + return err } // listMemberClusterResources lists resources to be deleted which created by karmada in member cluster -func (j *CommandUnregisterOption) listMemberClusterResources() []obj { - return []obj{ +func (j *CommandUnregisterOption) listMemberClusterResources() []register.Obj { + return []register.Obj{ // the rbac resource prepared for karmada-controller-manager to access member cluster's kube-apiserver {Kind: "ServiceAccount", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName(j.ClusterName)}, {Kind: "ClusterRole", Name: names.GenerateRoleName(names.GenerateServiceAccountName(j.ClusterName))},
pkg/karmadactl/unregister/unregister_test.go+1 −0 modified@@ -216,6 +216,7 @@ func TestCommandUnregisterOption_RunUnregisterCluster(t *testing.T) { } j.ControlPlaneClient = fakekarmadaclient.NewSimpleClientset(tt.clusterObject...) j.MemberClusterClient = fake.NewSimpleClientset(tt.clusterResources...) + j.rbacResources = register.GenerateRBACResources(j.ClusterName, j.ClusterNamespace) err := j.RunUnregisterCluster() if (err == nil && tt.wantErr) || (err != nil && !tt.wantErr) { t.Errorf("RunUnregisterCluster() error = %v, wantErr %v", err, tt.wantErr)
pkg/karmadactl/util/work.go+55 −0 added@@ -0,0 +1,55 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" +) + +// EnsureWorksDeleted ensures that all Work resources in the specified namespace are deleted. +func EnsureWorksDeleted(controlPlaneKarmadaClient karmadaclientset.Interface, namespace string, + timeout time.Duration) error { + // make sure the works object under the given namespace has been deleted. + err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(context.Context) (done bool, err error) { + list, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("failed to list work in namespace %s", namespace) + } + + if len(list.Items) == 0 { + return true, nil + } + for i := range list.Items { + work := &list.Items[i] + err = controlPlaneKarmadaClient.WorkV1alpha1().Works(namespace).Delete(context.TODO(), work.GetName(), metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("failed to delete the work(%s/%s)", namespace, work.GetName()) + } + } + return false, nil + }) + + return err +}
pkg/util/rbac.go+46 −0 modified@@ -191,3 +191,49 @@ func GenerateImpersonationRules(subjects []rbacv1.Subject) []rbacv1.PolicyRule { return rules } + +// CreateRole just try to create the Role. +func CreateRole(client kubeclient.Interface, roleObj *rbacv1.Role) (*rbacv1.Role, error) { + createdObj, err := client.RbacV1().Roles(roleObj.GetNamespace()).Create(context.TODO(), roleObj, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + return roleObj, nil + } + + return nil, err + } + + return createdObj, nil +} + +// CreateRoleBinding just try to create the RoleBinding. +func CreateRoleBinding(client kubeclient.Interface, roleBindingObj *rbacv1.RoleBinding) (*rbacv1.RoleBinding, error) { + createdObj, err := client.RbacV1().RoleBindings(roleBindingObj.GetNamespace()).Create(context.TODO(), roleBindingObj, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + return roleBindingObj, nil + } + + return nil, err + } + + return createdObj, nil +} + +// DeleteRole just try to delete the Role. +func DeleteRole(client kubeclient.Interface, namespace, name string) error { + err := client.RbacV1().Roles(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +// DeleteRoleBinding just try to delete the RoleBinding. +func DeleteRoleBinding(client kubeclient.Interface, namespace, name string) error { + err := client.RbacV1().RoleBindings(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +}
Vulnerability mechanics
Generated by null/stub on May 9, 2026. Inputs: CWE entries + fix-commit diffs from this CVE's patches. Citations validated against bundle.
References
6- github.com/advisories/GHSA-mg7w-c9x2-xh7rghsaADVISORY
- nvd.nist.gov/vuln/detail/CVE-2024-56513ghsaADVISORY
- github.com/karmada-io/karmada/commit/2c82055c4c7f469411b1ba48c4dba4841df04831nvdWEB
- github.com/karmada-io/karmada/pull/5793nvdWEB
- github.com/karmada-io/karmada/security/advisories/GHSA-mg7w-c9x2-xh7rnvdWEB
- karmada.io/docs/administrator/security/component-permissionnvdWEB
News mentions
0No linked articles in our index yet.