CVE-2025-54470

@@ -3865,13 +3865,14 @@ type RESTUserRoleConfigData struct {
 
 // Import task
 type RESTImportTask struct {
-	TID            string    `json:"tid"`
-	CtrlerID       string    `json:"ctrler_id"`
-	LastUpdateTime time.Time `json:"last_update_time,omitempty"`
-	Percentage     int       `json:"percentage"`
-	TriggeredBy    string    `json:"triggered_by,omitempty"` // fullname of the user who triggers import
-	Status         string    `json:"status,omitempty"`
-	TempToken      string    `json:"temp_token,omitempty"`
+	TID                    string              `json:"tid"`
+	CtrlerID               string              `json:"ctrler_id"`
+	LastUpdateTime         time.Time           `json:"last_update_time,omitempty"`
+	Percentage             int                 `json:"percentage"`
+	TriggeredBy            string              `json:"triggered_by,omitempty"` // fullname of the user who triggers import
+	Status                 string              `json:"status,omitempty"`
+	TempToken              string              `json:"temp_token,omitempty"`
+	FailToDecryptKeyFields map[string][]string `json:"fail_to_decrypt_key_fields"` // key : []fields
 }
 
 type RESTImportTaskData struct {

@@ -7519,6 +7519,14 @@ definitions:
       temp_token:
         type: string
         example: ""
+      fail_to_decrypt_key_fields:
+        type: object
+        description: Object key is kv key and value is array of cloaked fields that cannot be decrypted
+        additionalProperties:
+          type: array
+          items:
+            type: string
+          example: ["x509_cert", "signing_cert"]
   RESTImportTaskData:
     type: object
     required:

@@ -180,6 +180,7 @@ type AlertType string
 const (
 	AlertTypeRBAC           AlertType = "RBAC"
 	AlertTypeTlsCertificate AlertType = "TLS_CERTIFICATE"
+	AlertTypeOthers         AlertType = "Others"
 )
 
 type RESTNvAlertGroup struct {

@@ -161,6 +161,10 @@ const (
 	EventNameGroupMetricViolation        = "Group.Metric.Violation"
 	EventNameKvRestored                  = "Configuration.Restore"
 	EventNameScanDataRestored            = "Scan.Data.Restore"
+	EventNameMismatchedDEKSeed           = "Security.DEK.Seed.Mismatch"
+	EventNameDEKSeedUnavailable          = "Security.DEK.Seed.Unavailable"
+	EventNameReEncryptWithDEK            = "Security.DEK.Encrypt"
+	EventNameEncryptionSecretSet         = "Security.Encryption.Secret.Set"
 )
 
 // TODO: these are not events but incidents

@@ -1,12 +1,15 @@
 package cache
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"net"
 	"os"
 	"reflect"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -178,11 +181,13 @@ const (
 type Context struct {
 	k8sVersion               string
 	ocVersion                string
-	RancherEP                string // from yaml/helm chart
-	RancherSSO               bool   // from yaml/helm chart
-	TelemetryFreq            uint   // from yaml
-	CheckDefAdminFreq        uint   // from yaml, in minutes
-	CspPauseInterval         uint   // from yaml, in minutes
+	RancherEP                string        // from yaml/helm chart
+	RancherSSO               bool          // from yaml/helm chart
+	TelemetryFreq            uint          // from yaml
+	KeyRotationDuration      time.Duration // from yaml
+	CheckKeyRotationDuration time.Duration // from yaml
+	CheckDefAdminFreq        uint          // from yaml, in minutes
+	CspPauseInterval         uint          // from yaml, in minutes
 	LocalDev                 *common.LocalDevice
 	EvQueue                  cluster.ObjectQueueInterface
 	AuditQueue               cluster.ObjectQueueInterface
@@ -1679,6 +1684,9 @@ func startWorkerThread(ctx *Context) {
 	groupMetricCheckTicker := time.NewTicker(groupMetricCheckPeriod)
 	policyMetricCheckTicker := time.NewTicker(policyMetricCheckPeriod)
 
+	keyRotationCheckTicker := time.NewTicker(cctx.CheckKeyRotationDuration)
+	log.WithFields(log.Fields{"CheckKeyRotationDuration": cctx.CheckKeyRotationDuration, "KeyRotationDuration": cctx.KeyRotationDuration}).Info()
+
 	wlSuspected := utils.NewSet() // supicious workload ids
 	pruneKvTicker := time.NewTicker(pruneKVPeriod)
 	pruneWorkloadKV(wlSuspected) // the first scan
@@ -1742,6 +1750,8 @@ func startWorkerThread(ctx *Context) {
 				cacheMutexUnlock()
 			case <-pruneKvTicker.C:
 				pruneWorkloadKV(wlSuspected)
+			case <-keyRotationCheckTicker.C:
+				checkKeyRotation()
 			case <-scannerTicker.C:
 				if isScanner() {
 					// Remove stalled scanner
@@ -2031,6 +2041,33 @@ func startWorkerThread(ctx *Context) {
 							log.WithFields(log.Fields{"name": o.Name, "domain": o.Domain}).Info("deleted")
 						}
 					}
+				case resource.RscTypeSecret:
+					var err error
+					var lock cluster.LockInterface
+					for i := 0; i < 4; i++ {
+						var encKeys common.EncKeys
+						var currEncKeyVer string
+						lock, err = clusHelper.AcquireLock(share.CLUSStoreSecretKey, time.Duration(time.Second)*4)
+						if err == nil {
+							if encKeys, currEncKeyVer, _, err = resource.RetrieveStorePassphrases(); err == nil {
+								if len(encKeys) > 0 {
+									if err = common.InitAesGcmKey(encKeys, currEncKeyVer); err == nil {
+										log.Info("store passphrases reloaded")
+										clusHelper.ReleaseLock(lock)
+										break
+									}
+									log.WithFields(log.Fields{"i": i, "lead": isLeader(), "err": err}).Error("Failed to init AesGcm")
+								}
+							} else {
+								log.WithFields(log.Fields{"i": i, "err": err}).Error("Failed to reloaded store passphrases")
+							}
+							clusHelper.ReleaseLock(lock)
+						}
+						time.Sleep(time.Second)
+					}
+					if err != nil {
+						log.WithFields(log.Fields{"err": err}).Error("Failed to reloaded store passphrases")
+					}
 				/*
 						case resource.RscTypeMutatingWebhookConfiguration:
 							var n, o *resource.AdmissionWebhookConfiguration
@@ -2108,6 +2145,54 @@ func startWorkerThread(ctx *Context) {
 	}()
 }
 
+func checkKeyRotation() {
+	if !isLeader() {
+		return
+	}
+
+	if valueBackup, _ := cluster.Get(share.CLUSSystemEncMigratedKey); len(valueBackup) > 0 {
+		var cfgEncBackup share.CLUSSystemConfigEncMigrated
+		if err := json.Unmarshal(valueBackup, &cfgEncBackup); err == nil {
+			if time.Now().UTC().After(cfgEncBackup.EncDataExpirationTime) {
+				// the backup is expired
+				_ = cluster.Delete(share.CLUSSystemEncMigratedKey)
+				log.Info("encryption-migrated config backup is deleted")
+			}
+		}
+	}
+
+	value, _ := cluster.Get(share.CLUSNextKeyRotationTSKey)
+	if len(value) == 0 {
+		_ = kv.SetNextKeyRotationTime(cctx.KeyRotationDuration)
+		return
+	}
+	i, err := strconv.ParseInt(string(value), 10, 64)
+	if err != nil {
+		return
+	}
+	nextKeyRotationTime := time.Unix(i, 0)
+	if time.Now().After(nextKeyRotationTime) {
+		if err := resource.AddStorePassphrase(); err == nil {
+			if err = kv.SetNextKeyRotationTime(cctx.KeyRotationDuration); err == nil {
+				alert := "Important: Please backup the data in Kubernetes secret neuvector-store-secret reset by NeuVector"
+				b := sha256.Sum256([]byte(alert))
+				key := hex.EncodeToString(b[:])
+				allUser := clusHelper.GetAllUsers(access.NewReaderAccessControl())
+				for _, user := range allUser {
+					accepted := utils.NewSetFromStringSlice(user.AcceptedAlerts)
+					if accepted.Contains(key) {
+						accepted.Remove(key)
+						user.AcceptedAlerts = accepted.ToStringSlice()
+						if err = clusHelper.PutUser(user); err != nil {
+							log.WithFields(log.Fields{"": user.Fullname, "err": err}).Error()
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
 // handler of K8s resource watcher calls cbResourceWatcher() which sends to orchObjChan/objChan
 // [2021-02-15] CRD-related resource changes do not call this function.
 //

@@ -32,6 +32,7 @@ type webhookCache struct {
 
 const DefaultScannerConfigUpdateTimeout = time.Minute * 5
 
+var encMigratedConfigRestored bool
 var systemConfigCache share.CLUSSystemConfig = common.DefaultSystemConfig
 var fedSystemConfigCache share.CLUSSystemConfig = share.CLUSSystemConfig{CfgType: share.FederalCfg}
 var webhookCacheMap map[string]*webhookCache = make(map[string]*webhookCache, 0)    // Only the enabled webhooks
@@ -467,6 +468,42 @@ func (m CacheMethod) GetSystemConfigClusterName(acc *access.AccessControl) strin
 	return systemConfigCache.ClusterName
 }
 
+func encMigrateSystemConfig(valueBackup, value []byte) {
+	controllers := cacher.GetAllControllers(access.NewAdminAccessControl())
+	for _, ctrler := range controllers {
+		if ctrler.Ver != cctx.CtrlerVersion {
+			log.WithFields(log.Fields{"target": ctrler.Ver, "me": cctx.CtrlerVersion}).Info()
+			// the system config backup for variant encryption key migration is found.
+			// it means sensitive fields in system config are re-encrypted for variant encryption key migration.
+			var dec common.DecryptUnmarshaller
+			var cfg share.CLUSSystemConfig
+			if err := dec.Unmarshal(value, &cfg); err == nil && dec.GetDecryptedFieldsNumber() == 0 {
+				// however, it's unexpected that no sensitive field is decrypted in dec.Unmarshal() call.
+				// it means all sensitive fields in current system config(cfg) were set to empty string that we need to restore system config from the system config backup
+				var cfgEncBackup share.CLUSSystemConfigEncMigrated
+				if err = json.Unmarshal(valueBackup, &cfgEncBackup); err == nil {
+					if time.Now().UTC().Before(cfgEncBackup.EncDataExpirationTime) {
+						// the backup is not expired yet
+						if err = cluster.PutBinary(share.CLUSConfigSystemKey, []byte(cfgEncBackup.EncMigratedSystemConfig)); err == nil {
+							log.Info("encryption-migrated config is restored")
+							encMigratedConfigRestored = true
+						}
+					} else {
+						_ = cluster.Delete(share.CLUSSystemEncMigratedKey)
+						log.Info("encryption-migrated config backup is deleted")
+					}
+				} else {
+					_ = cluster.Delete(share.CLUSSystemEncMigratedKey)
+					log.WithFields(log.Fields{"err": err}).Error("failed to json unmarshal encryption-migrated config")
+				}
+			} else if err != nil {
+				log.WithFields(log.Fields{"err": err}).Error("failed to dec unmarshal system config")
+			}
+			break
+		}
+	}
+}
+
 func systemConfigUpdate(nType cluster.ClusterNotifyType, key string, value []byte) {
 	log.WithFields(log.Fields{"type": cluster.ClusterNotifyName[nType], "key": key}).Debug()
 
@@ -477,6 +514,14 @@ func systemConfigUpdate(nType cluster.ClusterNotifyType, key string, value []byt
 		_ = json.Unmarshal(value, &cfg)
 		log.WithFields(log.Fields{"config": cfg}).Debug()
 
+		if nType == cluster.ClusterNotifyModify {
+			if valueBackup, _ := cluster.Get(share.CLUSSystemEncMigratedKey); len(valueBackup) > 0 {
+				if !encMigratedConfigRestored {
+					encMigrateSystemConfig(valueBackup, value)
+				}
+			}
+		}
+
 		if cfg.IBMSAConfigNV.EpEnabled && cfg.IBMSAConfigNV.EpStart == 1 {
 			if isLeader() {
 				var param interface{} = &cfg.IBMSAConfig

@@ -1646,6 +1646,7 @@ func ObjectUpdateHandler(nType cluster.ClusterNotifyType, key string, value []by
 	case "uniconf":
 		uniconfUpdate(nType, key, value)
 	case "cert":
+		value, _, _ = kv.UpgradeAndConvert(key, value)
 		certObjectUpdate(nType, key, value)
 	case "throttled", "telemetry":
 	default:

@@ -141,7 +141,7 @@ func leadChangeHandler(newLead, oldLead string) {
 			}
 
 			if emptyKvFound {
-				if _, _, restored, restoredKvVersion, err := kv.GetConfigHelper().Restore(); restored && err == nil {
+				if _, _, restored, restoredKvVersion, err := kv.GetConfigHelper().Restore(Host, Ctrler); restored && err == nil {
 					clog := share.CLUSEventLog{
 						Event:          share.CLUSEvKvRestored,
 						HostID:         Host.ID,
@@ -216,7 +216,7 @@ func ctlrMemberUpdateHandler(nType cluster.ClusterNotifyType, memberAddr string,
 			selfRejoin = false
 
 			ctlrPutLocalInfo()
-			logController(share.CLUSEvControllerJoin)
+			logController(share.CLUSEvControllerJoin, "")
 		}
 	} else if nType == cluster.ClusterNotifyDelete {
 		if Ctrler.ClusterIP == memberAddr {
@@ -247,13 +247,14 @@ func clusterStart(clusterCfg *cluster.ClusterConfig) (string, string, error) {
 	return cluster.GetSelfAddress(), lead, nil
 }
 
-func logController(ev share.TLogEvent) {
+func logController(ev share.TLogEvent, msg string) {
 	clog := share.CLUSEventLog{
 		Event:          ev,
 		HostID:         Host.ID,
 		HostName:       Host.Name,
 		ControllerID:   Ctrler.ID,
 		ControllerName: Ctrler.Name,
+		Msg:            msg,
 	}
 	switch ev {
 	case share.CLUSEvControllerStart:

@@ -82,6 +82,9 @@ var ErrObjectExists error = errors.New("Object exists")
 var ErrAtomicWriteFail error = errors.New("Atomic write failed")
 var ErrUnsupported error = errors.New("Unsupported action")
 var ErrClusterWriteFail error = errors.New("Failed to write cluster")
+var ErrInvalidPassphrase error = errors.New("Invalid passphrase for data encryption key")
+var ErrDEKSeedUnavailable error = errors.New("store passphrase unavailable")
+var ErrEmptyValue error = errors.New("Empty value")
 
 var defaultWebhookCategory []string = []string{}
 var defaultSyslogCategory []string = []string{
@@ -340,6 +343,10 @@ var LogEventMap = map[share.TLogEvent]LogEventInfo{
 	share.CLUSEvGroupMetricViolation:        {api.EventNameGroupMetricViolation, api.EventCatGroup, api.LogLevelWARNING},
 	share.CLUSEvKvRestored:                  {api.EventNameKvRestored, api.EventCatConfig, api.LogLevelINFO},
 	share.CLUSEvScanDataRestored:            {api.EventNameScanDataRestored, api.EventCatScan, api.LogLevelINFO},
+	share.CLUSEvMismatchedDEKSeed:           {api.EventNameMismatchedDEKSeed, api.EventCatConfig, api.LogLevelERR},
+	share.CLUSEvDEKSeedUnavailable:          {api.EventNameDEKSeedUnavailable, api.EventCatConfig, api.LogLevelWARNING},
+	share.CLUSEvReEncryptWithDEK:            {api.EventNameReEncryptWithDEK, api.EventCatConfig, api.LogLevelINFO},
+	share.CLUSEvEncryptionSecretSet:         {api.EventNameEncryptionSecretSet, api.EventCatConfig, api.LogLevelNOTICE},
 }
 
 type LogIncidentInfo struct {

@@ -1,6 +1,8 @@
 package common
 
 import (
+	"crypto/aes"
+	"crypto/cipher"
 	"crypto/pbkdf2"
 	"crypto/rand"
 	"crypto/sha256"
@@ -9,18 +11,25 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
+	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/neuvector/neuvector/controller/api"
 	"github.com/neuvector/neuvector/share/utils"
+	log "github.com/sirupsen/logrus"
 )
 
 type Marshaller interface {
 	Marshal(data interface{}) ([]byte, error)
+	GetEmptyFieldsToEncrypt() utils.Set
 }
 type Unmarshaller interface {
 	Unmarshal(raw []byte, data interface{}) error
 	Uncloak(data interface{}) error
+	GetEmptyEncryptedFields() utils.Set
+	GetFailToDecryptFields() utils.Set
+	GetDecryptedFieldsNumber() int
 }
 
 const (
@@ -32,18 +41,197 @@ const (
 )
 
 const (
-	saltSize = 16
+	saltSize  = 16
+	nonceSize = 12
+
+	saltHexSize  = 32
+	nonceHexSize = 24
 
 	keyLength = 32
 	keyIter   = 600000
 
-	saltedHashPrefix = "s-"
+	DekSeedLength = 32
+
+	saltedHashPrefix   = "s-"
+	cipherTypeA        = "a"
+	cipherBundleFormat = "%s-%s-%s-%s-%s" // "{cipherTypeA}-{keyVersion}-{hex(salt)}-{hex(nonce)}-{hex(cipherText)}"
+	cipherBundleParts  = 5
 )
 
+type tMarshallResult struct {
+	emptyFieldsToEncrypt utils.Set // all to-cloak fields that have empty string value
+}
+
+func (md *tMarshallResult) Reset() {
+	if md.emptyFieldsToEncrypt == nil {
+		md.emptyFieldsToEncrypt = utils.NewSet()
+	} else {
+		md.emptyFieldsToEncrypt.Clear()
+	}
+}
+
+func (md *tMarshallResult) AddEmptyFieldToEncrypt(field string) {
+	if md.emptyFieldsToEncrypt != nil {
+		md.emptyFieldsToEncrypt.Add(field)
+	}
+}
+
+func (md *tMarshallResult) GetEmptyFieldsToEncrypt() utils.Set {
+	if md.emptyFieldsToEncrypt != nil && md.emptyFieldsToEncrypt.Cardinality() > 0 {
+		return md.emptyFieldsToEncrypt.Clone()
+	}
+	return nil
+}
+
+type tUnmarshallResult struct {
+	emptyEncryptedFields  utils.Set // all cloaked fields that have empty string value
+	failedToDecryptFields utils.Set // all cloaked fields that cannot be decrypted
+	decryptedFieldsNumber int
+}
+
+func (ud *tUnmarshallResult) Reset() {
+	if ud.emptyEncryptedFields == nil {
+		ud.emptyEncryptedFields = utils.NewSet()
+	} else {
+		ud.emptyEncryptedFields.Clear()
+	}
+
+	if ud.failedToDecryptFields == nil {
+		ud.failedToDecryptFields = utils.NewSet()
+	} else {
+		ud.failedToDecryptFields.Clear()
+	}
+}
+
+func (ud *tUnmarshallResult) AddEmptyEncryptedField(field string) {
+	if ud.emptyEncryptedFields != nil {
+		ud.emptyEncryptedFields.Add(field)
+	}
+}
+
+func (ud *tUnmarshallResult) AddFailedToDecryptField(field string) {
+	if ud.failedToDecryptFields != nil {
+		ud.failedToDecryptFields.Add(field)
+	}
+}
+
+func (ud *tUnmarshallResult) IncreaseDecryptedFields() {
+	ud.decryptedFieldsNumber++
+}
+
+func (ud *tUnmarshallResult) GetEmptyEncryptedFields() utils.Set {
+	if ud.emptyEncryptedFields != nil && ud.emptyEncryptedFields.Cardinality() > 0 {
+		return ud.emptyEncryptedFields.Clone()
+	}
+	return nil
+}
+
+func (ud *tUnmarshallResult) GetFailToDecryptFields() utils.Set {
+	if ud.failedToDecryptFields != nil && ud.failedToDecryptFields.Cardinality() > 0 {
+		return ud.failedToDecryptFields.Clone()
+	}
+	return nil
+}
+
+func (ud *tUnmarshallResult) GetDecryptedFieldsNumber() int {
+	return ud.decryptedFieldsNumber
+}
+
 type EmptyMarshaller struct{}
 type MaskMarshaller struct{}
-type EncryptMarshaller struct{}
-type DecryptUnmarshaller struct{}
+type EncryptMarshaller struct {
+	result tMarshallResult
+}
+type DecryptUnmarshaller struct {
+	result tUnmarshallResult
+}
+
+// specifically for import / restore purpose
+type MigrateDecryptUnmarshaller struct {
+	ReEncryptRequired bool // set to true when any sensitive field needs to be re-encrypted by the new encryption mechanism
+	result            tUnmarshallResult
+}
+
+type EncKeys map[string][]byte // map key is enc key version(the bigger the newer). value is the passphrase
+type nvDEK struct {
+	version string
+	dekSeed string
+}
+
+func (m nvDEK) isAvailable() bool {
+	return len(m.dekSeed) == keyLength
+}
+
+var dekSeedMutex sync.RWMutex
+var currentDekSeed nvDEK                                      // for the current dekSeed for encrypting data
+var dekSeedsCache map[string]string = make(map[string]string) // hash values of all passphrases in k8s secret neuvector-store-secret
+
+func getCurrentDekSeed() nvDEK {
+	dekSeedMutex.RLock()
+	defer dekSeedMutex.RUnlock()
+	return currentDekSeed
+}
+
+func getVersionedDekSeed(version string) string {
+	dekSeedMutex.RLock()
+	defer dekSeedMutex.RUnlock()
+	return dekSeedsCache[version]
+}
+
+func InitAesGcmKey(encKeys EncKeys, currEncKeyVer string) error {
+	if len(encKeys) == 0 {
+		return ErrInvalidPassphrase
+	}
+	dekSeedsCacheTemp := make(map[string]string, len(encKeys))
+	if passphrase, ok := encKeys[currEncKeyVer]; !ok || len(passphrase) < DekSeedLength {
+		return ErrInvalidPassphrase
+	}
+
+	for keyVersion, passphrase := range encKeys {
+		if len(passphrase) >= DekSeedLength {
+			h := sha256.New()
+			h.Write(passphrase)
+			dekSeedsCacheTemp[keyVersion] = string(h.Sum(nil))
+		}
+	}
+
+	if len(dekSeedsCacheTemp[currEncKeyVer]) < DekSeedLength {
+		return ErrInvalidPassphrase
+	}
+
+	dekSeedMutex.Lock()
+	dekSeedsCache = dekSeedsCacheTemp
+	currentDekSeed = nvDEK{
+		version: currEncKeyVer,
+		dekSeed: dekSeedsCache[currEncKeyVer],
+	}
+	dekSeedMutex.Unlock()
+
+	return nil
+}
+
+func AddAesGcmKey(keyVersion string, passphrase []byte) error {
+	if len(passphrase) >= DekSeedLength {
+		h := sha256.New()
+		h.Write(passphrase)
+		dekSeed := string(h.Sum(nil))
+
+		dekSeedMutex.Lock()
+		dekSeedsCache[keyVersion] = dekSeed
+		currentDekSeed = nvDEK{
+			version: keyVersion,
+			dekSeed: dekSeed,
+		}
+		dekSeedMutex.Unlock()
+		return nil
+	}
+	return ErrDEKSeedUnavailable
+}
+
+func IsDEKSeedAvailable() bool {
+	dekSeed := getCurrentDekSeed()
+	return dekSeed.isAvailable()
+}
 
 func HashPassword(password string, salt []byte) (string, error) {
 	if len(salt) == 0 {
@@ -64,6 +252,105 @@ func HashPassword(password string, salt []byte) (string, error) {
 	return cipherBundle, nil
 }
 
+func aesGcmEncrypt(plaintext string) (string, error) {
+	if plaintext == "" {
+		return "", ErrEmptyValue
+	}
+	dekSeed := getCurrentDekSeed()
+	if !dekSeed.isAvailable() {
+		return "", ErrDEKSeedUnavailable
+	}
+
+	salt := make([]byte, saltSize) // http://www.ietf.org/rfc/rfc2898.txt
+	if _, err := rand.Read(salt); err != nil {
+		return "", fmt.Errorf("failed to generate salt: %w", err)
+	}
+	dek, err := pbkdf2.Key(sha256.New, currentDekSeed.dekSeed, salt, keyIter, keyLength)
+	if err != nil {
+		return "", fmt.Errorf("failed to derive DEK: %w", err)
+	}
+
+	nonce := make([]byte, nonceSize)
+	if _, err := rand.Read(nonce); err != nil {
+		return "", fmt.Errorf("failed to generate nonce: %w", err)
+	}
+
+	block, err := aes.NewCipher(dek)
+	if err != nil {
+		return "", fmt.Errorf("failed to create cipher: %w", err)
+	}
+
+	aesGCM, err := cipher.NewGCM(block)
+	if err != nil {
+		return "", fmt.Errorf("failed to create GCM: %w", err)
+	}
+
+	additionalData := append(salt, nonce...)
+	cipherText := aesGCM.Seal(nil, nonce, []byte(plaintext), additionalData)
+	cipherBundle := fmt.Sprintf(cipherBundleFormat, cipherTypeA,
+		currentDekSeed.version, hex.EncodeToString(salt), hex.EncodeToString(nonce), hex.EncodeToString(cipherText))
+
+	return cipherBundle, nil
+}
+
+func aesGcmDecrypt(cipherBundle string) (string, error) {
+	if cipherBundle == "" {
+		return "", ErrEmptyValue
+	}
+
+	ss := strings.Split(cipherBundle, "-")
+	if len(ss) != cipherBundleParts || ss[0] != cipherTypeA || (len(ss[2]) != saltHexSize || len(ss[3]) != nonceHexSize) {
+		return "", ErrUnsupported
+	}
+
+	if _, err := strconv.ParseUint(ss[1], 10, 64); err != nil {
+		return "", fmt.Errorf("invalid version: %s", ss[1])
+	}
+
+	dekSeed := getVersionedDekSeed(ss[1])
+	if len(dekSeed) != keyLength {
+		return "", ErrDEKSeedUnavailable
+	}
+
+	salt, err := hex.DecodeString(ss[2])
+	if err != nil || len(salt) != saltSize {
+		return "", fmt.Errorf("invalid salt: %v(len=%d)", err, len(salt))
+	}
+
+	nonce, err := hex.DecodeString(ss[3])
+	if err != nil || len(nonce) != nonceSize {
+		return "", fmt.Errorf("invalid nonce: %v(len=%d)", err, len(nonce))
+	}
+
+	cipherText, err := hex.DecodeString(ss[4])
+	if err != nil {
+		return "", fmt.Errorf("invalid data: %w", err)
+	}
+
+	dek, err := pbkdf2.Key(sha256.New, dekSeed, salt, keyIter, keyLength)
+	if err != nil {
+		return "", fmt.Errorf("failed to derive DEK: %w", err)
+	}
+
+	block, err := aes.NewCipher(dek)
+	if err != nil {
+		return "", fmt.Errorf("failed to create cipher: %w", err)
+	}
+
+	aesGCM, err := cipher.NewGCM(block)
+	if err != nil {
+		return "", fmt.Errorf("failed to create GCM: %w", err)
+	}
+
+	additionalData := append(salt, nonce...)
+	data, err := aesGCM.Open(nil, nonce, cipherText, additionalData)
+	if err != nil {
+		return "", fmt.Errorf("failed to decrypt data: %w", err)
+	}
+
+	return string(data), nil
+}
+
 func IsSaltedPasswordHash(hash string) bool {
 	if strings.HasPrefix(hash, saltedHashPrefix) {
 		return len(strings.Split(hash, "-")) == 3
@@ -73,39 +360,88 @@ func IsSaltedPasswordHash(hash string) bool {
 }
 
 func (m EmptyMarshaller) Marshal(data interface{}) ([]byte, error) {
-	if u, err := marshal(emptyMask, data); err != nil {
+	if u, err := marshal(emptyMask, data, tMarshallResult{}); err != nil {
 		return nil, err
 	} else {
 		return json.Marshal(u)
 	}
 }
 
+func (m EmptyMarshaller) GetEmptyFieldsToEncrypt() utils.Set {
+	return nil
+}
+
 func (m MaskMarshaller) Marshal(data interface{}) ([]byte, error) {
-	if u, err := marshal(cloakMask, data); err != nil {
+	if u, err := marshal(cloakMask, data, tMarshallResult{}); err != nil {
 		return nil, err
 	} else {
 		return json.Marshal(u)
 	}
 }
 
-func (m EncryptMarshaller) Marshal(data interface{}) ([]byte, error) {
-	if u, err := marshal(cloakEncrypt, data); err != nil {
+func (m *EncryptMarshaller) Marshal(data interface{}) ([]byte, error) {
+	m.result.Reset()
+	if u, err := marshal(cloakEncrypt, data, m.result); err != nil {
 		return nil, err
 	} else {
 		return json.Marshal(u)
 	}
 }
 
-func (m DecryptUnmarshaller) Unmarshal(raw []byte, data interface{}) error {
+func (m *EncryptMarshaller) GetEmptyFieldsToEncrypt() utils.Set {
+	return m.result.GetEmptyFieldsToEncrypt()
+}
+
+func (m *DecryptUnmarshaller) Unmarshal(raw []byte, data interface{}) error {
 	if err := json.Unmarshal(raw, data); err != nil {
 		return err
 	} else {
-		return unmarshal(cloakDecrypt, data)
+		m.result.Reset()
+		return unmarshal(cloakDecrypt, data, nil, m.result)
 	}
 }
 
-func (m DecryptUnmarshaller) Uncloak(data interface{}) error {
-	return unmarshal(cloakDecrypt, data)
+func (m *DecryptUnmarshaller) Uncloak(data interface{}) error {
+	m.result.Reset()
+	return unmarshal(cloakDecrypt, data, nil, m.result)
+}
+
+func (m *DecryptUnmarshaller) GetEmptyEncryptedFields() utils.Set {
+	return m.result.GetEmptyEncryptedFields()
+}
+
+func (m *DecryptUnmarshaller) GetFailToDecryptFields() utils.Set {
+	return m.result.GetFailToDecryptFields()
+}
+
+func (m *DecryptUnmarshaller) GetDecryptedFieldsNumber() int {
+	return m.result.decryptedFieldsNumber
+}
+
+func (m *MigrateDecryptUnmarshaller) Unmarshal(raw []byte, data interface{}) error {
+	if err := json.Unmarshal(raw, data); err != nil {
+		return err
+	} else {
+		m.result.Reset()
+		return unmarshal(cloakDecrypt, data, &m.ReEncryptRequired, m.result)
+	}
+}
+
+func (m *MigrateDecryptUnmarshaller) Uncloak(data interface{}) error {
+	m.result.Reset()
+	return unmarshal(cloakDecrypt, data, &m.ReEncryptRequired, m.result)
+}
+
+func (m *MigrateDecryptUnmarshaller) GetEmptyEncryptedFields() utils.Set {
+	return m.result.GetEmptyEncryptedFields()
+}
+
+func (m *MigrateDecryptUnmarshaller) GetFailToDecryptFields() utils.Set {
+	return m.result.GetFailToDecryptFields()
+}
+
+func (m *MigrateDecryptUnmarshaller) GetDecryptedFieldsNumber() int {
+	return m.result.decryptedFieldsNumber
 }
 
 type MarshalInvalidTypeError struct {
@@ -154,7 +490,7 @@ func parseTag(tag string) (string, utils.Set) {
 	return tokens[0], utils.NewSetFromSliceKind(tokens[1:])
 }
 
-func unmarshal(cloak string, data interface{}) error {
+func unmarshal(cloak string, data interface{}, reEncryptRequired *bool, unmarshalResult tUnmarshallResult) error {
 	v := reflect.ValueOf(data)
 	t := v.Type()
 
@@ -168,7 +504,7 @@ func unmarshal(cloak string, data interface{}) error {
 	}
 
 	if t.Kind() != reflect.Struct {
-		return unmarshalValue(cloak, v)
+		return unmarshalValue(cloak, v, reEncryptRequired, unmarshalResult)
 	}
 
 	for i := 0; i < t.NumField(); i++ {
@@ -200,15 +536,44 @@ func unmarshal(cloak string, data interface{}) error {
 				switch cloak {
 				case cloakDecrypt:
 					if val.CanSet() {
-						s := utils.DecryptPassword(val.Interface().(string))
+						var s string
+						var err error
+						if strVal := val.Interface().(string); strVal != "" {
+							if idx := strings.Index(strVal, "-"); idx < 0 {
+								// it's encrypted by the fixed default key.
+								if s = utils.DecryptPassword(strVal); s != "" {
+									if reEncryptRequired != nil {
+										*reEncryptRequired = true
+									}
+									unmarshalResult.IncreaseDecryptedFields()
+								} else {
+									s = strVal
+								}
+							} else {
+								// it's encrypted by variant DEK
+								if s, err = aesGcmDecrypt(strVal); err != nil {
+									if err != ErrDEKSeedUnavailable && err != ErrEmptyValue {
+										log.WithFields(log.Fields{"error": err, "jsonTag": jsonTag}).Error()
+									}
+									if err == ErrEmptyValue {
+										unmarshalResult.AddEmptyEncryptedField(jsonTag)
+									} else {
+										unmarshalResult.AddFailedToDecryptField(jsonTag)
+									}
+									s = strVal
+								} else {
+									unmarshalResult.IncreaseDecryptedFields()
+								}
+							}
+						}
 						val.SetString(s)
 					}
 				}
 			}
 		}
 
 		if val.CanAddr() {
-			err := unmarshalValue(cloak, val.Addr())
+			err := unmarshalValue(cloak, val.Addr(), reEncryptRequired, unmarshalResult)
 			if err != nil {
 				return err
 			}
@@ -218,7 +583,7 @@ func unmarshal(cloak string, data interface{}) error {
 	return nil
 }
 
-func unmarshalValue(cloak string, v reflect.Value) error {
+func unmarshalValue(cloak string, v reflect.Value, reEncryptRequired *bool, unmarshalResult tUnmarshallResult) error {
 	// return nil on nil pointer struct fields
 	if !v.IsValid() || !v.CanInterface() {
 		return nil
@@ -232,7 +597,7 @@ func unmarshalValue(cloak string, v reflect.Value) error {
 
 	if k == reflect.Interface || k == reflect.Struct {
 		if v.CanAddr() {
-			return unmarshal(cloak, v.Addr().Interface())
+			return unmarshal(cloak, v.Addr().Interface(), reEncryptRequired, unmarshalResult)
 		} else {
 			return nil
 		}
@@ -241,7 +606,7 @@ func unmarshalValue(cloak string, v reflect.Value) error {
 	if k == reflect.Slice {
 		l := v.Len()
 		for i := 0; i < l; i++ {
-			err := unmarshalValue(cloak, v.Index(i))
+			err := unmarshalValue(cloak, v.Index(i), reEncryptRequired, unmarshalResult)
 			if err != nil {
 				return err
 			}
@@ -257,7 +622,7 @@ func unmarshalValue(cloak string, v reflect.Value) error {
 			return MarshalInvalidTypeError{t: mapKeys[0].Kind(), data: v.Interface()}
 		}
 		for _, key := range mapKeys {
-			err := unmarshalValue(cloak, v.MapIndex(key))
+			err := unmarshalValue(cloak, v.MapIndex(key), reEncryptRequired, unmarshalResult)
 			if err != nil {
 				return err
 			}
@@ -267,7 +632,7 @@ func unmarshalValue(cloak string, v reflect.Value) error {
 	return nil
 }
 
-func marshal(cloak string, data interface{}) (interface{}, error) {
+func marshal(cloak string, data interface{}, marshalResult tMarshallResult) (interface{}, error) {
 	v := reflect.ValueOf(data)
 	t := v.Type()
 
@@ -281,7 +646,7 @@ func marshal(cloak string, data interface{}) (interface{}, error) {
 	}
 
 	if t.Kind() != reflect.Struct {
-		return marshalValue(cloak, v)
+		return marshalValue(cloak, v, marshalResult)
 	}
 
 	dest := make(map[string]interface{})
@@ -326,13 +691,29 @@ func marshal(cloak string, data interface{}) (interface{}, error) {
 					m := api.RESTMaskedValue
 					val = reflect.ValueOf(m)
 				case cloakEncrypt:
-					m := utils.EncryptPassword(val.Interface().(string))
+					var m string
+					if strVal := val.Interface().(string); strVal != "" {
+						if currentDekSeed.isAvailable() {
+							var err error
+							if m, err = aesGcmEncrypt(strVal); err != nil {
+								if err != ErrDEKSeedUnavailable && err != ErrEmptyValue {
+									log.WithFields(log.Fields{"err": err, "jsonTag": jsonTag}).Error()
+								}
+								if err == ErrEmptyValue {
+									marshalResult.AddEmptyFieldToEncrypt(jsonTag)
+								}
+								m = utils.EncryptPassword(strVal)
+							}
+						} else {
+							m = utils.EncryptPassword(strVal)
+						}
+					}
 					val = reflect.ValueOf(m)
 				}
 			}
 		}
 
-		v, err := marshalValue(cloak, val)
+		v, err := marshalValue(cloak, val, marshalResult)
 		if err != nil {
 			return nil, err
 		}
@@ -350,7 +731,7 @@ func marshal(cloak string, data interface{}) (interface{}, error) {
 	return dest, nil
 }
 
-func marshalValue(cloak string, v reflect.Value) (interface{}, error) {
+func marshalValue(cloak string, v reflect.Value, marshalResult tMarshallResult) (interface{}, error) {
 	// return nil on nil pointer struct fields
 	if !v.IsValid() || !v.CanInterface() {
 		return nil, nil
@@ -374,7 +755,7 @@ func marshalValue(cloak string, v reflect.Value) (interface{}, error) {
 	}
 
 	if k == reflect.Interface || k == reflect.Struct {
-		return marshal(cloak, val)
+		return marshal(cloak, val, marshalResult)
 	}
 	if k == reflect.Slice {
 		if isEmptyValue(v) {
@@ -384,7 +765,7 @@ func marshalValue(cloak string, v reflect.Value) (interface{}, error) {
 		l := v.Len()
 		dest := make([]interface{}, l)
 		for i := 0; i < l; i++ {
-			d, err := marshalValue(cloak, v.Index(i))
+			d, err := marshalValue(cloak, v.Index(i), marshalResult)
 			if err != nil {
 				return nil, err
 			}
@@ -411,7 +792,7 @@ func marshalValue(cloak string, v reflect.Value) (interface{}, error) {
 			return nil, MarshalInvalidTypeError{t: mapKeys[0].Kind(), data: val}
 		}
 		for _, key := range mapKeys {
-			d, err := marshalValue(cloak, v.MapIndex(key))
+			d, err := marshalValue(cloak, v.MapIndex(key), marshalResult)
 			if err != nil {
 				return nil, err
 			}

@@ -1,6 +1,7 @@
 package common
 
 import (
+	"strings"
 	"testing"
 
 	"encoding/json"
@@ -123,6 +124,322 @@ func TestEncrypt(t *testing.T) {
 	}
 }
 
+func resetDekSeeds() {
+	dekSeedMutex.Lock()
+	defer dekSeedMutex.Unlock()
+
+	currentDekSeed = nvDEK{}
+	dekSeedsCache = make(map[string]string)
+}
+
+// just for unit test
+func _deleteAesGcmKey(keyVersion string) {
+	dekSeedMutex.Lock()
+	defer dekSeedMutex.Unlock()
+
+	delete(dekSeedsCache, keyVersion)
+	if currentDekSeed.version == keyVersion {
+		if len(dekSeedsCache) > 0 {
+			for k, v := range dekSeedsCache {
+				currentDekSeed = nvDEK{
+					version: k,
+					dekSeed: v,
+				}
+				break
+			}
+		} else {
+			currentDekSeed = nvDEK{}
+		}
+	}
+}
+
+func TestAesGcmEncrypt(t *testing.T) {
+	var enc EncryptMarshaller
+	var dec DecryptUnmarshaller
+
+	secret := "gary321"
+	u1 := maskUser{Username: "gary", Password: "gary123", Secret: &secret}
+	u2 := maskUser{Username: "mary", Password: "mary123", Secret: &secret}
+
+	var user maskUser
+	keyVersion := "1"
+	if err := InitAesGcmKey(map[string][]byte{keyVersion: []byte("abcdefghijklmnopqrstuvwxyz123456")}, keyVersion); err != nil {
+		t.Errorf("InitAesGcmKey failed: error=%v", err)
+	}
+
+	body, err := enc.Marshal(&u1) // enc marshal with DEK
+	if u1.Secret == nil || err != nil {
+		t.Errorf("enc.Marshal error: %v (u1.Secret=%v)", err, u1.Secret)
+		return
+	}
+
+	// now sensitive fields in 'body' is encrypted with DEK.
+
+	if err := json.Unmarshal(body, &user); err != nil { // sensitive fields(encrypted) are not decrypted after json unmarshal
+		t.Errorf("json.Unmarshal error: %v", err)
+	} else {
+		if ss := strings.Split(*user.Secret, "-"); len(ss) != cipherBundleParts {
+			t.Errorf("Unexpected json.Unmarshal result: %v (user.Secret=%v)", err, *user.Secret)
+		}
+	}
+
+	if err := dec.Unmarshal(body, &user); err != nil { // dec unmarshal with DEK
+		t.Errorf("dec.Unmarshal error: %v", err)
+	} else if failed := dec.GetFailToDecryptFields(); failed != nil && failed.Cardinality() > 0 {
+		t.Errorf("Failed to decrypt %v", failed)
+	} else if !reflect.DeepEqual(user, u1) {
+		t.Errorf("Incorrect mask marshal: marshal=%s, user=%v, u1=%v", string(body[:]), user, u1)
+		body, _ := json.Marshal(&user)
+		t.Errorf("Incorrect mask marshal: unmarshal=%s", string(body[:]))
+	}
+
+	var pair maskUserPair
+	p := maskUserPair{User1: u1, User2: &u2}
+	body, err = enc.Marshal(&p) // enc marshal with DEK
+	if p.User1.Secret == nil || p.User2.Secret == nil || err != nil {
+		t.Errorf("enc.Marshal error: %v (p.User1.Secret=%v, p.User2.Secret=%v)", err, p.User1.Secret, p.User2.Secret)
+	} else {
+		if err := json.Unmarshal(body, &pair); err != nil { // sensitive fields(encrypted) are not decrypted after json unmarshal
+			t.Errorf("json.Unmarshal error: %v", err)
+		} else {
+			if ss := strings.Split(*pair.User1.Secret, "-"); len(ss) != cipherBundleParts {
+				t.Errorf("Unexpected pair.User1 Marshal result: %v (pair.User1.Secret=%v)", err, *pair.User1.Secret)
+			} else if ss := strings.Split(*pair.User2.Secret, "-"); len(ss) != cipherBundleParts {
+				t.Errorf("Unexpected pair.User2 Marshal result: %v (pair.User2.Secret=%v)", err, *pair.User2.Secret)
+			}
+		}
+
+		if err := dec.Unmarshal(body, &pair); err != nil { // dec unmarshal with DEK
+			t.Errorf("dec.Unmarshal error: %v", err)
+		} else if failed := dec.GetFailToDecryptFields(); failed != nil && failed.Cardinality() > 0 {
+			t.Errorf("Failed to decrypt %v", failed)
+		} else if !reflect.DeepEqual(pair.User1, u1) || !reflect.DeepEqual(*pair.User2, u2) {
+			t.Errorf("Incorrect mask marshal: marshal=%s", string(body[:]))
+			body, _ := json.Marshal(&pair)
+			t.Errorf("Incorrect mask marshal: unmarshal=%s", string(body[:]))
+		}
+	}
+
+	resetDekSeeds()
+}
+
+func TestAesGcmDecryptWithRotation(t *testing.T) {
+	var enc EncryptMarshaller
+	var dec DecryptUnmarshaller
+
+	secret := "gary321"
+	u1 := maskUser{Username: "gary", Password: "gary123", Secret: &secret}
+
+	var user maskUser
+	keyVersion1 := "1"
+	if err := InitAesGcmKey(map[string][]byte{keyVersion1: []byte("11cdefghijklmnopqrstuvwxyz123456")}, keyVersion1); err != nil {
+		t.Errorf("InitAesGcmKey failed: error=%v", err)
+	}
+	currDekSeed1 := getCurrentDekSeed()
+	if currDekSeed1.version != keyVersion1 || !currDekSeed1.isAvailable() {
+		t.Errorf("currDekSeed1 error: %v", currDekSeed1)
+	}
+
+	body, err := enc.Marshal(&u1) // enc marshal with DEK (current DEK is dekSeed-v1)
+	if u1.Secret == nil || err != nil {
+		t.Errorf("enc.Marshal error: %v (u1.Secret=%v)", err, u1.Secret)
+		return
+	}
+
+	// now sensitive fields in 'body' is encrypted with dekSeed-v1.
+
+	if err := json.Unmarshal(body, &user); err != nil { // sensitive fields(encrypted) are not decrypted after json unmarshal
+		t.Errorf("json.Unmarshal error: %v", err)
+	} else {
+		if ss := strings.Split(*user.Secret, "-"); len(ss) != cipherBundleParts {
+			t.Errorf("Unexpected json.Unmarshal result: %v (user.Secret=%v)", err, *user.Secret)
+		}
+	}
+
+	if err := dec.Unmarshal(body, &user); err != nil { // dec unmarshal (current DEK is dekSeed-v1)
+		t.Errorf("dec.Unmarshal error: %v", err)
+	} else if failed := dec.GetFailToDecryptFields(); failed != nil && failed.Cardinality() > 0 {
+		t.Errorf("Failed to decrypt %v", failed)
+	} else if !reflect.DeepEqual(user, u1) {
+		t.Errorf("Incorrect mask marshal: marshal=%s, user=%v, u1=%v", string(body[:]), user, u1)
+		body, _ := json.Marshal(&user)
+		t.Errorf("Incorrect mask marshal: unmarshal=%s", string(body[:]))
+	}
+
+	// now currentDekSeed is set to version 2
+	keyVersion2 := "2"
+	if err := AddAesGcmKey(keyVersion2, []byte("22cdefghijklmnopqrstuvwxyz123456")); err != nil {
+		t.Errorf("AddAesGcmKey failed: error=%v", err)
+	}
+	currDekSeed2 := getCurrentDekSeed()
+	if currDekSeed2 == currDekSeed1 || currDekSeed2.version != keyVersion2 || !currDekSeed2.isAvailable() {
+		t.Errorf("currDekSeed2 error: %v", currDekSeed2)
+	}
+
+	// try dec.Unmarshal again to see whether the sensitive data encrypted with dekSeed-v1 can be decrypted when the currentDekSeed is version 2.
+	user = maskUser{}
+	if err := dec.Unmarshal(body, &user); err != nil { // dec unmarshal (current DEK is dekSeed-v2 but dekSeed-v1 is used for this decryption)
+		t.Errorf("dec.Unmarshal error: %v", err)
+	} else if failed := dec.GetFailToDecryptFields(); failed != nil && failed.Cardinality() > 0 {
+		t.Errorf("Failed to decrypt %v", failed)
+	} else if !reflect.DeepEqual(user, u1) {
+		t.Errorf("Incorrect mask marshal: marshal=%s, user=%v, u1=%v", string(body[:]), user, u1)
+		body, _ := json.Marshal(&user)
+		t.Errorf("Incorrect mask marshal: unmarshal=%s", string(body[:]))
+	}
+
+	// now intentionally delete dekSeed-v1 from cache (ideally should not happen in real world)
+	_deleteAesGcmKey(keyVersion1)
+
+	// try dec.Unmarshal again. currentDekSeed is version 2 & there is no dekSeed-v1 in the cache.
+	// it's expected that the sensitive data encrypted by dekSeed-v1 can not be decrypted anymore.
+	user = maskUser{}
+	if err := dec.Unmarshal(body, &user); err != nil { // dec unmarshal (current DEK is dekSeed-v2 and there is no dekSeed-v1 in the cache)
+		t.Errorf("dec.Unmarshal error: %v", err)
+	} else if failed := dec.GetFailToDecryptFields(); failed == nil || failed.Cardinality() == 0 {
+		t.Errorf("Expected to fail to decrypt sensitive fields in the object")
+	} else if failed.Cardinality() != 2 {
+		t.Errorf("Expected to fail to decrypt 2, not %d, sensitive fields in the object", failed.Cardinality())
+	} else if !failed.Contains("password") || !failed.Contains("secret") {
+		t.Errorf("Expected to fail to decrypt sensitive fields, %v, in the object", failed.ToStringSlice())
+	}
+
+	// Now try enc.Marshal again. Because current DEK is dekSeed-v2, it's expected the marshaled data is different
+	body2, err2 := enc.Marshal(&u1) // enc marshal with DEK (current DEK is dekSeed-v2)
+	if u1.Secret == nil || err2 != nil {
+		t.Errorf("enc.Marshal error: %v (u1.Secret=%v)", err, u1.Secret)
+	} else if string(body) == string(body2) {
+		t.Errorf("Expected to have different marshaled data because sensitive fields in the objects are encrypted with different dekSeed")
+	}
+
+	resetDekSeeds()
+}
+
+func TestAesGcmDecryptNegative(t *testing.T) {
+	var enc EncryptMarshaller
+	var dec DecryptUnmarshaller
+
+	secret := "gary321"
+	u1 := maskUser{Username: "gary", Password: "gary123", Secret: &secret}
+
+	var user maskUser
+	keyVersion := "1"
+	if err := InitAesGcmKey(map[string][]byte{keyVersion: []byte("abcdefghijklmnopqrstuvwxyz123456")}, keyVersion); err != nil {
+		t.Errorf("[1] InitAesGcmKey failed: error=%v", err)
+	}
+
+	body, err := enc.Marshal(&u1) // enc marshal with DEK
+	if u1.Secret == nil || err != nil {
+		t.Errorf("enc.Marshal error: %v (u1.Secret=%v)", err, u1.Secret)
+		return
+	}
+
+	// now sensitive fields in 'body' is encrypted with DEK.
+
+	// change dekSeed to different value
+	if err := InitAesGcmKey(map[string][]byte{keyVersion: []byte("abcdefghijklmnopqrstuvwxyz123457")}, keyVersion); err != nil {
+		t.Errorf("[2] InitAesGcmKey failed: error=%v", err)
+	}
+
+	// try to decrypt with different dekSeed
+	if err := dec.Unmarshal(body, &user); err != nil { // dec unmarshal with DEK
+		t.Errorf("dec.Unmarshal error: %v", err)
+	} else if failed := dec.GetFailToDecryptFields(); failed.Cardinality() == 0 {
+		t.Errorf("Unexpected that all sensitive fields can be decrypt")
+	} else if reflect.DeepEqual(user, u1) {
+		t.Errorf("Incorrect mask marshal: marshal=%s", string(body[:]))
+		body, _ := json.Marshal(&user)
+		t.Errorf("Incorrect mask marshal: unmarshal=%s", string(body[:]))
+	}
+
+	resetDekSeeds()
+}
+
+func TestAesGcmMigrateDecryptUnmarshaller(t *testing.T) {
+	var enc EncryptMarshaller
+	var dec DecryptUnmarshaller
+
+	secret := "gary321"
+	u1 := maskUser{Username: "gary", Password: "gary123", Secret: &secret}
+
+	resetDekSeeds()
+
+	var user maskUser
+	body, err := enc.Marshal(&u1) // enc marshal with fixed default key
+	if u1.Secret == nil || err != nil {
+		t.Errorf("enc.Marshal error: %v (u1.Secret=%v)", err, u1.Secret)
+		return
+	}
+
+	// now sensitive fields in 'body' is encrypted with fixed default key.
+
+	if err := json.Unmarshal(body, &user); err != nil { // sensitive fields(encrypted) are not decrypted after json unmarshal
+		t.Errorf("json.Unmarshal error: %v", err)
+	} else if ss := strings.Split(*user.Secret, "-"); len(ss) != 1 {
+		t.Errorf("Unexpected json.Marshal result: %v (user.Secret=%v)", err, *user.Secret)
+	}
+
+	if err := dec.Unmarshal(body, &user); err != nil { // dec unmarshal with fixed default key
+		t.Errorf("dec.Unmarshal error: %v", err)
+	} else if !reflect.DeepEqual(user, u1) {
+		t.Errorf("Incorrect mask marshal: marshal=%s", string(body[:]))
+		body, _ := json.Marshal(&user)
+		t.Errorf("Incorrect mask marshal: unmarshal=%s", string(body[:]))
+	}
+
+	keyVersion := "1"
+	if err := InitAesGcmKey(map[string][]byte{keyVersion: []byte("abcdefghijklmnopqrstuvwxyz123456")}, keyVersion); err != nil {
+		t.Errorf("InitAesGcmKey failed: error=%v", err)
+	}
+
+	// now DEK & fixed default key are available
+
+	var user1 maskUser
+	var dec2 MigrateDecryptUnmarshaller
+	if err := dec2.Unmarshal(body, &user1); err != nil { // dec unmarshal with fixed default key & set dec2.ReEncryptRequired to true
+		t.Errorf("Unmarshal error: %v", err)
+	} else if !dec2.ReEncryptRequired {
+		t.Errorf("Expect dec2.ReEncryptRequired=true but not see that")
+	} else {
+		if !reflect.DeepEqual(user1, u1) {
+			t.Errorf("Incorrect mask marshal: marshal=%s", string(body[:]))
+			body, _ := json.Marshal(&user)
+			t.Errorf("Incorrect mask marshal: unmarshal=%s", string(body[:]))
+		}
+	}
+
+	var user2 maskUser
+	if err := dec.Unmarshal(body, &user2); err != nil { // dec unmarshal with fixed default key
+		t.Errorf("dec.Unmarshal error: %v", err)
+	} else {
+		if !reflect.DeepEqual(user1, user2) {
+			t.Errorf("Incorrect mask marshal: marshal=%s", string(body[:]))
+			body, _ := json.Marshal(&user2)
+			t.Errorf("Incorrect mask marshal: unmarshal=%s", string(body[:]))
+		}
+	}
+
+	var user3 maskUser
+	if err := json.Unmarshal(body, &user3); err != nil { // sensitive fields(encrypted) are not decrypted after json unmarshal
+		t.Errorf("json.Unmarshal error: %v", err)
+	} else {
+		var dec3 MigrateDecryptUnmarshaller
+		if err := dec3.Uncloak(&user3); err != nil { // uncloak sensitive fields with fixed default key & set dec3.ReEncryptRequired to true
+			t.Errorf("dec3.Uncloak error: %v", err)
+		} else if !dec3.ReEncryptRequired {
+			t.Errorf("Expect dec3.ReEncryptRequired=true but not see that")
+		} else {
+			if !reflect.DeepEqual(user3, u1) {
+				t.Errorf("Incorrect mask marshal: marshal=%s", string(body[:]))
+				body, _ := json.Marshal(&user3)
+				t.Errorf("Incorrect mask marshal: unmarshal=%s", string(body[:]))
+			}
+		}
+	}
+
+	resetDekSeeds()
+}
+
 type aType struct {
 	IP    net.IP   `json:"ip"`
 	Array []byte   `json:"array"`

@@ -263,9 +263,12 @@ func main() {
 	pwdValidUnit := flag.Uint("pwd_valid_unit", 1440, "")
 	rancherEP := flag.String("rancher_ep", "", "Rancher endpoint URL")
 	rancherSSO := flag.Bool("rancher_sso", false, "Rancher SSO integration")
+	insecureSkipTeleTLSVerification := flag.Bool("insecure_skip_telemetry_tls_verification", false, "Do not verify Telemetry server's certificate chain and host name")
 	teleNeuvectorEP := flag.String("telemetry_neuvector_ep", "", "")                   // for testing only
 	teleCurrentVer := flag.String("telemetry_current_ver", "", "")                     // in the format {major}.{minor}.{patch}[-s{#}], for testing only
 	telemetryFreq := flag.Uint("telemetry_freq", 60, "")                               // in minutes, for testing only
+	keyRotationPeriod := flag.String("key_rotation_period", "", "")                    // for testing only
+	checkKeyRotationPeriod := flag.String("check_key_rotation_period", "", "")         // for testing only
 	noDefAdmin := flag.Bool("no_def_admin", false, "Do not create default admin user") // for new install only
 	cspEnv := flag.String("csp_env", "", "")                                           // "" or "aws"
 	cspPauseInterval := flag.Uint("csp_pause_interval", 240, "")                       // in minutes, for testing only
@@ -484,6 +487,34 @@ func main() {
 
 	if platform == share.PlatformKubernetes {
 		resource.AdjustAdmWebhookName(cache.QueryK8sVersion, cspType)
+
+		resource.GetNvCtrlerServiceAccount()
+		if !nvHasK8sSecretRbac(true) {
+			goodRBAC := false
+			ticker := time.Tick(time.Second * time.Duration(5))
+			c_sig := make(chan os.Signal, 1)
+			signal.Notify(c_sig, os.Interrupt, syscall.SIGTERM)
+		exit_controller:
+			for {
+				select {
+				case <-ticker:
+					logLevel := log.GetLevel()
+					log.SetLevel(log.WarnLevel)
+					goodRBAC = nvHasK8sSecretRbac(false)
+					log.SetLevel(logLevel)
+					if goodRBAC {
+						log.Info("Required Kubernetes RBAC for secrets are found")
+						break exit_controller
+					}
+				case <-c_sig:
+					break exit_controller
+				}
+			}
+			if !goodRBAC {
+				log.Error("Required Kubernetes RBAC for secrets are not found")
+				os.Exit(-2)
+			}
+		}
 	}
 
 	// Assign controller interface/IP scope
@@ -582,7 +613,21 @@ func main() {
 		log.WithFields(log.Fields{"error": err}).Error("CreateVulAssetDb")
 	}
 
-	kv.Init(Ctrler.ID, dev.Ctrler.Ver, Host.Platform, Host.Flavor, *persistConfig, isGroupMember, getConfigKvData, evqueue)
+	keyRotationDuration := time.Duration(time.Hour * 24 * 30 * 3)
+	checkKeyRotationDuration := time.Duration(time.Hour)
+	if *keyRotationPeriod != "" {
+		if duration, err := time.ParseDuration(*keyRotationPeriod); err == nil && duration.Seconds() != 0 {
+			keyRotationDuration = duration
+		}
+	}
+	if *checkKeyRotationPeriod != "" {
+		if duration, err := time.ParseDuration(*checkKeyRotationPeriod); err == nil && duration.Seconds() != 0 {
+			checkKeyRotationDuration = duration
+		}
+	}
+
+	kv.Init(Ctrler.ID, dev.Ctrler.Ver, Host.Platform, Host.Flavor, *persistConfig, isGroupMember,
+		getConfigKvData, evqueue, keyRotationDuration)
 	ruleid.Init()
 
 	// Start cluster
@@ -664,13 +709,52 @@ func main() {
 		log.WithError(err).Warn("installation id is not readable. Will retry later.")
 	}
 
+	var dekSeedEvent share.TLogEvent = share.CLUSEvDEKSeedUnavailable
+	var dekSeedEventMsg string
+	if platform == share.PlatformKubernetes {
+		var err error
+		var lock cluster.LockInterface
+		var encKeys common.EncKeys
+		var currEncKeyVer string
+		dekSeedEventMsg = "Failed to generate store passphrase for DEK seed"
+		for i := 0; i < 4; i++ {
+			lock, err = clusHelper.AcquireLock(share.CLUSStoreSecretKey, time.Duration(time.Second)*4)
+			if err == nil {
+				var msg string
+				if encKeys, currEncKeyVer, msg, err = resource.RetrieveStorePassphrases(); err == nil {
+					if err = common.InitAesGcmKey(encKeys, currEncKeyVer); err == nil {
+						if msg != "" {
+							dekSeedEvent = share.CLUSEvEncryptionSecretSet
+							dekSeedEventMsg = msg
+							log.Info(msg)
+						}
+						log.Info("store passphrase generate/read")
+					} else {
+						log.WithFields(log.Fields{"err": err, "len": len(encKeys)}).Error("Failed to init AesGcm")
+					}
+				} else {
+					log.WithFields(log.Fields{"err": err}).Error("Failed to generate/read store passphrase")
+				}
+				clusHelper.ReleaseLock(lock)
+				break
+			}
+			log.WithFields(log.Fields{"i": i, "err": err}).Info("retry for store passphrase")
+			time.Sleep(time.Second)
+		}
+		if err != nil || len(encKeys) == 0 {
+			log.WithFields(log.Fields{"err": err}).Error("Failed to read store passphrases")
+			os.Exit(-2)
+		}
+	}
+
 	emptyKvFound := false
 	ver := kv.GetControlVersion()
 	if ver.CtrlVersion == "" && ver.KVVersion == "" {
 		emptyKvFound = true
 	}
 	log.WithFields(log.Fields{"emptyKvFound": emptyKvFound, "ver": ver}).Info()
 
+	var restoreEvent string
 	if Ctrler.Leader || emptyKvFound {
 		// See [NVSHAS-5490]:
 		// clusterHelper.AcquireLock() may fail with error "failed to create session: Unexpected response code: 500 (Missing node registration)".
@@ -694,20 +778,10 @@ func main() {
 		var restored bool
 		var restoredKvVersion string
 		var errRestore error
-		restoredFedRole, defAdminRestored, restored, restoredKvVersion, errRestore = kv.GetConfigHelper().Restore()
+		restoredFedRole, defAdminRestored, restored, restoredKvVersion, errRestore = kv.GetConfigHelper().Restore(Host, Ctrler)
+		log.WithFields(log.Fields{"restored": restored, "errRestore": errRestore}).Info()
 		if restored && errRestore == nil {
-			clog := share.CLUSEventLog{
-				Event:          share.CLUSEvKvRestored,
-				HostID:         Host.ID,
-				HostName:       Host.Name,
-				ControllerID:   Ctrler.ID,
-				ControllerName: Ctrler.Name,
-				ReportedAt:     time.Now().UTC(),
-				Msg:            fmt.Sprintf("Restored kv version: %s", restoredKvVersion),
-			}
-			if err := evqueue.Append(&clog); err != nil {
-				log.WithFields(log.Fields{"error": err}).Error("evqueue.Append")
-			}
+			restoreEvent = fmt.Sprintf("Restored kv version: %s", restoredKvVersion)
 		}
 		if restoredFedRole == api.FedRoleJoint {
 			// fed rules are not restored on joint cluster but there might be fed rules left in kv so
@@ -838,6 +912,8 @@ func main() {
 		RancherEP:                *rancherEP,
 		RancherSSO:               *rancherSSO,
 		TelemetryFreq:            *telemetryFreq,
+		KeyRotationDuration:      keyRotationDuration,
+		CheckKeyRotationDuration: checkKeyRotationDuration,
 		CheckDefAdminFreq:        checkDefAdminFreq,
 		LocalDev:                 dev,
 		EvQueue:                  evqueue,
@@ -885,7 +961,7 @@ func main() {
 
 	if platform == share.PlatformKubernetes {
 		// k8s rbac watcher won't know anything about non-existing resources
-		resource.GetNvCtrlerServiceAccount(cache.CacheEvent)
+		resource.SetCacheEventFunc(cache.CacheEvent)
 
 		clusterRoleErrors, clusterRoleBindingErrors, roleErrors, roleBindingErrors := resource.VerifyNvK8sRBAC(dev.Host.Flavor, "", true)
 		if len(clusterRoleErrors) > 0 || len(roleErrors) > 0 || len(clusterRoleBindingErrors) > 0 || len(roleBindingErrors) > 0 {
@@ -903,24 +979,25 @@ func main() {
 	opa.InitOpaServer()
 
 	rctx := rest.Context{
-		LocalDev:           dev,
-		EvQueue:            evqueue,
-		AuditQueue:         auditQueue,
-		Messenger:          messenger,
-		Cacher:             cacher,
-		Scanner:            scanner,
-		RESTPort:           *restPort,
-		FedPort:            *fedPort,
-		PwdValidUnit:       *pwdValidUnit,
-		TeleNeuvectorURL:   *teleNeuvectorEP,
-		SearchRegistries:   *searchRegistries,
-		TeleFreq:           *telemetryFreq,
-		NvAppFullVersion:   nvAppFullVersion,
-		NvSemanticVersion:  nvSemanticVersion,
-		CspType:            cspType,
-		CspPauseInterval:   *cspPauseInterval,
-		CustomCheckControl: *custom_check_control,
-		CheckCrdSchemaFunc: nvcrd.CheckCrdSchema,
+		LocalDev:                dev,
+		EvQueue:                 evqueue,
+		AuditQueue:              auditQueue,
+		Messenger:               messenger,
+		Cacher:                  cacher,
+		Scanner:                 scanner,
+		RESTPort:                *restPort,
+		FedPort:                 *fedPort,
+		PwdValidUnit:            *pwdValidUnit,
+		TeleNeuvectorURL:        *teleNeuvectorEP,
+		TeleSkipTlsVerification: *insecureSkipTeleTLSVerification,
+		SearchRegistries:        *searchRegistries,
+		TeleFreq:                *telemetryFreq,
+		NvAppFullVersion:        nvAppFullVersion,
+		NvSemanticVersion:       nvSemanticVersion,
+		CspType:                 cspType,
+		CspPauseInterval:        *cspPauseInterval,
+		CustomCheckControl:      *custom_check_control,
+		CheckCrdSchemaFunc:      nvcrd.CheckCrdSchema,
 	}
 	// rest.PreInitContext() must be called before orch connector because existing CRD handling could happen right after orch connecter starts
 	rest.PreInitContext(&rctx)
@@ -1033,8 +1110,14 @@ func main() {
 	c_sig := make(chan os.Signal, 1)
 	signal.Notify(c_sig, os.Interrupt, syscall.SIGTERM)
 
-	logController(share.CLUSEvControllerStart)
-	logController(share.CLUSEvControllerJoin)
+	if dekSeedEventMsg != "" {
+		logController(dekSeedEvent, dekSeedEventMsg)
+	}
+	if restoreEvent != "" {
+		logController(share.CLUSEvKvRestored, restoreEvent)
+	}
+	logController(share.CLUSEvControllerStart, "")
+	logController(share.CLUSEvControllerJoin, "")
 
 	cache.PopulateRulesToOpa()
 
@@ -1088,7 +1171,7 @@ func main() {
 					memorySnapshot(mStats.WorkingSet)
 				}
 			case <-c_sig:
-				logController(share.CLUSEvControllerStop)
+				logController(share.CLUSEvControllerStop, "")
 				flushEventQueue()
 				done <- true
 			}
@@ -1171,3 +1254,10 @@ func amendNotPrivilegedMode() error {
 	}
 	return nil
 }
+
+func nvHasK8sSecretRbac(logging bool) bool {
+	k8sRbacRequried := []string{resource.NvSecretRole, resource.NvSecretControllerRole}
+	errs, _ := resource.VerifyNvRbacRoles(k8sRbacRequried, false, logging)
+	errs2, _ := resource.VerifyNvRbacRoleBindings(k8sRbacRequried, false, logging)
+	return len(errs) == 0 && len(errs2) == 0
+}

@@ -32,7 +32,7 @@ type PauseResumeStoreWatcherFunc func(ip string, port uint16, req share.CLUSStor
 type ConfigHelper interface {
 	NotifyConfigChange(endpoint string)
 	BackupAll()
-	Restore() (string, bool, bool, string, error)
+	Restore(host share.CLUSHost, ctrler share.CLUSController) (string, bool, bool, string, error)
 	Export(w *bufio.Writer, sections utils.Set) error
 	Import(eps []*common.RPCEndpoint, localCtrlerID, localCtrlerIP string, loginDomainRoles access.DomainRole, importTask share.CLUSImportTask,
 		tempToken string, revertFedRoles RevertFedRolesFunc, postImportOp PostImportFunc, pauseResumeStoreWatcher PauseResumeStoreWatcherFunc,
@@ -92,15 +92,15 @@ func GetConfigHelper() ConfigHelper {
 }
 
 func Init(id, version, platform, flavor string, persist bool, isGroupMember FuncIsGroupMember, getConfigData FuncGetConfigKVData,
-	evQueue cluster.ObjectQueueInterface) {
+	evQueue cluster.ObjectQueueInterface, keyRotationDuration time.Duration) {
 
 	evqueue = evQueue
 	for _, ep := range cfgEndpoints {
 		cfgEndpointMap[ep.name] = ep
 	}
 
 	newConfigHelper(id, version, persist)
-	clusHelper = newClusterHelper(id, version, persist)
+	clusHelper = newClusterHelper(id, version, persist, keyRotationDuration)
 
 	orchPlatform = platform
 	orchFlavor = flavor
@@ -341,11 +341,10 @@ func (c *configHelper) BackupAll() {
 func restoreEP(ep *cfgEndpoint, ch chan<- error, importInfo *fedRulesRevInfo) error {
 	var rc error
 
+	ep.backupReEncrypted = false
 	txn := cluster.Transact()
 	if rc = ep.restore(importInfo, txn); rc == errDone {
 		rc = nil
-	} else if rc != nil {
-		log.WithFields(log.Fields{"endpoint": ep.name, "rc": rc}).Error()
 	}
 	applyTransaction(txn, nil, false, 0)
 
@@ -374,7 +373,7 @@ func restoreEPs(eps utils.Set, ch chan error, importInfo *fedRulesRevInfo) error
 	return err
 }
 
-func (c *configHelper) Restore() (string, bool, bool, string, error) {
+func (c *configHelper) Restore(host share.CLUSHost, ctrler share.CLUSController) (string, bool, bool, string, error) {
 	log.Info()
 
 	if !c.persist {
@@ -425,6 +424,9 @@ func (c *configHelper) Restore() (string, bool, bool, string, error) {
 	var err error
 	ch := make(chan error)
 	eps := utils.NewSetFromSliceKind(cfgEndpoints)
+	for _, ep := range cfgEndpoints {
+		ep.errRestoreKeys = utils.NewSet()
+	}
 
 	// restore federation endpoint
 	if rc := restoreEP(fedCfgEndpoint, nil, &importInfo); rc != nil {
@@ -451,6 +453,47 @@ func (c *configHelper) Restore() (string, bool, bool, string, error) {
 
 	go restoreRegistry(ch, importInfo)
 
+	epsReEncrypted := make([]string, 0, len(cfgEndpoints))
+	epsMismatchedKey := make([]string, 0, len(cfgEndpoints))
+	for _, ep := range cfgEndpoints {
+		if ep.errRestoreKeys.Cardinality() > 0 {
+			epsMismatchedKey = append(epsMismatchedKey, ep.errRestoreKeys.ToStringSlice()...)
+			ep.errRestoreKeys = nil
+		} else if ep.backupReEncrypted {
+			epsReEncrypted = append(epsReEncrypted, ep.name)
+			ep.backupReEncrypted = false
+		}
+	}
+	log.WithFields(log.Fields{"names": epsReEncrypted}).Info("re-encrypt")
+	messages := map[share.TLogEvent]string{
+		share.CLUSEvMismatchedDEKSeed: fmt.Sprintf("Sensitive data in these kv keys cannot be restored because of decryption failure:\n%s",
+			strings.Join(epsMismatchedKey, ", ")),
+		share.CLUSEvReEncryptWithDEK: fmt.Sprintf("Sensitive data in the backup file for these endpoint(s) are re-encrypted by DEK:\n%s",
+			strings.Join(epsReEncrypted, ", ")),
+	}
+	for evt, eps := range map[share.TLogEvent][]string{share.CLUSEvMismatchedDEKSeed: epsMismatchedKey} {
+		if len(eps) > 0 {
+			clog := share.CLUSEventLog{
+				Event:          evt,
+				HostID:         host.ID,
+				HostName:       host.Name,
+				ControllerID:   ctrler.ID,
+				ControllerName: ctrler.Name,
+				ReportedAt:     time.Now().UTC(),
+				Msg:            messages[evt],
+			}
+			if err := evqueue.Append(&clog); err != nil {
+				log.WithFields(log.Fields{"error": err, "msg": clog.Msg}).Error("evqueue.Append")
+			}
+		}
+	}
+
+	c.cfgMutex.Lock()
+	for _, ep := range epsReEncrypted {
+		c.cfgChanged.Add(ep)
+	}
+	c.cfgMutex.Unlock()
+
 	ver := getBackupVersion()
 	_ = putControlVersion(&ver)
 	log.WithFields(log.Fields{"version": ver}).Info("Done")
@@ -475,6 +518,7 @@ type configHeader struct {
 	CreatedAt        string   `json:"created_at"`
 	Sections         []string `json:"sections"`
 	ExportedFromRole string   `json:"exported_from_role"`
+	DEKEncrypted     string   `json:"dek_encrypted"`
 }
 
 func getFedRole() (string, *share.CLUSFedRulesRevision) {
@@ -508,6 +552,7 @@ func (c *configHelper) Export(w *bufio.Writer, sections utils.Set) error {
 			sections.Add(ep.section)
 		}
 	}
+
 	// Fill header.Sections with order
 	added := utils.NewSet()
 	for _, ep := range cfgEndpoints {
@@ -783,8 +828,15 @@ func (c *configHelper) importInternal(rpcEps []*common.RPCEndpoint, localCtrlerI
 			// Value can be empty if a key was never been written when it's exported. No need to
 			// write empty string because keys have been purged.
 			if len(value) != 0 {
-				array, err := upgrade(key, []byte(value))
-				if err != nil {
+				array, _, failToDecryptFields, err := upgrade(key, []byte(value))
+				if failToDecryptFields != nil && failToDecryptFields.Cardinality() > 0 {
+					log.WithFields(log.Fields{"error": err, "key": key, "fields": failToDecryptFields}).Error("Failed to decrypt fields")
+					if importTask.FailToDecryptKeyFields == nil {
+						importTask.FailToDecryptKeyFields = make(map[string][]string)
+					}
+					fields := importTask.FailToDecryptKeyFields[key]
+					importTask.FailToDecryptKeyFields[key] = append(fields, failToDecryptFields.ToStringSlice()...)
+				} else if err != nil {
 					log.WithFields(log.Fields{"error": err, "key": key, "value": value}).Error("Failed to upgrade key/value")
 					return ErrInvalidFileFormat
 				}

@@ -25,12 +25,14 @@ import (
 type purgeFilterFunc func(epName, key string) bool
 
 type cfgEndpoint struct {
-	name        string
-	key         string
-	section     string
-	lock        string
-	isStore     bool
-	purgeFilter purgeFilterFunc
+	name              string
+	key               string
+	section           string
+	lock              string
+	isStore           bool
+	backupReEncrypted bool
+	errRestoreKeys    utils.Set
+	purgeFilter       purgeFilterFunc
 }
 
 const (
@@ -383,7 +385,7 @@ func (ep cfgEndpoint) backup(fedRole string) error {
 }
 
 // value of each key in the file is always in text format (i.e. non-gzip format). Compress it if it's >= 512k before restoring to kv
-func (ep cfgEndpoint) restore(importInfo *fedRulesRevInfo, txn *cluster.ClusterTransact) error {
+func (ep *cfgEndpoint) restore(importInfo *fedRulesRevInfo, txn *cluster.ClusterTransact) error {
 	fedEndpointCfg := false
 	source := ep.getBackupFilename()
 	if ep.name == share.CFGEndpointFederation {
@@ -444,6 +446,8 @@ func (ep cfgEndpoint) restore(importInfo *fedRulesRevInfo, txn *cluster.ClusterT
 
 	// Restore key/value from files
 	count := 0
+
+	log.WithFields(log.Fields{"name": ep.name}).Info()
 	r := bufio.NewReader(f)
 	policyZipRuleListKey := share.CLUSPolicyZipRuleListKey(share.DefaultPolicyName)
 	for {
@@ -537,11 +541,17 @@ func (ep cfgEndpoint) restore(importInfo *fedRulesRevInfo, txn *cluster.ClusterT
 
 		// Value can be empty if a key was never been written when it's exported
 		if !skip && len(value) != 0 {
-			array, err := upgrade(key, []byte(value))
-			if err != nil {
+			array, wrtForReEncrypt, failToDecryptFields, err := upgrade(key, []byte(value))
+			if failToDecryptFields != nil && failToDecryptFields.Cardinality() > 0 {
+				log.WithFields(log.Fields{"error": err, "key": key}).Error("Failed to upgrade key")
+				ep.errRestoreKeys.Add(key)
+			} else if err != nil {
 				log.WithFields(log.Fields{"error": err, "key": key, "value": value}).Error("Failed to upgrade key/value")
 				return ErrInvalidFileFormat
 			}
+			if wrtForReEncrypt {
+				ep.backupReEncrypted = true
+			}
 			if key == policyZipRuleListKey {
 				applyTransaction(txn, nil, false, 0)
 				//zip rulelist before put to cluster during restore

@@ -316,26 +316,44 @@ var (
 )
 
 type clusterHelper struct {
-	id      string
-	version string
-	persist bool
+	id                  string
+	version             string
+	persist             bool
+	keyRotationDuration time.Duration
 }
 
 var clusHelperImpl *clusterHelper
 var clusHelper ClusterHelper
 
-func newClusterHelper(id, version string, persist bool) ClusterHelper {
+func newClusterHelper(id, version string, persist bool, keyRotationDuration time.Duration) ClusterHelper {
 	clusHelperImpl = new(clusterHelper)
 	clusHelperImpl.id = id
 	clusHelperImpl.version = version
 	clusHelperImpl.persist = persist
+	clusHelperImpl.keyRotationDuration = keyRotationDuration
 	return clusHelperImpl
 }
 
 func GetClusterHelper() ClusterHelper {
 	return clusHelperImpl
 }
 
+func SetNextKeyRotationTime(keyRotationDuration time.Duration) error {
+	var err error
+	var nextRotationTime int64
+
+	key := share.CLUSNextKeyRotationTSKey
+	nextRotationTime = time.Now().Add(keyRotationDuration).Unix()
+	nextValue := fmt.Sprintf("%d", nextRotationTime)
+	if err = cluster.Put(key, []byte(nextValue)); err != nil {
+		log.WithFields(log.Fields{"err": err}).Error()
+	} else {
+		log.WithFields(log.Fields{"nextRotationTime": nextRotationTime}).Info()
+	}
+
+	return err
+}
+
 func nvJsonUnmarshal(key string, data []byte, v any) error {
 	var err error
 
@@ -351,6 +369,59 @@ func nvJsonUnmarshal(key string, data []byte, v any) error {
 	return err
 }
 
+// for switching from fixed default key to variant DEK.
+// [1]. If there is cloaked-by-fixed-default-key field in the json object represented by parameter 'data':
+// 1-1. 'data' is unmarshalled to 'obj' and cloaked-by-fixed-default-key fields are decrypted with the fixed default key
+// 1-2. 'obj' is marshalled to 'data' and sensitive fields are encrypted with variant DEK (no change on 'obj')
+// 1-3. 'data' is json unmarshalled to 'obj' (i.e. keep sensitive fields cloaked-by-DEK in 'obj')
+// [2]. If no field is cloaked by the fixed default key in the json object represented by parameter 'data', this function is equivalent to nvJsonUnmarshal()
+//
+// parameters:
+// 1. key:  kv key
+// 2. data: kv value
+// 3. obj:  pointer of the object for json unmarshalling.
+// when this function returns, sensitive fields in 'obj' are encrypted by DEK
+func nvJsonUnmarshalReEncrypt(key string, data []byte, obj any) (bool, utils.Set, error) {
+	if !common.IsDEKSeedAvailable() {
+		_ = nvJsonUnmarshal(key, data, obj)
+		return false, nil, nil
+	}
+
+	if obj == nil {
+		err := fmt.Errorf("nil target")
+		log.WithFields(log.Fields{"error": err, "key": key}).Error()
+		return false, nil, err
+	}
+
+	var dec common.MigrateDecryptUnmarshaller
+
+	err := dec.Unmarshal(data, obj)
+	if err == nil {
+		// it reaches here because dec.Unmarshal() return nil error & any of following conditions:
+		// 1. there is no sensitive field in 'obj'. No re-encryption required.
+		// 2. there is any sensitive field encrypted by the fixed default key in 'obj' that 'obj' need to be re-encrypted by DEK
+		// 3. all sensitive fields are encrypted with a DEK and can be DEK-decypted in 'obj'.
+		if dec.ReEncryptRequired { // condition #2
+			// data re-encryption is required because of switching from the fixed default key to variant DEK
+			// notice: when this function returns, the sensitive fields in 'obj' are encrypted by DEK
+			var dataReEncrypted []byte
+			var enc common.EncryptMarshaller
+			if dataReEncrypted, err = enc.Marshal(obj); err == nil {
+				if err = nvJsonUnmarshal(key, dataReEncrypted, obj); err == nil {
+					return true, dec.GetFailToDecryptFields(), nil
+				}
+			} else {
+				log.WithFields(log.Fields{"error": err, "key": key}).Error("re-encrypt object failed")
+			}
+		}
+	} else {
+		log.WithFields(log.Fields{"error": err, "key": key}).Error("dec.Unmarshal")
+	}
+	err = nvJsonUnmarshal(key, data, obj)
+
+	return false, dec.GetFailToDecryptFields(), err
+}
+
 func getAllSubKeys(scope, store string) utils.Set {
 	groups := utils.NewSet()
 
@@ -451,6 +522,7 @@ func (m clusterHelper) get(key string) ([]byte, uint64, error) {
 	} else {
 		var wrt bool
 		if value, err, wrt = UpgradeAndConvert(key, value); wrt {
+			// wrt being true means value is out-of-date & it needs to get from kv again
 			value, rev, err = cluster.GetRev(key)
 			// [31, 139] is the first 2 bytes of gzip-format data
 			if needToUnzip(key, value) {
@@ -571,10 +643,15 @@ func (m clusterHelper) GetOrCreateInstallationID() (string, error) {
 			return err
 		}
 
+		if value, _ := cluster.Get(share.CLUSNextKeyRotationTSKey); len(value) == 0 {
+			_ = SetNextKeyRotationTime(m.keyRotationDuration)
+		}
+
 		if err = cluster.PutRev(key, []byte(id), index); err != nil {
 			// Return the error. CASError will be automatically retried.
 			return err
 		}
+
 		return nil
 	}); err != nil {
 		return "", err
@@ -761,13 +838,13 @@ func (m clusterHelper) GetDomain(name string, acc *access.AccessControl) (*share
 
 func (m clusterHelper) PutDomainIfNotExist(domain *share.CLUSDomain) error {
 	key := share.CLUSDomainKey(domain.Name)
-	value, _ := enc.Marshal(domain)
+	value, _ := json.Marshal(domain)
 	return cluster.PutIfNotExist(key, value, true)
 }
 
 func (m clusterHelper) PutDomain(domain *share.CLUSDomain, rev *uint64) error {
 	key := share.CLUSDomainKey(domain.Name)
-	value, _ := enc.Marshal(domain)
+	value, _ := json.Marshal(domain)
 	if rev == nil {
 		return cluster.Put(key, value)
 	} else {
@@ -1030,19 +1107,19 @@ func (m clusterHelper) DeletePolicyRuleTxn(txn *cluster.ClusterTransact, id uint
 
 func (m clusterHelper) PutPolicyVer(s *share.CLUSGroupIPPolicyVer) error {
 	key := share.CLUSPolicyIPRulesKey(s.Key)
-	value, _ := enc.Marshal(s)
+	value, _ := json.Marshal(s)
 	return cluster.Put(key, value)
 }
 
 func (m clusterHelper) PutPolicyVerNode(s *share.CLUSGroupIPPolicyVer) error {
 	key := share.CLUSPolicyIPRulesKeyNode(s.Key, s.NodeId)
-	value, _ := enc.Marshal(s)
+	value, _ := json.Marshal(s)
 	return cluster.Put(key, value)
 }
 
 func (m clusterHelper) PutDlpVer(s *share.CLUSDlpRuleVer) error {
 	key := share.CLUSDlpWorkloadRulesKey(s.Key)
-	value, _ := enc.Marshal(s)
+	value, _ := json.Marshal(s)
 	return cluster.Put(key, value)
 }
 
@@ -1351,7 +1428,7 @@ func (m clusterHelper) GetAllProcessProfileSubKeys(scope string) utils.Set {
 // Scanner
 func (m clusterHelper) PutScannerTxn(txn *cluster.ClusterTransact, s *share.CLUSScanner) error {
 	key := share.CLUSScannerKey(s.ID)
-	value, err := enc.Marshal(s)
+	value, err := json.Marshal(s)
 	if err != nil {
 		return err
 	}
@@ -2151,8 +2228,11 @@ func (m clusterHelper) GetAdmissionCertRev(svcName string) (*share.CLUSAdmission
 
 func (m clusterHelper) GetObjectCertRev(cn string) (*share.CLUSX509Cert, uint64, error) {
 	key := share.CLUSObjectCertKey(cn)
-	value, rev, err := cluster.GetRev(key)
-	if err != nil || value == nil {
+	value, rev, err := m.get(key)
+	if err == nil && value == nil {
+		err = fmt.Errorf("cert %s is not set", key)
+	}
+	if err != nil {
 		log.WithFields(log.Fields{"cn": cn, "error": err}).Error()
 		return nil, rev, err
 	} else {

@@ -412,9 +412,24 @@ func upgradeAdmissionCert(value []byte) (*share.CLUSAdmissionCertCloaked, bool,
 	return nil, false, false
 }
 
-func doUpgrade(key string, value []byte) (interface{}, bool) {
+// This is called when
+// 1. restore persisted config into kv store
+// 2. import configurations into kv store
+// 3. read from kv store
+// 4. get notified by kv changes.
+//
+// returned values:
+// #1. pointer to the upgraded object(sensitive fields in 'obj' are encrypted by DEK). nil if no upgrade happened
+// #2. true to advise caller to write the upgraded object back to data source
+// #3. true to advise caller to write the re-marshalled(because of using DEK) object back to data source
+// #4. error: ErrMismatchedKey or other error
+func doUpgrade(key string, value []byte) (interface{}, bool, bool, utils.Set, error) {
 	object := share.CLUSObjectKey2Object(key)
 
+	var err error
+	var wrtForReEncrypt bool
+	var failToDecryptFields utils.Set
+
 	switch object {
 	case "config":
 		config := share.CLUSConfigKey2Config(key)
@@ -423,71 +438,74 @@ func doUpgrade(key string, value []byte) (interface{}, bool) {
 		case share.CFGEndpointUser:
 			var user share.CLUSUser
 			_ = nvJsonUnmarshal(key, value, &user)
-			if upd, wrt := upgradeUser(&user); upd {
-				return &user, wrt
+			if upd, wrtForUpgrade := upgradeUser(&user); upd {
+				return &user, wrtForUpgrade, false, nil, nil
 			}
 		case share.CFGEndpointSystem:
 			var cfg share.CLUSSystemConfig
-			_ = nvJsonUnmarshal(key, value, &cfg)
-			if upd, wrt := upgradeSystemConfig(&cfg); upd {
-				return &cfg, wrt
+			if wrtForReEncrypt, failToDecryptFields, err = nvJsonUnmarshalReEncrypt(key, value, &cfg); err == nil {
+				if upd, wrtForUpgrade := upgradeSystemConfig(&cfg); upd || wrtForReEncrypt {
+					return &cfg, wrtForUpgrade, wrtForReEncrypt, failToDecryptFields, nil
+				}
 			}
 		case share.CFGEndpointServer:
 			var cfg share.CLUSServer
-			_ = nvJsonUnmarshal(key, value, &cfg)
-			if upd, wrt := upgradeServer(&cfg); upd {
-				return &cfg, wrt
+			if wrtForReEncrypt, failToDecryptFields, err = nvJsonUnmarshalReEncrypt(key, value, &cfg); err == nil {
+				if upd, wrtForUpgrade := upgradeServer(&cfg); upd || wrtForReEncrypt {
+					return &cfg, wrtForUpgrade, wrtForReEncrypt, failToDecryptFields, nil
+				}
 			}
 		case share.CFGEndpointGroup:
 			var cfg share.CLUSGroup
 			_ = nvJsonUnmarshal(key, value, &cfg)
-			if upd, wrt := upgradeGroup(&cfg); upd {
-				return &cfg, wrt
+			if upd, wrtForUpgrade := upgradeGroup(&cfg); upd {
+				return &cfg, wrtForUpgrade, false, nil, nil
 			}
 		case share.CFGEndpointPolicy:
 			if share.CLUSIsPolicyRuleKey(key) {
 				var cfg share.CLUSPolicyRule
 				_ = nvJsonUnmarshal(key, value, &cfg)
-				if upd, wrt := upgradePolicyRule(&cfg); upd {
-					return &cfg, wrt
+				if upd, wrtForUpgrade := upgradePolicyRule(&cfg); upd {
+					return &cfg, wrtForUpgrade, false, nil, nil
 				}
 			} else if share.CLUSIsPolicyZipRuleListKey(key) {
 				var cfg []*share.CLUSRuleHead
 				_ = nvJsonUnmarshal(key, value, &cfg)
-				if upd, wrt := upgradePolicyRuleHead(cfg); upd {
-					return &cfg, wrt
+				if upd, wrtForUpgrade := upgradePolicyRuleHead(cfg); upd {
+					return &cfg, wrtForUpgrade, false, nil, nil
 				}
 
 			}
 		case share.CFGEndpointRegistry:
 			var cfg share.CLUSRegistryConfig
-			_ = nvJsonUnmarshal(key, value, &cfg)
-			if upd, wrt := upgradeRegistry(&cfg); upd {
-				return &cfg, wrt
+			if wrtForReEncrypt, failToDecryptFields, err = nvJsonUnmarshalReEncrypt(key, value, &cfg); err == nil {
+				if upd, wrtForUpgrade := upgradeRegistry(&cfg); upd || wrtForReEncrypt {
+					return &cfg, wrtForUpgrade, wrtForReEncrypt, failToDecryptFields, nil
+				}
 			}
 		case share.CFGEndpointProcessProfile:
 			var cfg share.CLUSProcessProfile
 			_ = nvJsonUnmarshal(key, value, &cfg)
-			if upd, wrt := upgradeProcessProfile(&cfg); upd {
-				return &cfg, wrt
+			if upd, wrtForUpgrade := upgradeProcessProfile(&cfg); upd {
+				return &cfg, wrtForUpgrade, false, nil, nil
 			}
 		case share.CFGEndpointFileMonitor:
 			var cfg share.CLUSFileMonitorProfile
 			_ = nvJsonUnmarshal(key, value, &cfg)
-			if upd, wrt := upgradeFileMonitorProfile(&cfg); upd {
-				return &cfg, wrt
+			if upd, wrtForUpgrade := upgradeFileMonitorProfile(&cfg); upd {
+				return &cfg, wrtForUpgrade, false, nil, nil
 			}
 		case share.CFGEndpointDlpGroup:
 			var cfg share.CLUSDlpGroup
 			_ = nvJsonUnmarshal(key, value, &cfg)
-			if upd, wrt := upgradeDlpGroup(&cfg); upd {
-				return &cfg, wrt
+			if upd, wrtForUpgrade := upgradeDlpGroup(&cfg); upd {
+				return &cfg, wrtForUpgrade, false, nil, nil
 			}
 		case share.CFGEndpointDlpRule:
 			var cfg share.CLUSDlpSensor
 			_ = nvJsonUnmarshal(key, value, &cfg)
-			if upd, wrt := upgradeDlpSensor(&cfg); upd {
-				return &cfg, wrt
+			if upd, wrtForUpgrade := upgradeDlpSensor(&cfg); upd {
+				return &cfg, wrtForUpgrade, false, nil, nil
 			}
 		case share.CFGEndpointAdmissionControl, share.CFGEndpointCrd:
 			scope := share.CLUSPolicyKey2AdmCfgPolicySubkey(key, false)
@@ -496,53 +514,62 @@ func doUpgrade(key string, value []byte) (interface{}, bool) {
 				if token == share.CLUSAdmissionCfgState {
 					var state share.CLUSAdmissionState
 					_ = nvJsonUnmarshal(key, value, &state)
-					if upd, wrt := upgradeAdmCtrlState(config, &state); upd {
-						return &state, wrt
+					if upd, wrtForUpgrade := upgradeAdmCtrlState(config, &state); upd {
+						return &state, wrtForUpgrade, false, nil, nil
 					}
 				} else {
 					if config == share.CFGEndpointAdmissionControl {
 						if token == share.CLUSAdmissionCfgRule {
 							var rule share.CLUSAdmissionRule
 							_ = nvJsonUnmarshal(key, value, &rule)
-							if upd, wrt := upgradeAdmCtrlRule(&rule); upd {
-								return &rule, wrt
+							if upd, wrtForUpgrade := upgradeAdmCtrlRule(&rule); upd {
+								return &rule, wrtForUpgrade, false, nil, nil
 							}
 						} else if token == share.CLUSAdmissionCfgRuleList {
 							var cfg []*share.CLUSRuleHead
 							_ = nvJsonUnmarshal(key, value, &cfg)
-							if upd, wrt := upgradeRuleHead(cfg); upd {
-								return &cfg, wrt
+							if upd, wrtForUpgrade := upgradeRuleHead(cfg); upd {
+								return &cfg, wrtForUpgrade, false, nil, nil
 							}
 						} else if token == share.CLUSAdmissionCfgCert {
 							var cert share.CLUSAdmissionCertCloaked
-							err := dec.Unmarshal(value, &cert)
-							if err != nil || !cert.Cloaked {
+							var dec common.MigrateDecryptUnmarshaller
+							err2 := dec.Unmarshal(value, &cert)
+							if err2 != nil || !cert.Cloaked {
 								if cert, upd, wrt := upgradeAdmissionCert(value); upd {
-									return cert, wrt
+									return cert, wrt, false, nil, nil
+								}
+							} else if dec.ReEncryptRequired {
+								if dataReEncrypted, err2 := enc.Marshal(&cert); err2 == nil {
+									_ = nvJsonUnmarshal(key, dataReEncrypted, &cert)
+									return &cert, false, true, nil, nil
+								} else {
+									log.WithFields(log.Fields{"error": err2, "key": key}).Error("re-encrypt object failed")
 								}
 							}
+							err = err2
 						}
 					}
 				}
 			} else if config == share.CFGEndpointCrd && scope == resource.NvSecurityRuleKind {
 				var cfg share.CLUSCrdSecurityRule
 				_ = nvJsonUnmarshal(key, value, &cfg)
-				if upd, wrt := upgradeCrdSecurityRule(&cfg); upd {
-					return &cfg, wrt
+				if upd, wrtForUpgrade := upgradeCrdSecurityRule(&cfg); upd {
+					return &cfg, wrtForUpgrade, false, nil, nil
 				}
 			}
 		case share.CFGEndpointResponseRule:
 			if share.CLUSIsPolicyRuleKey(key) {
 				var cfg share.CLUSResponseRule
 				_ = nvJsonUnmarshal(key, value, &cfg)
-				if upd, wrt := upgradeResponseRule(&cfg); upd {
-					return &cfg, wrt
+				if upd, wrtForUpgrade := upgradeResponseRule(&cfg); upd {
+					return &cfg, wrtForUpgrade, false, nil, nil
 				}
 			} else if share.CLUSIsPolicyRuleListKey(key) {
 				var cfg []*share.CLUSRuleHead
 				_ = nvJsonUnmarshal(key, value, &cfg)
-				if upd, wrt := upgradeRuleHead(cfg); upd {
-					return &cfg, wrt
+				if upd, wrtForUpgrade := upgradeRuleHead(cfg); upd {
+					return &cfg, wrtForUpgrade, false, nil, nil
 				}
 			}
 		case share.CFGEndpointVulnerability:
@@ -555,7 +582,7 @@ func doUpgrade(key string, value []byte) (interface{}, bool) {
 						upd = true
 					}
 					if upd {
-						return &cfg, upd
+						return &cfg, upd, false, nil, nil
 					}
 				}
 			}
@@ -569,28 +596,47 @@ func doUpgrade(key string, value []byte) (interface{}, bool) {
 						upd = true
 					}
 					if upd {
-						return &cfg, upd
+						return &cfg, upd, false, nil, nil
 					}
 				}
 			}
+		case share.CFGEndpointFederation:
+			if key == share.CLUSFedKey(share.CLUSFedMembershipSubKey) {
+				var cfg share.CLUSFedMembership
+				if wrtForReEncrypt, failToDecryptFields, err = nvJsonUnmarshalReEncrypt(key, value, &cfg); err == nil && wrtForReEncrypt {
+					return &cfg, false, wrtForReEncrypt, failToDecryptFields, nil
+				}
+			} else if share.CLUSFedKey2CfgKey(key) == share.CLUSFedClustersSubKey {
+				var cfg share.CLUSFedJointClusterInfo
+				if wrtForReEncrypt, failToDecryptFields, err = nvJsonUnmarshalReEncrypt(key, value, &cfg); err == nil && wrtForReEncrypt {
+					return &cfg, false, wrtForReEncrypt, failToDecryptFields, nil
+				}
+			}
+		}
+	case "cert":
+		var cert share.CLUSX509Cert
+		if wrtForReEncrypt, failToDecryptFields, err = nvJsonUnmarshalReEncrypt(key, value, &cert); err == nil && wrtForReEncrypt {
+			return &cert, false, wrtForReEncrypt, failToDecryptFields, nil
 		}
 	}
 
-	return nil, false
+	return nil, false, false, failToDecryptFields, err
 }
 
-// This is called when we restore the persisted config into kv store
-func upgrade(key string, value []byte) ([]byte, error) {
+// This is called when we restore the persisted config or import configurations into kv store
+func upgrade(key string, value []byte) ([]byte, bool, utils.Set, error) {
 	if len(value) == 0 {
-		return value, nil
+		return value, false, nil, nil
 	}
 
-	v, _ := doUpgrade(key, value)
+	v, _, wrtForReEncrypt, failToDecryptFields, err := doUpgrade(key, value)
 	if v == nil {
-		return value, nil
+		return value, false, failToDecryptFields, err
 	}
 
-	return json.Marshal(v)
+	data, err := json.Marshal(v)
+
+	return data, wrtForReEncrypt, failToDecryptFields, err
 }
 
 // This is called whenever we read from kv store or get notified by kv changes.
@@ -599,8 +645,11 @@ func UpgradeAndConvert(key string, value []byte) ([]byte, error, bool) {
 		return value, nil, false
 	}
 	var v interface{}
-	var wrt bool
+	var wrtForUpgrade bool
+	var wrtForReEncrypt bool
 	var policyListKey bool
+	var needToUncloak bool
+	var failToDecryptFields utils.Set
 
 	if key == share.CLUSPolicyZipRuleListKey(share.DefaultPolicyName) {
 		policyListKey = true
@@ -612,7 +661,10 @@ func UpgradeAndConvert(key string, value []byte) ([]byte, error, bool) {
 			return value, nil, false
 		}
 	}
-	v, wrt = doUpgrade(key, value)
+	v, wrtForUpgrade, wrtForReEncrypt, failToDecryptFields, _ = doUpgrade(key, value)
+	// v being nil means no need to upgrade/convert the obj represented by value at all
+	wrt := wrtForUpgrade || wrtForReEncrypt
+	// wrt means need to write v to kv or not (sensitive fields in v are still encrypted)
 
 	if v != nil && wrt {
 		var err error
@@ -624,6 +676,23 @@ func UpgradeAndConvert(key string, value []byte) ([]byte, error, bool) {
 		} else {
 			// Write back to the cluster if needed.
 			err = cluster.Put(key, newv)
+			if err == nil && wrtForReEncrypt && key == share.CLUSConfigSystemKey && (failToDecryptFields == nil || failToDecryptFields.Cardinality() == 0) {
+				if cfg, ok := v.(*share.CLUSSystemConfig); ok && cfg != nil {
+					cfgBackup := share.CLUSSystemConfigEncMigrated{
+						EncMigratedSystemConfig: string(newv),
+						EncDataExpirationTime:   time.Now().UTC().Add(time.Hour),
+					}
+					value, err := json.Marshal(&cfgBackup)
+					if err == nil {
+						if err = cluster.PutBinary(share.CLUSSystemEncMigratedKey, value); err == nil {
+							log.Info("backup encryption-migrated config")
+						}
+					}
+					if err != nil {
+						log.WithFields(log.Fields{"err": err}).Error("failed to backup encryption-migrated config")
+					}
+				}
+			}
 		}
 		if err != nil {
 			log.WithFields(log.Fields{"key": key}).Error(err)
@@ -646,9 +715,7 @@ func UpgradeAndConvert(key string, value []byte) ([]byte, error, bool) {
 					_ = nvJsonUnmarshal(key, value, &r)
 					v = &r
 				}
-				if err := dec.Uncloak(v); err != nil {
-					log.WithFields(log.Fields{"err": err, "key": key}).Error("Uncloak")
-				}
+				needToUncloak = true
 			}
 		}
 	case "config":
@@ -661,37 +728,36 @@ func UpgradeAndConvert(key string, value []byte) ([]byte, error, bool) {
 				_ = nvJsonUnmarshal(key, value, &cfg)
 				v = &cfg
 			}
-			if err := dec.Uncloak(v); err != nil {
-				log.WithFields(log.Fields{"err": err, "key": key}).Error("Uncloak")
-			}
+			needToUncloak = true
 		case share.CFGEndpointServer:
 			if v == nil {
 				var cfg share.CLUSServer
 				_ = nvJsonUnmarshal(key, value, &cfg)
 				v = &cfg
 			}
-			if err := dec.Uncloak(v); err != nil {
-				log.WithFields(log.Fields{"err": err, "key": key}).Error("Uncloak")
-			}
+			needToUncloak = true
 		case share.CFGEndpointRegistry:
 			if v == nil {
 				var cfg share.CLUSRegistryConfig
 				_ = nvJsonUnmarshal(key, value, &cfg)
 				v = &cfg
 			}
-			if err := dec.Uncloak(v); err != nil {
-				log.WithFields(log.Fields{"err": err, "key": key}).Error("Uncloak")
-			}
+			needToUncloak = true
 		case share.CFGEndpointCloud:
 			if v == nil {
 				// Currently the only data structure
 				var cfg share.CLUSAwsProjectCfg
 				_ = nvJsonUnmarshal(key, value, &cfg)
 				v = &cfg
 			}
-			if err := dec.Uncloak(v); err != nil {
-				log.WithFields(log.Fields{"err": err, "key": key}).Error("Uncloak")
-			}
+			needToUncloak = true
+		}
+	}
+
+	if v != nil && needToUncloak {
+		// sensitive fields in v are still encrypted
+		if err := dec.Uncloak(v); err != nil {
+			log.WithFields(log.Fields{"err": err, "key": key}).Error("Uncloak")
 		}
 	}
 

@@ -154,7 +154,7 @@ func (c *orchConn) cbResourceWatcher(rt string, event string, res interface{}, o
 		}
 	default:
 		logFields := log.Fields{"event": event, "type": rt}
-		if rt != resource.RscTypeConfigMap {
+		if rt != resource.RscTypeConfigMap && rt != resource.RscTypeSecret {
 			logFields["object"] = res
 		}
 		k8sResLog.WithFields(logFields).Debug("Event received")
@@ -203,11 +203,14 @@ func (c *orchConn) Start(ocImageRegistered bool, cspType share.TCspType) {
 		resource.RscTypeValidatingWebhookConfiguration, resource.RscTypePersistentVolumeClaim, resource.RscTypeConfigMap}
 	for _, r := range rscTypes {
 		if err := global.ORCH.StartWatchResource(r, k8s.AllNamespaces, c.cbResourceWatcher, nil); err != nil {
-			log.WithFields(log.Fields{"error": err}).Error("StartWatchResource")
+			log.WithFields(log.Fields{"type": r, "error": err}).Error("StartWatchResource")
 		}
 	}
-	if err := global.ORCH.StartWatchResource(resource.RscTypeDeployment, Ctrler.Domain, c.cbResourceWatcher, nil); err != nil {
-		log.WithFields(log.Fields{"error": err}).Error("StartWatchResource")
+
+	for _, r := range []string{resource.RscTypeDeployment, resource.RscTypeSecret} {
+		if err := global.ORCH.StartWatchResource(r, Ctrler.Domain, c.cbResourceWatcher, nil); err != nil {
+			log.WithFields(log.Fields{"type": r, "error": err}).Error("StartWatchResource")
+		}
 	}
 
 	rscTypes = []string{

@@ -1975,8 +1975,7 @@ func GetSaFromJwtToken(tokenStr string) (string, error) {
 	return sa, err
 }
 
-func GetNvCtrlerServiceAccount(objFunc common.CacheEventFunc) {
-	cacheEventFunc = objFunc
+func GetNvCtrlerServiceAccount() {
 	// controller pod runs as "controller" sa if it's deployed with least privilge enabled
 	filePath := "/var/run/secrets/kubernetes.io/serviceaccount/token"
 	if data, err := os.ReadFile(filePath); err == nil {
@@ -1994,6 +1993,10 @@ func GetNvCtrlerServiceAccount(objFunc common.CacheEventFunc) {
 	log.WithFields(log.Fields{"nvControllerSA": ctrlerSubjectWanted}).Info()
 }
 
+func SetCacheEventFunc(objFunc common.CacheEventFunc) {
+	cacheEventFunc = objFunc
+}
+
 func getSubjectsString(ns string, subjects []string) string {
 	subjectSet := utils.NewSet()
 	fullSubjects := make([]string, 0, len(subjects))

@@ -133,6 +133,7 @@ const (
 	nvCspUsageRoleBinding = nvCspUsageRole
 
 	nvBootstrapSecret = "neuvector-bootstrap-secret"
+	nvStoreSecret     = "neuvector-store-secret"
 
 	secretKeyResetByNV         = "resetByNV"
 	secretKeyBootstrapPassword = "bootstrapPassword"
@@ -707,7 +708,7 @@ var resourceMakers map[string]k8sResource = map[string]k8sResource{
 				"v1",
 				func() metav1.Object { return new(corev1.Secret) },
 				func() metav1.ListInterface { return new(corev1.SecretList) },
-				nil, // xlateSecret,
+				xlateSecret,
 				nil,
 			},
 		},
@@ -1204,6 +1205,10 @@ func xlateConfigMap(obj metav1.Object) (string, interface{}) {
 	return "", nil
 }
 
+func xlateSecret(obj metav1.Object) (string, interface{}) {
+	return string(obj.GetUID()), obj
+}
+
 // func xlateMutatingWebhookConfiguration(obj metav1.Object) (string, interface{}) {
 // 	var name string
 // 	var guid string
@@ -2243,27 +2248,29 @@ func xlatePersistentVolumeClaim(obj metav1.Object) (string, interface{}) {
 	return "", nil
 }
 
-func retrieveSecretData(secretName, key string) ([]byte, bool, error) {
-	var objSecret *corev1.Secret
-	var foundSecret bool
+func retrieveSecretData(secretName, key string) ([]byte, map[string][]byte, string, bool, error) {
 	obj, err := global.ORCH.GetResource(RscTypeSecret, NvAdmSvcNamespace, secretName)
 	if obj != nil && err == nil {
-		objSecret, foundSecret = obj.(*corev1.Secret)
+		objSecret, foundSecret := obj.(*corev1.Secret)
 		if foundSecret && objSecret != nil {
 			if objSecret.Data != nil {
-				if v, ok := objSecret.Data[key]; ok {
-					return v, foundSecret, nil
+				if key != "" {
+					if v, ok := objSecret.Data[key]; ok {
+						return v, nil, "", foundSecret, nil
+					}
+				} else {
+					return nil, objSecret.Data, objSecret.ResourceVersion, foundSecret, nil
 				}
 			}
 		} else {
-			err = errors.New("type conversion failed")
+			err = fmt.Errorf("type conversion failed")
 		}
 	}
 	if err != nil {
 		log.WithFields(log.Fields{"err": err, "secretName": secretName}).Error()
 	}
 
-	return nil, foundSecret, err
+	return nil, nil, "", false, err
 }
 
 func GenRandomString(length int) (string, error) {
@@ -2281,7 +2288,7 @@ func GenRandomString(length int) (string, error) {
 }
 
 func RetrieveBootstrapPassword() (string, error) {
-	data, foundSecret, err := retrieveSecretData(nvBootstrapSecret, secretKeyBootstrapPassword)
+	data, _, _, foundSecret, err := retrieveSecretData(nvBootstrapSecret, secretKeyBootstrapPassword)
 	if err == nil && len(data) > 0 {
 		return string(data), nil
 	}
@@ -2329,6 +2336,124 @@ func RetrieveBootstrapPassword() (string, error) {
 	return "", err
 }
 
+// Caller must acquire/lease CLUSStoreSecretKey lock befor/after calling this function
+func RetrieveStorePassphrases() (common.EncKeys, string, string, error) {
+	var err error
+	var msg string
+	var currEncKeyIdx uint64
+	var currEncKeyVer string
+
+	_, secretData, rscVersion, foundSecret, _ := retrieveSecretData(nvStoreSecret, "")
+	encKeys := make(common.EncKeys, len(secretData))
+	for k, v := range secretData {
+		if ui64, err := strconv.ParseUint(k, 10, 64); err == nil {
+			if ui64 > currEncKeyIdx {
+				currEncKeyIdx = ui64
+			}
+			encKeys[k] = v
+		}
+	}
+
+	if len(encKeys) == 0 {
+		var action string
+		randomPass := make([]byte, common.DekSeedLength)
+		if _, err := rand.Read(randomPass); err != nil {
+			return nil, "", "", fmt.Errorf("failed to generate random password: %w", err)
+		}
+		encKeys = make(common.EncKeys, 1)
+		currEncKeyIdx = 1
+		currEncKeyVer = fmt.Sprintf("%v", currEncKeyIdx)
+		encKeys[currEncKeyVer] = randomPass
+		objSecret := corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      nvStoreSecret,
+				Namespace: NvAdmSvcNamespace,
+			},
+			Data: map[string][]byte{
+				strconv.Itoa(int(currEncKeyIdx)): randomPass,
+				secretKeyResetByNV:               []byte("true"),
+			},
+			Type: corev1.SecretTypeOpaque,
+		}
+		if !foundSecret {
+			err = global.ORCH.AddResource(RscTypeSecret, &objSecret)
+			action = "created"
+		} else {
+			objSecret.ResourceVersion = rscVersion
+			err = global.ORCH.UpdateResource(RscTypeSecret, &objSecret)
+			action = "updated"
+		}
+		if err != nil {
+			log.WithFields(log.Fields{"err": err, "rscVersion": rscVersion, "foundSecret": foundSecret}).Error()
+		} else {
+			msg = fmt.Sprintf("Kubernetes secret %s is %s", nvStoreSecret, action)
+		}
+	} else {
+		err = nil
+	}
+
+	return encKeys, fmt.Sprintf("%d", currEncKeyIdx), msg, err
+}
+
+func AddStorePassphrase() error {
+	var err error
+	var currEncKeyIdx uint64
+
+	_, secretData, rscVersion, foundSecret, _ := retrieveSecretData(nvStoreSecret, "")
+	for k := range secretData {
+		if ui64, err := strconv.ParseUint(k, 10, 64); err == nil {
+			if ui64 > currEncKeyIdx {
+				currEncKeyIdx = ui64
+			}
+		}
+	}
+	if secretData == nil {
+		secretData = make(map[string][]byte, 2)
+		secretData[secretKeyResetByNV] = []byte("true")
+	}
+
+	randomPass := make([]byte, common.DekSeedLength)
+	if _, err := rand.Read(randomPass); err != nil {
+		return fmt.Errorf("failed to generate random password: %w", err)
+	}
+	currEncKeyIdx++
+	keyVersion := fmt.Sprintf("%d", currEncKeyIdx)
+	secretData[keyVersion] = randomPass
+	objSecret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      nvStoreSecret,
+			Namespace: NvAdmSvcNamespace,
+		},
+		Data: secretData,
+		Type: corev1.SecretTypeOpaque,
+	}
+	if !foundSecret {
+		err = global.ORCH.AddResource(RscTypeSecret, &objSecret)
+	} else {
+		objSecret.ResourceVersion = rscVersion
+		err = global.ORCH.UpdateResource(RscTypeSecret, &objSecret)
+	}
+	if err == nil {
+		err = common.AddAesGcmKey(keyVersion, randomPass)
+	}
+	if err != nil {
+		log.WithFields(log.Fields{"err": err, "foundSecret": foundSecret}).Error()
+	}
+
+	return err
+}
+
+func IsStoreSecretUpdatedByNV() bool {
+	secretResetByNV := false
+
+	data, _, _, found, err := retrieveSecretData(nvStoreSecret, secretKeyResetByNV)
+	if err == nil && found && string(data) == "true" {
+		secretResetByNV = true
+	}
+
+	return secretResetByNV
+}
+
 func GetNvControllerPodsNumber() {
 	var requestMemory string
 	var limitMemory string

@@ -154,6 +154,7 @@ const (
 	userTooMany
 	userKeyError
 	userNoPlatformAuth
+	userTokenGenError
 )
 
 // const (
@@ -207,7 +208,10 @@ func newLoginSessionFromUser(user *share.CLUSUser, domainRoles access.DomainRole
 	remote, mainSessionID, mainSessionUser string, sso *SsoSession) (*loginSession, int) {
 
 	// Note: JWT keys should be loaded in initJWTSignKey() before calling this function.
-	id, token, claims := jwtGenerateToken(user, domainRoles, extraDomainPermits, remote, mainSessionID, mainSessionUser, sso)
+	id, token, claims, err := jwtGenerateToken(user, domainRoles, extraDomainPermits, remote, mainSessionID, mainSessionUser, sso)
+	if err != nil {
+		return nil, userTokenGenError
+	}
 	now := user.LastLoginAt // Already updated
 	s := &loginSession{
 		id:                 id,
@@ -1279,14 +1283,16 @@ func resetFedJointKeys() {
 func setJointKeysInCache(callerFedRole string, jointCluster *share.CLUSFedJointClusterInfo) error {
 	var err error
 	var data []byte
+	var token string
 	var rsaPrivateKey *rsa.PrivateKey
 	var rsaPublicKey *rsa.PublicKey
 	if data, err = base64.StdEncoding.DecodeString(jointCluster.ClientKey); err == nil {
 		if rsaPrivateKey, err = jwt.ParseRSAPrivateKeyFromPEM(data); err == nil && rsaPrivateKey != nil {
 			if data, err = base64.StdEncoding.DecodeString(jointCluster.ClientCert); err == nil {
 				if rsaPublicKey, err = jwt.ParseRSAPublicKeyFromPEM(data); err == nil && rsaPublicKey != nil {
-					if token := jwtGenFedPingToken(callerFedRole, jointCluster.ID, jointCluster.Secret, rsaPrivateKey); token != "" {
-						if _, err := jwtValidateToken(token, jointCluster.Secret, rsaPublicKey); err == nil {
+					token, err = jwtGenFedPingToken(callerFedRole, jointCluster.ID, jointCluster.Secret, rsaPrivateKey)
+					if token != "" && err == nil {
+						if _, err = jwtValidateToken(token, jointCluster.Secret, rsaPublicKey); err == nil {
 							_setFedJointPrivateKey(jointCluster.ID, rsaPrivateKey)
 							if callerFedRole == api.FedRoleJoint {
 								_setFedJointPublicKey(rsaPublicKey)
@@ -1352,7 +1358,11 @@ func jwtValidateToken(encryptedToken, secret string, rsaPublicKey *rsa.PublicKey
 	}
 
 	if secret == "" {
-		tokenString = utils.DecryptUserToken(encryptedToken, []byte(installID))
+		if tokenString, err = utils.DecryptUserToken(encryptedToken, []byte(installID)); err != nil {
+			err = fmt.Errorf("failed to decrypt token: %w", err)
+			log.WithError(err).Error()
+			return nil, err
+		}
 	} else {
 		tokenString = utils.DecryptSensitive(encryptedToken, []byte(secret))
 	}
@@ -1447,13 +1457,13 @@ func jwtValidateFedJoinTicket(encryptedTicket, secret string) error {
 
 // permits is for Rancher SSO only
 func jwtGenerateToken(user *share.CLUSUser, domainRoles access.DomainRole, extraDomainPermits access.DomainPermissions,
-	remote, mainSessionID, mainSessionUser string, sso *SsoSession) (string, string, *tokenClaim) {
+	remote, mainSessionID, mainSessionUser string, sso *SsoSession) (string, string, *tokenClaim, error) {
 
 	id := utils.GetRandomID(idLength, "")
 	installID, err := clusHelper.GetInstallationID()
 	if err != nil {
 		log.WithError(err).Error("failed to get installation ID")
-		return "", "", &tokenClaim{}
+		return "", "", nil, err
 	}
 	now := time.Now()
 	c := tokenClaim{
@@ -1494,10 +1504,15 @@ func jwtGenerateToken(user *share.CLUSUser, domainRoles access.DomainRole, extra
 	jwtCert := GetJWTSigningKey()
 	token := jwt.NewWithClaims(jwt.SigningMethodRS256, c)
 	tokenString, err := token.SignedString(jwtCert.jwtPrivateKey)
-	if tokenString == "" || err != nil {
-		log.WithFields(log.Fields{"err": err}).Error()
+	if err != nil {
+		log.WithFields(log.Fields{"err": err}).Error("failed to sign token")
+		return "", "", nil, err
+	}
+	if tokenString, err = utils.EncryptUserToken(tokenString, []byte(installID)); err != nil {
+		log.WithError(err).Error("failed to encrypt token")
+		return "", "", nil, err
 	}
-	return id, utils.EncryptUserToken(tokenString, []byte(installID)), &c
+	return id, tokenString, &c, nil
 }
 
 func jwtGenFedJoinToken(masterCluster *api.RESTFedMasterClusterInfo, duration time.Duration) []byte {
@@ -1521,7 +1536,7 @@ func jwtGenFedTicket(secret string, duration time.Duration) string {
 	return utils.EncryptSensitive(string(tokenBytes), []byte(secret))
 }
 
-func _genFedJwtToken(c *tokenClaim, callerFedRole, clusterID, secret string, rsaPrivateKey *rsa.PrivateKey) string {
+func _genFedJwtToken(c *tokenClaim, callerFedRole, clusterID, secret string, rsaPrivateKey *rsa.PrivateKey) (string, error) {
 	// rsaPrivateKey being non-nil is for validating new public/private keys purpose
 	token := jwt.NewWithClaims(jwt.SigningMethodRS256, *c)
 	var privateKey *rsa.PrivateKey
@@ -1537,24 +1552,33 @@ func _genFedJwtToken(c *tokenClaim, callerFedRole, clusterID, secret string, rsa
 	} else {
 		privateKey = rsaPrivateKey
 	}
+
+	var tokenString string
+	var err error
 	if privateKey != nil {
-		tokenString, _ := token.SignedString(privateKey)
-		return utils.EncryptSensitive(tokenString, []byte(secret))
+		tokenString, err = token.SignedString(privateKey)
+		if tokenString == "" || err != nil {
+			log.WithFields(log.Fields{"id": clusterID, "err": err}).Error("failed to sign token")
+			return "", err
+		}
+		return utils.EncryptSensitive(tokenString, []byte(secret)), nil
 	} else {
-		log.WithFields(log.Fields{"id": clusterID}).Error("empty private key")
+		err = errors.New("empty private key")
+		log.WithFields(log.Fields{"id": clusterID, "err": err}).Error()
 	}
-	return ""
+
+	return "", err
 }
 
-func jwtGenFedMasterToken(user *share.CLUSUser, login *loginSession, clusterID, secret string) string {
+func jwtGenFedMasterToken(user *share.CLUSUser, login *loginSession, clusterID, secret string) (string, error) {
 
 	if !login.hasFedPermission() {
 		// caller needs to have fed role or fed permission
-		return ""
+		return "", common.ErrObjectAccessDenied
 	}
 
 	if user.RemoteRolePermits == nil || (len(user.RemoteRolePermits.DomainRole) == 0 && len(user.RemoteRolePermits.ExtraPermits) == 0) {
-		return ""
+		return "", common.ErrObjectAccessDenied
 	}
 
 	id := utils.GetRandomID(idLength, "")
@@ -1585,7 +1609,7 @@ func jwtGenFedMasterToken(user *share.CLUSUser, login *loginSession, clusterID,
 	return _genFedJwtToken(&c, api.FedRoleMaster, clusterID, secret, nil)
 }
 
-func jwtGenFedPingToken(callerFedRole, clusterID, secret string, rsaPrivateKey *rsa.PrivateKey) string {
+func jwtGenFedPingToken(callerFedRole, clusterID, secret string, rsaPrivateKey *rsa.PrivateKey) (string, error) {
 	// rsaPrivateKey being non-nil is for validating new public/private keys purpose
 	id := utils.GetRandomID(idLength, "")
 

@@ -1043,7 +1043,10 @@ func TestJWTSignValidate(t *testing.T) {
 	}
 	remote := "10.1.2.3"
 
-	_, tokenString, _ := jwtGenerateToken(user, roles, nil, remote, "", "", nil)
+	_, tokenString, _, err := jwtGenerateToken(user, roles, nil, remote, "", "", nil)
+	if err != nil {
+		t.Errorf("Failed to generate jwt token: user=%+v error=%+v", user, err)
+	}
 
 	token, _ := jwtValidateToken(tokenString, "", nil)
 	if token.Fullname != user.Fullname {

@@ -47,9 +47,10 @@ type cmdResponse struct {
 }
 
 type tNvHttpClient struct {
-	httpClient  *http.Client
-	proxyUrlStr string // non-empty for connection thru proxy
-	basicAuth   string // non-empty for proxy that requires auth
+	httpClient         *http.Client // HTTP client with TLS verification enabled.
+	insecureHttpClient *http.Client // HTTP client with TLS verification skipped.
+	proxyUrlStr        string       // non-empty for connection thru proxy
+	basicAuth          string       // non-empty for proxy that requires auth
 }
 
 const (
@@ -414,21 +415,40 @@ func getProxyURL(r *http.Request) (*url.URL, error) {
 	return nil, nil
 }
 
-func createHttpClient(proxyOption int8, timeout time.Duration) (*http.Client, string, string) {
-	var proxyUrlStr string
-	var basicAuth string
-	var proxy share.CLUSProxy
-
+func createHttpClient(basicAuth string, timeout time.Duration, tlsInsecureSkipVerify bool) *http.Client {
 	// refer to http.DefaultTransport
 	transport := &http.Transport{
 		Proxy: getProxyURL,
 		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true,
+			InsecureSkipVerify: tlsInsecureSkipVerify,
 		},
 		MaxIdleConns:       100,
 		IdleConnTimeout:    90 * time.Second,
 		DisableCompression: true,
 	}
+	if basicAuth != "" {
+		transport.ProxyConnectHeader = http.Header{}
+		transport.ProxyConnectHeader.Add("Proxy-Authorization", basicAuth)
+	}
+	httpClient := &http.Client{
+		Transport: transport,
+		Timeout:   timeout,
+	}
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		log.WithFields(log.Fields{"tlsInsecureSkipVerify": tlsInsecureSkipVerify, "err": err}).Error("creating cookie jar")
+	} else {
+		httpClient.Jar = jar
+	}
+
+	return httpClient
+}
+
+func createNvHttpClient(proxyOption int8, timeout time.Duration) *tNvHttpClient {
+	var proxyUrlStr string
+	var basicAuth string
+	var proxy share.CLUSProxy
+
 	if proxyOption != const_no_proxy {
 		_sysProxyMutex.RLock()
 		if proxyOption == const_http_proxy {
@@ -443,22 +463,15 @@ func createHttpClient(proxyOption int8, timeout time.Duration) (*http.Client, st
 		if proxy.Username != "" {
 			auth := fmt.Sprintf("%s:%s", proxy.Username, proxy.Password)
 			basicAuth = "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
-			transport.ProxyConnectHeader = http.Header{}
-			transport.ProxyConnectHeader.Add("Proxy-Authorization", basicAuth)
 		}
 	}
-	httpClient := &http.Client{
-		Transport: transport,
-		Timeout:   timeout,
-	}
-	jar, err := cookiejar.New(nil)
-	if err != nil {
-		log.WithFields(log.Fields{"proxyOption": proxyOption, "err": err}).Error("creating cookie jar")
-	} else {
-		httpClient.Jar = jar
-	}
 
-	return httpClient, proxyUrlStr, basicAuth
+	return &tNvHttpClient{
+		httpClient:         createHttpClient(basicAuth, timeout, false), // http.Transport.TLSClientConfig.InsecureSkipVerify is false
+		insecureHttpClient: createHttpClient(basicAuth, timeout, true),  // http.Transport.TLSClientConfig.InsecureSkipVerify is true
+		proxyUrlStr:        proxyUrlStr,
+		basicAuth:          basicAuth,
+	}
 }
 
 func getProxyOptions(id string, useProxy int8) []int8 {
@@ -511,7 +524,6 @@ func getProxyOptions(id string, useProxy int8) []int8 {
 
 func sendRestRequest(idTarget string, method, urlStr, token, cntType, jointTicket, jointID string, cookie *http.Cookie,
 	body []byte, logError bool, specificProxy *int8, acc *access.AccessControl) ([]byte, int, bool, error) {
-
 	var useProxy int8
 	var data []byte
 	var statusCode int
@@ -540,19 +552,13 @@ func sendRestRequest(idTarget string, method, urlStr, token, cntType, jointTicke
 		nvHttpClient = _nvHttpClients[proxyOption]
 		_httpClientMutex.RUnlock()
 		if nvHttpClient == nil {
-			httpClient, proxyUrlStr, basicAuth := createHttpClient(proxyOption, clusterAuthTimeout)
-			nvHttpClient = &tNvHttpClient{
-				httpClient:  httpClient,
-				proxyUrlStr: proxyUrlStr,
-				basicAuth:   basicAuth,
-			}
+			nvHttpClient = createNvHttpClient(proxyOption, clusterAuthTimeout)
 			_httpClientMutex.Lock()
 			_nvHttpClients[proxyOption] = nvHttpClient
 			_httpClientMutex.Unlock()
 		}
-		if data, statusCode, err = sendRestReqInternal(nvHttpClient, method, urlStr, token, cntType,
+		if data, statusCode, err = sendRestReqInternal(nvHttpClient, idTarget, method, urlStr, token, cntType,
 			jointTicket, jointID, proxyOption, cookie, body, logError); err == nil {
-
 			_httpClientMutex.Lock()
 			_proxyOptionHistory[idTarget] = proxyOption
 			_httpClientMutex.Unlock()
@@ -566,14 +572,17 @@ func sendRestRequest(idTarget string, method, urlStr, token, cntType, jointTicke
 	return data, statusCode, usedProxy, err
 }
 
-func sendRestReqInternal(nvHttpClient *tNvHttpClient, method, urlStr, token, cntType, jointTicket, jointID string,
+func sendRestReqInternal(nvHttpClient *tNvHttpClient, idTarget, method, urlStr, token, cntType, jointTicket, jointID string,
 	proxyOption int8, cookie *http.Cookie, body []byte, logError bool) ([]byte, int, error) {
-
-	var httpClient *http.Client = nvHttpClient.httpClient
+	var httpClient *http.Client = nvHttpClient.insecureHttpClient
 	var req *http.Request
 	var gzipped bool
 	var err error
 
+	if idTarget == "telemetry" && !cctx.TeleSkipTlsVerification {
+		httpClient = nvHttpClient.httpClient
+	}
+
 	if jointTicket != "" && jointID != "" {
 		if len(body) > gzipThreshold {
 			body = utils.GzipBytes(body)
@@ -636,7 +645,23 @@ func sendRestReqInternal(nvHttpClient *tNvHttpClient, method, urlStr, token, cnt
 	}
 	defer resp.Body.Close()
 
-	data, err := io.ReadAll(resp.Body)
+	var data []byte
+	if idTarget == "telemetry" {
+		// because neuvector_versions.json only has one version entry, the expected len of returned neuvector_versions.json should be < 256
+		var dataBuffer [256]byte
+		reader := io.LimitReader(resp.Body, int64(len(dataBuffer)))
+		n, err2 := reader.Read(dataBuffer[:])
+		if err2 != io.EOF {
+			err = fmt.Errorf("unexpected %d byes of data read, error: %v", n, err2)
+			if _, err2 = io.Copy(io.Discard, resp.Body); err2 != nil {
+				log.WithFields(log.Fields{"error": err2}).Error("Discard remaining data fail")
+			}
+		} else {
+			data = dataBuffer[:n]
+		}
+	} else {
+		data, err = io.ReadAll(resp.Body)
+	}
 	if err != nil {
 		log.WithFields(log.Fields{"url": urlStr, "status": resp.Status, "proxyOption": proxyOption}).Error("Read data fail")
 		return nil, 0, err
@@ -696,17 +721,17 @@ func RestConfig(cmd, interval uint32, param1 interface{}, param2 interface{}) er
 					// no need to reconstruct a new http client when only proxy url changes.
 					// however, if proxy auth changes, we need to reconstruct a new http client because transport ProxyConnectHeader is a map
 					if nvHttpClient == nil || newBasicAuth != nvHttpClient.basicAuth {
-						newHttpClient, proxyUrlStr, basicAuth := createHttpClient(proxyOption, clusterAuthTimeout)
-						newNvHttpClient := &tNvHttpClient{
-							httpClient:  newHttpClient,
-							proxyUrlStr: proxyUrlStr,
-							basicAuth:   basicAuth,
-						}
+						newNvHttpClient := createNvHttpClient(proxyOption, clusterAuthTimeout)
 						_httpClientMutex.Lock()
 						_nvHttpClients[proxyOption] = newNvHttpClient
 						_httpClientMutex.Unlock()
-						if nvHttpClient != nil && nvHttpClient.httpClient != nil {
-							nvHttpClient.httpClient.CloseIdleConnections()
+						if nvHttpClient != nil {
+							if nvHttpClient.httpClient != nil {
+								nvHttpClient.httpClient.CloseIdleConnections()
+							}
+							if nvHttpClient.insecureHttpClient != nil {
+								nvHttpClient.insecureHttpClient.CloseIdleConnections()
+							}
 						}
 					} else {
 						if nvHttpClient.proxyUrlStr != newProxy.URL {
@@ -730,14 +755,10 @@ func initHttpClients() {
 		_sysProxyMutex.Unlock()
 	}
 	for _, proxyOption := range []int8{const_no_proxy, const_https_proxy, const_http_proxy} {
-		httpClient, proxyUrlStr, basicAuth := createHttpClient(proxyOption, clusterAuthTimeout)
+		nvHttpClient := createNvHttpClient(proxyOption, clusterAuthTimeout)
 		_httpClientMutex.Lock()
 		if _nvHttpClients[proxyOption] == nil {
-			_nvHttpClients[proxyOption] = &tNvHttpClient{
-				httpClient:  httpClient,
-				proxyUrlStr: proxyUrlStr,
-				basicAuth:   basicAuth,
-			}
+			_nvHttpClients[proxyOption] = nvHttpClient
 		}
 		_httpClientMutex.Unlock()
 	}
@@ -769,12 +790,12 @@ func sendReqToJointCluster(rc share.CLUSRestServerInfo, clusterID, token, method
 		nvHttpClient = _nvHttpClients[proxyOption]
 		_httpClientMutex.RUnlock()
 		if scanRepository {
-			nvHttpClient.httpClient.Timeout = repoScanLingeringDuration + time.Duration(30*time.Second)
+			nvHttpClient.insecureHttpClient.Timeout = repoScanLingeringDuration + time.Duration(30*time.Second)
 		}
 		headers, statusCode, data, err = sendReqToJointClusterInternal(nvHttpClient, method, urlStr, token, contentType, tag, txnID,
 			proxyOption, body, gzipped, forward, remoteExport, logError)
 		if scanRepository {
-			nvHttpClient.httpClient.Timeout = clusterAuthTimeout
+			nvHttpClient.insecureHttpClient.Timeout = clusterAuthTimeout
 		}
 		if err == nil {
 			_httpClientMutex.Lock()
@@ -793,7 +814,7 @@ func sendReqToJointCluster(rc share.CLUSRestServerInfo, clusterID, token, method
 func sendReqToJointClusterInternal(nvHttpClient *tNvHttpClient, method, urlStr, token, contentType, tag, txnID string,
 	proxyOption int8, body []byte, gzipped, forward, remoteExport, logError bool) (map[string]string, int, []byte, error) {
 
-	var httpClient *http.Client = nvHttpClient.httpClient
+	var httpClient *http.Client = nvHttpClient.insecureHttpClient
 
 	req, err := http.NewRequest(method, urlStr, bytes.NewBuffer(body))
 	if err != nil {
@@ -882,18 +903,21 @@ func getJointClusterToken(rc *share.CLUSFedJointClusterInfo, clusterID string, u
 	if !refreshToken {
 		return cacher.GetFedJoinedClusterToken(clusterID, login.id, acc)
 	} else {
+		masterToken, err := jwtGenFedMasterToken(user, login, rc.ID, rc.Secret)
+		if err != nil {
+			return "", err
+		}
 		reqTo := &api.RESTFedAuthData{
 			ClientIP:       _masterClusterIP,
 			MasterUsername: login.fullname,
 			JointUsername:  common.DefaultAdminUser,
 			// master token is for requesting regular jwt token from joint cluster. It can be validated by joint cluster based on shared secret/key/cert between master & joint clusters
-			MasterToken: jwtGenFedMasterToken(user, login, rc.ID, rc.Secret),
+			MasterToken: masterToken,
 		}
 		if reqTo.MasterToken == "" {
 			return "", common.ErrObjectAccessDenied
 		}
 
-		var err error
 		var data []byte
 		var statusCode int
 		var proxyUsed bool
@@ -1262,7 +1286,10 @@ func pingJointCluster(tag, urlStr string, jointCluster share.CLUSFedJointCluster
 		FedKvVersion: kv.GetFedKvVer(),
 	}
 	if id != "" {
-		reqTo.Token = jwtGenFedPingToken(api.FedRoleMaster, id, jointCluster.Secret, nil)
+		reqTo.Token, err = jwtGenFedPingToken(api.FedRoleMaster, id, jointCluster.Secret, nil)
+		if err != nil {
+			return 0, false, err
+		}
 	}
 	bodyTo, _ := json.Marshal(&reqTo)
 	cmdResp := cmdResponse{id: id, result: _fedClusterDisconnected}
@@ -2301,9 +2328,9 @@ func handlerJoinFedInternal(w http.ResponseWriter, r *http.Request, ps httproute
 	// verify if joint cluster is reachable from master cluster
 	var jointCluster share.CLUSFedJointClusterInfo
 	jointCluster.RestInfo = reqData.JointCluster.RestInfo
-	statusCode, proxyUsed, _ := pingJointCluster(_tagVerifyJointCluster, "v1/fed/joint_test_internal", jointCluster, nil, accReadAll)
-	if statusCode != http.StatusOK {
-		log.WithFields(log.Fields{"statusCode": statusCode, "rest": reqData.JointCluster.RestInfo}).Error("Managed cluster unreachable")
+	statusCode, proxyUsed, err2 := pingJointCluster(_tagVerifyJointCluster, "v1/fed/joint_test_internal", jointCluster, nil, accReadAll)
+	if statusCode != http.StatusOK || err2 != nil {
+		log.WithFields(log.Fields{"statusCode": statusCode, "rest": reqData.JointCluster.RestInfo, "error": err2}).Error("Managed cluster unreachable")
 		restRespError(w, http.StatusBadRequest, api.RESTErrFedJointUnreachable)
 		return
 	}

@@ -174,6 +174,7 @@ var restErrMessage = []string{
 	api.RESTErrRemoteExportFail:      "Failed to export to remote repository",
 	api.RESTErrInvalidQueryToken:     "Invalid or expired query token",
 	api.RESTErrPollJobNotFoundError:  "Job not found in the Job Queue",
+	api.RESTErrServerError:           "Server Error",
 }
 
 func restRespForward(w http.ResponseWriter, r *http.Request, statusCode int, headers map[string]string, data []byte, remoteExport, remoteRegScanTest bool) {
@@ -1297,24 +1298,25 @@ func (l restLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 type Context struct {
-	LocalDev           *common.LocalDevice
-	EvQueue            cluster.ObjectQueueInterface
-	AuditQueue         cluster.ObjectQueueInterface
-	Messenger          cluster.MessengerInterface
-	Cacher             cache.CacheInterface
-	Scanner            scan.ScanInterface
-	SearchRegistries   string
-	FedPort            uint
-	RESTPort           uint
-	PwdValidUnit       uint
-	TeleNeuvectorURL   string
-	TeleFreq           uint
-	NvAppFullVersion   string
-	NvSemanticVersion  string
-	CspType            share.TCspType
-	CspPauseInterval   uint   // in minutes
-	CustomCheckControl string // disable / strict / loose
-	CheckCrdSchemaFunc func(lead, init, crossCheck bool, cspType share.TCspType) []string
+	LocalDev                *common.LocalDevice
+	EvQueue                 cluster.ObjectQueueInterface
+	AuditQueue              cluster.ObjectQueueInterface
+	Messenger               cluster.MessengerInterface
+	Cacher                  cache.CacheInterface
+	Scanner                 scan.ScanInterface
+	SearchRegistries        string
+	FedPort                 uint
+	RESTPort                uint
+	PwdValidUnit            uint
+	TeleNeuvectorURL        string
+	TeleSkipTlsVerification bool // default value is false, meaning it is secure by default
+	TeleFreq                uint
+	NvAppFullVersion        string
+	NvSemanticVersion       string
+	CspType                 share.TCspType
+	CspPauseInterval        uint   // in minutes
+	CustomCheckControl      string // disable / strict / loose
+	CheckCrdSchemaFunc      func(lead, init, crossCheck bool, cspType share.TCspType) []string
 }
 
 var cctx *Context

@@ -2324,6 +2324,7 @@ func getNvUpgradeInfo() *api.RESTCheckUpgradeInfo {
 
 func getAcceptableAlerts(acc *access.AccessControl, login *loginSession) ([]string, []string, []string, []string, []string, map[string]string, utils.Set) {
 	var clusterRoleErrors, clusterRoleBindingErrors, roleErrors, roleBindingErrors, nvCrdSchemaErrors []string
+	otherAlerts := map[string]string{}
 	if k8sPlatform {
 		clusterRoleErrors, clusterRoleBindingErrors, roleErrors, roleBindingErrors =
 			resource.VerifyNvK8sRBAC(localDev.Host.Flavor, "", false)
@@ -2345,8 +2346,18 @@ func getAcceptableAlerts(acc *access.AccessControl, login *loginSession) ([]stri
 	}
 	acceptedAlerts := utils.NewSetFromStringSlice(accepted)
 
+	if k8sPlatform {
+		if secretResetByNV := resource.IsStoreSecretUpdatedByNV(); secretResetByNV {
+			alert := "Important: Please backup the data in Kubernetes secret neuvector-store-secret reset by NeuVector"
+			b := sha256.Sum256([]byte(alert))
+			key := hex.EncodeToString(b[:])
+			if !acceptedAlerts.Contains(key) {
+				otherAlerts[key] = alert
+			}
+		}
+	}
+
 	fedRole := cacher.GetFedMembershipRoleNoAuth()
-	otherAlerts := map[string]string{}
 	if (fedRole == api.FedRoleMaster && (acc.IsFedReader() || acc.IsFedAdmin() || acc.HasPermFed())) ||
 		(fedRole == api.FedRoleJoint && acc.HasGlobalPermissions(share.PERMS_CLUSTER_READ, 0)) {
 		// _fedClusterLeft(206), _fedClusterDisconnected(204)
@@ -2478,9 +2489,14 @@ func handlerSystemGetAlerts(w http.ResponseWriter, r *http.Request, ps httproute
 	if NvCrdSchemaAlertGroup := getAlertGroup(nvCrdSchemaAlerts, api.AlertTypeRBAC, acceptedAlerts); NvCrdSchemaAlertGroup != nil {
 		resp.AcceptableAlerts.NvCrdSchemaAlerts = NvCrdSchemaAlertGroup
 	}
+
+	if NvCrdSchemaAlertGroup := getAlertGroup(nvCrdSchemaAlerts, api.AlertTypeRBAC, acceptedAlerts); NvCrdSchemaAlertGroup != nil {
+		resp.AcceptableAlerts.NvCrdSchemaAlerts = NvCrdSchemaAlertGroup
+	}
+
 	if otherAlerts != nil {
 		otherAlertGroup := &api.RESTNvAlertGroup{
-			Type: api.AlertTypeRBAC,
+			Type: api.AlertTypeOthers,
 		}
 		for id, msg := range otherAlerts {
 			otherAlertGroup.Data = append(otherAlertGroup.Data, &api.RESTNvAlert{
@@ -2806,6 +2822,10 @@ func _importHandler(w http.ResponseWriter, r *http.Request, tid, importType, tem
 				restRespErrorMessageEx(w, status, api.RESTErrFailImport, importTask.Status, resp)
 			} else {
 				// import is not running and caller tries to query the last import status
+				resp.Data.FailToDecryptKeyFields = importTask.FailToDecryptKeyFields
+				if len(importTask.FailToDecryptKeyFields) > 0 {
+					log.WithFields(log.Fields{"FailToDecryptKeyFields": importTask.FailToDecryptKeyFields}).Warn()
+				}
 				restRespSuccess(w, r, &resp, acc, login, nil, "")
 				if importType == share.IMPORT_TYPE_CONFIG {
 					if importTask.Status == share.IMPORT_DONE {
@@ -2872,16 +2892,6 @@ func _importHandler(w http.ResponseWriter, r *http.Request, tid, importType, tem
 		}
 		if err == nil {
 			var tempToken string
-			if importType == share.IMPORT_TYPE_CONFIG {
-				user := &share.CLUSUser{
-					Fullname: login.fullname,
-					Username: login.fullname,
-					Server:   login.server,
-				}
-				domainRoles := access.DomainRole{access.AccessDomainGlobal: api.UserRoleImportStatus}
-				_, tempToken, _ = jwtGenerateToken(user, domainRoles, nil, login.remote, login.mainSessionID, "", nil)
-			}
-
 			importTask.TotalLines = lines
 			importTask.Percentage = 3
 			importTask.LastUpdateTime = time.Now().UTC()
@@ -2890,14 +2900,23 @@ func _importHandler(w http.ResponseWriter, r *http.Request, tid, importType, tem
 			eps := cacher.GetAllControllerRPCEndpoints(access.NewReaderAccessControl())
 			switch importType {
 			case share.IMPORT_TYPE_CONFIG:
-				value := r.Header.Get("X-As-Standalone")
-				ignoreFed, _ := strconv.ParseBool(value)
-				go func() {
-					if err := cfgHelper.Import(eps, localDev.Ctrler.ID, localDev.Ctrler.ClusterIP, login.domainRoles, importTask,
-						tempToken, revertFedRoles, postImportOp, rpc.PauseResumeStoreWatcher, ignoreFed); err != nil {
-						log.WithFields(log.Fields{"error": err}).Error("Import")
-					}
-				}()
+				user := &share.CLUSUser{
+					Fullname: login.fullname,
+					Username: login.fullname,
+					Server:   login.server,
+				}
+				domainRoles := access.DomainRole{access.AccessDomainGlobal: api.UserRoleImportStatus}
+				_, tempToken, _, err = jwtGenerateToken(user, domainRoles, nil, login.remote, login.mainSessionID, "", nil)
+				if err == nil {
+					value := r.Header.Get("X-As-Standalone")
+					ignoreFed, _ := strconv.ParseBool(value)
+					go func() {
+						if err := cfgHelper.Import(eps, localDev.Ctrler.ID, localDev.Ctrler.ClusterIP, login.domainRoles, importTask,
+							tempToken, revertFedRoles, postImportOp, rpc.PauseResumeStoreWatcher, ignoreFed); err != nil {
+							log.WithFields(log.Fields{"error": err}).Error("Import")
+						}
+					}()
+				}
 			case share.IMPORT_TYPE_GROUP_POLICY:
 				go func() {
 					if err := importGroupPolicy(share.ScopeLocal, login.domainRoles, importTask, postImportOp); err != nil {

@@ -52,16 +52,20 @@
 #define ENV_PWD_VALID_UNIT     "PWD_VALID_UNIT"
 #define ENV_RANCHER_EP         "RANCHER_EP"
 #define ENV_RANCHER_SSO        "RANCHER_SSO"
-#define ENV_TELE_NEUVECTOR_EP  "TELEMETRY_NEUVECTOR_EP"
-#define ENV_TELE_CURRENT_VER   "TELEMETRY_CURRENT_VER"
-#define ENV_TELEMETRY_FREQ     "TELEMETRY_FREQ"
 #define ENV_NO_DEFAULT_ADMIN   "NO_DEFAULT_ADMIN"
 #define ENV_CSP_ENV            "CSP_ENV"
 #define ENV_CSP_PAUSE_INTERVAL "CSP_PAUSE_INTERVAL"
 #define ENV_AUTOPROFILE_CLT    "AUTO_PROFILE_COLLECT"
 #define ENV_SET_CUSTOM_BENCH   "CUSTOM_CHECK_CONTROL"
 #define ENV_SHOW_ALL_CMD       "SHOW_ALL_COMMAND"
 
+#define ENV_KEY_ROTATION_PERIOD           "KEY_ROTATION_PERIOD"
+#define ENV_CHECK_ROTATE_PERIOD           "CHECK_KEY_ROTATION_PERIOD"
+#define ENV_TELE_NEUVECTOR_EP             "TELEMETRY_NEUVECTOR_EP"
+#define ENV_TELE_CURRENT_VER              "TELEMETRY_CURRENT_VER"
+#define ENV_TELEMETRY_FREQ                "TELEMETRY_FREQ"
+#define ENV_INSECURE_SKIP_TELE_TLS_VERIFY "INSECURE_SKIP_TELEMETRY_TLS_VERIFICATION"
+
 #define ENV_SCANNER_DOCKER_URL  "SCANNER_DOCKER_URL"
 #define ENV_SCANNER_LICENSE     "SCANNER_LICENSE"
 #define ENV_SCANNER_ON_DEMAND   "SCANNER_ON_DEMAND"
@@ -242,7 +246,7 @@ static pid_t fork_exec(int i)
     char *registry, *repository, *tag, *user, *pass, *base, *api_user, *api_pass, *enable;
     char *pwd_valid_unit, *rancher_ep, *debug_level, *policy_pull_period, *search_regs;
     char *telemetry_neuvector_ep, *telemetry_current_ver, *telemetry_freq, *csp_env, *csp_pause_interval;
-    char *custom_check_control, *log_level;
+    char *custom_check_control, *log_level, *key_rotation_period, *check_key_rotation_period;
     char *max_scanner_tasks, *max_concurrent_repo_scan_tasks, *scanner_lb_max, *scan_job_queue_capacity, *scan_job_fail_retry_max, *repo_scan_long_poll_timeout, *stale_scan_job_cleanup_interval_hour;
     int a;
 
@@ -449,6 +453,17 @@ static pid_t fork_exec(int i)
             args[a++] = "-telemetry_freq";
             args[a++] = telemetry_freq;
         }
+        if ((key_rotation_period = getenv(ENV_KEY_ROTATION_PERIOD)) != NULL) {
+            args[a++] = "-key_rotation_period";
+            args[a++] = key_rotation_period;
+        }
+        if ((check_key_rotation_period = getenv(ENV_CHECK_ROTATE_PERIOD)) != NULL) {
+            args[a++] = "-check_key_rotation_period";
+            args[a++] = check_key_rotation_period;
+        }
+        if (getenv(ENV_INSECURE_SKIP_TELE_TLS_VERIFY)) {
+            args[a++] = "-insecure_skip_telemetry_tls_verification";
+        }
         if ((enable = getenv(ENV_NO_DEFAULT_ADMIN)) != NULL) {
             if (checkImplicitEnableFlag(enable) == 1) {
                 args[a ++] = "-no_def_admin";
@@ -499,16 +514,16 @@ static pid_t fork_exec(int i)
         if ((stale_scan_job_cleanup_interval_hour = getenv(ENV_STALE_SCAN_JOB_CLEANUP_INTERVAL_HOUR)) != NULL) {
             args[a++] = "-stale_scan_job_cleanup_interval_hour";
             args[a++] = stale_scan_job_cleanup_interval_hour;
-        }
+        }        
         if ((repo_scan_long_poll_timeout = getenv(ENV_REPO_SCAN_LONG_POLL_TIMEOUT)) != NULL) {
             args[a++] = "-repo_scan_long_poll_timeout";
             args[a++] = repo_scan_long_poll_timeout;
-        }
+        }    
         if ((scanner_lb_max = getenv(ENV_SCANNER_LB_MAX)) != NULL) {
             args[a++] = "-scanner_lb_max";
             args[a++] = scanner_lb_max;
         }
-
+        
         //  debug("Start %s, pid=%d\n", g_procs[i].name, g_procs[i].pid);
         args[a] = NULL;
         break;
@@ -681,6 +696,16 @@ static void stop_proc(int i, int sig, int wait)
 #define DEFAULT_RPC_PORT "18300"
 #define DEFAULT_LAN_PORT "18301"
 
+static bool is_valid_port(const char *strPort) {
+    if ((strPort == NULL) || (strlen(strPort) != strspn(strPort, "0123456789"))) {
+        return false; // Handle NULL string and empty string case
+    }
+
+    int port = atoi(strPort);
+
+    return ((port > 0) && (port <= 65535));
+}
+
 static int check_consul_ports(void)
 {
     FILE *fp;
@@ -698,6 +723,14 @@ static int check_consul_ports(void)
     if (lan_port == NULL) {
         lan_port = DEFAULT_LAN_PORT;
     }
+    if (!is_valid_port(rpc_port)) {
+        debug("invalid consul rpc port %s\n", rpc_port);
+        return -1;
+    }
+    if (!is_valid_port(lan_port)) {
+        debug("invalid consul lan port %s\n", lan_port);
+        return -1;
+    }
     sprintf(shbuf,"ss -lnp|grep '%s\\|%s'",rpc_port, lan_port);
 
     fp = popen(shbuf, "r");

@@ -39,6 +39,7 @@ const CLUSLockFedScanDataKey string = CLUSLockStore + "fed_scan_data"
 const CLUSLockApikeyKey string = CLUSLockStore + "apikey"
 const CLUSLockVulnKey string = CLUSLockStore + "vulnerability"
 const CLUSLockCompKey string = CLUSLockStore + "compliance"
+const CLUSStoreSecretKey string = CLUSLockStore + "store_secret"
 
 //const CLUSLockResponseRuleKey string = CLUSLockStore + "response_rule"
 
@@ -163,6 +164,8 @@ const CLUSNodeCommonProfileStore string = CLUSNodeCommonStoreKey + CLUSWorkloadP
 // state
 const CLUSCtrlEnabledValue string = "ok"
 
+const CLUSSystemEncMigratedKey string = CLUSStateStore + "enc_migrated"
+
 // cluster key represent one installation, which will remain unchanged when controllers
 // come and go, and rolling upgrade. It is not part of system configuration.
 const CLUSCtrlInstallationKey string = CLUSStateStore + "installation"
@@ -174,7 +177,7 @@ const CLUSCtrlVerKey string = CLUSStateStore + "ctrl_ver"
 const CLUSKvRestoreKey string = CLUSStateStore + "kv_restore"
 const CLUSExpiredTokenStore string = CLUSStateStore + "expired_token/"
 const CLUSImportStore string = CLUSStateStore + "import/"
-
+const CLUSNextKeyRotationTSKey string = CLUSStateStore + "next_key_rotation_ts"
 const CLUSConfigSecretPatternsKey string = CLUSConfigCustomRuleStore + "secret_patterns"
 
 func CLUSExpiredTokenKey(token string) string {
@@ -834,6 +837,11 @@ type CLUSSystemConfig struct {
 	AllowNsUserExportNetPolicy bool                      `json:"allow_ns_user_export_net_policy,omitempty"`
 }
 
+type CLUSSystemConfigEncMigrated struct {
+	EncMigratedSystemConfig string    `json:"enc_migrated_system_config"`
+	EncDataExpirationTime   time.Time `json:"enc_data_expiration_time"`
+}
+
 type CLUSSystemConfigAutoscale struct {
 	Strategy         string `json:"strategy"`
 	MinPods          uint32 `json:"min_pods"`
@@ -1478,6 +1486,10 @@ const (
 	CLUSEvGroupMetricViolation       //network metric violation per group level
 	CLUSEvKvRestored                 // kv is restored from pvc
 	CLUSEvScanDataRestored           // scan data is restored from pvc
+	CLUSEvMismatchedDEKSeed          // mismatched dekSeed for backup from pvc
+	CLUSEvDEKSeedUnavailable         // dekSeed unavailable (most likely because of RBAC neuvector-binding-secret-controller)
+	CLUSEvReEncryptWithDEK           // re-encrypt sensitive data in backup files with variant DEK
+	CLUSEvEncryptionSecretSet        // neuvector-store-secret secret is set
 )
 
 const (
@@ -2984,18 +2996,19 @@ func CLUSImportOpKey(name string) string {
 }
 
 type CLUSImportTask struct {
-	TID            string    `json:"tid"`
-	ImportType     string    `json:"import_type"`
-	CtrlerID       string    `json:"ctrler_id"`
-	TempFilename   string    `json:"temp_filename"`
-	Status         string    `json:"status"`
-	Percentage     int       `json:"percentage"`
-	TotalLines     int       `json:"total_lines"`
-	LastUpdateTime time.Time `json:"last_update_time"`
-	CallerFullname string    `json:"caller_fullname"`
-	CallerRemote   string    `json:"caller_remote"`
-	CallerID       string    `json:"caller_id"`
-	Overwrite      string    `json:"overwrite"`
+	TID                    string              `json:"tid"`
+	ImportType             string              `json:"import_type"`
+	CtrlerID               string              `json:"ctrler_id"`
+	TempFilename           string              `json:"temp_filename"`
+	Status                 string              `json:"status"`
+	Percentage             int                 `json:"percentage"`
+	TotalLines             int                 `json:"total_lines"`
+	LastUpdateTime         time.Time           `json:"last_update_time"`
+	CallerFullname         string              `json:"caller_fullname"`
+	CallerRemote           string              `json:"caller_remote"`
+	CallerID               string              `json:"caller_id"`
+	Overwrite              string              `json:"overwrite"`
+	FailToDecryptKeyFields map[string][]string `json:"fail_to_decrypt_key_fields"` // key_path : []fields
 }
 
 func CLUSNodeProfileStoreKey(nodeID string) string {

@@ -1058,34 +1058,28 @@ func EncryptSensitive(data string, key []byte) string {
 	return encrypted
 }
 
-func DecryptUserToken(encrypted string, key []byte) string {
+func DecryptUserToken(encrypted string, key []byte) (string, error) {
+	if len(key) == 0 {
+		return "", errors.New("empty encryption key")
+	}
 	if encrypted == "" {
-		return ""
+		return "", nil
 	}
 
-	encrypted = strings.ReplaceAll(encrypted, "_", "/")
-	if key == nil {
-		key = getPasswordSymKey()
-	}
-	token, _ := DecryptFromRawStdBase64(key, encrypted)
-	return token
+	return DecryptFromRawURLBase64(key, encrypted)
 }
 
-// User token cannot have / in it and cannot have - as the first char.
-func EncryptUserToken(token string, key []byte) string {
-	if token == "" {
-		return ""
+func EncryptUserToken(token string, key []byte) (string, error) {
+	if len(key) == 0 {
+		return "", errors.New("empty encryption key")
 	}
-
-	if key == nil {
-		key = getPasswordSymKey()
+	if token == "" {
+		return "", nil
 	}
 
 	// Std base64 encoding has + and /, instead of - and _ (url encoding)
-	// token can be part of kv key, so we replace / with _
-	encrypted, _ := EncryptToRawStdBase64(key, []byte(token))
-	encrypted = strings.ReplaceAll(encrypted, "/", "_")
-	return encrypted
+	// encrypted token can be used as part of kv key string, so we replace / with _
+	return EncryptToRawURLBase64(key, []byte(token))
 }
 
 func DecryptURLSafe(encrypted string) string {

@@ -184,12 +184,23 @@ func TestPlatformEnv(t *testing.T) {
 	}
 }
 
-func TestBase64Encrypt(t *testing.T) {
+func TestUserTokenEncrypt(t *testing.T) {
 	token := "123456"
-	encrypt := EncryptUserToken(token, nil)
-	decrypt := DecryptUserToken(encrypt, nil)
-	if decrypt != token {
-		t.Errorf("Token encrypt error: token=%v decrypt=%v\n", token, decrypt)
+	encrypt, err := EncryptUserToken(token, nil)
+	if encrypt != "" {
+		t.Errorf("Token encrypt unexpected: token=%v encrypt=%v err=%v\n", token, encrypt, err)
+	}
+	key, err := GetGuid()
+	if err != nil {
+		t.Errorf("failed to call GetGuid: %v", err)
+	}
+	encrypt, err = EncryptUserToken(token, []byte(key))
+	if err != nil {
+		t.Errorf("Token encrypt unexpected: token=%v encrypt=%v err=%v\n", token, encrypt, err)
+	}
+	decrypt, err := DecryptUserToken(encrypt, []byte(key))
+	if decrypt != token || err != nil {
+		t.Errorf("Token encrypt error: token=%v encrypt=%v decrypt=%v err=%v\n", token, encrypt, decrypt, err)
 	}
 }
 

@@ -249,6 +249,7 @@ func main() {
 	pwdValidUnit := flag.Uint("pwd_valid_unit", 1440, "")
 	rancherEP := flag.String("rancher_ep", "", "Rancher endpoint URL")
 	rancherSSO := flag.Bool("rancher_sso", false, "Rancher SSO integration")
+	insecureSkipTeleTLSVerification := flag.Bool("insecure_skip_telemetry_tls_verification", false, "Do not verify Telemetry server's certificate chain and host name")
 	teleNeuvectorEP := flag.String("telemetry_neuvector_ep", "", "")                   // for testing only
 	teleCurrentVer := flag.String("telemetry_current_ver", "", "")                     // in the format {major}.{minor}.{patch}[-s{#}], for testing only
 	telemetryFreq := flag.Uint("telemetry_freq", 60, "")                               // in minutes, for testing only
@@ -752,23 +753,24 @@ func main() {
 	opa.InitOpaServer()
 
 	rctx := rest.Context{
-		LocalDev:           dev,
-		EvQueue:            evqueue,
-		AuditQueue:         auditQueue,
-		Messenger:          messenger,
-		Cacher:             cacher,
-		Scanner:            scanner,
-		RESTPort:           *restPort,
-		FedPort:            *fedPort,
-		PwdValidUnit:       *pwdValidUnit,
-		TeleNeuvectorURL:   *teleNeuvectorEP,
-		TeleFreq:           *telemetryFreq,
-		NvAppFullVersion:   nvAppFullVersion,
-		NvSemanticVersion:  nvSemanticVersion,
-		CspType:            cspType,
-		CspPauseInterval:   *cspPauseInterval,
-		CustomCheckControl: *custom_check_control,
-		CheckCrdSchemaFunc: nvcrd.CheckCrdSchema,
+		LocalDev:                dev,
+		EvQueue:                 evqueue,
+		AuditQueue:              auditQueue,
+		Messenger:               messenger,
+		Cacher:                  cacher,
+		Scanner:                 scanner,
+		RESTPort:                *restPort,
+		FedPort:                 *fedPort,
+		PwdValidUnit:            *pwdValidUnit,
+		TeleNeuvectorURL:        *teleNeuvectorEP,
+		TeleSkipTlsVerification: *insecureSkipTeleTLSVerification,
+		TeleFreq:                *telemetryFreq,
+		NvAppFullVersion:        nvAppFullVersion,
+		NvSemanticVersion:       nvSemanticVersion,
+		CspType:                 cspType,
+		CspPauseInterval:        *cspPauseInterval,
+		CustomCheckControl:      *custom_check_control,
+		CheckCrdSchemaFunc:      nvcrd.CheckCrdSchema,
 	}
 	// rest.PreInitContext() must be called before orch connector because existing CRD handling could happen right after orch connecter starts
 	rest.PreInitContext(&rctx)

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"mime"
 	"net/http"
@@ -47,9 +48,10 @@ type cmdResponse struct {
 }
 
 type tNvHttpClient struct {
-	httpClient  *http.Client
-	proxyUrlStr string // non-empty for connection thru proxy
-	basicAuth   string // non-empty for proxy that requires auth
+	httpClient         *http.Client // HTTP client with TLS verification enabled.
+	insecureHttpClient *http.Client // HTTP client with TLS verification skipped.
+	proxyUrlStr        string       // non-empty for connection thru proxy
+	basicAuth          string       // non-empty for proxy that requires auth
 }
 
 const (
@@ -410,21 +412,40 @@ func getProxyURL(r *http.Request) (*url.URL, error) {
 	return nil, nil
 }
 
-func createHttpClient(proxyOption int8, timeout time.Duration) (*http.Client, string, string) {
-	var proxyUrlStr string
-	var basicAuth string
-	var proxy share.CLUSProxy
-
+func createHttpClient(basicAuth string, timeout time.Duration, tlsInsecureSkipVerify bool) *http.Client {
 	// refer to http.DefaultTransport
 	transport := &http.Transport{
 		Proxy: getProxyURL,
 		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true,
+			InsecureSkipVerify: tlsInsecureSkipVerify,
 		},
 		MaxIdleConns:       100,
 		IdleConnTimeout:    90 * time.Second,
 		DisableCompression: true,
 	}
+	if basicAuth != "" {
+		transport.ProxyConnectHeader = http.Header{}
+		transport.ProxyConnectHeader.Add("Proxy-Authorization", basicAuth)
+	}
+	httpClient := &http.Client{
+		Transport: transport,
+		Timeout:   timeout,
+	}
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		log.WithFields(log.Fields{"tlsInsecureSkipVerify": tlsInsecureSkipVerify, "err": err}).Error("creating cookie jar")
+	} else {
+		httpClient.Jar = jar
+	}
+
+	return httpClient
+}
+
+func createNvHttpClient(proxyOption int8, timeout time.Duration) *tNvHttpClient {
+	var proxyUrlStr string
+	var basicAuth string
+	var proxy share.CLUSProxy
+
 	if proxyOption != const_no_proxy {
 		_sysProxyMutex.RLock()
 		if proxyOption == const_http_proxy {
@@ -439,22 +460,15 @@ func createHttpClient(proxyOption int8, timeout time.Duration) (*http.Client, st
 		if proxy.Username != "" {
 			auth := fmt.Sprintf("%s:%s", proxy.Username, proxy.Password)
 			basicAuth = "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
-			transport.ProxyConnectHeader = http.Header{}
-			transport.ProxyConnectHeader.Add("Proxy-Authorization", basicAuth)
 		}
 	}
-	httpClient := &http.Client{
-		Transport: transport,
-		Timeout:   timeout,
-	}
-	jar, err := cookiejar.New(nil)
-	if err != nil {
-		log.WithFields(log.Fields{"proxyOption": proxyOption, "err": err}).Error("creating cookie jar")
-	} else {
-		httpClient.Jar = jar
-	}
 
-	return httpClient, proxyUrlStr, basicAuth
+	return &tNvHttpClient{
+		httpClient:         createHttpClient(basicAuth, timeout, false), // http.Transport.TLSClientConfig.InsecureSkipVerify is false
+		insecureHttpClient: createHttpClient(basicAuth, timeout, true),  // http.Transport.TLSClientConfig.InsecureSkipVerify is true
+		proxyUrlStr:        proxyUrlStr,
+		basicAuth:          basicAuth,
+	}
 }
 
 func getProxyOptions(id string, useProxy int8) []int8 {
@@ -536,17 +550,12 @@ func sendRestRequest(idTarget string, method, urlStr, token, cntType, jointTicke
 		nvHttpClient = _nvHttpClients[proxyOption]
 		_httpClientMutex.RUnlock()
 		if nvHttpClient == nil {
-			httpClient, proxyUrlStr, basicAuth := createHttpClient(proxyOption, clusterAuthTimeout)
-			nvHttpClient = &tNvHttpClient{
-				httpClient:  httpClient,
-				proxyUrlStr: proxyUrlStr,
-				basicAuth:   basicAuth,
-			}
+			nvHttpClient = createNvHttpClient(proxyOption, clusterAuthTimeout)
 			_httpClientMutex.Lock()
 			_nvHttpClients[proxyOption] = nvHttpClient
 			_httpClientMutex.Unlock()
 		}
-		if data, statusCode, err = sendRestReqInternal(nvHttpClient, method, urlStr, token, cntType,
+		if data, statusCode, err = sendRestReqInternal(nvHttpClient, idTarget, method, urlStr, token, cntType,
 			jointTicket, jointID, proxyOption, cookie, body, logError); err == nil {
 
 			_httpClientMutex.Lock()
@@ -562,14 +571,17 @@ func sendRestRequest(idTarget string, method, urlStr, token, cntType, jointTicke
 	return data, statusCode, usedProxy, err
 }
 
-func sendRestReqInternal(nvHttpClient *tNvHttpClient, method, urlStr, token, cntType, jointTicket, jointID string,
+func sendRestReqInternal(nvHttpClient *tNvHttpClient, idTarget, method, urlStr, token, cntType, jointTicket, jointID string,
 	proxyOption int8, cookie *http.Cookie, body []byte, logError bool) ([]byte, int, error) {
-
-	var httpClient *http.Client = nvHttpClient.httpClient
+	var httpClient *http.Client = nvHttpClient.insecureHttpClient
 	var req *http.Request
 	var gzipped bool
 	var err error
 
+	if idTarget == "telemetry" && !cctx.TeleSkipTlsVerification {
+		httpClient = nvHttpClient.httpClient
+	}
+
 	if jointTicket != "" && jointID != "" {
 		if len(body) > gzipThreshold {
 			body = utils.GzipBytes(body)
@@ -632,7 +644,23 @@ func sendRestReqInternal(nvHttpClient *tNvHttpClient, method, urlStr, token, cnt
 	}
 	defer resp.Body.Close()
 
-	data, err := ioutil.ReadAll(resp.Body)
+	var data []byte
+	if idTarget == "telemetry" {
+		// because neuvector_versions.json only has one version entry, the expected len of returned neuvector_versions.json should be < 256
+		var dataBuffer [256]byte
+		reader := io.LimitReader(resp.Body, int64(len(dataBuffer)))
+		n, err2 := reader.Read(dataBuffer[:])
+		if err2 != io.EOF {
+			err = fmt.Errorf("unexpected %d byes of data read, error: %v", n, err2)
+			if _, err2 = io.Copy(io.Discard, resp.Body); err2 != nil {
+				log.WithFields(log.Fields{"error": err2}).Error("Discard remaining data fail")
+			}
+		} else {
+			data = dataBuffer[:n]
+		}
+	} else {
+		data, err = io.ReadAll(resp.Body)
+	}
 	if err != nil {
 		log.WithFields(log.Fields{"url": urlStr, "status": resp.Status, "proxyOption": proxyOption}).Error("Read data fail")
 		return nil, 0, err
@@ -692,17 +720,17 @@ func RestConfig(cmd, interval uint32, param1 interface{}, param2 interface{}) er
 					// no need to reconstruct a new http client when only proxy url changes.
 					// however, if proxy auth changes, we need to reconstruct a new http client because transport ProxyConnectHeader is a map
 					if nvHttpClient == nil || newBasicAuth != nvHttpClient.basicAuth {
-						newHttpClient, proxyUrlStr, basicAuth := createHttpClient(proxyOption, clusterAuthTimeout)
-						newNvHttpClient := &tNvHttpClient{
-							httpClient:  newHttpClient,
-							proxyUrlStr: proxyUrlStr,
-							basicAuth:   basicAuth,
-						}
+						newNvHttpClient := createNvHttpClient(proxyOption, clusterAuthTimeout)
 						_httpClientMutex.Lock()
 						_nvHttpClients[proxyOption] = newNvHttpClient
 						_httpClientMutex.Unlock()
-						if nvHttpClient != nil && nvHttpClient.httpClient != nil {
-							nvHttpClient.httpClient.CloseIdleConnections()
+						if nvHttpClient != nil {
+							if nvHttpClient.httpClient != nil {
+								nvHttpClient.httpClient.CloseIdleConnections()
+							}
+							if nvHttpClient.insecureHttpClient != nil {
+								nvHttpClient.insecureHttpClient.CloseIdleConnections()
+							}
 						}
 					} else {
 						if nvHttpClient.proxyUrlStr != newProxy.URL {
@@ -726,14 +754,10 @@ func initHttpClients() {
 		_sysProxyMutex.Unlock()
 	}
 	for _, proxyOption := range []int8{const_no_proxy, const_https_proxy, const_http_proxy} {
-		httpClient, proxyUrlStr, basicAuth := createHttpClient(proxyOption, clusterAuthTimeout)
+		nvHttpClient := createNvHttpClient(proxyOption, clusterAuthTimeout)
 		_httpClientMutex.Lock()
 		if _nvHttpClients[proxyOption] == nil {
-			_nvHttpClients[proxyOption] = &tNvHttpClient{
-				httpClient:  httpClient,
-				proxyUrlStr: proxyUrlStr,
-				basicAuth:   basicAuth,
-			}
+			_nvHttpClients[proxyOption] = nvHttpClient
 		}
 		_httpClientMutex.Unlock()
 	}
@@ -765,12 +789,12 @@ func sendReqToJointCluster(rc share.CLUSRestServerInfo, clusterID, token, method
 		nvHttpClient = _nvHttpClients[proxyOption]
 		_httpClientMutex.RUnlock()
 		if scanRepository {
-			nvHttpClient.httpClient.Timeout = repoScanLingeringDuration + time.Duration(30*time.Second)
+			nvHttpClient.insecureHttpClient.Timeout = repoScanLingeringDuration + time.Duration(30*time.Second)
 		}
 		headers, statusCode, data, err = sendReqToJointClusterInternal(nvHttpClient, method, urlStr, token, contentType, tag, txnID,
 			proxyOption, body, gzipped, forward, remoteExport, logError)
 		if scanRepository {
-			nvHttpClient.httpClient.Timeout = clusterAuthTimeout
+			nvHttpClient.insecureHttpClient.Timeout = clusterAuthTimeout
 		}
 		if err == nil {
 			_httpClientMutex.Lock()
@@ -789,7 +813,7 @@ func sendReqToJointCluster(rc share.CLUSRestServerInfo, clusterID, token, method
 func sendReqToJointClusterInternal(nvHttpClient *tNvHttpClient, method, urlStr, token, contentType, tag, txnID string,
 	proxyOption int8, body []byte, gzipped, forward, remoteExport, logError bool) (map[string]string, int, []byte, error) {
 
-	var httpClient *http.Client = nvHttpClient.httpClient
+	var httpClient *http.Client = nvHttpClient.insecureHttpClient
 
 	req, err := http.NewRequest(method, urlStr, bytes.NewBuffer(body))
 	if err != nil {

@@ -1250,24 +1250,24 @@ func (l restLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 type Context struct {
-	LocalDev           *common.LocalDevice
-	EvQueue            cluster.ObjectQueueInterface
-	AuditQueue         cluster.ObjectQueueInterface
-	Messenger          cluster.MessengerInterface
-	Cacher             cache.CacheInterface
-	Scanner            scan.ScanInterface
-	FedPort            uint
-	RESTPort           uint
-	PwdValidUnit       uint
-	TeleNeuvectorURL   string
-	TeleFreq           uint
-	NvAppFullVersion   string
-	NvSemanticVersion  string
-	CspType            share.TCspType
-	CspPauseInterval   uint   // in minutes
-	CustomCheckControl string // disable / strict / loose
-	CheckCrdSchemaFunc func(lead, init, crossCheck bool, cspType share.TCspType) []string
-	CertManager        *kv.CertManager
+	LocalDev                *common.LocalDevice
+	EvQueue                 cluster.ObjectQueueInterface
+	AuditQueue              cluster.ObjectQueueInterface
+	Messenger               cluster.MessengerInterface
+	Cacher                  cache.CacheInterface
+	Scanner                 scan.ScanInterface
+	FedPort                 uint
+	RESTPort                uint
+	PwdValidUnit            uint
+	TeleNeuvectorURL        string
+	TeleSkipTlsVerification bool // default value is false, meaning it is secure by default
+	TeleFreq                uint
+	NvAppFullVersion        string
+	NvSemanticVersion       string
+	CspType                 share.TCspType
+	CspPauseInterval        uint   // in minutes
+	CustomCheckControl      string // disable / strict / loose
+	CheckCrdSchemaFunc      func(lead, init, crossCheck bool, cspType share.TCspType) []string
 }
 
 var cctx *Context

@@ -50,15 +50,17 @@
 #define ENV_PWD_VALID_UNIT     "PWD_VALID_UNIT"
 #define ENV_RANCHER_EP         "RANCHER_EP"
 #define ENV_RANCHER_SSO        "RANCHER_SSO"
-#define ENV_TELE_NEUVECTOR_EP  "TELEMETRY_NEUVECTOR_EP"
-#define ENV_TELE_CURRENT_VER   "TELEMETRY_CURRENT_VER"
-#define ENV_TELEMETRY_FREQ     "TELEMETRY_FREQ"
 #define ENV_NO_DEFAULT_ADMIN   "NO_DEFAULT_ADMIN"
 #define ENV_CSP_ENV            "CSP_ENV"
 #define ENV_CSP_PAUSE_INTERVAL "CSP_PAUSE_INTERVAL"
 #define ENV_AUTOPROFILE_CLT    "AUTO_PROFILE_COLLECT"
 #define ENV_SET_CUSTOM_BENCH   "CUSTOM_CHECK_CONTROL"
 
+#define ENV_TELE_NEUVECTOR_EP             "TELEMETRY_NEUVECTOR_EP"
+#define ENV_TELE_CURRENT_VER              "TELEMETRY_CURRENT_VER"
+#define ENV_TELEMETRY_FREQ                "TELEMETRY_FREQ"
+#define ENV_INSECURE_SKIP_TELE_TLS_VERIFY "INSECURE_SKIP_TELEMETRY_TLS_VERIFICATION"
+
 #define ENV_SCANNER_DOCKER_URL  "SCANNER_DOCKER_URL"
 #define ENV_SCANNER_LICENSE     "SCANNER_LICENSE"
 #define ENV_SCANNER_ON_DEMAND   "SCANNER_ON_DEMAND"
@@ -411,6 +413,9 @@ static pid_t fork_exec(int i)
             args[a++] = "-telemetry_freq";
             args[a++] = telemetry_freq;
         }
+        if (getenv(ENV_INSECURE_SKIP_TELE_TLS_VERIFY)) {
+            args[a++] = "-insecure_skip_telemetry_tls_verification";
+        }
         if ((enable = getenv(ENV_NO_DEFAULT_ADMIN)) != NULL) {
             if (checkImplicitEnableFlag(enable) == 1) {
                 args[a ++] = "-no_def_admin";
@@ -606,6 +611,16 @@ static void stop_proc(int i, int sig, int wait)
 #define DEFAULT_RPC_PORT "18300"
 #define DEFAULT_LAN_PORT "18301"
 
+static bool is_valid_port(const char *strPort) {
+    if ((strPort == NULL) || (strlen(strPort) != strspn(strPort, "0123456789"))) {
+        return false; // Handle NULL string and empty string case
+    }
+
+    int port = atoi(strPort);
+
+    return ((port > 0) && (port <= 65535));
+}
+
 static int check_consul_ports(void)
 {
     FILE *fp;
@@ -623,7 +638,15 @@ static int check_consul_ports(void)
     if (lan_port == NULL) {
         lan_port = DEFAULT_LAN_PORT;
     }
-    sprintf(shbuf,"netstat -lnp|grep '%s\\|%s'",rpc_port, lan_port);
+    if (!is_valid_port(rpc_port)) {
+        debug("invalid consul rpc port %s\n", rpc_port);
+        return -1;
+    }
+    if (!is_valid_port(lan_port)) {
+        debug("invalid consul lan port %s\n", lan_port);
+        return -1;
+    }
+    sprintf(shbuf,"ss -lnp|grep '%s\\|%s'",rpc_port, lan_port);
 
     fp = popen(shbuf, "r");
     if (fp == NULL) {