diff --git a/common/collect/validity_map.go b/common/collect/validity_map.go
index 0d48b82d5..faf19b5ae 100644
--- a/common/collect/validity_map.go
+++ b/common/collect/validity_map.go
@@ -2,7 +2,7 @@ package collect
 
 import (
 	"sync"
-	"time"
+	"sync/atomic"
 
 	"github.com/v2ray/v2ray-core/common/serial"
 )
@@ -18,39 +18,35 @@ type entry struct {
 
 type ValidityMap struct {
 	sync.RWMutex
-	cache              map[string]Validity
-	cleanupIntervalSec int
+	cache   map[string]Validity
+	opCount int32
 }
 
 func NewValidityMap(cleanupIntervalSec int) *ValidityMap {
 	instance := &ValidityMap{
-		cache:              make(map[string]Validity),
-		cleanupIntervalSec: cleanupIntervalSec,
+		cache: make(map[string]Validity),
 	}
-	go instance.cleanup()
 	return instance
 }
 
 func (this *ValidityMap) cleanup() {
-	for range time.Tick(time.Duration(this.cleanupIntervalSec) * time.Second) {
-		entry2Remove := make([]entry, 0, 128)
-		this.RLock()
-		for key, value := range this.cache {
-			if !value.IsValid() {
-				entry2Remove = append(entry2Remove, entry{
-					key:   key,
-					value: value,
-				})
-			}
+	entry2Remove := make([]entry, 0, 128)
+	this.RLock()
+	for key, value := range this.cache {
+		if !value.IsValid() {
+			entry2Remove = append(entry2Remove, entry{
+				key:   key,
+				value: value,
+			})
 		}
-		this.RUnlock()
+	}
+	this.RUnlock()
 
-		for _, entry := range entry2Remove {
-			if !entry.value.IsValid() {
-				this.Lock()
-				delete(this.cache, entry.key)
-				this.Unlock()
-			}
+	for _, entry := range entry2Remove {
+		if !entry.value.IsValid() {
+			this.Lock()
+			delete(this.cache, entry.key)
+			this.Unlock()
 		}
 	}
 }
@@ -59,6 +55,11 @@ func (this *ValidityMap) Set(key serial.String, value Validity) {
 	this.Lock()
 	this.cache[key.String()] = value
 	this.Unlock()
+	opCount := atomic.AddInt32(&this.opCount, 1)
+	if opCount > 1000 {
+		atomic.StoreInt32(&this.opCount, 0)
+		go this.cleanup()
+	}
 }
 
 func (this *ValidityMap) Get(key serial.String) Validity {