type GoRoutineMap interface {
	// Run adds operation name to the list of running operations and spawns a
	// new go routine to execute the operation.
	// If an operation with the same operation name already exists, an
	// AlreadyExists or ExponentialBackoff error is returned.
	// Once the operation is complete, the go routine is terminated and the
	// operation name is removed from the list of executing operations allowing
	// a new operation to be started with the same operation name without error.
	Run(operationName string, operationFunc func() error) error

	// Wait blocks until operations map is empty. This is typically
	// necessary during tests - the test should wait until all operations finish
	// and evaluate results after that.
	Wait()

	// WaitForCompletion blocks until either all operations have successfully completed
	// or have failed but are not pending. The test should wait until operations are either
	// complete or have failed.
	WaitForCompletion()

	IsOperationPending(operationName string) bool
}

// goRoutineMap結(jié)構(gòu)體實(shí)現(xiàn)GoRoutineMap接口，
type goRoutineMap struct {
	// 用于記錄goRoutineMap維護(hù)協(xié)程的狀態(tài)
	operations                map[string]operation
	// 發(fā)生錯(cuò)誤時(shí)是否指數(shù)級(jí)補(bǔ)償
	exponentialBackOffOnError bool
	// 用在多個(gè) Goroutine 等待，一個(gè) Goroutine 通知（事件發(fā)生）的場(chǎng)景
	cond                      *sync.Cond
	lock                      sync.RWMutex
}

// operation結(jié)構(gòu)體對(duì)象維護(hù)單個(gè)goroutine的狀態(tài)。
type operation struct {
	// 是否操作掛起
	operationPending bool
	// 單個(gè)goroutine執(zhí)行邏輯報(bào)錯(cuò)時(shí)，實(shí)現(xiàn)以指數(shù)退避方式
	expBackoff       exponentialbackoff.ExponentialBackoff
}

// ExponentialBackoff contains the last occurrence of an error and the duration
// that retries are not permitted.
type ExponentialBackoff struct {
	lastError           error
	lastErrorTime       time.Time
	durationBeforeRetry time.Duration
}

// NewGoRoutineMap returns a new instance of GoRoutineMap.
func NewGoRoutineMap(exponentialBackOffOnError bool) GoRoutineMap {
	g := &goRoutineMap{
		operations:                make(map[string]operation),
		exponentialBackOffOnError: exponentialBackOffOnError,
	}

	g.cond = sync.NewCond(&g.lock)
	return g
}

// Run函數(shù)是外部函數(shù)，是goRoutineMap核心方法，其能夠防止創(chuàng)建具有相同名稱的多個(gè)goroutine，并使用協(xié)程的方式運(yùn)行函數(shù)邏輯
// 如果函數(shù)成功執(zhí)行，則退出該協(xié)程；如果函數(shù)執(zhí)行報(bào)錯(cuò)，在指數(shù)退避的時(shí)間內(nèi)禁止再次執(zhí)行該函數(shù)邏輯。
func (grm *goRoutineMap) Run(
	operationName string,
	operationFunc func() error) error {
	grm.lock.Lock()
	defer grm.lock.Unlock()

	// 判斷grm.operations這個(gè)map中是否存在具有operationName名稱的協(xié)程
	existingOp, exists := grm.operations[operationName]
	if exists {
		// 如果grm.operations這個(gè)map中已經(jīng)存在operationName名稱的協(xié)程，并且existingOp.operationPending==true，說明grm.operations中operationName名稱這個(gè)協(xié)程正在執(zhí)行函數(shù)邏輯，在這期間又有一個(gè)同名的
		// operationName希望加入grm.operations這個(gè)map，此時(shí)加入map失敗并報(bào)AlreadyExistsError錯(cuò)誤
		if existingOp.operationPending {
			return NewAlreadyExistsError(operationName)
		}

		// 到這步說明名稱為operationName名稱的協(xié)程執(zhí)行函數(shù)邏輯失敗，此時(shí)判斷此協(xié)程最后一次失敗時(shí)間 + 指數(shù)退避的時(shí)間 >= 當(dāng)前時(shí)間，如果不符合條件的話禁止執(zhí)行該協(xié)程函數(shù)邏輯。
		if err := existingOp.expBackoff.SafeToRetry(operationName); err != nil {
			return err
		}
	}

	// 如果grm.operations這個(gè)map中不存在operationName名稱的協(xié)程 或者 此協(xié)程最后一次失敗時(shí)間 + 指數(shù)退避的時(shí)間 < 當(dāng)前時(shí)間，則在grm.operations這個(gè)map中重新維護(hù)此協(xié)程（注意，operationPending=true）
	grm.operations[operationName] = operation{
		operationPending: true,
		expBackoff:       existingOp.expBackoff,
	}

	// 以協(xié)程方式執(zhí)行函數(shù)邏輯operationFunc()
	go func() (err error) {
		// 捕獲崩潰并記錄錯(cuò)誤，默認(rèn)不傳參的話，在程序發(fā)送崩潰時(shí)，在控制臺(tái)打印一下崩潰日志后再崩潰，方便技術(shù)人員排查程序錯(cuò)誤。
		defer k8sRuntime.HandleCrash()

		// 如果執(zhí)行operationFunc()函數(shù)邏輯不報(bào)錯(cuò)或者grm.exponentialBackOffOnError=false的話，將從grm.operations這個(gè)map中移除此operationName名稱協(xié)程；
		// 如果執(zhí)行operationFunc()函數(shù)邏輯報(bào)錯(cuò)并且grm.exponentialBackOffOnError=true，則將產(chǎn)生指數(shù)級(jí)補(bǔ)償，到達(dá)補(bǔ)償時(shí)間后才能再調(diào)用此operationName名稱協(xié)程的函數(shù)邏輯
		// Handle completion of and error, if any, from operationFunc()
		defer grm.operationComplete(operationName, &err)
		// 處理operationFunc()函數(shù)發(fā)生的panic錯(cuò)誤，以便defer grm.operationComplete(operationName, &err)能執(zhí)行
		// Handle panic, if any, from operationFunc()
		defer k8sRuntime.RecoverFromPanic(&err)
		return operationFunc()
	}()

	return nil
}

// SafeToRetry returns an error if the durationBeforeRetry period for the given
// lastErrorTime has not yet expired. Otherwise it returns nil.
func (expBackoff *ExponentialBackoff) SafeToRetry(operationName string) error {
	if time.Since(expBackoff.lastErrorTime) <= expBackoff.durationBeforeRetry {
		return NewExponentialBackoffError(operationName, *expBackoff)
	}

	return nil
}

// operationComplete是一個(gè)內(nèi)部函數(shù)，用于處理在goRoutineMap中已經(jīng)運(yùn)行完函數(shù)邏輯的協(xié)程
// 如果執(zhí)行operationFunc()函數(shù)邏輯不報(bào)錯(cuò)或者grm.exponentialBackOffOnError=false的話，將從grm.operations這個(gè)map中移除此operationName名稱協(xié)程；
// 如果執(zhí)行operationFunc()函數(shù)邏輯報(bào)錯(cuò)并且grm.exponentialBackOffOnError=true，則將產(chǎn)生指數(shù)級(jí)補(bǔ)償，到達(dá)補(bǔ)償時(shí)間后才能再調(diào)用此operationName名稱協(xié)程的函數(shù)邏輯
// operationComplete handles the completion of a goroutine run in the
// goRoutineMap.
func (grm *goRoutineMap) operationComplete(
	operationName string, err *error) {
	// Defer operations are executed in Last-In is First-Out order. In this case
	// the lock is acquired first when operationCompletes begins, and is
	// released when the method finishes, after the lock is released cond is
	// signaled to wake waiting goroutine.
	defer grm.cond.Signal()
	grm.lock.Lock()
	defer grm.lock.Unlock()

	if *err == nil || !grm.exponentialBackOffOnError {
		// 函數(shù)邏輯執(zhí)行完成無錯(cuò)誤或已禁用錯(cuò)誤指數(shù)級(jí)補(bǔ)償，將從grm.operations這個(gè)map中移除此operationName名稱協(xié)程；
		// Operation completed without error, or exponentialBackOffOnError disabled
		delete(grm.operations, operationName)
		if *err != nil {
			// Log error
			klog.Errorf("operation for %q failed with: %v",
				operationName,
				*err)
		}
	} else {
		//  函數(shù)邏輯執(zhí)行完成有錯(cuò)誤則將產(chǎn)生指數(shù)級(jí)補(bǔ)償，到達(dá)補(bǔ)償時(shí)間后才能再調(diào)用此operationName名稱協(xié)程的函數(shù)邏輯（注意，指數(shù)補(bǔ)充的協(xié)程，operationPending=false）
		// Operation completed with error and exponentialBackOffOnError Enabled
		existingOp := grm.operations[operationName]
		existingOp.expBackoff.Update(err)
		existingOp.operationPending = false
		grm.operations[operationName] = existingOp

		// Log error
		klog.Errorf("%v",
			existingOp.expBackoff.GenerateNoRetriesPermittedMsg(operationName))
	}
}

const (
   initialDurationBeforeRetry time.Duration = 500 * time.Millisecond
   maxDurationBeforeRetry time.Duration = 2*time.Minute + 2*time.Second
)
func (expBackoff *ExponentialBackoff) Update(err *error) {
	if expBackoff.durationBeforeRetry == 0 {
		expBackoff.durationBeforeRetry = initialDurationBeforeRetry
	} else {
		expBackoff.durationBeforeRetry = 2 * expBackoff.durationBeforeRetry
		if expBackoff.durationBeforeRetry > maxDurationBeforeRetry {
			expBackoff.durationBeforeRetry = maxDurationBeforeRetry
		}
	}

	expBackoff.lastError = *err
	expBackoff.lastErrorTime = time.Now()
}