fix: make etcd errors transient (argoproj#12567)

Signed-off-by: Alan Clucas <[email protected]>

util/errors/errors.go

@@ -28,7 +28,15 @@ func IsTransientErr(err error) bool {
 		return false
 	}
 	err = argoerrs.Cause(err)
-	isTransient := isExceededQuotaErr(err) || apierr.IsTooManyRequests(err) || isResourceQuotaConflictErr(err) || isResourceQuotaTimeoutErr(err) || isTransientNetworkErr(err) || apierr.IsServerTimeout(err) || apierr.IsServiceUnavailable(err) || matchTransientErrPattern(err) ||
+	isTransient := isExceededQuotaErr(err) ||
+		apierr.IsTooManyRequests(err) ||
+		isResourceQuotaConflictErr(err) ||
+		isResourceQuotaTimeoutErr(err) ||
+		isTransientNetworkErr(err) ||
+		apierr.IsServerTimeout(err) ||
+		apierr.IsServiceUnavailable(err) ||
+		isTransientEtcdErr(err) ||
+		matchTransientErrPattern(err) ||
 		errors.Is(err, NewErrTransient(""))
 	if isTransient {
 		log.Infof("Transient error: %v", err)
@@ -61,6 +69,16 @@ func isResourceQuotaTimeoutErr(err error) bool {
 	return apierr.IsInternalError(err) && strings.Contains(err.Error(), "resource quota evaluation timed out")
 }
 
+func isTransientEtcdErr(err error) bool {
+	// Some clusters expose these (transient) etcd errors to the caller
+	if strings.Contains(err.Error(), "etcdserver: leader changed") {
+		return true
+	} else if strings.Contains(err.Error(), "etcdserver: request timed out") {
+		return true
+	}
+	return false
+}
+
 func isTransientNetworkErr(err error) bool {
 	switch err.(type) {
 	case *net.DNSError, *net.OpError, net.UnknownNetworkError: