framework.ExpectNoError(err) }) <ins> ginkgo.It("should list, patch and delete a LimitRange by collection", func() { ns := f.Namespace.Name lrClient := f.ClientSet.CoreV1().LimitRanges(ns) lrName := "e2e-limitrange-" + utilrand.String(5) e2eLabelSelector := "e2e-test=" + lrName patchedLabelSelector := lrName + "=patched" min := getResourceList("50m", "100Mi", "100Gi") max := getResourceList("500m", "500Mi", "500Gi") defaultLimit := getResourceList("500m", "500Mi", "500Gi") defaultRequest := getResourceList("100m", "200Mi", "200Gi") maxLimitRequestRatio := v1.ResourceList{} limitRange := &v1.LimitRange{ ObjectMeta: metav1.ObjectMeta{ Name: lrName, Labels: map[string]string{ "e2e-test": lrName, lrName:     "created", }, }, Spec: v1.LimitRangeSpec{ Limits: []v1.LimitRangeItem{ { Type:                 v1.LimitTypeContainer, Min:                  min, Max:                  max, Default:              defaultLimit, DefaultRequest:       defaultRequest, MaxLimitRequestRatio: maxLimitRequestRatio, }, }, }, } // Create a copy to be used in a second namespace limitRange2 := &v1.LimitRange{} *limitRange2 = *limitRange ctx, cancelCtx := context.WithTimeout(context.Background(), wait.ForeverTestTimeout) defer cancelCtx() ginkgo.By(fmt.Sprintf("Creating LimitRange %q in namespace %q", lrName, f.Namespace.Name)) limitRange, err := lrClient.Create(ctx, limitRange, metav1.CreateOptions{}) framework.ExpectNoError(err, "Failed to create limitRange %q", lrName) ginkgo.By("Creating another limitRange in another namespace") lrNamespace, err := f.CreateNamespace(lrName, nil) framework.ExpectNoError(err, "failed creating Namespace") framework.Logf("Namespace %q created", lrNamespace.ObjectMeta.Name) framework.Logf(fmt.Sprintf("Creating LimitRange %q in namespace %q", lrName, lrNamespace.Name)) _, err = f.ClientSet.CoreV1().LimitRanges(lrNamespace.ObjectMeta.Name).Create(ctx, limitRange2, metav1.CreateOptions{}) framework.ExpectNoError(err, "Failed to create limitRange %q in %q namespace", lrName, lrNamespace.ObjectMeta.Name) // Listing across all namespaces to verify api endpoint: listCoreV1LimitRangeForAllNamespaces ginkgo.By(fmt.Sprintf("Listing all LimitRanges with label %q", e2eLabelSelector)) limitRangeList, err := f.ClientSet.CoreV1().LimitRanges("").List(ctx, metav1.ListOptions{LabelSelector: e2eLabelSelector}) framework.ExpectNoError(err, "Failed to list any limitRanges: %v", err) framework.ExpectEqual(len(limitRangeList.Items), 2, "Failed to find the correct limitRange count") framework.Logf("Found %d limitRanges", len(limitRangeList.Items)) ginkgo.By(fmt.Sprintf("Patching LimitRange %q in %q namespace", lrName, ns)) newMin := getResourceList("9m", "49Mi", "49Gi") limitRange.Spec.Limits[0].Min = newMin limitRangePayload, err := json.Marshal(v1.LimitRange{ ObjectMeta: metav1.ObjectMeta{ CreationTimestamp: limitRange.CreationTimestamp, Labels: map[string]string{ lrName: "patched", }, }, Spec: v1.LimitRangeSpec{ Limits: limitRange.Spec.Limits, }, }) framework.ExpectNoError(err, "Failed to marshal limitRange JSON") patchedLimitRange, err := lrClient.Patch(ctx, lrName, types.StrategicMergePatchType, []byte(limitRangePayload), metav1.PatchOptions{}) framework.ExpectNoError(err, "Failed to patch limitRange %q", lrName) framework.ExpectEqual(patchedLimitRange.Labels[lrName], "patched", "%q label didn't have value 'patched' for this limitRange. Current labels: %v", lrName, patchedLimitRange.Labels) checkMinLimitRange := apiequality.Semantic.DeepEqual(patchedLimitRange.Spec.Limits[0].Min, newMin) framework.ExpectEqual(checkMinLimitRange, true, "LimitRange does not have the correct min limitRange. Currently is %#v ", patchedLimitRange.Spec.Limits[0].Min) framework.Logf("LimitRange %q has been patched", lrName) ginkgo.By(fmt.Sprintf("Delete LimitRange %q by Collection with labelSelector: %q", lrName, patchedLabelSelector)) err = lrClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: patchedLabelSelector}) framework.ExpectNoError(err, "failed to delete the LimitRange by Collection") ginkgo.By(fmt.Sprintf("Confirm that the limitRange %q has been deleted", lrName)) err = wait.PollImmediate(1*time.Second, 10*time.Second, checkLimitRangeListQuantity(f, patchedLabelSelector, 0)) framework.ExpectNoError(err, "failed to count the required limitRanges") framework.Logf("LimitRange %q has been deleted.", lrName) ginkgo.By(fmt.Sprintf("Confirm that a single LimitRange still exists with label %q", e2eLabelSelector)) limitRangeList, err = f.ClientSet.CoreV1().LimitRanges("").List(ctx, metav1.ListOptions{LabelSelector: e2eLabelSelector}) framework.ExpectNoError(err, "Failed to list any limitRanges: %v", err) framework.ExpectEqual(len(limitRangeList.Items), 1, "Failed to find the correct limitRange count") framework.Logf("Found %d limitRange", len(limitRangeList.Items)) }) </ins> }) func equalResourceRequirement(expected v1.ResourceRequirements, actual v1.ResourceRequirements) error {


}, } } <ins> func checkLimitRangeListQuantity(f *framework.Framework, label string, quantity int) func() (bool, error) { return func() (bool, error) { framework.Logf("Requesting list of LimitRange to confirm quantity") list, err := f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{LabelSelector: label}) if err != nil { return false, err } if len(list.Items) != quantity { return false, nil } framework.Logf("Found %d LimitRange with label %q", quantity, label) return true, nil } } </ins>


validate the pod resources are applied to the Limitrange release: v1.18 file: test/e2e/scheduling/limit_range.go <ins> - testname: LimitRange, list, patch and delete a LimitRange by collection codename: '[sig-scheduling] LimitRange should list, patch and delete a LimitRange by collection [Conformance]' description: When two limitRanges are created in different namespaces, both MUST succeed. Listing limitRanges across all namespaces with a labelSelector MUST find both limitRanges. When patching the first limitRange it MUST succeed and the fields MUST equal the new values. When deleting the limitRange by collection with a labelSelector it MUST delete only one limitRange. release: v1.26 file: test/e2e/scheduling/limit_range.go </ins> - testname: Scheduler, resource limits codename: '[sig-scheduling] SchedulerPredicates [Serial] validates resource limits of pods that are allowed to run  [Conformance]'


framework.ExpectNoError(err) }) <del> ginkgo.It("should list, patch and delete a LimitRange by collection", func() { </del> <ins> /* Release: v1.26 Testname: LimitRange, list, patch and delete a LimitRange by collection Description: When two limitRanges are created in different namespaces, both MUST succeed. Listing limitRanges across all namespaces with a labelSelector MUST find both limitRanges. When patching the first limitRange it MUST succeed and the fields MUST equal the new values. When deleting the limitRange by collection with a labelSelector it MUST delete only one limitRange. */ framework.ConformanceIt("should list, patch and delete a LimitRange by collection", func() { </ins> ns := f.Namespace.Name lrClient := f.ClientSet.CoreV1().LimitRanges(ns)


if _, err := dc.coreClient.CoreV1().Pods(tc.pod.Namespace).Create(ctx, tc.pod, metav1.CreateOptions{}); err != nil { t.Fatalf("Failed to create pod: %v", err) } <ins> dc.clock.Sleep(tc.timePassed) </ins> if err := dc.informerFactory.Core().V1().Pods().Informer().GetIndexer().Add(tc.pod); err != nil { t.Fatalf("Failed adding pod to indexer: %v", err) } <del> dc.clock.Sleep(tc.timePassed) </del> <ins> diff := "" </ins> if err := wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { <del> return dc.stalePodDisruptionQueue.Len() == 0, nil </del> <ins> pod, err := dc.kubeClient.CoreV1().Pods(tc.pod.Namespace).Get(ctx, tc.pod.Name, metav1.GetOptions{}) if err != nil { t.Fatalf("Failed getting updated pod: %v", err) } diff = cmp.Diff(tc.wantConditions, pod.Status.Conditions, cmpopts.IgnoreFields(v1.PodCondition{}, "LastTransitionTime")) return diff == "", nil </ins> }); err != nil { <del> t.Fatalf("Failed waiting for worker to sync: %v", err) } pod, err := dc.kubeClient.CoreV1().Pods(tc.pod.Namespace).Get(ctx, tc.pod.Name, metav1.GetOptions{}) if err != nil { t.Fatalf("Failed getting updated pod: %v", err) } diff := cmp.Diff(tc.wantConditions, pod.Status.Conditions, cmpopts.IgnoreFields(v1.PodCondition{}, "LastTransitionTime")) if diff != "" { t.Errorf("Obtained pod conditions (-want,+got):n%s", diff) </del> <ins> t.Fatalf("Failed waiting for worker to sync: %v, (-want,+got):n%s", err, diff) </ins> } }) }


} else { klog.V(3).InfoS("Skipped creating Hns LoadBalancer for loadBalancer Ingress resources", "lbIngressIP", lbIngressIP) } <del> lbIngressIP.hnsID = hnsLoadBalancer.hnsID klog.V(3).InfoS("Hns LoadBalancer resource created for loadBalancer Ingress resources", "lbIngressIP", lbIngressIP) </del> if proxier.forwardHealthCheckVip && gatewayHnsendpoint != nil { nodeport := proxier.healthzPort


return false, nil } <del> completionMode := string(batch.NonIndexedCompletion) if isIndexedJob(&job) { completionMode = string(batch.IndexedCompletion) } </del> <ins> completionMode := getCompletionMode(&job) </ins> action := metrics.JobSyncActionReconciling defer func() {


job.Status.CompletedIndexes = succeededIndexes.String() } job.Status.UncountedTerminatedPods = nil <del> jm.enactJobFinished(&job, finishedCondition) </del> <ins> jobFinished := jm.enactJobFinished(&job, finishedCondition) </ins> if _, err := jm.updateStatusHandler(ctx, &job); err != nil { return forget, err } <ins> if jobFinished { jm.recordJobFinished(&job, finishedCondition) } </ins> if jobHasNewFailure && !IsJobFinished(&job) { // returning an error will re-enqueue Job after the backoff period


if job, needsFlush, err = jm.flushUncountedAndRemoveFinalizers(ctx, job, podsToRemoveFinalizer, uidsWithFinalizer, &oldCounters, needsFlush); err != nil { return err } <del> if jm.enactJobFinished(job, finishedCond) { </del> <ins> jobFinished := jm.enactJobFinished(job, finishedCond) if jobFinished { </ins> needsFlush = true } if needsFlush { if _, err := jm.updateStatusHandler(ctx, job); err != nil { return fmt.Errorf("removing uncounted pods from status: %w", err) } <ins> if jobFinished { jm.recordJobFinished(job, finishedCond) } </ins> recordJobPodFinished(job, oldCounters) } return nil


return false } } <del> completionMode := string(batch.NonIndexedCompletion) if isIndexedJob(job) { completionMode = string(*job.Spec.CompletionMode) } </del> job.Status.Conditions, _ = ensureJobConditionStatus(job.Status.Conditions, finishedCond.Type, finishedCond.Status, finishedCond.Reason, finishedCond.Message) if finishedCond.Type == batch.JobComplete { <ins> job.Status.CompletionTime = &finishedCond.LastTransitionTime } return true } // recordJobFinished records events and the job_finished_total metric for a finished job. func (jm *Controller) recordJobFinished(job *batch.Job, finishedCond *batch.JobCondition) bool { completionMode := getCompletionMode(job) if finishedCond.Type == batch.JobComplete { </ins> if job.Spec.Completions != nil && job.Status.Succeeded > *job.Spec.Completions { jm.recorder.Event(job, v1.EventTypeWarning, "TooManySucceededPods", "Too many succeeded pods running after completion count reached") } <del> job.Status.CompletionTime = &finishedCond.LastTransitionTime </del> jm.recorder.Event(job, v1.EventTypeNormal, "Completed", "Job completed") metrics.JobFinishedNum.WithLabelValues(completionMode, "succeeded").Inc() } else {


return result } <ins> // getCompletionMode returns string representation of the completion mode. Used as a label value for metrics. func getCompletionMode(job *batch.Job) string { if isIndexedJob(job) { return string(batch.IndexedCompletion) } return string(batch.NonIndexedCompletion) } </ins> func trackingUncountedPods(job *batch.Job) bool { return feature.DefaultFeatureGate.Enabled(features.JobTrackingWithFinalizers) && hasJobTrackingAnnotation(job) }


"k8s.io/client-go/restmapper" "k8s.io/client-go/util/retry" featuregatetesting "k8s.io/component-base/featuregate/testing" <ins> basemetrics "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/testutil" </ins> "k8s.io/controller-manager/pkg/informerfactory" "k8s.io/klog/v2" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/controller/garbagecollector" jobcontroller "k8s.io/kubernetes/pkg/controller/job" <ins> "k8s.io/kubernetes/pkg/controller/job/metrics" </ins> "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/integration/framework" "k8s.io/utils/pointer"


const waitInterval = time.Second <ins> type metricLabelsWithValue struct { Labels []string Value  int } func TestMetrics(t *testing.T) { nonIndexedCompletion := batchv1.NonIndexedCompletion indexedCompletion := batchv1.IndexedCompletion wFinalizers := true defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobTrackingWithFinalizers, wFinalizers)() // setup the job controller closeFn, restConfig, clientSet, ns := setup(t, "simple") defer closeFn() ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer func() { cancel() }() testCases := map[string]struct { job                            *batchv1.Job wantJobFinishedNumMetricDelta  metricLabelsWithValue wantJobPodsFinishedMetricDelta metricLabelsWithValue }{ "non-indexed job": { job: &batchv1.Job{ Spec: batchv1.JobSpec{ Completions:    pointer.Int32(2), Parallelism:    pointer.Int32(2), CompletionMode: &nonIndexedCompletion, }, }, wantJobFinishedNumMetricDelta: metricLabelsWithValue{ Labels: []string{"NonIndexed", "succeeded"}, Value:  1, }, wantJobPodsFinishedMetricDelta: metricLabelsWithValue{ Labels: []string{"NonIndexed", "succeeded"}, Value:  2, }, }, "indexed job": { job: &batchv1.Job{ Spec: batchv1.JobSpec{ Completions:    pointer.Int32(2), Parallelism:    pointer.Int32(2), CompletionMode: &indexedCompletion, }, }, wantJobFinishedNumMetricDelta: metricLabelsWithValue{ Labels: []string{"Indexed", "succeeded"}, Value:  1, }, wantJobPodsFinishedMetricDelta: metricLabelsWithValue{ Labels: []string{"Indexed", "succeeded"}, Value:  2, }, }, } job_index := 0 // job index to avoid collisions between job names created by different test cases for name, tc := range testCases { t.Run(name, func(t *testing.T) { // record the metrics after the job is created jobFinishedNumBefore, err := getCounterMetricValueForLabels(metrics.JobFinishedNum, tc.wantJobFinishedNumMetricDelta.Labels) if err != nil { t.Fatalf("Failed to collect the JobFinishedNum metric before creating the job: %q", err) } jobPodsFinishedBefore, err := getCounterMetricValueForLabels(metrics.JobPodsFinished, tc.wantJobPodsFinishedMetricDelta.Labels) if err != nil { t.Fatalf("Failed to collect the JobPodsFinished metric before creating the job: %q", err) } // create a single job and wait for its completion job := tc.job.DeepCopy() job.Name = fmt.Sprintf("job-%v", job_index) job_index++ jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, job) if err != nil { t.Fatalf("Failed to create Job: %v", err) } validateJobPodsStatus(ctx, t, clientSet, jobObj, podsByStatus{ Active: int(*jobObj.Spec.Parallelism), Ready:  pointer.Int32(0), }, wFinalizers) if err, _ := setJobPodsPhase(ctx, clientSet, jobObj, v1.PodSucceeded, int(*jobObj.Spec.Parallelism)); err != nil { t.Fatalf("Failed setting phase %s on Job Pod: %v", v1.PodSucceeded, err) } validateJobSucceeded(ctx, t, clientSet, jobObj) // verify metric values after the job is finished validateMetricValueDeltas(t, metrics.JobFinishedNum, tc.wantJobFinishedNumMetricDelta, jobFinishedNumBefore) validateMetricValueDeltas(t, metrics.JobPodsFinished, tc.wantJobPodsFinishedMetricDelta, jobPodsFinishedBefore) }) } } func validateMetricValueDeltas(t *testing.T, counterVer *basemetrics.CounterVec, wantMetricDelta metricLabelsWithValue, metricValuesBefore metricLabelsWithValue) { t.Helper() var cmpErr error err := wait.PollImmediate(10*time.Millisecond, 10*time.Second, func() (bool, error) { cmpErr = nil metricValuesAfter, err := getCounterMetricValueForLabels(counterVer, wantMetricDelta.Labels) if err != nil { return true, fmt.Errorf("Failed to collect the %q metric after the job is finished: %q", counterVer.Name, err) } wantDelta := wantMetricDelta.Value gotDelta := metricValuesAfter.Value - metricValuesBefore.Value if wantDelta != gotDelta { cmpErr = fmt.Errorf("Unexepected metric delta for %q metric with labels %q. want: %v, got: %v", counterVer.Name, wantMetricDelta.Labels, wantDelta, gotDelta) return false, nil } return true, nil }) if err != nil { t.Errorf("Failed waiting for expected metric delta: %q", err) } if cmpErr != nil { t.Error(cmpErr) } } func getCounterMetricValueForLabels(counterVec *basemetrics.CounterVec, labels []string) (metricLabelsWithValue, error) { var result metricLabelsWithValue = metricLabelsWithValue{Labels: labels} value, err := testutil.GetCounterMetricValue(counterVec.WithLabelValues(labels...)) if err != nil { return result, err } result.Value = int(value) return result, nil } </ins> // TestJobPodFailurePolicyWithFailedPodDeletedDuringControllerRestart verifies that the job is properly marked as Failed // in a scenario when the job controller crashes between removing pod finalizers and marking the job as Failed (based on // the pod failure policy). After the finalizer for the failed pod is removed we remove the failed pod. This step is


componentbaseconfig "k8s.io/component-base/config" "k8s.io/component-base/configz" "k8s.io/component-base/logs" <ins> metricsfeatures "k8s.io/component-base/metrics/features" </ins> "k8s.io/component-base/metrics/legacyregistry" <ins> "k8s.io/component-base/metrics/prometheus/slis" </ins> "k8s.io/component-base/version" "k8s.io/component-base/version/verflag" "k8s.io/klog/v2"


"k8s.io/utils/pointer" ) <ins> func init() { utilruntime.Must(metricsfeatures.AddFeatureGates(utilfeature.DefaultMutableFeatureGate)) } </ins> // proxyRun defines the interface to run a specified ProxyServer type proxyRun interface { Run() error


proxyMux := mux.NewPathRecorderMux("kube-proxy") healthz.InstallHandler(proxyMux) <ins> if utilfeature.DefaultFeatureGate.Enabled(metricsfeatures.ComponentSLIs) { slis.SLIMetricsWithReset{}.Install(proxyMux) } </ins> proxyMux.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("X-Content-Type-Options", "nosniff")


"k8s.io/component-base/configz" "k8s.io/component-base/logs" compbasemetrics "k8s.io/component-base/metrics" <ins> metricsfeatures "k8s.io/component-base/metrics/features" </ins> "k8s.io/component-base/metrics/legacyregistry" <ins> "k8s.io/component-base/metrics/prometheus/slis" </ins> runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" podresourcesapiv1alpha1 "k8s.io/kubelet/pkg/apis/podresources/v1alpha1"


"k8s.io/kubernetes/pkg/kubelet/util" ) <ins> func init() { utilruntime.Must(metricsfeatures.AddFeatureGates(utilfeature.DefaultMutableFeatureGate)) } </ins> const ( metricsPath         = "/metrics" cadvisorMetricsPath = "/metrics/cadvisor"


healthz.NamedCheck("syncloop", s.syncLoopHealthCheck), ) <ins> if utilfeature.DefaultFeatureGate.Enabled(metricsfeatures.ComponentSLIs) { slis.SLIMetricsWithReset{}.Install(s.restfulCont) } </ins> s.addMetricsBucketMatcher("pods") ws := new(restful.WebService) ws.


func (t *prefixTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, error) { start := time.Now() transformer := t.transformers[0] <del> prefixedData := make([]byte, len(transformer.Prefix), len(data)+len(transformer.Prefix)) copy(prefixedData, transformer.Prefix) </del> result, err := transformer.Transformer.TransformToStorage(ctx, data, dataCtx) RecordTransformation("to_storage", string(transformer.Prefix), start, err) if err != nil { return nil, err } <ins> prefixedData := make([]byte, len(transformer.Prefix), len(result)+len(transformer.Prefix)) copy(prefixedData, transformer.Prefix) </ins> prefixedData = append(prefixedData, result...) return prefixedData, nil }


testDeploymentDefaultReplicas := int32(2) testDeploymentMinimumReplicas := int32(1) testDeploymentNoReplicas := int32(0) <ins> testDeploymentAvailableReplicas := int32(0) </ins> testDeploymentLabels := map[string]string{"test-deployment-static": "true"} testDeploymentLabelsFlat := "test-deployment-static=true" w := &cache.ListWatch{


"labels": map[string]string{"test-deployment": "patched-status"}, }, "status": map[string]interface{}{ <del> "readyReplicas": testDeploymentNoReplicas, </del> <ins> "readyReplicas":     testDeploymentNoReplicas, "availableReplicas": testDeploymentAvailableReplicas, </ins> }, }) framework.ExpectNoError(err, "failed to Marshal Deployment JSON patch") <del> // This test is broken, patching fails with: // Deployment.apps "test-deployment" is invalid: status.availableReplicas: Invalid value: 2: cannot be greater than readyReplicas // https://github.com/kubernetes/kubernetes/issues/113259 _, _ = dc.Resource(deploymentResource).Namespace(testNamespaceName).Patch(ctx, testDeploymentName, types.StrategicMergePatchType, []byte(deploymentStatusPatch), metav1.PatchOptions{}, "status") </del> <ins> _, err = dc.Resource(deploymentResource).Namespace(testNamespaceName).Patch(ctx, testDeploymentName, types.StrategicMergePatchType, []byte(deploymentStatusPatch), metav1.PatchOptions{}, "status") framework.ExpectNoError(err) </ins> ctxUntil, cancel = context.WithTimeout(ctx, 30*time.Second) defer cancel()


limitRange := validLimitRangeNoDefaults() limitRanges := []corev1.LimitRange{limitRange} <del> var count int64 </del> mockClient := &fake.Clientset{} <del> unhold := make(chan struct{}) </del> <ins> var ( testCount  int64 test1Count int64 ) </ins> mockClient.AddReactor("list", "limitranges", func(action core.Action) (bool, runtime.Object, error) { <del> atomic.AddInt64(&count, 1) </del> <ins> switch action.GetNamespace() { case "test": atomic.AddInt64(&testCount, 1) case "test1": atomic.AddInt64(&test1Count, 1) default: t.Error("unexpected namespace") } </ins> limitRangeList := &corev1.LimitRangeList{ ListMeta: metav1.ListMeta{


value.Namespace = action.GetNamespace() limitRangeList.Items = append(limitRangeList.Items, value) } <del> // he always blocking before sending the signal <-unhold </del> <ins> // make the handler slow so concurrent calls exercise the singleflight time.Sleep(time.Second) </ins> return true, limitRangeList, nil })


} }() } <del> // unhold all the calls with the same namespace handler.GetLimitRanges(attributes) calls, that have to be aggregated unhold <- struct{}{} go func() { unhold <- struct{}{} }() </del> // and here we wait for all the goroutines wg.Wait() // since all the calls with the same namespace will be holded, they must be catched on the singleflight group, // There are two different sets of namespace calls // hence only 2 <del> if count != 2 { t.Errorf("Expected 1 limit range, got %d", count) </del> <ins> if testCount != 1 { t.Errorf("Expected 1 limit range call, got %d", testCount) } if test1Count != 1 { t.Errorf("Expected 1 limit range call, got %d", test1Count) </ins> } // invalidate the cache handler.liveLookupCache.Remove(attributes.GetNamespace()) <del> go func() { // unhold it is blocking until GetLimitRanges is executed unhold <- struct{}{} }() </del> _, err = handler.GetLimitRanges(attributes) if err != nil { t.Errorf("unexpected error: %v", err) } <del> close(unhold) if count != 3 { t.Errorf("Expected 2 limit range, got %d", count) </del> <ins> if testCount != 2 { t.Errorf("Expected 2 limit range call, got %d", testCount) } if test1Count != 1 { t.Errorf("Expected 1 limit range call, got %d", test1Count) </ins> } }


eventsRule(), }, }) <del> addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "disruption-controller"}, Rules: []rbacv1.PolicyRule{ rbacv1helpers.NewRule("get", "list", "watch").Groups(extensionsGroup, appsGroup).Resources("deployments").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups(appsGroup, extensionsGroup).Resources("replicasets").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups(legacyGroup).Resources("replicationcontrollers").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups(policyGroup).Resources("poddisruptionbudgets").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups(appsGroup).Resources("statefulsets").RuleOrDie(), rbacv1helpers.NewRule("update").Groups(policyGroup).Resources("poddisruptionbudgets/status").RuleOrDie(), rbacv1helpers.NewRule("get").Groups("*").Resources("*/scale").RuleOrDie(), eventsRule(), }, }) </del> <ins> addControllerRole(&controllerRoles, &controllerRoleBindings, func() rbacv1.ClusterRole { role := rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "disruption-controller"}, Rules: []rbacv1.PolicyRule{ rbacv1helpers.NewRule("get", "list", "watch").Groups(extensionsGroup, appsGroup).Resources("deployments").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups(appsGroup, extensionsGroup).Resources("replicasets").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups(legacyGroup).Resources("replicationcontrollers").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups(policyGroup).Resources("poddisruptionbudgets").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups(appsGroup).Resources("statefulsets").RuleOrDie(), rbacv1helpers.NewRule("update").Groups(policyGroup).Resources("poddisruptionbudgets/status").RuleOrDie(), rbacv1helpers.NewRule("get").Groups("*").Resources("*/scale").RuleOrDie(), eventsRule(), }, } if utilfeature.DefaultFeatureGate.Enabled(features.PodDisruptionConditions) { role.Rules = append(role.Rules, rbacv1helpers.NewRule("patch").Groups(legacyGroup).Resources("pods/status").RuleOrDie()) } return role }()) </ins> addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "endpoint-controller"}, Rules: []rbacv1.PolicyRule{


"description": "CustomResourceConversion describes how to convert different versions of a CR.", "properties": { "strategy": { <del> "description": "strategy specifies how custom resources are converted between versions. Allowed values are: - `None`: The converter only change the apiVersion and would not touch any other field in the custom resource. - `Webhook`: API Server will call to an external webhook to do the conversion. Additional informationn  is needed for this option. This requires spec.preserveUnknownFields to be false, and spec.conversion.webhook to be set.", </del> <ins> "description": "strategy specifies how custom resources are converted between versions. Allowed values are: - `"None"`: The converter only change the apiVersion and would not touch any other field in the custom resource. - `"Webhook"`: API Server will call to an external webhook to do the conversion. Additional informationn  is needed for this option. This requires spec.preserveUnknownFields to be false, and spec.conversion.webhook to be set.", </ins> "type": "string" }, "webhook": { "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.WebhookConversion", <del> "description": "webhook describes how to call the conversion webhook. Required when `strategy` is set to `Webhook`." </del> <ins> "description": "webhook describes how to call the conversion webhook. Required when `strategy` is set to `"Webhook"`." </ins> } }, "required": [


"properties": { "strategy": { "default": "", <del> "description": "strategy specifies how custom resources are converted between versions. Allowed values are: - `None`: The converter only change the apiVersion and would not touch any other field in the custom resource. - `Webhook`: API Server will call to an external webhook to do the conversion. Additional informationn  is needed for this option. This requires spec.preserveUnknownFields to be false, and spec.conversion.webhook to be set.", </del> <ins> "description": "strategy specifies how custom resources are converted between versions. Allowed values are: - `"None"`: The converter only change the apiVersion and would not touch any other field in the custom resource. - `"Webhook"`: API Server will call to an external webhook to do the conversion. Additional informationn  is needed for this option. This requires spec.preserveUnknownFields to be false, and spec.conversion.webhook to be set.", </ins> "type": "string" }, "webhook": {


"$ref": "#/components/schemas/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.WebhookConversion" } ], <del> "description": "webhook describes how to call the conversion webhook. Required when `strategy` is set to `Webhook`." </del> <ins> "description": "webhook describes how to call the conversion webhook. Required when `strategy` is set to `"Webhook"`." </ins> } }, "required": [


<ins> /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "fmt" "os" "regexp" "strings" flag "github.com/spf13/pflag" kruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" ) var ( typeSrc = flag.StringP("type-src", "s", "", "From where we are going to read the types") re      = regexp.MustCompile("`(bw+b)`") ) // kubeTypesMap is a map from field name to its tag name and doc. type kubeTypesMap map[string]kruntime.Pair func main() { flag.Parse() if *typeSrc == "" { klog.Fatalf("Please define -s flag as it is the api type file") } docsForTypes := kruntime.ParseDocumentationFrom(*typeSrc) rc := false for _, ks := range docsForTypes { typesMap := make(kubeTypesMap) for _, p := range ks[1:] { // skip the field with no tag name if p.Name != "" { typesMap[strings.ToLower(p.Name)] = p } } structName := ks[0].Name rc = checkFieldNameAndDoc(structName, "", ks[0].Doc, typesMap) for _, p := range ks[1:] { rc = checkFieldNameAndDoc(structName, p.Name, p.Doc, typesMap) } } if rc { os.Exit(1) } } func checkFieldNameAndDoc(structName, fieldName, doc string, typesMap kubeTypesMap) bool { rc := false visited := sets.Set[string]{} // The rule is: // 1. Get all back-tick quoted names in the doc // 2. Skip the name which is already found mismatched. // 3. Skip the name whose lowercase is different from the lowercase of tag names, //    because some docs use back-tick to quote field value or nil // 4. Check if the name is different from its tag name // TODO: a manual pass adding back-ticks to the doc strings, then update the linter to // check the existence of back-ticks nameGroups := re.FindAllStringSubmatch(doc, -1) for _, nameGroup := range nameGroups { name := nameGroup[1] if visited.Has(name) { continue } if p, ok := typesMap[strings.ToLower(name)]; ok && p.Name != name { rc = true visited.Insert(name) fmt.Fprintf(os.Stderr, "Error: doc for %s", structName) if fieldName != "" { fmt.Fprintf(os.Stderr, ".%s", fieldName) } fmt.Fprintf(os.Stderr, " contains: %s, which should be: %sn", name, p.Name) } } return rc } </ins>


<ins> #!/usr/bin/env bash # Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This script checks API-related files for mismatch in docs and field names, # and outputs a list of fields that their docs and field names are mismatched. # Usage: `hack/verify-fieldname-docs.sh`. set -o errexit set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" source "${KUBE_ROOT}/hack/lib/util.sh" kube::golang::setup_env make -C "${KUBE_ROOT}" WHAT=cmd/fieldnamedocscheck # Find binary fieldnamedocscheck=$(kube::util::find-binary "fieldnamedocscheck") result=0 find_files() { find . -not (  (  -wholename './output'  -o -wholename './_output'  -o -wholename './_gopath'  -o -wholename './release'  -o -wholename './target'  -o -wholename '*/third_party/*'  -o -wholename '*/vendor/*'  -o -wholename './pkg/*'  ) -prune  )  ( -wholename './staging/src/k8s.io/api/*/v*/types.go'  -o -wholename './staging/src/k8s.io/kube-aggregator/pkg/apis/*/v*/types.go'  -o -wholename './staging/src/k8s.io/apiextensions-apiserver/pkg/apis/*/v*/types.go'  ) } versioned_api_files=$(find_files) || true for file in ${versioned_api_files}; do package="${file%"/types.go"}" echo "Checking ${package}" ${fieldnamedocscheck} -s "${file}" || result=$? done exit ${result} </ins>


Properties: map[string]spec.Schema{ "strategy": { SchemaProps: spec.SchemaProps{ <del> Description: "strategy specifies how custom resources are converted between versions. Allowed values are: - `None`: The converter only change the apiVersion and would not touch any other field in the custom resource. - `Webhook`: API Server will call to an external webhook to do the conversion. Additional informationn  is needed for this option. This requires spec.preserveUnknownFields to be false, and spec.conversion.webhook to be set.", </del> <ins> Description: "strategy specifies how custom resources are converted between versions. Allowed values are: - `"None"`: The converter only change the apiVersion and would not touch any other field in the custom resource. - `"Webhook"`: API Server will call to an external webhook to do the conversion. Additional informationn  is needed for this option. This requires spec.preserveUnknownFields to be false, and spec.conversion.webhook to be set.", </ins> Default:     "", Type:        []string{"string"}, Format:      "",


}, "webhook": { SchemaProps: spec.SchemaProps{ <del> Description: "webhook describes how to call the conversion webhook. Required when `strategy` is set to `Webhook`.", </del> <ins> Description: "webhook describes how to call the conversion webhook. Required when `strategy` is set to `"Webhook"`.", </ins> Ref:         ref("k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1.WebhookConversion"), }, },


// CustomResourceConversion describes how to convert different versions of a CR. message CustomResourceConversion { // strategy specifies how custom resources are converted between versions. Allowed values are: <del> // - `None`: The converter only change the apiVersion and would not touch any other field in the custom resource. // - `Webhook`: API Server will call to an external webhook to do the conversion. Additional information </del> <ins> // - `"None"`: The converter only change the apiVersion and would not touch any other field in the custom resource. // - `"Webhook"`: API Server will call to an external webhook to do the conversion. Additional information </ins> //   is needed for this option. This requires spec.preserveUnknownFields to be false, and spec.conversion.webhook to be set. optional string strategy = 1; <del> // webhook describes how to call the conversion webhook. Required when `strategy` is set to `Webhook`. </del> <ins> // webhook describes how to call the conversion webhook. Required when `strategy` is set to `"Webhook"`. </ins> // +optional optional WebhookConversion webhook = 2; }


// CustomResourceConversion describes how to convert different versions of a CR. type CustomResourceConversion struct { // strategy specifies how custom resources are converted between versions. Allowed values are: <del> // - `None`: The converter only change the apiVersion and would not touch any other field in the custom resource. // - `Webhook`: API Server will call to an external webhook to do the conversion. Additional information </del> <ins> // - `"None"`: The converter only change the apiVersion and would not touch any other field in the custom resource. // - `"Webhook"`: API Server will call to an external webhook to do the conversion. Additional information </ins> //   is needed for this option. This requires spec.preserveUnknownFields to be false, and spec.conversion.webhook to be set. Strategy ConversionStrategyType `json:"strategy" protobuf:"bytes,1,name=strategy"` <del> // webhook describes how to call the conversion webhook. Required when `strategy` is set to `Webhook`. </del> <ins> // webhook describes how to call the conversion webhook. Required when `strategy` is set to `"Webhook"`. </ins> // +optional Webhook *WebhookConversion `json:"webhook,omitempty" protobuf:"bytes,2,opt,name=webhook"` }


"k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" resourcev1alpha2listers "k8s.io/client-go/listers/resource/v1alpha2" <del> corev1helpers "k8s.io/component-helpers/scheduling/corev1" </del> "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" "k8s.io/dynamic-resource-allocation/resourceclaim" "k8s.io/klog/v2"


// Empty if the Pod has no claims. claims []*resourcev1alpha2.ResourceClaim <del> // The AvailableOnNodes node filters of the claims converted from the // v1 API to nodeaffinity.NodeSelector by PreFilter for repeated // evaluation in Filter. Nil for claims which don't have it. availableOnNodes []*nodeaffinity.NodeSelector </del> // The indices of all claims that: // - are allocated // - use delayed allocation


podSchedulingDirty bool mutex sync.Mutex <ins> informationsForClaim []informationForClaim } type informationForClaim struct { // The availableOnNode node filter of the claim converted from the // v1 API to nodeaffinity.NodeSelector by PreFilter for repeated // evaluation in Filter. Nil for claim which don't have it. availableOnNode *nodeaffinity.NodeSelector // The status of the claim got from the // schedulingCtx by PreFilter for repeated // evaluation in Filter. Nil for claim which don't have it. status *resourcev1alpha2.ResourceClaimSchedulingStatus </ins> } func (d *stateData) Clone() framework.StateData {


return nil, framework.NewStatus(framework.Skip) } <del> s.availableOnNodes = make([]*nodeaffinity.NodeSelector, len(claims)) </del> <ins> s.informationsForClaim = make([]informationForClaim, len(claims)) </ins> for index, claim := range claims { if claim.Spec.AllocationMode == resourcev1alpha2.AllocationModeImmediate && claim.Status.Allocation == nil {


if err != nil { return nil, statusError(logger, err) } <del> s.availableOnNodes[index] = nodeSelector </del> <ins> s.informationsForClaim[index].availableOnNode = nodeSelector } if claim.Status.Allocation == nil && claim.Spec.AllocationMode == resourcev1alpha2.AllocationModeWaitForFirstConsumer { // The ResourceClass might have a node filter. This is // useful for trimming the initial set of potential // nodes before we ask the driver(s) for information // about the specific pod. class, err := pl.classLister.Get(claim.Spec.ResourceClassName) if err != nil { // If the class does not exist, then allocation cannot proceed. return nil, statusError(logger, fmt.Errorf("look up resource class: %v", err)) } if class.SuitableNodes != nil { selector, err := nodeaffinity.NewNodeSelector(class.SuitableNodes) if err != nil { return nil, statusError(logger, err) } s.informationsForClaim[index].availableOnNode = selector } // Now we need information from drivers. schedulingCtx, err := s.initializePodSchedulingContexts(ctx, pod, pl.podSchedulingContextLister) if err != nil { return nil, statusError(logger, err) } s.informationsForClaim[index].status = statusForClaim(schedulingCtx, pod.Spec.ResourceClaims[index].Name) </ins> } }


logger.V(10).Info("filtering based on resource claims of the pod", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim)) switch { case claim.Status.Allocation != nil: <del> if nodeSelector := state.availableOnNodes[index]; nodeSelector != nil { </del> <ins> if nodeSelector := state.informationsForClaim[index].availableOnNode; nodeSelector != nil { </ins> if !nodeSelector.Match(node) { logger.V(5).Info("AvailableOnNodes does not match", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim)) unavailableClaims = append(unavailableClaims, index)


// We shouldn't get here. PreFilter already checked this. return statusUnschedulable(logger, "resourceclaim must be reallocated", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim)) case claim.Spec.AllocationMode == resourcev1alpha2.AllocationModeWaitForFirstConsumer: <del> // The ResourceClass might have a node filter. This is // useful for trimming the initial set of potential // nodes before we ask the driver(s) for information // about the specific pod. class, err := pl.classLister.Get(claim.Spec.ResourceClassName) if err != nil { // If the class does not exist, then allocation cannot proceed. return statusError(logger, fmt.Errorf("look up resource class: %v", err)) } if class.SuitableNodes != nil { // TODO (#113700): parse class.SuitableNodes once in PreFilter, reuse result. matches, err := corev1helpers.MatchNodeSelectorTerms(node, class.SuitableNodes) if err != nil { return statusError(logger, fmt.Errorf("potential node filter: %v", err)) </del> <ins> if selector := state.informationsForClaim[index].availableOnNode; selector != nil { if matches := selector.Match(node); !matches { return statusUnschedulable(logger, "excluded by resource class node filter", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclassName", claim.Spec.ResourceClassName) </ins> } <del> if !matches { return statusUnschedulable(logger, "excluded by resource class node filter", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclass", klog.KObj(class)) } } // Now we need information from drivers. schedulingCtx, err := state.initializePodSchedulingContexts(ctx, pod, pl.podSchedulingContextLister) if err != nil { return statusError(logger, err) </del> } <del> status := statusForClaim(schedulingCtx, pod.Spec.ResourceClaims[index].Name) if status != nil { </del> <ins> if status := state.informationsForClaim[index].status; status != nil { </ins> for _, unsuitableNode := range status.UnsuitableNodes { if node.Name == unsuitableNode { return statusUnschedulable(logger, "resourceclaim cannot be allocated for the node (unsuitable)", "pod", klog.KObj(pod), "node", klog.KObj(node), "resourceclaim", klog.KObj(claim), "unsuitablenodes", status.UnsuitableNodes)


pod:    podWithClaimName, claims: []*resourcev1alpha2.ResourceClaim{pendingDelayedClaim}, want: want{ <del> filter: perNodeResult{ workerNode.Name: { status: framework.AsStatus(fmt.Errorf(`look up resource class: resourceclass.resource.k8s.io "%s" not found`, className)), }, </del> <ins> prefilter: result{ status: framework.AsStatus(fmt.Errorf(`look up resource class: resourceclass.resource.k8s.io "%s" not found`, className)), </ins> }, postfilter: result{ <del> status: framework.NewStatus(framework.Unschedulable, `still not schedulable`), </del> <ins> status: framework.NewStatus(framework.Unschedulable, `no new claims to deallocate`), </ins> }, }, },


kubeReserved, err := parseResourceList(s.KubeReserved) if err != nil { <del> return err </del> <ins> return fmt.Errorf("--kube-reserved value failed to parse: %w", err) </ins> } systemReserved, err := parseResourceList(s.SystemReserved) if err != nil { <del> return err </del> <ins> return fmt.Errorf("--system-reserved value failed to parse: %w", err) </ins> } var hardEvictionThresholds []evictionapi.Threshold // If the user requested to ignore eviction thresholds, then do not set valid values for hardEvictionThresholds here.


} experimentalQOSReserved, err := cm.ParseQOSReserved(s.QOSReserved) if err != nil { <del> return err </del> <ins> return fmt.Errorf("--qos-reserved value failed to parse: %w", err) </ins> } var cpuManagerPolicyOptions map[string]string


case v1.ResourceCPU, v1.ResourceMemory, v1.ResourceEphemeralStorage, pidlimit.PIDs: q, err := resource.ParseQuantity(v) if err != nil { <del> return nil, err </del> <ins> return nil, fmt.Errorf("failed to parse quantity %q for %q resource: %w", v, k, err) </ins> } if q.Sign() == -1 { return nil, fmt.Errorf("resource quantity for %q cannot be negative: %v", k, v)


case v1.ResourceMemory: q, err := parsePercentage(v) if err != nil { <del> return nil, err </del> <ins> return nil, fmt.Errorf("failed to parse percentage %q for %q resource: %w", v, k, err) </ins> } reservations[v1.ResourceName(k)] = q default:


package kuberuntime import ( <del> "runtime" </del> v1 "k8s.io/api/core/v1" <ins> "k8s.io/apimachinery/pkg/api/resource" </ins> runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/klog/v2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" <ins> "k8s.io/kubernetes/pkg/kubelet/winstats" </ins> "k8s.io/kubernetes/pkg/securitycontext" )


cpuLimit := container.Resources.Limits.Cpu() if !cpuLimit.IsZero() { <del> // Note that sysinfo.NumCPU() is limited to 64 CPUs on Windows due to Processor Groups, // as only 64 processors are available for execution by a given process. This causes // some oddities on systems with more than 64 processors. // Refer https://msdn.microsoft.com/en-us/library/windows/desktop/dd405503(v=vs.85).aspx. </del> // Since Kubernetes doesn't have any notion of weight in the Pod/Container API, only limits/reserves, then applying CpuMaximum only // will better follow the intent of the user. At one point CpuWeights were set, but this prevented limits from having any effect.


//   https://github.com/kubernetes/kubernetes/blob/56d1c3b96d0a544130a82caad33dd57629b8a7f8/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto#L681-L682 //   https://github.com/opencontainers/runtime-spec/blob/ad53dcdc39f1f7f7472b10aa0a45648fe4865496/config-windows.md#cpu //   If both CpuWeight and CpuMaximum are set - ContainerD catches this invalid case and returns an error instead. <del> cpuMaximum := 10000 * cpuLimit.MilliValue() / int64(runtime.NumCPU()) / 1000 // ensure cpuMaximum is in range [1, 10000]. if cpuMaximum < 1 { cpuMaximum = 1 } else if cpuMaximum > 10000 { cpuMaximum = 10000 } wc.Resources.CpuMaximum = cpuMaximum </del> <ins> wc.Resources.CpuMaximum = calculateCPUMaximum(cpuLimit, int64(winstats.ProcessorCount())) </ins> } // The processor resource controls are mutually exclusive on


return wc, nil } <ins> // calculateCPUMaximum calculates the maximum CPU given a limit and a number of cpus while ensuring it's in range [1,10000]. func calculateCPUMaximum(cpuLimit *resource.Quantity, cpuCount int64) int64 { cpuMaximum := 10 * cpuLimit.MilliValue() / cpuCount // ensure cpuMaximum is in range [1, 10000]. if cpuMaximum < 1 { cpuMaximum = 1 } else if cpuMaximum > 10000 { cpuMaximum = 10000 } return cpuMaximum } </ins>


package kuberuntime import ( <del> "runtime" </del> "testing" "github.com/stretchr/testify/assert"


v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <del> utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" </del> runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" <del> "k8s.io/kubernetes/pkg/features" </del> <ins> "k8s.io/kubernetes/pkg/kubelet/winstats" </ins> ) func TestApplyPlatformSpecificContainerConfig(t *testing.T) {


err = fakeRuntimeSvc.applyPlatformSpecificContainerConfig(containerConfig, &pod.Spec.Containers[0], pod, new(int64), "foo", nil) require.NoError(t, err) <del> expectedCpuMax := ((10000 * 3000) / int64(runtime.NumCPU()) / 1000) </del> <ins> limit := int64(3000) expectedCpuMax := 10 * limit / int64(winstats.ProcessorCount()) </ins> expectedWindowsConfig := &runtimeapi.WindowsContainerConfig{ Resources: &runtimeapi.WindowsContainerResources{ CpuMaximum:         expectedCpuMax,


} assert.Equal(t, expectedWindowsConfig, containerConfig.Windows) } <ins> func TestCalculateCPUMaximum(t *testing.T) { tests := []struct { name     string cpuLimit resource.Quantity cpuCount int64 want     int64 }{ { name:     "max range when same amount", cpuLimit: resource.MustParse("1"), cpuCount: 1, want:     10000, }, { name:     "percentage calculation is working as intended", cpuLimit: resource.MustParse("94"), cpuCount: 96, want:     9791, }, { name:     "half range when half amount", cpuLimit: resource.MustParse("1"), cpuCount: 2, want:     5000, }, { name:     "max range when more requested than available", cpuLimit: resource.MustParse("2"), cpuCount: 1, want:     10000, }, { name:     "min range when less than minimum", cpuLimit: resource.MustParse("1m"), cpuCount: 100, want:     1, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { assert.Equal(t, tt.want, calculateCPUMaximum(&tt.cpuLimit, tt.cpuCount)) }) } } </ins>


} return &cadvisorapi.MachineInfo{ <del> NumCores:       processorCount(), </del> <ins> NumCores:       ProcessorCount(), </ins> MemoryCapacity: p.nodeInfo.memoryPhysicalCapacityBytes, MachineID:      hostname, SystemUUID:     systemUUID,


// more notes for this issue: // same issue in moby: https://github.com/moby/moby/issues/38935#issuecomment-744638345 // solution in hcsshim: https://github.com/microsoft/hcsshim/blob/master/internal/processorinfo/processor_count.go <del> func processorCount() int { </del> <ins> func ProcessorCount() int { </ins> if amount := getActiveProcessorCount(allProcessorGroups); amount != 0 { return int(amount) }


}) ginkgo.It("should work with the pod containing more than 6 DNS search paths and longer than 256 search list characters", func(ctx context.Context) { <del> ginkgo.By("Getting the kube-dns IP") svc, err := f.ClientSet.CoreV1().Services("kube-system").Get(ctx, "kube-dns", metav1.GetOptions{}) framework.ExpectNoError(err, "Failed to get kube-dns service") kubednsIP := svc.Spec.ClusterIP </del> // All the names we need to be able to resolve. namesToResolve := []string{ "kubernetes.default",


pod := createDNSPod(f.Namespace.Name, wheezyProbeCmd, jessieProbeCmd, dnsTestPodHostName, dnsTestServiceName) pod.Spec.DNSPolicy = v1.DNSClusterFirst pod.Spec.DNSConfig = &v1.PodDNSConfig{ <del> Nameservers: []string{kubednsIP}, Searches:    testSearchPaths, </del> <ins> Searches: testSearchPaths, </ins> Options: []v1.PodDNSConfigOption{ { Name:  "ndots",


v1.LabelTopologyRegion, } <ins> func translateToGALabel(label string) string { if label == v1.LabelFailureDomainBetaRegion { return v1.LabelTopologyRegion } if label == v1.LabelFailureDomainBetaZone { return v1.LabelTopologyZone } return label } </ins> // Name returns name of the plugin. It is used in logs, etc. func (pl *VolumeZone) Name() string { return Name


for _, pvTopology := range podPVTopologies { v, ok := node.Labels[pvTopology.key] <ins> if !ok { // if we can't match the beta label, try to match pv's beta label with node's ga label v, ok = node.Labels[translateToGALabel(pvTopology.key)] } </ins> if !ok || !pvTopology.values.Has(v) { logger.V(10).Info("Won't schedule pod onto node due to volume (mismatch on label key)", "pod", klog.KObj(pod), "node", klog.KObj(node), "PV", klog.KRef("", pvTopology.pvName), "PVLabelKey", pvTopology.key) return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonConflict)


}, wantFilterStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonConflict), }, <ins> { name: "pv with beta label,node with ga label, matched", Pod:  createPodWithVolume("pod_1", "Vol_1", "PVC_1"), Node: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "host1", Labels: map[string]string{ v1.LabelTopologyZone: "us-west1-a", }, }, }, }, { name: "pv with beta label,node with ga label, don't match", Pod:  createPodWithVolume("pod_1", "vol_1", "PVC_1"), Node: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "host1", Labels: map[string]string{ v1.LabelTopologyZone: "us-west1-b", }, }, }, wantFilterStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonConflict), }, </ins> } for _, test := range tests {


return } <del> // enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided // timeout inside the parent context is lower than requestTimeoutUpperBound. ctx, cancel := context.WithTimeout(ctx, requestTimeoutUpperBound) defer cancel() </del> <ins> // DELETECOLLECTION can be a lengthy operation, // we should not impose any 34s timeout here. // NOTE: This is similar to LIST which does not enforce a 34s timeout. </ins> ctx = request.WithNamespace(ctx, namespace) outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)


"context" "io" "net/http" <ins> "net/http/httptest" </ins> "strings" <ins> "sync/atomic" </ins> "testing" <ins> metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" </ins> metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime"


auditapis "k8s.io/apiserver/pkg/apis/audit" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" <ins> "k8s.io/apiserver/pkg/registry/rest" </ins> "k8s.io/utils/pointer" )


}) } } <ins> func TestDeleteCollectionWithNoContextDeadlineEnforced(t *testing.T) { var invokedGot, hasDeadlineGot int32 fakeDeleterFn := func(ctx context.Context, _ rest.ValidateObjectFunc, _ *metav1.DeleteOptions, _ *metainternalversion.ListOptions) (runtime.Object, error) { // we expect CollectionDeleter to be executed once atomic.AddInt32(&invokedGot, 1) // we don't expect any context deadline to be set if _, hasDeadline := ctx.Deadline(); hasDeadline { atomic.AddInt32(&hasDeadlineGot, 1) } return nil, nil } // do the minimum setup to ensure that it gets as far as CollectionDeleter scope := &RequestScope{ Namer: &mockNamer{}, Serializer: &fakeSerializer{ serializer: runtime.NewCodec(runtime.NoopEncoder{}, runtime.NoopDecoder{}), }, } handler := DeleteCollection(fakeCollectionDeleterFunc(fakeDeleterFn), false, scope, nil) request, err := http.NewRequest("GET", "/test", nil) if err != nil { t.Fatalf("unexpected error: %v", err) } // the request context should not have any deadline by default if _, hasDeadline := request.Context().Deadline(); hasDeadline { t.Fatalf("expected request context to not have any deadline") } recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, request) if atomic.LoadInt32(&invokedGot) != 1 { t.Errorf("expected collection deleter to be invoked") } if atomic.LoadInt32(&hasDeadlineGot) > 0 { t.Errorf("expected context to not have any deadline") } } type fakeCollectionDeleterFunc func(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) func (f fakeCollectionDeleterFunc) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) { return f(ctx, deleteValidation, options, listOptions) } type fakeSerializer struct { serializer runtime.Serializer } func (n *fakeSerializer) SupportedMediaTypes() []runtime.SerializerInfo { return []runtime.SerializerInfo{ { MediaType:        "application/json", MediaTypeType:    "application", MediaTypeSubType: "json", }, } } func (n *fakeSerializer) EncoderForVersion(serializer runtime.Encoder, gv runtime.GroupVersioner) runtime.Encoder { return n.serializer } func (n *fakeSerializer) DecoderToVersion(serializer runtime.Decoder, gv runtime.GroupVersioner) runtime.Decoder { return n.serializer } </ins>


var _ storagelisters.CSINodeLister = CSINodeLister{} // CSINodeLister declares a storagev1.CSINode type for testing. <del> type CSINodeLister storagev1.CSINode </del> <ins> type CSINodeLister []storagev1.CSINode </ins> // Get returns a fake CSINode object. func (n CSINodeLister) Get(name string) (*storagev1.CSINode, error) { <del> csiNode := storagev1.CSINode(n) return &csiNode, nil </del> <ins> for _, cn := range n { if cn.Name == name { return &cn, nil } } return nil, fmt.Errorf("csiNode %q not found", name) </ins> } // List lists all CSINodes in the indexer.


return fmt.Errorf("looking up provisioner name for volume %v: %w", vol, err) } if !isCSIMigrationOn(csiNode, inTreeProvisionerName) { <ins> csiNodeName := "" if csiNode != nil { csiNodeName = csiNode.Name } </ins> klog.V(5).InfoS("CSI Migration is not enabled for provisioner", "provisioner", inTreeProvisionerName, <del> "pod", klog.KObj(pod), "csiNode", csiNode.Name) </del> <ins> "pod", klog.KObj(pod), "csiNode", csiNodeName) </ins> return nil } // Do translation for the in-tree volume.


wantStatus:       framework.NewStatus(framework.Unschedulable, ErrReasonMaxVolumeCountExceeded), }, { <ins> newPod:           inTreeInlineVolPod, existingPods:     []*v1.Pod{inTreeTwoVolPod}, filterName:       "csi", maxVols:          2, driverNames:      []string{csilibplugins.AWSEBSInTreePluginName, ebsCSIDriverName}, migrationEnabled: true, limitSource:      "node", test:             "nil csi node", }, { </ins> newPod:           pendingVolumePod, existingPods:     []*v1.Pod{inTreeTwoVolPod}, filterName:       "csi",


} pvLister = append(pvLister, pv) } } return pvLister }


} func getFakeCSINodeLister(csiNode *storagev1.CSINode) fakeframework.CSINodeLister { <ins> csiNodeLister := fakeframework.CSINodeLister{} </ins> if csiNode != nil { <del> return fakeframework.CSINodeLister(*csiNode) </del> <ins> csiNodeLister = append(csiNodeLister, *csiNode.DeepCopy()) </ins> } <del> return fakeframework.CSINodeLister{} </del> <ins> return csiNodeLister </ins> } func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int64, driverNames ...string) (*framework.NodeInfo, *storagev1.CSINode) {


initCSINode := func() { csiNode = &storagev1.CSINode{ <del> ObjectMeta: metav1.ObjectMeta{Name: "csi-node-for-max-pd-test-1"}, </del> <ins> ObjectMeta: metav1.ObjectMeta{Name: "node-for-max-pd-test-1"}, </ins> Spec: storagev1.CSINodeSpec{ Drivers: []storagev1.CSINodeDriver{}, },


API Server using the current Aggregator [Conformance]' description: Ensure that the sample-apiserver code from 1.17 and compiled against 1.17 will work on the current Aggregator/API-Server. <del> release: v1.17, v1.21 </del> <ins> release: v1.17, v1.21, v1.27 </ins> file: test/e2e/apimachinery/aggregator.go - testname: Custom Resource Definition Conversion Webhook, convert mixed version list codename: '[sig-api-machinery] CustomResourceConversionWebhook [Privileged:ClusterAdmin]


apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" <ins> "k8s.io/apimachinery/pkg/labels" </ins> "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/discovery" clientset "k8s.io/client-go/kubernetes" <ins> "k8s.io/client-go/util/retry" </ins> apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1" "k8s.io/kubernetes/test/e2e/framework" <ins> e2eauth "k8s.io/kubernetes/test/e2e/framework/auth" </ins> e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" imageutils "k8s.io/kubernetes/test/utils/image"


const ( aggregatorServicePort = 7443 <ins> apiServiceRetryPeriod  = 1 * time.Second apiServiceRetryTimeout = 2 * time.Minute </ins> ) var _ = SIGDescribe("Aggregator", func() {


}) /* <del> Release: v1.17, v1.21 </del> <ins> Release: v1.17, v1.21, v1.27 </ins> Testname: aggregator-supports-the-sample-apiserver Description: Ensure that the sample-apiserver code from 1.17 and compiled against 1.17 will work on the current Aggregator/API-Server.


_, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating secret %s in namespace %s", secretName, namespace) <del> // kubectl create -f clusterrole.yaml _, err = client.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{ </del> <ins> if e2eauth.IsRBACEnabled(ctx, client.RbacV1()) { // kubectl create -f clusterrole.yaml _, err = client.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{ </ins> <del> ObjectMeta: metav1.ObjectMeta{Name: "sample-apiserver-reader"}, Rules: []rbacv1.PolicyRule{ rbacv1helpers.NewRule("get", "list", "watch").Groups("").Resources("namespaces").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups("admissionregistration.k8s.io").Resources("*").RuleOrDie(), }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role %s", "sample-apiserver-reader") </del> <ins> ObjectMeta: metav1.ObjectMeta{Name: "sample-apiserver-reader"}, Rules: []rbacv1.PolicyRule{ rbacv1helpers.NewRule("get", "list", "watch").Groups("").Resources("namespaces").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups("admissionregistration.k8s.io").Resources("*").RuleOrDie(), }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role %s", "sample-apiserver-reader") </ins> <del> _, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler:" + namespace + ":sample-apiserver-reader", }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind:     "ClusterRole", Name:     "sample-apiserver-reader", }, Subjects: []rbacv1.Subject{ { APIGroup:  "", Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, </del> <ins> _, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler:" + namespace + ":sample-apiserver-reader", </ins> }, <del> }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+namespace+":sample-apiserver-reader") </del> <ins> RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind:     "ClusterRole", Name:     "sample-apiserver-reader", }, Subjects: []rbacv1.Subject{ { APIGroup:  "", Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, }, }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+namespace+":sample-apiserver-reader") </ins> <del> // kubectl create -f authDelegator.yaml _, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler:" + namespace + ":auth-delegator", }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind:     "ClusterRole", Name:     "system:auth-delegator", }, Subjects: []rbacv1.Subject{ { APIGroup:  "", Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, </del> <ins> // kubectl create -f authDelegator.yaml _, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler:" + namespace + ":auth-delegator", </ins> }, <del> }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+namespace+":auth-delegator") </del> <ins> RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind:     "ClusterRole", Name:     "system:auth-delegator", }, Subjects: []rbacv1.Subject{ { APIGroup:  "", Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, }, }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+namespace+":auth-delegator") } </ins> // kubectl create -f deploy.yaml deploymentName := "sample-apiserver-deployment"


_, err = client.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating service account %s in namespace %s", "sample-apiserver", namespace) <del> // kubectl create -f auth-reader.yaml _, err = client.RbacV1().RoleBindings("kube-system").Create(ctx, &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler-auth-reader", Annotations: map[string]string{ rbacv1.AutoUpdateAnnotationKey: "true", </del> <ins> if e2eauth.IsRBACEnabled(ctx, client.RbacV1()) { // kubectl create -f auth-reader.yaml _, err = client.RbacV1().RoleBindings("kube-system").Create(ctx, &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "wardler-auth-reader", Annotations: map[string]string{ rbacv1.AutoUpdateAnnotationKey: "true", }, </ins> }, <del> }, RoleRef: rbacv1.RoleRef{ APIGroup: "", Kind:     "Role", Name:     "extension-apiserver-authentication-reader", }, Subjects: []rbacv1.Subject{ { Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, </del> <ins> RoleRef: rbacv1.RoleRef{ APIGroup: "", Kind:     "Role", Name:     "extension-apiserver-authentication-reader", </ins> }, <del> }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating role binding %s in namespace %s", "wardler-auth-reader", "kube-system") </del> <ins> Subjects: []rbacv1.Subject{ { Kind:      "ServiceAccount", Name:      "default", Namespace: namespace, }, }, }, metav1.CreateOptions{}) framework.ExpectNoError(err, "creating role binding %s in namespace %s", "wardler-auth-reader", "kube-system") } </ins> // Wait for the extension apiserver to be up and healthy // kubectl get deployments -n <aggregated-api-namespace> && status == Running


Service: &apiregistrationv1.ServiceReference{ Namespace: namespace, Name:      "sample-api", <del> Port:      pointer.Int32Ptr(aggregatorServicePort), </del> <ins> Port:      pointer.Int32(aggregatorServicePort), </ins> }, Group:                "wardle.example.com", Version:              "v1alpha1",


framework.Failf("Unable to find v1alpha1.wardle.example.com in APIServiceList") } <ins> // As the APIService doesn't have any labels currently set we need to // set one so that we can select it later when we call deleteCollection ginkgo.By("Adding a label to the APIService") apiServiceName := "v1alpha1.wardle.example.com" apiServiceClient := aggrclient.ApiregistrationV1().APIServices() apiServiceLabel := map[string]string{"e2e-apiservice": "patched"} apiServiceLabelSelector := labels.SelectorFromSet(apiServiceLabel).String() apiServicePatch, err := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ "labels": apiServiceLabel, }, }) framework.ExpectNoError(err, "failed to Marshal APIService JSON patch") _, err = apiServiceClient.Patch(ctx, apiServiceName, types.StrategicMergePatchType, []byte(apiServicePatch), metav1.PatchOptions{}) framework.ExpectNoError(err, "failed to patch APIService") patchedApiService, err := apiServiceClient.Get(ctx, apiServiceName, metav1.GetOptions{}) framework.ExpectNoError(err, "Unable to retrieve api service %s", apiServiceName) framework.Logf("APIService labels: %v", patchedApiService.Labels) ginkgo.By("Updating APIService Status") var updatedStatus, wardle *apiregistrationv1.APIService err = retry.RetryOnConflict(retry.DefaultRetry, func() error { var statusToUpdate *apiregistrationv1.APIService statusContent, err = restClient.Get(). AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). SetHeader("Accept", "application/json").DoRaw(ctx) framework.ExpectNoError(err, "No response for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) err = json.Unmarshal([]byte(statusContent), &statusToUpdate) framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) statusToUpdate.Status.Conditions = append(statusToUpdate.Status.Conditions, apiregistrationv1.APIServiceCondition{ Type:    "StatusUpdated", Status:  "True", Reason:  "E2E", Message: "Set from e2e test", }) updatedStatus, err = apiServiceClient.UpdateStatus(ctx, statusToUpdate, metav1.UpdateOptions{}) return err }) framework.ExpectNoError(err, "Failed to update status. %v", err) framework.Logf("updatedStatus.Conditions: %#v", updatedStatus.Status.Conditions) ginkgo.By("Confirm that v1alpha1.wardle.example.com /status was updated") statusContent, err = restClient.Get(). AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). SetHeader("Accept", "application/json").DoRaw(ctx) framework.ExpectNoError(err, "No response for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) err = json.Unmarshal([]byte(statusContent), &wardle) framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) foundUpdatedStatusCondition := false for _, cond := range wardle.Status.Conditions { if cond.Type == "StatusUpdated" && cond.Reason == "E2E" && cond.Message == "Set from e2e test" { framework.Logf("Found APIService %v with Labels: %v & Condition: %v", wardle.ObjectMeta.Name, wardle.Labels, cond) foundUpdatedStatusCondition = true break } else { framework.Logf("Observed APIService %v with Labels: %v & Condition: %v", wardle.ObjectMeta.Name, wardle.Labels, cond) } } framework.ExpectEqual(foundUpdatedStatusCondition, true, "The updated status condition was not found. %#v", wardle.Status.Conditions) framework.Logf("Found updated status condition for %s", wardle.ObjectMeta.Name) ginkgo.By(fmt.Sprintf("Replace APIService %s", apiServiceName)) var updatedApiService *apiregistrationv1.APIService err = retry.RetryOnConflict(retry.DefaultRetry, func() error { currentApiService, err := apiServiceClient.Get(ctx, apiServiceName, metav1.GetOptions{}) framework.ExpectNoError(err, "Unable to get APIService %s", apiServiceName) currentApiService.Labels = map[string]string{ apiServiceName: "updated", } updatedApiService, err = apiServiceClient.Update(ctx, currentApiService, metav1.UpdateOptions{}) return err }) framework.ExpectNoError(err) framework.ExpectEqual(updatedApiService.Labels[apiServiceName], "updated", "should have the updated label but have %q", updatedApiService.Labels[apiServiceName]) framework.Logf("Found updated apiService label for %q", apiServiceName) </ins> // kubectl delete flunder test-flunder <ins> ginkgo.By(fmt.Sprintf("Delete APIService %q", flunderName)) </ins> err = dynamicClient.Delete(ctx, flunderName, metav1.DeleteOptions{}) validateErrorWithDebugInfo(ctx, f, err, pods, "deleting flunders(%v) using dynamic client", unstructuredList.Items)


framework.Failf("failed to get back the correct deleted flunders list %v from the dynamic client", unstructuredList) } <ins> ginkgo.By("Recreating test-flunder before removing endpoint via deleteCollection") jsonFlunder, err = json.Marshal(testFlunder) framework.ExpectNoError(err, "marshalling test-flunder for create using dynamic client") unstruct = &unstructured.Unstructured{} err = unstruct.UnmarshalJSON(jsonFlunder) framework.ExpectNoError(err, "unmarshalling test-flunder as unstructured for create using dynamic client") _, err = dynamicClient.Create(ctx, unstruct, metav1.CreateOptions{}) framework.ExpectNoError(err, "listing flunders using dynamic client") // kubectl get flunders unstructuredList, err = dynamicClient.List(ctx, metav1.ListOptions{}) framework.ExpectNoError(err, "listing flunders using dynamic client") if len(unstructuredList.Items) != 1 { framework.Failf("failed to get back the correct flunders list %v from the dynamic client", unstructuredList) } ginkgo.By("Read v1alpha1.wardle.example.com /status before patching it") statusContent, err = restClient.Get(). AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). SetHeader("Accept", "application/json").DoRaw(ctx) framework.ExpectNoError(err, "No response for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) wardle.Reset() err = json.Unmarshal([]byte(statusContent), &wardle) framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) ginkgo.By("Patch APIService Status") patch := map[string]interface{}{ "status": map[string]interface{}{ "conditions": append(wardle.Status.Conditions, apiregistrationv1.APIServiceCondition{ Type:    "StatusPatched", Status:  "True", Reason:  "E2E", Message: "Set by e2e test", }), }, } payload, err := json.Marshal(patch) framework.ExpectNoError(err, "Failed to marshal JSON. %v", err) _, err = restClient.Patch(types.MergePatchType). AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). SetHeader("Accept", "application/json"). Body([]byte(payload)). DoRaw(ctx) framework.ExpectNoError(err, "Patch failed for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) ginkgo.By("Confirm that v1alpha1.wardle.example.com /status was patched") statusContent, err = restClient.Get(). AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). SetHeader("Accept", "application/json").DoRaw(ctx) framework.ExpectNoError(err, "No response for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) wardle.Reset() err = json.Unmarshal([]byte(statusContent), &wardle) framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) foundPatchedStatusCondition := false for _, cond := range wardle.Status.Conditions { if cond.Type == "StatusPatched" && cond.Reason == "E2E" && cond.Message == "Set by e2e test" { framework.Logf("Found APIService %v with Labels: %v & Conditions: %v", wardle.ObjectMeta.Name, wardle.Labels, cond) foundPatchedStatusCondition = true break } else { framework.Logf("Observed APIService %v with Labels: %v & Conditions: %v", wardle.ObjectMeta.Name, wardle.Labels, cond) } } framework.ExpectEqual(foundPatchedStatusCondition, true, "The patched status condition was not found. %#v", wardle.Status.Conditions) framework.Logf("Found patched status condition for %s", wardle.ObjectMeta.Name) ginkgo.By(fmt.Sprintf("APIService deleteCollection with labelSelector: %q", apiServiceLabelSelector)) err = aggrclient.ApiregistrationV1().APIServices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: apiServiceLabelSelector}) framework.ExpectNoError(err, "Unable to delete apiservice %s", apiServiceName) ginkgo.By("Confirm that the generated APIService has been deleted") err = wait.PollImmediate(apiServiceRetryPeriod, apiServiceRetryTimeout, checkApiServiceListQuantity(ctx, aggrclient, apiServiceLabelSelector, 0)) framework.ExpectNoError(err, "failed to count the required APIServices") framework.Logf("APIService %s has been deleted.", apiServiceName) </ins> cleanTest(ctx, client, aggrclient, namespace) }


} return fmt.Sprintf("%s-%d", base, id) } <ins> func checkApiServiceListQuantity(ctx context.Context, aggrclient *aggregatorclient.Clientset, label string, quantity int) func() (bool, error) { return func() (bool, error) { var err error framework.Logf("Requesting list of APIServices to confirm quantity") list, err := aggrclient.ApiregistrationV1().APIServices().List(ctx, metav1.ListOptions{LabelSelector: label}) if err != nil { return false, err } if len(list.Items) != quantity { return false, err } framework.Logf("Found %d APIService with label %q", quantity, label) return true, nil } } </ins>


isMigratedOnNode := mpaSet.Has(pluginName) <del> if isMigratedOnNode { installed := false driverName, err := csiMigratedPluginManager.GetCSINameFromInTreeName(pluginName) if err != nil { return isMigratedOnNode, err } for _, driver := range csiNode.Spec.Drivers { if driver.Name == driverName { installed = true break } } if !installed { return true, fmt.Errorf("in-tree plugin %s is migrated on node %s but driver %s is not installed", pluginName, string(nodeName), driverName) } } </del> return isMigratedOnNode, nil }


<ins> /* Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package util import ( "os" "reflect" "strings" "testing" "time" "github.com/google/go-cmp/cmp" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubetypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/informers" csitrans "k8s.io/csi-translation-lib" fakeframework "k8s.io/kubernetes/pkg/scheduler/framework/fake" "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/kubernetes/pkg/volume/fc" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes/fake" utiltesting "k8s.io/client-go/util/testing" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" ) const ( testHostName      = "test-hostname" socketPath        = "/var/run/kmsplugin" migratedVolume    = "migrated-volume-name" nonMigratedVolume = "non-migrated-volume-name" testNodeName      = "test-node-name" ) var ( dirOrCreate = v1.HostPathType(v1.HostPathDirectoryOrCreate) nodeName    = kubetypes.NodeName(testNodeName) hostPath    = &v1.HostPathVolumeSource{ Path: socketPath, Type: &dirOrCreate, } migratedObjectReference    = v1.ObjectReference{Namespace: "default", Name: "migrated-pvc"} nonMigratedObjectReference = v1.ObjectReference{Namespace: "default", Name: "non-migrated-pvc"} fsVolumeMode               = new(v1.PersistentVolumeMode) ) type vaTest struct { desc                 string createNodeName       kubetypes.NodeName pod                  *v1.Pod wantVolume           *v1.Volume wantPersistentVolume *v1.PersistentVolume wantErrorMessage     string } func Test_CreateVolumeSpec(t *testing.T) { for _, test := range []vaTest{ { desc:           "inline volume type that does not support csi migration", createNodeName: nodeName, pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "kube-apiserver", Namespace: "default", }, Spec: v1.PodSpec{ Volumes: []v1.Volume{ { Name: migratedVolume, VolumeSource: v1.VolumeSource{ HostPath: hostPath, }, }, }, }, }, wantVolume: &v1.Volume{ Name: migratedVolume, VolumeSource: v1.VolumeSource{ HostPath: hostPath, }, }, }, { desc:           "inline volume type that supports csi migration", createNodeName: nodeName, pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "kube-apiserver", Namespace: "default", }, Spec: v1.PodSpec{ Volumes: []v1.Volume{ { Name: migratedVolume, VolumeSource: v1.VolumeSource{ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ PDName:    "test-disk", FSType:    "ext4", Partition: 0, ReadOnly:  false, }, }, }, }, }, }, wantPersistentVolume: &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "pd.csi.storage.gke.io-test-disk", }, Spec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ CSI: &v1.CSIPersistentVolumeSource{ Driver:           "pd.csi.storage.gke.io", VolumeHandle:     "projects/UNSPECIFIED/zones/UNSPECIFIED/disks/test-disk", FSType:           "ext4", ReadOnly:         false, VolumeAttributes: map[string]string{"partition": ""}, }, }, AccessModes: []v1.PersistentVolumeAccessMode{"ReadWriteOnce"}, VolumeMode:  fsVolumeMode, }, }, }, { desc:           "pv type that does not support csi migration", createNodeName: nodeName, pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "kube-apiserver", Namespace: "default", }, Spec: v1.PodSpec{ Volumes: []v1.Volume{ { Name: nonMigratedVolume, VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ ClaimName: "non-migrated-pvc", ReadOnly:  false, }, }, }, }, }, }, wantPersistentVolume: &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: nonMigratedVolume, }, Spec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ ScaleIO: &v1.ScaleIOPersistentVolumeSource{}, }, ClaimRef: &nonMigratedObjectReference, }, }, }, { desc:           "pv type that supports csi migration", createNodeName: nodeName, pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "kube-apiserver", Namespace: "default", }, Spec: v1.PodSpec{ Volumes: []v1.Volume{ { Name: migratedVolume, VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ ClaimName: "migrated-pvc", ReadOnly:  false, }, }, }, }, }, }, wantPersistentVolume: &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: migratedVolume, }, Spec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ CSI: &v1.CSIPersistentVolumeSource{ Driver:           "pd.csi.storage.gke.io", VolumeHandle:     "projects/UNSPECIFIED/zones/UNSPECIFIED/disks/test-disk", FSType:           "ext4", ReadOnly:         false, VolumeAttributes: map[string]string{"partition": ""}, }, }, ClaimRef: &migratedObjectReference, }, }, }, { desc:           "CSINode not found for a volume type that supports csi migration", createNodeName: kubetypes.NodeName("another-node"), pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "kube-apiserver", Namespace: "default", }, Spec: v1.PodSpec{ Volumes: []v1.Volume{ { Name: migratedVolume, VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ ClaimName: "migrated-pvc", ReadOnly:  false, }, }, }, }, }, }, wantErrorMessage: "csiNode "another-node" not found", }, } { t.Run(test.desc, func(t *testing.T) { plugMgr, intreeToCSITranslator, csiTranslator, pvLister, pvcLister := setup(testNodeName, t) actualSpec, err := CreateVolumeSpec(test.pod.Spec.Volumes[0], test.pod, test.createNodeName, plugMgr, pvcLister, pvLister, intreeToCSITranslator, csiTranslator) if actualSpec == nil && (test.wantPersistentVolume != nil || test.wantVolume != nil) { t.Errorf("got volume spec is nil") } if (len(test.wantErrorMessage) > 0 && err == nil) || (err != nil && !strings.Contains(err.Error(), test.wantErrorMessage)) { t.Errorf("got err %v, want err with message %v", err, test.wantErrorMessage) } if test.wantPersistentVolume != nil { if actualSpec.PersistentVolume == nil { t.Errorf("gotVolumeWithCSIMigration is nil") } gotVolumeWithCSIMigration := *actualSpec.PersistentVolume if gotVolumeWithCSIMigration.Name != test.wantPersistentVolume.Name { t.Errorf("got volume name is %v, want volume name is %v", gotVolumeWithCSIMigration.Name, test.wantPersistentVolume.Name) } if !reflect.DeepEqual(gotVolumeWithCSIMigration.Spec, test.wantPersistentVolume.Spec) { t.Errorf("got volume.Spec and want.Spec diff is %s", cmp.Diff(gotVolumeWithCSIMigration.Spec, test.wantPersistentVolume.Spec)) } } if test.wantVolume != nil { if actualSpec.Volume == nil { t.Errorf("gotVolume is nil") } gotVolume := *actualSpec.Volume if !reflect.DeepEqual(gotVolume, *test.wantVolume) { t.Errorf("got volume and want diff is %s", cmp.Diff(gotVolume, test.wantVolume)) } } }) } } func setup(nodeName string, t *testing.T) (*volume.VolumePluginMgr, csimigration.PluginManager, csitrans.CSITranslator, fakeframework.PersistentVolumeLister, fakeframework.PersistentVolumeClaimLister) { tmpDir, err := utiltesting.MkTmpdir("csi-test") if err != nil { t.Fatalf("can't make a temp dir: %v", err) } defer os.RemoveAll(tmpDir) *fsVolumeMode = v1.PersistentVolumeFilesystem csiTranslator := csitrans.New() intreeToCSITranslator := csimigration.NewPluginManager(csiTranslator, utilfeature.DefaultFeatureGate) kubeClient := fake.NewSimpleClientset() factory := informers.NewSharedInformerFactory(kubeClient, time.Minute) csiDriverInformer := factory.Storage().V1().CSIDrivers() csiDriverLister := csiDriverInformer.Lister() volumeAttachmentInformer := factory.Storage().V1().VolumeAttachments() volumeAttachmentLister := volumeAttachmentInformer.Lister() plugMgr := &volume.VolumePluginMgr{} fakeAttachDetachVolumeHost := volumetest.NewFakeAttachDetachVolumeHostWithCSINodeName(t, tmpDir, kubeClient, fc.ProbeVolumePlugins(), nodeName, csiDriverLister, volumeAttachmentLister, ) plugMgr.Host = fakeAttachDetachVolumeHost pvLister := fakeframework.PersistentVolumeLister{ { ObjectMeta: metav1.ObjectMeta{Name: migratedVolume}, Spec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ PDName:    "test-disk", FSType:    "ext4", Partition: 0, ReadOnly:  false, }, }, ClaimRef: &migratedObjectReference, }, }, { ObjectMeta: metav1.ObjectMeta{Name: nonMigratedVolume}, Spec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ ScaleIO: &v1.ScaleIOPersistentVolumeSource{}, }, ClaimRef: &nonMigratedObjectReference, }, }, } pvcLister := fakeframework.PersistentVolumeClaimLister{ { ObjectMeta: metav1.ObjectMeta{Name: "migrated-pvc", Namespace: "default"}, Spec:       v1.PersistentVolumeClaimSpec{VolumeName: migratedVolume}, Status: v1.PersistentVolumeClaimStatus{ Phase: v1.ClaimBound, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "non-migrated-pvc", Namespace: "default"}, Spec:       v1.PersistentVolumeClaimSpec{VolumeName: nonMigratedVolume}, Status: v1.PersistentVolumeClaimStatus{ Phase: v1.ClaimBound, }, }, } return plugMgr, intreeToCSITranslator, csiTranslator, pvLister, pvcLister } </ins>


"fmt" "net" "path/filepath" <ins> "strings" </ins> "sync" "testing" "time" authenticationv1 "k8s.io/api/authentication/v1" v1 "k8s.io/api/core/v1" <ins> storagev1 "k8s.io/api/storage/v1" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <ins> "k8s.io/apimachinery/pkg/labels" </ins> "k8s.io/apimachinery/pkg/types" <ins> "k8s.io/apimachinery/pkg/util/sets" </ins> "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes"


"k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" cloudprovider "k8s.io/cloud-provider" <ins> csilibplugins "k8s.io/csi-translation-lib/plugins" </ins> proxyutil "k8s.io/kubernetes/pkg/proxy/util" . "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util/hostutil"


} func (f *fakeAttachDetachVolumeHost) CSINodeLister() storagelistersv1.CSINodeLister { <del> // not needed for testing return nil </del> <ins> csiNode := &storagev1.CSINode{ ObjectMeta: metav1.ObjectMeta{Name: f.nodeName}, Spec: storagev1.CSINodeSpec{ Drivers: []storagev1.CSINodeDriver{}, }, } enableMigrationOnNode(csiNode, csilibplugins.GCEPDInTreePluginName) return getFakeCSINodeLister(csiNode) } func enableMigrationOnNode(csiNode *storagev1.CSINode, pluginName string) { nodeInfoAnnotations := csiNode.GetAnnotations() if nodeInfoAnnotations == nil { nodeInfoAnnotations = map[string]string{} } newAnnotationSet := sets.NewString() newAnnotationSet.Insert(pluginName) nas := strings.Join(newAnnotationSet.List(), ",") nodeInfoAnnotations[v1.MigratedPluginsAnnotationKey] = nas csiNode.Annotations = nodeInfoAnnotations </ins> } func (f *fakeAttachDetachVolumeHost) CSIDriverLister() storagelistersv1.CSIDriverLister {


return f } <ins> type CSINodeLister []storagev1.CSINode // Get returns a fake CSINode object. func (n CSINodeLister) Get(name string) (*storagev1.CSINode, error) { for _, cn := range n { if cn.Name == name { return &cn, nil } } return nil, fmt.Errorf("csiNode %q not found", name) } // List lists all CSINodes in the indexer. func (n CSINodeLister) List(selector labels.Selector) (ret []*storagev1.CSINode, err error) { return nil, fmt.Errorf("not implemented") } func getFakeCSINodeLister(csiNode *storagev1.CSINode) CSINodeLister { csiNodeLister := CSINodeLister{} if csiNode != nil { csiNodeLister = append(csiNodeLister, *csiNode.DeepCopy()) } return csiNodeLister } </ins> func (f *fakeKubeletVolumeHost) SetKubeletError(err error) { f.mux.Lock() defer f.mux.Unlock()


utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apiserver/pkg/apis/apiserver" egressmetrics "k8s.io/apiserver/pkg/server/egressselector/metrics" <del> compbasemetrics "k8s.io/component-base/metrics" </del> <ins> "k8s.io/component-base/metrics/legacyregistry" </ins> "k8s.io/component-base/tracing" "k8s.io/klog/v2" client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client"


var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext func init() { <del> client.Metrics.RegisterMetrics(compbasemetrics.NewKubeRegistry().Registerer()) </del> <ins> client.Metrics.RegisterMetrics(legacyregistry.Registerer()) </ins> } // EgressSelector is the map of network context type to context dialer, for network egress.


import ( "context" <ins> "errors" </ins> "fmt" "net" "strings"


"k8s.io/component-base/metrics/legacyregistry" "k8s.io/component-base/metrics/testutil" testingclock "k8s.io/utils/clock/testing" <ins> clientmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" ccmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" </ins> ) type fakeEgressSelection struct {


} }) } <ins> } func TestKonnectivityClientMetrics(t *testing.T) { testcases := []struct { name    string metrics []string trigger func() want    string }{ { name:    "stream packets", metrics: []string{"konnectivity_network_proxy_client_stream_packets_total"}, trigger: func() { clientmetrics.Metrics.ObservePacket(ccmetrics.SegmentFromClient, client.PacketType_DIAL_REQ) }, want: ` # HELP konnectivity_network_proxy_client_stream_packets_total Count of packets processed, by segment and packet type (example: from_client, DIAL_REQ) # TYPE konnectivity_network_proxy_client_stream_packets_total counter konnectivity_network_proxy_client_stream_packets_total{packet_type="DIAL_REQ",segment="from_client"} 1 `, }, { name:    "stream errors", metrics: []string{"konnectivity_network_proxy_client_stream_errors_total"}, trigger: func() { clientmetrics.Metrics.ObserveStreamError(ccmetrics.SegmentToClient, errors.New("example"), client.PacketType_DIAL_RSP) }, want: ` # HELP konnectivity_network_proxy_client_stream_errors_total Count of gRPC stream errors, by segment, grpc Code, packet type. (example: from_agent, Code.Unavailable, DIAL_RSP) # TYPE konnectivity_network_proxy_client_stream_errors_total counter konnectivity_network_proxy_client_stream_errors_total{code="Unknown",packet_type="DIAL_RSP",segment="to_client"} 1 `, }, { name:    "dial failure", metrics: []string{"konnectivity_network_proxy_client_dial_failure_total"}, trigger: func() { clientmetrics.Metrics.ObserveDialFailure(clientmetrics.DialFailureTimeout) }, want: ` # HELP konnectivity_network_proxy_client_dial_failure_total Number of dial failures observed, by reason (example: remote endpoint error) # TYPE konnectivity_network_proxy_client_dial_failure_total counter konnectivity_network_proxy_client_dial_failure_total{reason="timeout"} 1 `, }, { name:    "client connections", metrics: []string{"konnectivity_network_proxy_client_client_connections"}, trigger: func() { clientmetrics.Metrics.GetClientConnectionsMetric().WithLabelValues("dialing").Inc() }, want: ` # HELP konnectivity_network_proxy_client_client_connections Number of open client connections, by status (Example: dialing) # TYPE konnectivity_network_proxy_client_client_connections gauge konnectivity_network_proxy_client_client_connections{status="dialing"} 1 `, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { tc.trigger() if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.want), tc.metrics...); err != nil { t.Errorf("GatherAndCompare error: %v", err) } }) } </ins> }


// Register registers a collectable metric but uses the global registry Register = defaultRegistry.Register <ins> // Registerer exposes the global registerer Registerer = defaultRegistry.Registerer </ins> ) func init() {


endpointsAvailableForLB := !allEndpointsTerminating && !allEndpointsNonServing proxier.deleteExistingLoadBalancer(hns, svcInfo.winProxyOptimization, &svcInfo.hnsID, sourceVip, Enum(svcInfo.Protocol()), uint16(svcInfo.targetPort), uint16(svcInfo.Port()), hnsEndpoints, queriedLoadBalancers) <del> if endpointsAvailableForLB { </del> <ins> // clusterIPEndpoints is the endpoint list used for creating ClusterIP loadbalancer. clusterIPEndpoints := hnsEndpoints if svcInfo.internalTrafficLocal { // Take local endpoints for clusterip loadbalancer when internal traffic policy is local. clusterIPEndpoints = hnsLocalEndpoints } </ins> <del> // clusterIPEndpoints is the endpoint list used for creating ClusterIP loadbalancer. clusterIPEndpoints := hnsEndpoints if svcInfo.internalTrafficLocal { // Take local endpoints for clusterip loadbalancer when internal traffic policy is local. clusterIPEndpoints = hnsLocalEndpoints } </del> <ins> if len(clusterIPEndpoints) > 0 { </ins> // If all endpoints are terminating, then no need to create Cluster IP LoadBalancer // Cluster IP LoadBalancer creation


} } <ins> // TestClusterIPLBInCreateDsrLoadBalancer tests, if the available endpoints are remote, // syncproxyrules only creates ClusterIP Loadbalancer and no NodePort, External IP or IngressIP // loadbalancers will be created. func TestClusterIPLBInCreateDsrLoadBalancer(t *testing.T) { syncPeriod := 30 * time.Second proxier := NewFakeProxier(syncPeriod, syncPeriod, clusterCIDR, "testhost", netutils.ParseIPSloppy("10.0.0.1"), NETWORK_TYPE_OVERLAY) if proxier == nil { t.Error() } svcIP := "10.20.30.41" svcPort := 80 svcNodePort := 3001 svcPortName := proxy.ServicePortName{ NamespacedName: makeNSN("ns1", "svc1"), Port:           "p80", Protocol:       v1.ProtocolTCP, } lbIP := "11.21.31.41" makeServiceMap(proxier, makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { svc.Spec.Type = "NodePort" svc.Spec.ClusterIP = svcIP svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal svc.Spec.Ports = []v1.ServicePort{{ Name:     svcPortName.Port, Port:     int32(svcPort), Protocol: v1.ProtocolTCP, NodePort: int32(svcNodePort), }} svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ IP: lbIP, }} }), ) tcpProtocol := v1.ProtocolTCP populateEndpointSlices(proxier, makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{epIpAddressRemote}, NodeName:  pointer.StringPtr("testhost2"), // This will make this endpoint as a remote endpoint }} eps.Ports = []discovery.EndpointPort{{ Name:     pointer.StringPtr(svcPortName.Port), Port:     pointer.Int32(int32(svcPort)), Protocol: &tcpProtocol, }} }), ) proxier.setInitialized(true) proxier.syncProxyRules() svc := proxier.svcPortMap[svcPortName] svcInfo, ok := svc.(*serviceInfo) if !ok { t.Errorf("Failed to cast serviceInfo %q", svcPortName.String()) } else { // Checking ClusterIP Loadbalancer is created if svcInfo.hnsID != guid { t.Errorf("%v does not match %v", svcInfo.hnsID, guid) } // Verifying NodePort Loadbalancer is not created if svcInfo.nodePorthnsID != "" { t.Errorf("NodePortHnsID %v is not empty.", svcInfo.nodePorthnsID) } // Verifying ExternalIP Loadbalancer is not created for _, externalIP := range svcInfo.externalIPs { if externalIP.hnsID != "" { t.Errorf("ExternalLBID %v is not empty.", externalIP.hnsID) } } // Verifying IngressIP Loadbalancer is not created for _, ingressIP := range svcInfo.loadBalancerIngressIPs { if ingressIP.hnsID != "" { t.Errorf("IngressLBID %v is not empty.", ingressIP.hnsID) } } } } </ins> func TestEndpointSlice(t *testing.T) { syncPeriod := 30 * time.Second proxier := NewFakeProxier(syncPeriod, syncPeriod, clusterCIDR, "testhost", netutils.ParseIPSloppy("10.0.0.1"), NETWORK_TYPE_OVERLAY)


register("storage-version-gc", startStorageVersionGCController) } if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.DynamicResourceAllocation) { <del> controllers["resource-claim-controller"] = startResourceClaimController </del> <ins> register("resource-claim-controller", startResourceClaimController) </ins> } return controllers


const ( // SampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework. <del> SampleDevicePluginDSYAML                    = "test/e2e/testing-manifests/sample-device-plugin.yaml" </del> <ins> SampleDevicePluginDSYAML                    = "test/e2e/testing-manifests/sample-device-plugin/sample-device-plugin.yaml" </ins> SampleDevicePluginControlRegistrationDSYAML = "test/e2e/testing-manifests/sample-device-plugin/sample-device-plugin-control-registration.yaml" // SampleDevicePluginName is the name of the device plugin pod


"k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/apis/example" examplev1 "k8s.io/apiserver/pkg/apis/example/v1" <ins> example2v1 "k8s.io/apiserver/pkg/apis/example2/v1" </ins> "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/etcd3"


metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion) utilruntime.Must(example.AddToScheme(scheme)) utilruntime.Must(examplev1.AddToScheme(scheme)) <ins> utilruntime.Must(example2v1.AddToScheme(scheme)) </ins> } func newTestCacher(s storage.Interface) (*Cacher, storage.Versioner, error) {


// test data newEtcdTestStorage := func(t *testing.T, prefix string) (*etcd3testing.EtcdTestServer, storage.Interface) { server, _ := etcd3testing.NewUnsecuredEtcd3TestClientServer(t) <del> storage := etcd3.New(server.V3Client, apitesting.TestCodec(codecs, example.SchemeGroupVersion), func() runtime.Object { return &example.Pod{} }, prefix, schema.GroupResource{Resource: "pods"}, identity.NewEncryptCheckTransformer(), true, etcd3.NewDefaultLeaseManagerConfig()) </del> <ins> storage := etcd3.New(server.V3Client, apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion, example2v1.SchemeGroupVersion), func() runtime.Object { return &example.Pod{} }, prefix, schema.GroupResource{Resource: "pods"}, identity.NewEncryptCheckTransformer(), true, etcd3.NewDefaultLeaseManagerConfig()) </ins> return server, storage } server, etcdStorage := newEtcdTestStorage(t, "")


require.NoError(t, err) return out } <del> makeReplicaSet := func(name string) *example.ReplicaSet { return &example.ReplicaSet{ </del> <ins> makeReplicaSet := func(name string) *example2v1.ReplicaSet { return &example2v1.ReplicaSet{ </ins> ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: name}, } } <del> createReplicaSet := func(obj *example.ReplicaSet) *example.ReplicaSet { </del> <ins> createReplicaSet := func(obj *example2v1.ReplicaSet) *example2v1.ReplicaSet { </ins> key := "replicasets/" + obj.Namespace + "/" + obj.Name <del> out := &example.ReplicaSet{} </del> <ins> out := &example2v1.ReplicaSet{} </ins> err := etcdStorage.Create(context.TODO(), key, obj, out, 0) require.NoError(t, err) return out