template: &api.PodTemplateSpec{ Spec: api.PodSpec{ TopologySpreadConstraints: []api.TopologySpreadConstraint{ <del> {TopologyKey: `foo`}, {TopologyKey: `beta.kubernetes.io/arch`}, {TopologyKey: `beta.kubernetes.io/os`}, {TopologyKey: `failure-domain.beta.kubernetes.io/region`}, {TopologyKey: `failure-domain.beta.kubernetes.io/zone`}, {TopologyKey: `beta.kubernetes.io/instance-type`}, </del> <ins> { TopologyKey:   `foo`, LabelSelector: &metav1.LabelSelector{}, }, { TopologyKey:   `beta.kubernetes.io/arch`, LabelSelector: &metav1.LabelSelector{}, }, { TopologyKey:   `beta.kubernetes.io/os`, LabelSelector: &metav1.LabelSelector{}, }, { TopologyKey:   `failure-domain.beta.kubernetes.io/region`, LabelSelector: &metav1.LabelSelector{}, }, { TopologyKey:   `failure-domain.beta.kubernetes.io/zone`, LabelSelector: &metav1.LabelSelector{}, }, { TopologyKey:   `beta.kubernetes.io/instance-type`, LabelSelector: &metav1.LabelSelector{}, }, </ins> }, }, },


`spec.terminationGracePeriodSeconds: must be >= 0; negative values are invalid and will be treated as 1`, }, }, <ins> { name: "null LabelSelector in topologySpreadConstraints", template: &api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{}, Spec: api.PodSpec{ TopologySpreadConstraints: []api.TopologySpreadConstraint{ { LabelSelector: &metav1.LabelSelector{}, }, { LabelSelector: nil, }, }, }, }, expected: []string{ `spec.topologySpreadConstraints[1].labelSelector: a null labelSelector results in matching no pod`, }, }, { name: "null LabelSelector in PodAffinity", template: &api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{}, Spec: api.PodSpec{ Affinity: &api.Affinity{ PodAffinity: &api.PodAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{ { LabelSelector: &metav1.LabelSelector{}, }, { LabelSelector: nil, }, }, PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{ { PodAffinityTerm: api.PodAffinityTerm{ LabelSelector: &metav1.LabelSelector{}, }, }, { PodAffinityTerm: api.PodAffinityTerm{ LabelSelector: nil, }, }, }, }, PodAntiAffinity: &api.PodAntiAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []api.PodAffinityTerm{ { LabelSelector: &metav1.LabelSelector{}, }, { LabelSelector: nil, }, }, PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{ { PodAffinityTerm: api.PodAffinityTerm{ LabelSelector: &metav1.LabelSelector{}, }, }, { PodAffinityTerm: api.PodAffinityTerm{ LabelSelector: nil, }, }, }, }, }, }, }, expected: []string{ `spec.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[1].labelSelector: a null labelSelector results in matching no pod`, `spec.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[1].podAffinityTerm.labelSelector: a null labelSelector results in matching no pod`, `spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[1].labelSelector: a null labelSelector results in matching no pod`, `spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[1].podAffinityTerm.labelSelector: a null labelSelector results in matching no pod`, }, }, </ins> } for _, tc := range testcases {


"description": "EndpointConditions represents the current condition of an endpoint.", "properties": { "ready": { <del> "description": "ready indicates that this endpoint is prepared to receive traffic, according to whatever system is managing the endpoint. A nil value indicates an unknown state. In most cases consumers should interpret this unknown state as ready. For compatibility reasons, ready should never be "true" for terminating endpoints.", </del> <ins> "description": "ready indicates that this endpoint is prepared to receive traffic, according to whatever system is managing the endpoint. A nil value indicates an unknown state. In most cases consumers should interpret this unknown state as ready. For compatibility reasons, ready should never be "true" for terminating endpoints, except when the normal readiness behavior is being explicitly overridden, for example when the associated Service has set the publishNotReadyAddresses flag.", </ins> "type": "boolean" }, "serving": {


// according to whatever system is managing the endpoint. A nil value // indicates an unknown state. In most cases consumers should interpret this // unknown state as ready. For compatibility reasons, ready should never be <del> // "true" for terminating endpoints. </del> <ins> // "true" for terminating endpoints, except when the normal readiness // behavior is being explicitly overridden, for example when the associated // Service has set the publishNotReadyAddresses flag. </ins> Ready *bool // serving is identical to ready except that it is set regardless of the


Properties: map[string]spec.Schema{ "ready": { SchemaProps: spec.SchemaProps{ <del> Description: "ready indicates that this endpoint is prepared to receive traffic, according to whatever system is managing the endpoint. A nil value indicates an unknown state. In most cases consumers should interpret this unknown state as ready. For compatibility reasons, ready should never be "true" for terminating endpoints.", </del> <ins> Description: "ready indicates that this endpoint is prepared to receive traffic, according to whatever system is managing the endpoint. A nil value indicates an unknown state. In most cases consumers should interpret this unknown state as ready. For compatibility reasons, ready should never be "true" for terminating endpoints, except when the normal readiness behavior is being explicitly overridden, for example when the associated Service has set the publishNotReadyAddresses flag.", </ins> Type:        []string{"boolean"}, Format:      "", },


// according to whatever system is managing the endpoint. A nil value // indicates an unknown state. In most cases consumers should interpret this // unknown state as ready. For compatibility reasons, ready should never be <del> // "true" for terminating endpoints. </del> <ins> // "true" for terminating endpoints, except when the normal readiness // behavior is being explicitly overridden, for example when the associated // Service has set the publishNotReadyAddresses flag. </ins> // +optional optional bool ready = 1;


// according to whatever system is managing the endpoint. A nil value // indicates an unknown state. In most cases consumers should interpret this // unknown state as ready. For compatibility reasons, ready should never be <del> // "true" for terminating endpoints. </del> <ins> // "true" for terminating endpoints, except when the normal readiness // behavior is being explicitly overridden, for example when the associated // Service has set the publishNotReadyAddresses flag. </ins> // +optional Ready *bool `json:"ready,omitempty" protobuf:"bytes,1,name=ready"`


var map_EndpointConditions = map[string]string{ "":            "EndpointConditions represents the current condition of an endpoint.", <del> "ready":       "ready indicates that this endpoint is prepared to receive traffic, according to whatever system is managing the endpoint. A nil value indicates an unknown state. In most cases consumers should interpret this unknown state as ready. For compatibility reasons, ready should never be "true" for terminating endpoints.", </del> <ins> "ready":       "ready indicates that this endpoint is prepared to receive traffic, according to whatever system is managing the endpoint. A nil value indicates an unknown state. In most cases consumers should interpret this unknown state as ready. For compatibility reasons, ready should never be "true" for terminating endpoints, except when the normal readiness behavior is being explicitly overridden, for example when the associated Service has set the publishNotReadyAddresses flag.", </ins> "serving":     "serving is identical to ready except that it is set regardless of the terminating state of endpoints. This condition should be set to true for a ready endpoint that is terminating. If nil, consumers should defer to the ready condition.", "terminating": "terminating indicates that this endpoint is terminating. A nil value indicates an unknown state. Consumers should interpret this unknown state to mean that the endpoint is not terminating.", }


event := recorder.makeEvent(ref, annotations, eventtype, reason, message) event.Source = recorder.source <ins> event.ReportingInstance = recorder.source.Host event.ReportingController = recorder.source.Component </ins> // NOTE: events should be a non-blocking operation, but we also need to not // put this in a goroutine, otherwise we'll race to write to a closed channel // when we go to shut down this broadcaster.  Just drop events if we get overloaded,


APIVersion: "v1", FieldPath:  "spec.containers[2]", }, <del> Reason:  "Started", Message: "some verbose message: 1", Source:  v1.EventSource{Component: "eventTest"}, Count:   1, Type:    v1.EventTypeNormal, </del> <ins> Reason:              "Started", Message:             "some verbose message: 1", Source:              v1.EventSource{Component: "eventTest"}, ReportingController: "eventTest", Count:               1, Type:                v1.EventTypeNormal, </ins> }, expectLog:    `Event(v1.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers[2]"}): type: 'Normal' reason: 'Started' some verbose message: 1`, expectUpdate: false,


UID:        "bar", APIVersion: "v1", }, <del> Reason:  "Killed", Message: "some other verbose message: 1", Source:  v1.EventSource{Component: "eventTest"}, Count:   1, Type:    v1.EventTypeNormal, </del> <ins> Reason:              "Killed", Message:             "some other verbose message: 1", Source:              v1.EventSource{Component: "eventTest"}, ReportingController: "eventTest", Count:               1, Type:                v1.EventTypeNormal, </ins> }, expectLog:    `Event(v1.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'Killed' some other verbose message: 1`, expectUpdate: false,


APIVersion: "v1", FieldPath:  "spec.containers[2]", }, <del> Reason:  "Started", Message: "some verbose message: 1", Source:  v1.EventSource{Component: "eventTest"}, Count:   2, Type:    v1.EventTypeNormal, </del> <ins> Reason:              "Started", Message:             "some verbose message: 1", Source:              v1.EventSource{Component: "eventTest"}, ReportingController: "eventTest", Count:               2, Type:                v1.EventTypeNormal, </ins> }, expectLog:    `Event(v1.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers[2]"}): type: 'Normal' reason: 'Started' some verbose message: 1`, expectUpdate: true,


APIVersion: "v1", FieldPath:  "spec.containers[3]", }, <del> Reason:  "Started", Message: "some verbose message: 1", Source:  v1.EventSource{Component: "eventTest"}, Count:   1, Type:    v1.EventTypeNormal, </del> <ins> Reason:              "Started", Message:             "some verbose message: 1", Source:              v1.EventSource{Component: "eventTest"}, ReportingController: "eventTest", Count:               1, Type:                v1.EventTypeNormal, </ins> }, expectLog:    `Event(v1.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"differentUid", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers[3]"}): type: 'Normal' reason: 'Started' some verbose message: 1`, expectUpdate: false,


APIVersion: "v1", FieldPath:  "spec.containers[2]", }, <del> Reason:  "Started", Message: "some verbose message: 1", Source:  v1.EventSource{Component: "eventTest"}, Count:   3, Type:    v1.EventTypeNormal, </del> <ins> Reason:              "Started", Message:             "some verbose message: 1", Source:              v1.EventSource{Component: "eventTest"}, ReportingController: "eventTest", Count:               3, Type:                v1.EventTypeNormal, </ins> }, expectLog:    `Event(v1.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers[2]"}): type: 'Normal' reason: 'Started' some verbose message: 1`, expectUpdate: true,


APIVersion: "v1", FieldPath:  "spec.containers[3]", }, <del> Reason:  "Stopped", Message: "some verbose message: 1", Source:  v1.EventSource{Component: "eventTest"}, Count:   1, Type:    v1.EventTypeNormal, </del> <ins> Reason:              "Stopped", Message:             "some verbose message: 1", Source:              v1.EventSource{Component: "eventTest"}, ReportingController: "eventTest", Count:               1, Type:                v1.EventTypeNormal, </ins> }, expectLog:    `Event(v1.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"differentUid", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers[3]"}): type: 'Normal' reason: 'Stopped' some verbose message: 1`, expectUpdate: false,


APIVersion: "v1", FieldPath:  "spec.containers[3]", }, <del> Reason:  "Stopped", Message: "some verbose message: 1", Source:  v1.EventSource{Component: "eventTest"}, Count:   2, Type:    v1.EventTypeNormal, </del> <ins> Reason:              "Stopped", Message:             "some verbose message: 1", Source:              v1.EventSource{Component: "eventTest"}, ReportingController: "eventTest", Count:               2, Type:                v1.EventTypeNormal, </ins> }, expectLog:    `Event(v1.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"differentUid", APIVersion:"v1", ResourceVersion:"", FieldPath:"spec.containers[3]"}): type: 'Normal' reason: 'Stopped' some verbose message: 1`, expectUpdate: true,


// Temp clear time stamps for comparison because actual values don't matter for comparison recvEvent.FirstTimestamp = expectedEvent.FirstTimestamp recvEvent.LastTimestamp = expectedEvent.LastTimestamp <ins> recvEvent.ReportingController = expectedEvent.ReportingController </ins> // Check that name has the right prefix. if n, en := recvEvent.Name, expectedEvent.Name; !strings.HasPrefix(n, en) { t.Errorf("%v - Name '%v' does not contain prefix '%v'", messagePrefix, n, en)


} for _, tc := range tests { <ins> tc := tc </ins> t.Run(tc.desc, func(t *testing.T) { fakeSystemBus := &fakeSystemDBus{} bus := DBusCon{


outBuiltIn := e2ekubectl.RunKubectlOrDie("", "get", "nodes", node.Name) ginkgo.By(fmt.Sprintf("calling kubectl get nodes %s --subresource=status", node.Name)) outStatusSubresource := e2ekubectl.RunKubectlOrDie("", "get", "nodes", node.Name, "--subresource=status") <del> gomega.Expect(outBuiltIn).To(gomega.Equal(outStatusSubresource)) </del> <ins> // Avoid comparing values of fields that might end up // changing between the two invocations of kubectl. requiredOutput := [][]string{ {"NAME"}, {"STATUS"}, {"ROLES"}, {"AGE"}, {"VERSION"}, {node.Name},                           // check for NAME {""},                                  // avoid comparing STATUS {""},                                  // avoid comparing ROLES {""},                                  // avoid comparing AGE {node.Status.NodeInfo.KubeletVersion}, // check for VERSION } checkOutput(outBuiltIn, requiredOutput) checkOutput(outStatusSubresource, requiredOutput) </ins> }) }) })


}, }, { <ins> name:    "pod that could not start still has a pending update and is tracked in metrics", wantErr: false, pods: []*v1.Pod{ staticPod(), }, prepareWorker: func(t *testing.T, w *podWorkers, records map[types.UID][]syncPodRecord) { // send a create of a static pod pod := staticPod() // block startup of the static pod due to full name collision w.startedStaticPodsByFullname[kubecontainer.GetPodFullName(pod)] = types.UID("2") w.UpdatePod(UpdatePodOptions{ UpdateType: kubetypes.SyncPodCreate, StartTime:  time.Unix(1, 0).UTC(), Pod:        pod, }) drainAllWorkers(w) if _, ok := records[pod.UID]; ok { t.Fatalf("unexpected records: %#v", records) } // pod worker is unaware of pod1 yet }, wantWorker: func(t *testing.T, w *podWorkers, records map[types.UID][]syncPodRecord) { uid := types.UID("1") if len(w.podSyncStatuses) != 1 { t.Fatalf("unexpected sync statuses: %#v", w.podSyncStatuses) } s, ok := w.podSyncStatuses[uid] if !ok || s.IsTerminationRequested() || s.IsTerminationStarted() || s.IsFinished() || s.IsWorking() || s.IsDeleted() || s.restartRequested || s.activeUpdate != nil || s.pendingUpdate == nil { t.Errorf("unexpected requested pod termination: %#v", s) } // expect that no sync calls are made, since the pod doesn't ever start if actual, expected := records[uid], []syncPodRecord(nil); !reflect.DeepEqual(expected, actual) { t.Fatalf("unexpected pod sync records: %s", cmp.Diff(expected, actual, cmp.AllowUnexported(syncPodRecord{}))) } }, expectMetrics: map[string]string{ metrics.DesiredPodCount.FQName(): `# HELP kubelet_desired_pods [ALPHA] The number of pods the kubelet is being instructed to run. static is true if the pod is not from the apiserver. # TYPE kubelet_desired_pods gauge kubelet_desired_pods{static=""} 0 kubelet_desired_pods{static="true"} 1 `, metrics.WorkingPodCount.FQName(): `# HELP kubelet_working_pods [ALPHA] Number of pods the kubelet is actually running, broken down by lifecycle phase, whether the pod is desired, orphaned, or runtime only (also orphaned), and whether the pod is static. An orphaned pod has been removed from local configuration or force deleted in the API and consumes resources that are not otherwise visible. # TYPE kubelet_working_pods gauge kubelet_working_pods{config="desired",lifecycle="sync",static=""} 0 kubelet_working_pods{config="desired",lifecycle="sync",static="true"} 1 kubelet_working_pods{config="desired",lifecycle="terminated",static=""} 0 kubelet_working_pods{config="desired",lifecycle="terminated",static="true"} 0 kubelet_working_pods{config="desired",lifecycle="terminating",static=""} 0 kubelet_working_pods{config="desired",lifecycle="terminating",static="true"} 0 kubelet_working_pods{config="orphan",lifecycle="sync",static=""} 0 kubelet_working_pods{config="orphan",lifecycle="sync",static="true"} 0 kubelet_working_pods{config="orphan",lifecycle="terminated",static=""} 0 kubelet_working_pods{config="orphan",lifecycle="terminated",static="true"} 0 kubelet_working_pods{config="orphan",lifecycle="terminating",static=""} 0 kubelet_working_pods{config="orphan",lifecycle="terminating",static="true"} 0 kubelet_working_pods{config="runtime_only",lifecycle="sync",static="unknown"} 0 kubelet_working_pods{config="runtime_only",lifecycle="terminated",static="unknown"} 0 kubelet_working_pods{config="runtime_only",lifecycle="terminating",static="unknown"} 0 `, }, }, { </ins> name:           "pod that could not start and is not in config is force terminated without runtime during pod cleanup", wantErr:        false, terminatingErr: errors.New("unable to terminate"),


// or can be started, and updates the cached pod state so that downstream components can observe what the // pod worker goroutine is currently attempting to do. If ok is false, there is no available event. If any // of the boolean values is false, ensure the appropriate cleanup happens before returning. <ins> // // This method should ensure that either status.pendingUpdate is cleared and merged into status.activeUpdate, // or when a pod cannot be started status.pendingUpdate remains the same. Pods that have not been started // should never have an activeUpdate because that is exposed to downstream components on started pods. </ins> func (p *podWorkers) startPodSync(podUID types.UID) (ctx context.Context, update podWork, canStart, canEverStart, ok bool) { p.podLock.Lock() defer p.podLock.Unlock()


klog.V(4).InfoS("Pod cannot start ever", "pod", klog.KObj(update.Options.Pod), "podUID", podUID, "updateType", update.WorkType) return ctx, update, canStart, canEverStart, true case !canStart: <ins> // this is the only path we don't start the pod, so we need to put the change back in pendingUpdate status.pendingUpdate = &update.Options </ins> status.working = false klog.V(4).InfoS("Pod cannot start yet", "pod", klog.KObj(update.Options.Pod), "podUID", podUID) return ctx, update, canStart, canEverStart, true


State:  status.WorkType(), Orphan: orphan, } <del> if status.activeUpdate != nil && status.activeUpdate.Pod != nil { sync.HasConfig = true sync.Static = kubetypes.IsStaticPod(status.activeUpdate.Pod) </del> <ins> switch { case status.activeUpdate != nil: if status.activeUpdate.Pod != nil { sync.HasConfig = true sync.Static = kubetypes.IsStaticPod(status.activeUpdate.Pod) } case status.pendingUpdate != nil: if status.pendingUpdate.Pod != nil { sync.HasConfig = true sync.Static = kubetypes.IsStaticPod(status.pendingUpdate.Pod) } </ins> } workers[uid] = sync }


// Iterate through all pods in desired state of world, and remove if they no // longer exist func (dswp *desiredStateOfWorldPopulator) findAndRemoveDeletedPods() { <ins> podsFromCache := make(map[volumetypes.UniquePodName]struct{}) </ins> for _, volumeToMount := range dswp.desiredStateOfWorld.GetVolumesToMount() { <ins> podsFromCache[volumetypes.UniquePodName(volumeToMount.Pod.UID)] = struct{}{} </ins> pod, podExists := dswp.podManager.GetPodByUID(volumeToMount.Pod.UID) if podExists {


dswp.deleteProcessedPod(volumeToMount.PodName) } <ins> // Cleanup orphanded entries from processedPods dswp.pods.Lock() orphanedPods := make([]volumetypes.UniquePodName, 0, len(dswp.pods.processedPods)) for k := range dswp.pods.processedPods { if _, ok := podsFromCache[k]; !ok { orphanedPods = append(orphanedPods, k) } } dswp.pods.Unlock() for _, orphanedPod := range orphanedPods { uid := types.UID(orphanedPod) _, podExists := dswp.podManager.GetPodByUID(uid) if !podExists && dswp.podStateProvider.ShouldPodRuntimeBeRemoved(uid) { dswp.deleteProcessedPod(orphanedPod) } } </ins> podsWithError := dswp.desiredStateOfWorld.GetPodsWithErrors() for _, podName := range podsWithError { if _, podExists := dswp.podManager.GetPodByUID(types.UID(podName)); !podExists {


t.Fatalf("Failed to remove pods from desired state of world since they no longer exist") } <ins> // podWorker may call volume_manager WaitForUnmount() after we processed the pod in findAndRemoveDeletedPods() dswp.ReprocessPod(podName) dswp.findAndRemoveDeletedPods() // findAndRemoveDeletedPods() above must detect orphaned pod and delete it from the map if _, ok := dswp.pods.processedPods[podName]; ok { t.Fatalf("Failed to remove orphanded pods from internal map") } </ins> volumeExists := dswp.desiredStateOfWorld.VolumeExists(expectedVolumeName, "" /* SELinuxContext */) if volumeExists { t.Fatalf(


// TODO: scope this to the kube-system namespace rbacv1helpers.NewRule("create").Groups(coordinationGroup).Resources("leases").RuleOrDie(), rbacv1helpers.NewRule("get", "update").Groups(coordinationGroup).Resources("leases").Names("kube-scheduler").RuleOrDie(), <del> // TODO: Remove once we fully migrate to lease in leader-election. rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("endpoints").RuleOrDie(), rbacv1helpers.NewRule("get", "update").Groups(legacyGroup).Resources("endpoints").Names("kube-scheduler").RuleOrDie(), </del> // Fundamental resources rbacv1helpers.NewRule(Read...).Groups(legacyGroup).Resources("nodes").RuleOrDie(),


- apiGroups: - "" resources: <del> - endpoints verbs: - create - apiGroups: - "" resourceNames: - kube-scheduler resources: - endpoints verbs: - get - update - apiGroups: - "" resources: </del> - nodes verbs: - get


Image:   imageutils.GetE2EImage(imageutils.BusyBox), Command: []string{"sh", "-c"}, Args: []string{` <ins> sleep 9999999 & PID=$! </ins> _term() { <ins> kill $PID </ins> echo "Caught SIGTERM signal!" <del> exit 0 </del> } trap _term SIGTERM <del> while true; do sleep 5; done </del> <ins> wait $PID exit 0 </ins> `, }, },


} return priority } <ins> // getGracePeriodOverrideTestPod returns a new Pod object containing a container // runs a shell script, hangs the process until a SIGTERM signal is received. // The script waits for $PID to ensure that the process does not exist. // If priorityClassName is scheduling.SystemNodeCritical, the Pod is marked as critical and a comment is added. </ins> func getGracePeriodOverrideTestPod(name string, node string, gracePeriod int64, priorityClassName string) *v1.Pod { pod := &v1.Pod{ TypeMeta: metav1.TypeMeta{


Image:   busyboxImage, Command: []string{"sh", "-c"}, Args: []string{` <del> _term() { echo "Caught SIGTERM signal!" while true; do sleep 5; done } trap _term SIGTERM while true; do sleep 5; done `}, </del> <ins> sleep 9999999 & PID=$! _term() { echo "Caught SIGTERM signal!" wait $PID } trap _term SIGTERM wait $PID `}, </ins> }, }, TerminationGracePeriodSeconds: &gracePeriod,


"fmt" "net" "net/url" <ins> "os" </ins> "path/filepath" "strings" "syscall"


// does NOT work in 1809 if the socket file is created within a bind mounted directory by a container // and the FSCTL is issued in the host by the kubelet. <ins> // If the file does not exist, it cannot be a Unix domain socket. if _, err := os.Stat(filePath); os.IsNotExist(err) { return false, fmt.Errorf("File %s not found. Err: %v", filePath, err) } </ins> klog.V(6).InfoS("Function IsUnixDomainSocket starts", "filePath", filePath) // As detailed in https://github.com/kubernetes/kubernetes/issues/104584 we cannot rely // on the Unix Domain socket working on the very first try, hence the potential need to


import ( "fmt" "math/rand" <ins> "net" "os" </ins> "reflect" "runtime" "sync"


// the PATCH end-point. Returns true if the query param is supported by the // spec for the passed GVK; false otherwise. func supportsQueryParamV3(doc *spec3.OpenAPI, gvk schema.GroupVersionKind, queryParam VerifiableQueryParam) bool { <ins> if doc == nil || doc.Paths == nil { return false } </ins> for _, path := range doc.Paths.Paths { // If operation is not PATCH, then continue. op := path.PathProps.Patch


"k8s.io/client-go/openapi/cached" "k8s.io/client-go/openapi/openapitest" "k8s.io/client-go/openapi3" <ins> "k8s.io/kube-openapi/pkg/spec3" </ins> ) func TestV3SupportsQueryParamBatchV1(t *testing.T) {


}) } } <ins> func TestInvalidOpenAPIV3Document(t *testing.T) { tests := map[string]struct { spec *spec3.OpenAPI }{ "nil document correctly returns Unsupported error": { spec: nil, }, "empty document correctly returns Unsupported error": { spec: &spec3.OpenAPI{}, }, "minimal document correctly returns Unsupported error": { spec: &spec3.OpenAPI{ Version: "openapi 3.0.0", Paths:   nil, }, }, "document with empty Paths correctly returns Unsupported error": { spec: &spec3.OpenAPI{ Version: "openapi 3.0.0", Paths:   &spec3.Paths{}, }, }, } gvk := schema.GroupVersionKind{ Group:   "batch", Version: "v1", Kind:    "Job", } for tn, tc := range tests { t.Run(tn, func(t *testing.T) { verifier := &queryParamVerifierV3{ finder: NewCRDFinder(func() ([]schema.GroupKind, error) { return []schema.GroupKind{}, nil }), root:       &fakeRoot{tc.spec}, queryParam: QueryParamFieldValidation, } err := verifier.HasSupport(gvk) if err == nil { t.Errorf("Expected not supports error, but none received.") } }) } } // fakeRoot implements Root interface; manually specifies the returned OpenAPI V3 document. type fakeRoot struct { spec *spec3.OpenAPI } func (f *fakeRoot) GroupVersions() ([]schema.GroupVersion, error) { // Unused return nil, nil } // GVSpec returns hard-coded OpenAPI V3 document. func (f *fakeRoot) GVSpec(gv schema.GroupVersion) (*spec3.OpenAPI, error) { return f.spec, nil } func (f *fakeRoot) GVSpecAsMap(gv schema.GroupVersion) (map[string]interface{}, error) { // Unused return nil, nil } </ins>


// back to the queue multiple times before it's successfully scheduled. // It shouldn't be updated once initialized. It's used to record the e2e scheduling // latency for a pod. <del> InitialAttemptTimestamp time.Time </del> <ins> InitialAttemptTimestamp *time.Time </ins> // If a Pod failed in a scheduling cycle, record the plugin names it failed by. UnschedulablePlugins sets.Set[string] // Whether the Pod is scheduling gated (by PreEnqueuePlugins) or not.


p.unschedulablePods.addOrUpdate(pInfo) return false, nil } <ins> if pInfo.InitialAttemptTimestamp == nil { now := p.clock.Now() pInfo.InitialAttemptTimestamp = &now } </ins> if err := p.activeQ.Add(pInfo); err != nil { logger.Error(err, "Error adding pod to the active queue", "pod", klog.KObj(pInfo.Pod)) return false, err


return &framework.QueuedPodInfo{ PodInfo:                 podInfo, Timestamp:               now, <del> InitialAttemptTimestamp: now, </del> <ins> InitialAttemptTimestamp: nil, </ins> UnschedulablePlugins:    sets.New(plugins...), } }


t.Fatalf("Failed to pop a pod %v", err) } checkPerPodSchedulingMetrics("Attempt twice with update", t, pInfo, 2, timestamp) <ins> // Case 4: A gated pod is created and scheduled after lifting gate. The queue operations are //  Add gated pod -> check unschedulablePods -> lift gate & update pod -> Pop. c = testingclock.NewFakeClock(timestamp) // Create a queue with PreEnqueuePlugin m := map[string][]framework.PreEnqueuePlugin{"": {&preEnqueuePlugin{allowlists: []string{"foo"}}}} queue = NewTestQueue(ctx, newDefaultQueueSort(), WithClock(c), WithPreEnqueuePluginMap(m), WithPluginMetricsSamplePercent(0)) // Create a pod without PreEnqueuePlugin label. gatedPod := st.MakePod().Name("gated-test-pod").Namespace("test-ns").UID("test-uid").Obj() err = queue.Add(logger, gatedPod) if err != nil { t.Fatalf("Failed to add a pod %v", err) } // Check pod is added to the unschedulablePods queue. if getUnschedulablePod(queue, gatedPod) != gatedPod { t.Errorf("Pod %v was not found in the unschedulablePods.", gatedPod.Name) } // Override clock to get different InitialAttemptTimestamp c.Step(1 * time.Minute) // Update pod with the required label to get it out of unschedulablePods queue. updateGatedPod := gatedPod.DeepCopy() updateGatedPod.Labels = map[string]string{"foo": ""} queue.Update(logger, gatedPod, updateGatedPod) pInfo, err = queue.Pop() if err != nil { t.Fatalf("Failed to pop a pod %v", err) } checkPerPodSchedulingMetrics("Attempt once/gated", t, pInfo, 1, timestamp.Add(1*time.Minute)) </ins> } func TestIncomingPodsMetrics(t *testing.T) {


if pInfo.Attempts != wantAttempts { t.Errorf("[%s] Pod schedule attempt unexpected, got %v, want %v", name, pInfo.Attempts, wantAttempts) } <del> if pInfo.InitialAttemptTimestamp != wantInitialAttemptTs { t.Errorf("[%s] Pod initial schedule attempt timestamp unexpected, got %v, want %v", name, pInfo.InitialAttemptTimestamp, wantInitialAttemptTs) </del> <ins> if *pInfo.InitialAttemptTimestamp != wantInitialAttemptTs { t.Errorf("[%s] Pod initial schedule attempt timestamp unexpected, got %v, want %v", name, *pInfo.InitialAttemptTimestamp, wantInitialAttemptTs) </ins> } }


logger.V(2).Info("Successfully bound pod to node", "pod", klog.KObj(assumedPod), "node", scheduleResult.SuggestedHost, "evaluatedNodes", scheduleResult.EvaluatedNodes, "feasibleNodes", scheduleResult.FeasibleNodes) metrics.PodScheduled(fwk.ProfileName(), metrics.SinceInSeconds(start)) metrics.PodSchedulingAttempts.Observe(float64(assumedPodInfo.Attempts)) <del> metrics.PodSchedulingDuration.WithLabelValues(getAttemptsLabel(assumedPodInfo)).Observe(metrics.SinceInSeconds(assumedPodInfo.InitialAttemptTimestamp)) </del> <ins> if assumedPodInfo.InitialAttemptTimestamp != nil { metrics.PodSchedulingDuration.WithLabelValues(getAttemptsLabel(assumedPodInfo)).Observe(metrics.SinceInSeconds(*assumedPodInfo.InitialAttemptTimestamp)) } </ins> // Run "postbind" plugins. fwk.RunPostBindPlugins(ctx, state, assumedPod, scheduleResult.SuggestedHost)


<ins> #!/bin/bash # Copyright 2014 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Bring up a Kubernetes cluster. # Usage: #   wget -q -O - https://get.k8s.io | sh # or #   curl -sS https://get.k8s.io | sh # # Advanced options #  Set KUBERNETES_PROVIDER to choose between different providers: #  Google Compute Engine [default] #   * export KUBERNETES_PROVIDER=gce; wget -q -O - https://get.k8s.io | sh #  Amazon EC2 #   * export KUBERNETES_PROVIDER=aws; wget -q -O - https://get.k8s.io | sh #  Microsoft Azure #   * export KUBERNETES_PROVIDER=azure; wget -q -O - https://get.k8s.io | sh #  Vagrant (local virtual machines) #   * export KUBERNETES_PROVIDER=vagrant; wget -q -O - https://get.k8s.io | sh #  VMWare VSphere #   * export KUBERNETES_PROVIDER=vsphere; wget -q -O - https://get.k8s.io | sh #  Rackspace #   * export KUBERNETES_PROVIDER=rackspace; wget -q -O - https://get.k8s.io | sh # #  Set KUBERNETES_SKIP_DOWNLOAD to non-empty to skip downloading a release. #  Set KUBERNETES_SKIP_CONFIRM to skip the installation confirmation prompt. set -o errexit set -o nounset set -o pipefail function create-cluster { echo "Creating a kubernetes on ${KUBERNETES_PROVIDER:-gce}..." ( cd kubernetes ./cluster/kube-up.sh echo "Kubernetes binaries at ${PWD}/kubernetes/cluster/" echo "You may want to add this directory to your PATH in $HOME/.profile" echo "Installation successful!" ) } if [[ "${KUBERNETES_SKIP_DOWNLOAD-}" ]]; then create-cluster exit 0 fi release=v0.7.2 release_url=https://storage.googleapis.com/kubernetes-release/release/${release}/kubernetes.tar.gz uname=$(uname) if [[ "${uname}" == "Darwin" ]]; then platform="darwin" elif [[ "${uname}" == "Linux" ]]; then platform="linux" else echo "Unknown, unsupported platform: (${uname})." echo "Supported platforms: Linux, Darwin." echo "Bailing out." exit 2 fi machine=$(uname -m) if [[ "${machine}" == "x86_64" ]]; then arch="amd64" elif [[ "${machine}" == "i686" ]]; then arch="386" elif [[ "${machine}" == "arm*" ]]; then arch="arm" else echo "Unknown, unsupported architecture (${machine})." echo "Supported architectures x86_64, i686, arm*" echo "Bailing out." exit 3 fi file=kubernetes.tar.gz echo "Downloading kubernetes release ${release} to ${PWD}/kubernetes.tar.gz" if [[ -n "${KUBERNETES_SKIP_CONFIRM-}" ]]; then echo "Is this ok? [Y]/n" read confirm if [[ "$confirm" == "n" ]]; then echo "Aborting." exit 0 fi fi if [[ $(which wget) ]]; then wget -O ${file} ${release_url} elif [[ $(which curl) ]]; then curl -L -o ${file} ${release_url} else echo "Couldn't find curl or wget.  Bailing out." exit 1 fi echo "Unpacking kubernetes release ${release}" tar -xzf ${file} rm ${file} create-cluster </ins>


}) err = clientretry.RetryOnConflict(UpdateNodeSpecBackoff, func() error { <del> var curNode *v1.Node if cnc.cloud.ProviderName() == "gce" { // TODO(wlan0): Move this logic to the route controller using the node taint instead of condition // Since there are node taints, do we still need this? // This condition marks the node as unusable until routes are initialized in the cloud provider if err := nodeutil.SetNodeCondition(cnc.kubeClient, types.NodeName(nodeName), v1.NodeCondition{ Type:               v1.NodeNetworkUnavailable, Status:             v1.ConditionTrue, Reason:             "NoRouteCreated", Message:            "Node created without a route", LastTransitionTime: metav1.Now(), }); err != nil { return err } // fetch latest node from API server since GCE-specific condition was set and informer cache may be stale curNode, err = cnc.kubeClient.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil { return err } } else { curNode, err = cnc.nodeInformer.Lister().Get(nodeName) if err != nil { return err } </del> <ins> curNode, err = cnc.nodeInformer.Lister().Get(nodeName) if err != nil { return err </ins> } newNode := curNode.DeepCopy()


} } <del> // test syncNode with instanceV2, same test case with TestGCECondition. func TestGCEConditionV2(t *testing.T) { existingNode := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name:              "node0", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:               v1.NodeReady, Status:             v1.ConditionUnknown, LastHeartbeatTime:  metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, Spec: v1.NodeSpec{ Taints: []v1.Taint{ { Key:    cloudproviderapi.TaintExternalCloudProvider, Value:  "true", Effect: v1.TaintEffectNoSchedule, }, }, }, } fakeCloud := &fakecloud.Cloud{ EnableInstancesV2: true, InstanceTypes: map[types.NodeName]string{ types.NodeName("node0"): "t1.micro", }, ProviderID: map[types.NodeName]string{ types.NodeName("node0"): "fake://12334", }, Addresses: []v1.NodeAddress{ { Type:    v1.NodeHostName, Address: "node0.cloud.internal", }, { Type:    v1.NodeInternalIP, Address: "10.0.0.1", }, { Type:    v1.NodeExternalIP, Address: "132.143.154.163", }, }, Provider: "gce", Err:      nil, } clientset := fake.NewSimpleClientset(existingNode) factory := informers.NewSharedInformerFactory(clientset, 0) eventBroadcaster := record.NewBroadcaster() cloudNodeController := &CloudNodeController{ kubeClient:                clientset, nodeInformer:              factory.Core().V1().Nodes(), nodesLister:               factory.Core().V1().Nodes().Lister(), cloud:                     fakeCloud, recorder:                  eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), nodeStatusUpdateFrequency: 1 * time.Second, } stopCh := make(chan struct{}) defer close(stopCh) factory.Start(stopCh) factory.WaitForCacheSync(stopCh) w := eventBroadcaster.StartLogging(klog.Infof) defer w.Stop() cloudNodeController.syncNode(context.TODO(), existingNode.Name) updatedNode, err := clientset.CoreV1().Nodes().Get(context.TODO(), existingNode.Name, metav1.GetOptions{}) if err != nil { t.Fatalf("error getting updated nodes: %v", err) } conditionAdded := false for _, cond := range updatedNode.Status.Conditions { if cond.Status == "True" && cond.Type == "NetworkUnavailable" && cond.Reason == "NoRouteCreated" { conditionAdded = true } } assert.True(t, conditionAdded, "Network Route Condition for GCE not added by external cloud initializer") } // This test checks that a node with the external cloud provider taint is cloudprovider initialized and // the GCE route condition is added if cloudprovider is GCE func TestGCECondition(t *testing.T) { existingNode := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name:              "node0", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:               v1.NodeReady, Status:             v1.ConditionUnknown, LastHeartbeatTime:  metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, Spec: v1.NodeSpec{ Taints: []v1.Taint{ { Key:    cloudproviderapi.TaintExternalCloudProvider, Value:  "true", Effect: v1.TaintEffectNoSchedule, }, }, }, } fakeCloud := &fakecloud.Cloud{ EnableInstancesV2: false, InstanceTypes: map[types.NodeName]string{ types.NodeName("node0"): "t1.micro", }, Addresses: []v1.NodeAddress{ { Type:    v1.NodeHostName, Address: "node0.cloud.internal", }, { Type:    v1.NodeInternalIP, Address: "10.0.0.1", }, { Type:    v1.NodeExternalIP, Address: "132.143.154.163", }, }, Provider: "gce", Err:      nil, } clientset := fake.NewSimpleClientset(existingNode) factory := informers.NewSharedInformerFactory(clientset, 0) eventBroadcaster := record.NewBroadcaster() cloudNodeController := &CloudNodeController{ kubeClient:                clientset, nodeInformer:              factory.Core().V1().Nodes(), nodesLister:               factory.Core().V1().Nodes().Lister(), cloud:                     fakeCloud, recorder:                  eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-node-controller"}), nodeStatusUpdateFrequency: 1 * time.Second, } stopCh := make(chan struct{}) defer close(stopCh) factory.Start(stopCh) factory.WaitForCacheSync(stopCh) w := eventBroadcaster.StartLogging(klog.Infof) defer w.Stop() cloudNodeController.syncNode(context.TODO(), existingNode.Name) updatedNode, err := clientset.CoreV1().Nodes().Get(context.TODO(), existingNode.Name, metav1.GetOptions{}) if err != nil { t.Fatalf("error getting updated nodes: %v", err) } conditionAdded := false for _, cond := range updatedNode.Status.Conditions { if cond.Status == "True" && cond.Type == "NetworkUnavailable" && cond.Reason == "NoRouteCreated" { conditionAdded = true } } assert.True(t, conditionAdded, "Network Route Condition for GCE not added by external cloud initializer") } </del> func Test_reconcileNodeLabels(t *testing.T) { testcases := []struct { name           string


name: "net.ipv4.tcp_keepalive_time", // refer to https://github.com/torvalds/linux/commit/13b287e8d1cad951634389f85b8c9b816bd3bb1e. kernel: "4.5", <ins> }, { // refer to https://github.com/torvalds/linux/commit/1e579caa18b96f9eb18f4f5416658cd15f37c062. name:   "net.ipv4.tcp_fin_timeout", kernel: "4.6", }, { // refer to https://github.com/torvalds/linux/commit/b840d15d39128d08ed4486085e5507d2617b9ae1. name:   "net.ipv4.tcp_keepalive_intvl", kernel: "4.5", }, { // refer to https://github.com/torvalds/linux/commit/9bd6861bd4326e3afd3f14a9ec8a723771fb20bb. name:   "net.ipv4.tcp_keepalive_probes", kernel: "4.5", </ins> }, }


}, }, { <del> name: "kernelVersion is 5.15.0, return safeSysctls with no kernelVersion limit and net.ipv4.ip_local_reserved_ports and net.ipv4.tcp_keepalive_time", </del> <ins> name: "kernelVersion is 5.15.0, return safeSysctls with no kernelVersion limit and kernelVersion below 5.15.0", </ins> getVersion: func() (*version.Version, error) { kernelVersionStr := "5.15.0-75-generic" return version.ParseGeneric(kernelVersionStr)


"net.ipv4.ip_unprivileged_port_start", "net.ipv4.ip_local_reserved_ports", "net.ipv4.tcp_keepalive_time", <ins> "net.ipv4.tcp_fin_timeout", "net.ipv4.tcp_keepalive_intvl", "net.ipv4.tcp_keepalive_probes", </ins> }, }, }


'net.ipv4.ip_unprivileged_port_start' 'net.ipv4.ip_local_reserved_ports' 'net.ipv4.tcp_keepalive_time' <ins> 'net.ipv4.tcp_fin_timeout' 'net.ipv4.tcp_keepalive_intvl' 'net.ipv4.tcp_keepalive_probes' </ins> */


"net.ipv4.ip_unprivileged_port_start", "net.ipv4.ip_local_reserved_ports", "net.ipv4.tcp_keepalive_time", <ins> "net.ipv4.tcp_fin_timeout", "net.ipv4.tcp_keepalive_intvl", "net.ipv4.tcp_keepalive_probes", </ins> ) )


expectReason: `forbidden sysctls`, expectDetail: `net.ipv4.tcp_keepalive_time`, }, <ins> { name: "new supported sysctls not supported: net.ipv4.tcp_fin_timeout", pod: &corev1.Pod{Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_fin_timeout", Value: "60"}}, }, }}, allowed:      false, expectReason: `forbidden sysctls`, expectDetail: `net.ipv4.tcp_fin_timeout`, }, { name: "new supported sysctls not supported: net.ipv4.tcp_keepalive_intvl", pod: &corev1.Pod{Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_intvl", Value: "75"}}, }, }}, allowed:      false, expectReason: `forbidden sysctls`, expectDetail: `net.ipv4.tcp_keepalive_intvl`, }, { name: "new supported sysctls not supported: net.ipv4.tcp_keepalive_probes", pod: &corev1.Pod{Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_probes", Value: "9"}}, }, }}, allowed:      false, expectReason: `forbidden sysctls`, expectDetail: `net.ipv4.tcp_keepalive_probes`, }, </ins> } for _, tc := range tests {


expectDetail: `a, b`, }, { <del> name: "new supported sysctls", </del> <ins> name: "new supported sysctls: net.ipv4.tcp_keepalive_time", </ins> pod: &corev1.Pod{Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_time", Value: "7200"}},


}}, allowed: true, }, <ins> { name: "new supported sysctls: net.ipv4.tcp_fin_timeout", pod: &corev1.Pod{Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_fin_timeout", Value: "60"}}, }, }}, allowed: true, }, { name: "new supported sysctls: net.ipv4.tcp_keepalive_intvl", pod: &corev1.Pod{Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_intvl", Value: "75"}}, }, }}, allowed: true, }, { name: "new supported sysctls: net.ipv4.tcp_keepalive_probes", pod: &corev1.Pod{Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ Sysctls: []corev1.Sysctl{{Name: "net.ipv4.tcp_keepalive_probes", Value: "9"}}, }, }}, allowed: true, }, </ins> } for _, tc := range tests {


{Name: "net.ipv4.ip_unprivileged_port_start", Value: "1024"}, {Name: "net.ipv4.ip_local_reserved_ports", Value: "1024-4999"}, {Name: "net.ipv4.tcp_keepalive_time", Value: "7200"}, <ins> {Name: "net.ipv4.tcp_fin_timeout", Value: "60"}, {Name: "net.ipv4.tcp_keepalive_intvl", Value: "75"}, {Name: "net.ipv4.tcp_keepalive_probes", Value: "9"}, </ins> } }), }


value: 1024-4999 - name: net.ipv4.tcp_keepalive_time value: "7200" <ins> - name: net.ipv4.tcp_fin_timeout value: "60" - name: net.ipv4.tcp_keepalive_intvl value: "75" - name: net.ipv4.tcp_keepalive_probes value: "9" </ins>


# See the License for the specific language governing permissions and # limitations under the License. <del> ARG BASEIMAGE </del> ARG RUNNERIMAGE <del> FROM ${BASEIMAGE} as debbase </del> FROM ${RUNNERIMAGE} <del> # This is a dependency for `kubectl diff` tests COPY --from=debbase /usr/bin/diff /usr/local/bin/ COPY --from=debbase /lib/x86_64-linux-gnu/libc.so.6 /lib/x86_64-linux-gnu COPY --from=debbase /lib/x86_64-linux-gnu/libpthread.so.0 /lib/x86_64-linux-gnu COPY --from=debbase /lib64/ld-linux-x86-64.so.2 /lib64 </del> COPY cluster /kubernetes/cluster COPY ginkgo /usr/local/bin/ COPY e2e.test /usr/local/bin/


# This is defined in root Makefile, but some build contexts do not refer to them KUBE_BASE_IMAGE_REGISTRY?=registry.k8s.io BASE_IMAGE_VERSION?=bookworm-v1.0.0 <del> BASEIMAGE?=${KUBE_BASE_IMAGE_REGISTRY}/build-image/debian-base-${ARCH}:${BASE_IMAGE_VERSION} # Keep debian releases (e.g. debian 11 == bullseye) consistent # between BASE_IMAGE_VERSION and DISTROLESS_IMAGE images DISTROLESS_IMAGE?=base-debian11 RUNNERIMAGE?=gcr.io/distroless/${DISTROLESS_IMAGE}:latest </del> <ins> RUNNERIMAGE?=${KUBE_BASE_IMAGE_REGISTRY}/build-image/debian-base-${ARCH}:${BASE_IMAGE_VERSION} </ins> TEMP_DIR:=$(shell mktemp -d -t conformance-XXXXXX)


--load  --pull  -t ${REGISTRY}/conformance-${ARCH}:${VERSION}  <del> --build-arg BASEIMAGE=$(BASEIMAGE)  </del> --build-arg RUNNERIMAGE=$(RUNNERIMAGE)  ${TEMP_DIR} rm -rf "${TEMP_DIR}"


allNodePluginScores := make([]framework.NodePluginScores, len(nodes)) numPlugins := len(f.scorePlugins) - state.SkipScorePlugins.Len() plugins := make([]framework.ScorePlugin, 0, numPlugins) <del> pluginToNodeScores := make([]framework.NodeScoreList, numPlugins) </del> <ins> pluginToNodeScores := make(map[string]framework.NodeScoreList, numPlugins) </ins> for _, pl := range f.scorePlugins { if state.SkipScorePlugins.Has(pl.Name()) { continue } plugins = append(plugins, pl) <ins> pluginToNodeScores[pl.Name()] = make(framework.NodeScoreList, len(nodes)) </ins> } ctx, cancel := context.WithCancel(ctx) defer cancel() errCh := parallelize.NewErrorChannel() <del> logger := klog.FromContext(ctx) logger = klog.LoggerWithName(logger, "Score") // TODO(knelasevero): Remove duplicated keys from log entry calls // When contextualized logging hits GA // https://github.com/kubernetes/kubernetes/issues/111672 logger = klog.LoggerWithValues(logger, "pod", klog.KObj(pod)) // Run Score method for each node in parallel. f.Parallelizer().Until(ctx, len(plugins), func(i int) { pl := plugins[i] logger := klog.LoggerWithName(logger, pl.Name()) nodeScores := make(framework.NodeScoreList, len(nodes)) for index, node := range nodes { nodeName := node.Name </del> <ins> if len(plugins) > 0 { logger := klog.FromContext(ctx) logger = klog.LoggerWithName(logger, "Score") // TODO(knelasevero): Remove duplicated keys from log entry calls // When contextualized logging hits GA // https://github.com/kubernetes/kubernetes/issues/111672 logger = klog.LoggerWithValues(logger, "pod", klog.KObj(pod)) // Run Score method for each node in parallel. f.Parallelizer().Until(ctx, len(nodes), func(index int) { nodeName := nodes[index].Name </ins> logger := klog.LoggerWithValues(logger, "node", klog.ObjectRef{Name: nodeName}) <del> ctx := klog.NewContext(ctx, logger) s, status := f.runScorePlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) errCh.SendErrorWithCancel(err, cancel) return } nodeScores[index] = framework.NodeScore{ Name:  nodeName, Score: s, </del> <ins> for _, pl := range plugins { logger := klog.LoggerWithName(logger, pl.Name()) ctx := klog.NewContext(ctx, logger) s, status := f.runScorePlugin(ctx, pl, state, pod, nodeName) if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) errCh.SendErrorWithCancel(err, cancel) return } pluginToNodeScores[pl.Name()][index] = framework.NodeScore{ Name:  nodeName, Score: s, } </ins> } <ins> }, metrics.Score) if err := errCh.ReceiveError(); err != nil { return nil, framework.AsStatus(fmt.Errorf("running Score plugins: %w", err)) </ins> } <ins> } // Run NormalizeScore method for each ScorePlugin in parallel. f.Parallelizer().Until(ctx, len(plugins), func(index int) { pl := plugins[index] </ins> if pl.ScoreExtensions() == nil { <del> pluginToNodeScores[i] = nodeScores </del> return } <del> status := f.runScoreExtension(ctx, pl, state, pod, nodeScores) </del> <ins> nodeScoreList := pluginToNodeScores[pl.Name()] status := f.runScoreExtension(ctx, pl, state, pod, nodeScoreList) </ins> if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) errCh.SendErrorWithCancel(err, cancel) return } <del> pluginToNodeScores[i] = nodeScores </del> }, metrics.Score) if err := errCh.ReceiveError(); err != nil { return nil, framework.AsStatus(fmt.Errorf("running Normalize on Score plugins: %w", err))


for i, pl := range plugins { weight := f.scorePluginWeight[pl.Name()] <del> nodeScoreList := pluginToNodeScores[i] </del> <ins> nodeScoreList := pluginToNodeScores[pl.Name()] </ins> score := nodeScoreList[index].Score if score > framework.MaxNodeScore || score < framework.MinNodeScore {


driverName: driverName, } if shouldTimeout { <del> timeout := plugin.PluginClientTimeout + time.Millisecond </del> <ins> timeout := plugin.PluginClientTimeout + 10*time.Millisecond </ins> fakeDRADriverGRPCServer.timeout = &timeout }


klog.V(2).InfoS("Preemptor Pod preempted victim Pod", "preemptor", klog.KObj(pod), "victim", klog.KObj(victim), "node", c.Name()) } <del> fh.EventRecorder().Eventf(victim, pod, v1.EventTypeNormal, "Preempted", "Preempting", "Preempted by a pod on node %v", c.Name()) </del> <ins> fh.EventRecorder().Eventf(victim, pod, v1.EventTypeNormal, "Preempted", "Preempting", "Preempted by pod %v on node %v", pod.UID, c.Name()) </ins> } fh.Parallelizer().Until(ctx, len(c.Victims().Pods), preemptPod, ev.PluginName)


requests := make(map[string]int64, len(pods)) for _, pod := range pods { podSum := int64(0) <del> for _, c := range pod.Spec.Containers { </del> <ins> // Calculate all regular containers and restartable init containers requests. containers := append([]v1.Container{}, pod.Spec.Containers...) for _, c := range pod.Spec.InitContainers { if c.RestartPolicy != nil && *c.RestartPolicy == v1.ContainerRestartPolicyAlways { containers = append(containers, c) } } for _, c := range containers { </ins> if container == "" || container == c.Name { if containerRequest, ok := c.Resources.Requests[resource]; ok { podSum += containerRequest.MilliValue()


}) } } <ins> func TestCalculatePodRequests(t *testing.T) { containerRestartPolicyAlways := v1.ContainerRestartPolicyAlways testPod := "test-pod" tests := []struct { name             string pods             []*v1.Pod container        string resource         v1.ResourceName expectedRequests map[string]int64 expectedError    error }{ { name:             "void", pods:             []*v1.Pod{}, container:        "", resource:         v1.ResourceCPU, expectedRequests: map[string]int64{}, expectedError:    nil, }, { name: "pod with regular containers", pods: []*v1.Pod{{ ObjectMeta: metav1.ObjectMeta{ Name:      testPod, Namespace: testNamespace, }, Spec: v1.PodSpec{ Containers: []v1.Container{ {Name: "container1", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI)}}}, {Name: "container2", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(50, resource.DecimalSI)}}}, }, }, }}, container:        "", resource:         v1.ResourceCPU, expectedRequests: map[string]int64{testPod: 150}, expectedError:    nil, }, { name: "calculate requests with special container", pods: []*v1.Pod{{ ObjectMeta: metav1.ObjectMeta{ Name:      testPod, Namespace: testNamespace, }, Spec: v1.PodSpec{ Containers: []v1.Container{ {Name: "container1", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI)}}}, {Name: "container2", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(50, resource.DecimalSI)}}}, }, }, }}, container:        "container1", resource:         v1.ResourceCPU, expectedRequests: map[string]int64{testPod: 100}, expectedError:    nil, }, { name: "container missing requests", pods: []*v1.Pod{{ ObjectMeta: metav1.ObjectMeta{ Name:      testPod, Namespace: testNamespace, }, Spec: v1.PodSpec{ Containers: []v1.Container{ {Name: "container1"}, }, }, }}, container:        "", resource:         v1.ResourceCPU, expectedRequests: nil, expectedError:    fmt.Errorf("missing request for %s in container %s of Pod %s", v1.ResourceCPU, "container1", testPod), }, { name: "pod with restartable init containers", pods: []*v1.Pod{{ ObjectMeta: metav1.ObjectMeta{ Name:      testPod, Namespace: testNamespace, }, Spec: v1.PodSpec{ Containers: []v1.Container{ {Name: "container1", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI)}}}, }, InitContainers: []v1.Container{ {Name: "init-container1", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(20, resource.DecimalSI)}}}, {Name: "restartable-container1", RestartPolicy: &containerRestartPolicyAlways, Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(50, resource.DecimalSI)}}}, }, }, }}, container:        "", resource:         v1.ResourceCPU, expectedRequests: map[string]int64{testPod: 150}, expectedError:    nil, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { requests, err := calculatePodRequests(tc.pods, tc.container, tc.resource) assert.Equal(t, tc.expectedRequests, requests, "requests should be as expected") assert.Equal(t, tc.expectedError, err, "error should be as expected") }) } } </ins>


import ( "fmt" "reflect" <del> "strings" </del> "testing" "github.com/google/go-cmp/cmp"


utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/features" <ins> st "k8s.io/kubernetes/pkg/scheduler/testing" </ins> ) func TestNewResource(t *testing.T) {


} } <del> type testingMode interface { Fatalf(format string, args ...interface{}) } func makeBasePod(t testingMode, nodeName, objName, cpu, mem, extended string, ports []v1.ContainerPort, volumes []v1.Volume) *v1.Pod { req := v1.ResourceList{} if cpu != "" { req = v1.ResourceList{ v1.ResourceCPU:    resource.MustParse(cpu), v1.ResourceMemory: resource.MustParse(mem), } if extended != "" { parts := strings.Split(extended, ":") if len(parts) != 2 { t.Fatalf("Invalid extended resource string: "%s"", extended) } req[v1.ResourceName(parts[0])] = resource.MustParse(parts[1]) } } return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ UID:       types.UID(objName), Namespace: "node_info_cache_test", Name:      objName, }, Spec: v1.PodSpec{ Containers: []v1.Container{{ Resources: v1.ResourceRequirements{ Requests: req, }, Ports: ports, }}, NodeName: nodeName, Volumes:  volumes, }, } } </del> func TestNewNodeInfo(t *testing.T) { nodeName := "test-node" pods := []*v1.Pod{ <del> makeBasePod(t, nodeName, "test-1", "100m", "500", "", []v1.ContainerPort{{HostIP: "127.0.0.1", HostPort: 80, Protocol: "TCP"}}, nil), makeBasePod(t, nodeName, "test-2", "200m", "1Ki", "", []v1.ContainerPort{{HostIP: "127.0.0.1", HostPort: 8080, Protocol: "TCP"}}, nil), </del> <ins> st.MakePod().UID("test-1").Namespace("node_info_cache_test").Name("test-1").Node(nodeName). Containers([]v1.Container{st.MakeContainer().ResourceRequests(map[v1.ResourceName]string{ v1.ResourceCPU:    "100m", v1.ResourceMemory: "500", }).ContainerPort([]v1.ContainerPort{{ HostIP:   "127.0.0.1", HostPort: 80, Protocol: "TCP", }}).Obj()}). Obj(), st.MakePod().UID("test-2").Namespace("node_info_cache_test").Name("test-2").Node(nodeName). Containers([]v1.Container{st.MakeContainer().ResourceRequests(map[v1.ResourceName]string{ v1.ResourceCPU:    "200m", v1.ResourceMemory: "1Ki", }).ContainerPort([]v1.ContainerPort{{ HostIP:   "127.0.0.1", HostPort: 8080, Protocol: "TCP", }}).Obj()}). Obj(), </ins> } expected := &NodeInfo{


func TestNodeInfoRemovePod(t *testing.T) { nodeName := "test-node" pods := []*v1.Pod{ <del> makeBasePod(t, nodeName, "test-1", "100m", "500", "", []v1.ContainerPort{{HostIP: "127.0.0.1", HostPort: 80, Protocol: "TCP"}}, []v1.Volume{{VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: "pvc-1"}}}}), makeBasePod(t, nodeName, "test-2", "200m", "1Ki", "", []v1.ContainerPort{{HostIP: "127.0.0.1", HostPort: 8080, Protocol: "TCP"}}, nil), </del> <ins> st.MakePod().UID("test-1").Namespace("node_info_cache_test").Name("test-1").Node(nodeName). Containers([]v1.Container{st.MakeContainer().ResourceRequests(map[v1.ResourceName]string{ v1.ResourceCPU:    "100m", v1.ResourceMemory: "500", }).ContainerPort([]v1.ContainerPort{{ HostIP:   "127.0.0.1", HostPort: 80, Protocol: "TCP", }}).Obj()}). Volumes([]v1.Volume{{VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: "pvc-1"}}}}). Obj(), st.MakePod().UID("test-2").Namespace("node_info_cache_test").Name("test-2").Node(nodeName). Containers([]v1.Container{st.MakeContainer().ResourceRequests(map[v1.ResourceName]string{ v1.ResourceCPU:    "200m", v1.ResourceMemory: "1Ki", }).ContainerPort([]v1.ContainerPort{{ HostIP:   "127.0.0.1", HostPort: 8080, Protocol: "TCP", }}).Obj()}). Obj(), </ins> } // add pod Overhead


expectedNodeInfo *NodeInfo }{ { <del> pod:         makeBasePod(t, nodeName, "non-exist", "0", "0", "", []v1.ContainerPort{{}}, []v1.Volume{}), </del> <ins> pod:         st.MakePod().UID("non-exist").Namespace("node_info_cache_test").Node(nodeName).Obj(), </ins> errExpected: true, expectedNodeInfo: &NodeInfo{ node: &v1.Node{


return p } <ins> // Volumes set the volumes and inject into the inner pod. func (p *PodWrapper) Volumes(volumes []v1.Volume) *PodWrapper { p.Spec.Volumes = volumes return p } </ins> // SchedulingGates sets `gates` as additional SchedulerGates of the inner pod. func (p *PodWrapper) SchedulingGates(gates []string) *PodWrapper { for _, gate := range gates {


// This will start reconciliation of node.status.volumesInUse. rc.updateLastSyncTime() } <del> if len(rc.volumesNeedReportedInUse) != 0 && rc.populatorHasAddedPods() { // Once DSW is populated, mark all reconstructed as reported in node.status, // so they can proceed with MountDevice / SetUp. rc.desiredStateOfWorld.MarkVolumesReportedInUse(rc.volumesNeedReportedInUse) rc.volumesNeedReportedInUse = nil } </del> }


timeOfLastSync:                  time.Time{}, volumesFailedReconstruction:     make([]podVolume, 0), volumesNeedUpdateFromNodeStatus: make([]v1.UniqueVolumeName, 0), <del> volumesNeedReportedInUse:        make([]v1.UniqueVolumeName, 0), </del> } }


timeOfLastSync                  time.Time volumesFailedReconstruction     []podVolume volumesNeedUpdateFromNodeStatus []v1.UniqueVolumeName <del> volumesNeedReportedInUse        []v1.UniqueVolumeName </del> } func (rc *reconciler) unmountVolumes() {


// Add the volumes to ASW rc.updateStates(reconstructedVolumes) <del> // The reconstructed volumes are mounted, hence a previous kubelet must have already put it into node.status.volumesInUse. // Remember to update DSW with this information. rc.volumesNeedReportedInUse = reconstructedVolumeNames </del> // Remember to update devicePath from node.status.volumesAttached rc.volumesNeedUpdateFromNodeStatus = reconstructedVolumeNames }


tests := []struct { name                                string volumePaths                         []string <del> expectedVolumesNeedReportedInUse    []string </del> expectedVolumesNeedDevicePath       []string expectedVolumesFailedReconstruction []string verifyFunc                          func(rcInstance *reconciler, fakePlugin *volumetesting.FakeVolumePlugin) error


filepath.Join("pod1", "volumes", "fake-plugin", "pvc-abcdef"), filepath.Join("pod2", "volumes", "fake-plugin", "pvc-abcdef"), }, <del> expectedVolumesNeedReportedInUse:    []string{"fake-plugin/pvc-abcdef", "fake-plugin/pvc-abcdef"}, </del> expectedVolumesNeedDevicePath:       []string{"fake-plugin/pvc-abcdef", "fake-plugin/pvc-abcdef"}, expectedVolumesFailedReconstruction: []string{}, verifyFunc: func(rcInstance *reconciler, fakePlugin *volumetesting.FakeVolumePlugin) error {


volumePaths: []string{ filepath.Join("pod1", "volumes", "missing-plugin", "pvc-abcdef"), }, <del> expectedVolumesNeedReportedInUse:    []string{}, </del> expectedVolumesNeedDevicePath:       []string{}, expectedVolumesFailedReconstruction: []string{"pvc-abcdef"}, },


t.Errorf("Expected expectedVolumesNeedDevicePath:n%vn got:n%v", expectedVolumes, rcInstance.volumesNeedUpdateFromNodeStatus) } <del> expectedVolumes = make([]v1.UniqueVolumeName, len(tc.expectedVolumesNeedReportedInUse)) for i := range tc.expectedVolumesNeedReportedInUse { expectedVolumes[i] = v1.UniqueVolumeName(tc.expectedVolumesNeedReportedInUse[i]) } if !reflect.DeepEqual(expectedVolumes, rcInstance.volumesNeedReportedInUse) { t.Errorf("Expected volumesNeedReportedInUse:n%vn got:n%v", expectedVolumes, rcInstance.volumesNeedReportedInUse) } </del> volumesFailedReconstruction := sets.NewString() for _, vol := range rcInstance.volumesFailedReconstruction { volumesFailedReconstruction.Insert(vol.volumeSpecName)


resourcev1alpha2 "k8s.io/api/resource/v1alpha2" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <ins> "k8s.io/apimachinery/pkg/labels" </ins> "k8s.io/client-go/kubernetes" "k8s.io/dynamic-resource-allocation/controller" "k8s.io/klog/v2" "k8s.io/kubernetes/test/e2e/dra/test-driver/app" "k8s.io/kubernetes/test/e2e/framework" <ins> e2enode "k8s.io/kubernetes/test/e2e/framework/node" </ins> e2epod "k8s.io/kubernetes/test/e2e/framework/pod" admissionapi "k8s.io/pod-security-admission/api" utilpointer "k8s.io/utils/pointer"


framework.ExpectNoError(err, "start pod") } }) <ins> // This test covers aspects of non graceful node shutdown by DRA controller // More details about this can be found in the KEP: // https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/2268-non-graceful-shutdown // NOTE: this test depends on kind. It will only work with kind cluster as it shuts down one of the // nodes by running `docker stop <node name>`, which is very kind-specific. ginkgo.It("[Serial] [Disruptive] [Slow] must deallocate on non graceful node shutdown", func(ctx context.Context) { ginkgo.By("create test pod") parameters := b.parameters() label := "app.kubernetes.io/instance" instance := f.UniqueName + "-test-app" pod := b.podExternal() pod.Labels[label] = instance claim := b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer) b.create(ctx, parameters, claim, pod) ginkgo.By("wait for test pod " + pod.Name + " to run") labelSelector := labels.SelectorFromSet(labels.Set(pod.Labels)) pods, err := e2epod.WaitForPodsWithLabelRunningReady(ctx, f.ClientSet, pod.Namespace, labelSelector, 1, framework.PodStartTimeout) framework.ExpectNoError(err, "start pod") runningPod := &pods.Items[0] nodeName := runningPod.Spec.NodeName // Prevent builder tearDown to fail waiting for unprepared resources delete(b.driver.Nodes, nodeName) ginkgo.By("stop node " + nodeName + " non gracefully") _, stderr, err := framework.RunCmd("docker", "stop", nodeName) gomega.Expect(stderr).To(gomega.BeEmpty()) framework.ExpectNoError(err) ginkgo.DeferCleanup(framework.RunCmd, "docker", "start", nodeName) if ok := e2enode.WaitForNodeToBeNotReady(ctx, f.ClientSet, nodeName, f.Timeouts.NodeNotReady); !ok { framework.Failf("Node %s failed to enter NotReady state", nodeName) } ginkgo.By("apply out-of-service taint on node " + nodeName) taint := v1.Taint{ Key:    v1.TaintNodeOutOfService, Effect: v1.TaintEffectNoExecute, } e2enode.AddOrUpdateTaintOnNode(ctx, f.ClientSet, nodeName, taint) e2enode.ExpectNodeHasTaint(ctx, f.ClientSet, nodeName, &taint) ginkgo.DeferCleanup(e2enode.RemoveTaintOffNode, f.ClientSet, nodeName, taint) ginkgo.By("waiting for claim to get deallocated") gomega.Eventually(ctx, framework.GetObject(b.f.ClientSet.ResourceV1alpha2().ResourceClaims(b.f.Namespace.Name).Get, claim.Name, metav1.GetOptions{})).WithTimeout(f.Timeouts.PodDelete).Should(gomega.HaveField("Status.Allocation", gomega.BeNil())) }) </ins> }) ginkgo.Context("with node-local resources", func() {


SystemPodsStartup:         10 * time.Minute, NodeSchedulable:           30 * time.Minute, SystemDaemonsetStartup:    5 * time.Minute, <ins> NodeNotReady:              3 * time.Minute, </ins> } // TimeoutContext contains timeout settings for several actions.


// SystemDaemonsetStartup is how long to wait for all system daemonsets to be ready. SystemDaemonsetStartup time.Duration <ins> // NodeNotReady is how long to wait for a node to be not ready. NodeNotReady time.Duration </ins> } // NewTimeoutContext returns a TimeoutContext with all values set either to


// implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod. // wait for config to be observed <del> verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, test) </del> <ins> verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, "", test) </ins> // run storage migration // get secrets


} // remove old KMS provider <ins> // verifyIfKMSTransformersSwapped sometimes passes even before the changes in the encryption config file are observed. // this causes the metrics tests to fail, which validate two config changes. // this may happen when an existing KMS provider is already running (e.g., new-kms-provider-for-secrets in this case). // to ensure that the changes are observed, we added one more provider (kms-provider-to-encrypt-all) and are validating it in verifyIfKMSTransformersSwapped. </ins> encryptionConfigWithoutOldProvider := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1


name: new-kms-provider-for-configmaps cachesize: 1000 endpoint: unix:///@new-kms-provider.sock <ins> - resources: - '*.*' providers: - kms: name: kms-provider-to-encrypt-all cachesize: 1000 endpoint: unix:///@new-encrypt-all-kms-provider.sock - identity: {} </ins> ` <ins> // start new KMS Plugin _ = mock.NewBase64Plugin(t, "@new-encrypt-all-kms-provider.sock") </ins> // update encryption config and wait for hot reload if err := os.WriteFile(filepath.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithoutOldProvider), 0644); err != nil { t.Fatalf("failed to update encryption config, err: %v", err) } <ins> wantPrefixForEncryptAll := "k8s:enc:kms:v1:kms-provider-to-encrypt-all:" </ins> // wait for config to be observed <del> verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, test) </del> <ins> verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, wantPrefixForEncryptAll, test) </ins> // confirm that reading secrets still works _, err = test.restClient.CoreV1().Secrets(testNamespace).Get(


func verifyPrefixOfSecretResource(t *testing.T, wantPrefix string, test *transformTest) { // implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod. // wait for config to be observed <del> verifyIfKMSTransformersSwapped(t, wantPrefix, test) </del> <ins> verifyIfKMSTransformersSwapped(t, wantPrefix, "", test) </ins> // run storage migration secretsList, err := test.restClient.CoreV1().Secrets("").List(


} } <del> func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix string, test *transformTest) { </del> <ins> func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix, wantPrefixForEncryptAll string, test *transformTest) { </ins> t.Helper() var swapErr error


return false, nil } <ins> if wantPrefixForEncryptAll != "" { deploymentName := fmt.Sprintf("deployment-%d", idx) _, err := test.createDeployment(deploymentName, "default") if err != nil { t.Fatalf("Failed to create test secret, error: %v", err) } rawEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", deploymentName, "default")) if err != nil { t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", deploymentName, "default"), err) } // check prefix if !bytes.HasPrefix(rawEnvelope.Kvs[0].Value, []byte(wantPrefixForEncryptAll)) { idx++ swapErr = fmt.Errorf("expected deployment to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawEnvelope.Kvs[0].Value) // return nil error to continue polling till timeout return false, nil } } </ins> return true, nil }) if pollErr == wait.ErrWaitTimeout {


// start new KMS Plugin _ = mock.NewBase64Plugin(t, "@new-kms-provider.sock") // update encryption config <del> if err := os.WriteFile(filepath.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithNewProvider), 0644); err != nil { t.Fatalf("failed to update encryption config, err: %v", err) } </del> <ins> updateFile(t, test.configDir, encryptionConfigFileName, []byte(encryptionConfigWithNewProvider)) </ins> wantPrefixForSecrets := "k8s:enc:kms:v1:new-kms-provider-for-secrets:" // implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod. // wait for config to be observed <del> verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, "", test) </del> <ins> verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, test) </ins> // run storage migration // get secrets


} // remove old KMS provider <del> // verifyIfKMSTransformersSwapped sometimes passes even before the changes in the encryption config file are observed. // this causes the metrics tests to fail, which validate two config changes. // this may happen when an existing KMS provider is already running (e.g., new-kms-provider-for-secrets in this case). // to ensure that the changes are observed, we added one more provider (kms-provider-to-encrypt-all) and are validating it in verifyIfKMSTransformersSwapped. </del> encryptionConfigWithoutOldProvider := ` kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1


name: new-kms-provider-for-configmaps cachesize: 1000 endpoint: unix:///@new-kms-provider.sock <del> - resources: - '*.*' providers: - kms: name: kms-provider-to-encrypt-all cachesize: 1000 endpoint: unix:///@new-encrypt-all-kms-provider.sock - identity: {} </del> ` <del> // start new KMS Plugin _ = mock.NewBase64Plugin(t, "@new-encrypt-all-kms-provider.sock") </del> // update encryption config and wait for hot reload <del> if err := os.WriteFile(filepath.Join(test.configDir, encryptionConfigFileName), []byte(encryptionConfigWithoutOldProvider), 0644); err != nil { t.Fatalf("failed to update encryption config, err: %v", err) } wantPrefixForEncryptAll := "k8s:enc:kms:v1:kms-provider-to-encrypt-all:" </del> <ins> updateFile(t, test.configDir, encryptionConfigFileName, []byte(encryptionConfigWithoutOldProvider)) </ins> // wait for config to be observed <del> verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, wantPrefixForEncryptAll, test) </del> <ins> verifyIfKMSTransformersSwapped(t, wantPrefixForSecrets, test) </ins> // confirm that reading secrets still works _, err = test.restClient.CoreV1().Secrets(testNamespace).Get(


func verifyPrefixOfSecretResource(t *testing.T, wantPrefix string, test *transformTest) { // implementing this brute force approach instead of fancy channel notification to avoid test specific code in prod. // wait for config to be observed <del> verifyIfKMSTransformersSwapped(t, wantPrefix, "", test) </del> <ins> verifyIfKMSTransformersSwapped(t, wantPrefix, test) </ins> // run storage migration secretsList, err := test.restClient.CoreV1().Secrets("").List(


} } <del> func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix, wantPrefixForEncryptAll string, test *transformTest) { </del> <ins> func verifyIfKMSTransformersSwapped(t *testing.T, wantPrefix string, test *transformTest) { </ins> t.Helper() var swapErr error


return false, nil } <del> if wantPrefixForEncryptAll != "" { deploymentName := fmt.Sprintf("deployment-%d", idx) _, err := test.createDeployment(deploymentName, "default") if err != nil { t.Fatalf("Failed to create test secret, error: %v", err) } rawEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", deploymentName, "default")) if err != nil { t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", deploymentName, "default"), err) } // check prefix if !bytes.HasPrefix(rawEnvelope.Kvs[0].Value, []byte(wantPrefixForEncryptAll)) { idx++ swapErr = fmt.Errorf("expected deployment to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawEnvelope.Kvs[0].Value) // return nil error to continue polling till timeout return false, nil } } </del> return true, nil }) if pollErr == wait.ErrWaitTimeout {


} } <ins> func updateFile(t *testing.T, configDir, filename string, newContent []byte) { t.Helper() // Create a temporary file tempFile, err := os.CreateTemp(configDir, "tempfile") if err != nil { t.Fatal(err) } defer tempFile.Close() // Write the new content to the temporary file _, err = tempFile.Write(newContent) if err != nil { t.Fatal(err) } // Atomically replace the original file with the temporary file err = os.Rename(tempFile.Name(), filepath.Join(configDir, filename)) if err != nil { t.Fatal(err) } } </ins> func TestKMSHealthz(t *testing.T) { encryptionConfig := ` kind: EncryptionConfiguration


"k8s.io/apimachinery/pkg/util/uuid" netutils "k8s.io/utils/net" <ins> utilfeature "k8s.io/apiserver/pkg/util/feature" </ins> kubefeatures "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/test/e2e/framework"


e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" <ins> e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" </ins> "k8s.io/kubernetes/test/e2e/network/common" imageutils "k8s.io/kubernetes/test/utils/image" <ins> admissionapi "k8s.io/pod-security-admission/api" </ins> ) <del> var _ = common.SIGDescribe("Dual Stack Host IP [Feature:PodHostIPs]", func() { </del> <ins> var _ = common.SIGDescribe("DualStack Host IP [Serial] [NodeFeature:PodHostIPs] [Feature:PodHostIPs]", func() { </ins> f := framework.NewDefaultFramework("dualstack") <ins> f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged </ins> ginkgo.Context("when creating a Pod, it has no PodHostIPs feature", func() { tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) {


gomega.Expect(p.Status.HostIPs).Should(gomega.BeNil()) ginkgo.By("deleting the pod") <del> err := podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(30)) </del> <ins> err := podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(1)) </ins> framework.ExpectNoError(err, "failed to delete pod") })


gomega.Expect(p.Status.HostIPs).Should(gomega.BeNil()) ginkgo.By("deleting the pod") <del> err := podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(30)) </del> <ins> err := podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(1)) </ins> framework.ExpectNoError(err, "failed to delete pod") }) })


podName := "pod-dualstack-host-ips" <del> pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:   podName, Labels: map[string]string{"test": "dualstack-host-ips"}, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:  "dualstack-host-ips", Image: imageutils.GetE2EImage(imageutils.Agnhost), }, }, }, } </del> <ins> pod := genPodHostIPs(podName+string(uuid.NewUUID()), false) </ins> ginkgo.By("submitting the pod to kubernetes") podClient := e2epod.NewPodClient(f)


} ginkgo.By("deleting the pod") <del> err = podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(30)) </del> <ins> err = podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(1)) </ins> framework.ExpectNoError(err, "failed to delete pod") })


podName := "pod-dualstack-host-ips" <del> pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:   podName, Labels: map[string]string{"test": "dualstack-host-ips"}, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:  "dualstack-host-ips", Image: imageutils.GetE2EImage(imageutils.Agnhost), }, }, HostNetwork: true, }, } </del> <ins> pod := genPodHostIPs(podName+string(uuid.NewUUID()), true) </ins> ginkgo.By("submitting the pod to kubernetes") podClient := e2epod.NewPodClient(f)


} ginkgo.By("deleting the pod") <del> err = podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(30)) </del> <ins> err = podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(1)) </ins> framework.ExpectNoError(err, "failed to delete pod") }) ginkgo.It("should provide hostIPs as an env var", func(ctx context.Context) { <ins> if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodHostIPs) { e2eskipper.Skipf("PodHostIPs feature is not enabled") } </ins> podName := "downward-api-" + string(uuid.NewUUID()) env := []v1.EnvVar{ {


testDownwardAPI(ctx, f, podName, env, expectations) }) }) <ins> ginkgo.Context("when feature rollback", func() { tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) { initialConfig.FeatureGates = map[string]bool{ string(kubefeatures.PodHostIPs): true, } }) ginkgo.It("should able upgrade and rollback", func(ctx context.Context) { podName := "pod-dualstack-host-ips" pod := genPodHostIPs(podName+string(uuid.NewUUID()), false) ginkgo.By("submitting the pod to kubernetes") podClient := e2epod.NewPodClient(f) p := podClient.CreateSync(ctx, pod) gomega.Expect(p.Status.HostIPs).ShouldNot(gomega.BeNil()) ginkgo.By("Disable PodHostIPs feature") cfg, err := getCurrentKubeletConfig(ctx) framework.ExpectNoError(err) newCfg := cfg.DeepCopy() newCfg.FeatureGates = map[string]bool{ string(kubefeatures.PodHostIPs): false, } updateKubeletConfig(ctx, f, newCfg, true) gomega.Expect(p.Status.HostIPs).ShouldNot(gomega.BeNil()) ginkgo.By("deleting the pod") err = podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(1)) framework.ExpectNoError(err, "failed to delete pod") ginkgo.By("recreate pod") pod = genPodHostIPs(podName+string(uuid.NewUUID()), false) p = podClient.CreateSync(ctx, pod) // Feature PodHostIPs is disabled, HostIPs should be nil gomega.Expect(p.Status.HostIPs).Should(gomega.BeNil()) newCfg.FeatureGates = map[string]bool{ string(kubefeatures.PodHostIPs): true, } updateKubeletConfig(ctx, f, newCfg, true) p, err = podClient.Get(ctx, pod.Name, metav1.GetOptions{}) framework.ExpectNoError(err) // Feature PodHostIPs is enabled, HostIPs should not be nil gomega.Expect(p.Status.HostIPs).ShouldNot(gomega.BeNil()) ginkgo.By("deleting the pod") err = podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(1)) framework.ExpectNoError(err, "failed to delete pod") }) }) </ins> }) <ins> func genPodHostIPs(podName string, hostNetwork bool) *v1.Pod { return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:   podName, Labels: map[string]string{"test": "dualstack-host-ips"}, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:  "test-container", Image: imageutils.GetE2EImage(imageutils.Agnhost), }, }, RestartPolicy: v1.RestartPolicyNever, HostNetwork:   hostNetwork, }, } } </ins> func testDownwardAPI(ctx context.Context, f *framework.Framework, podName string, env []v1.EnvVar, expectations []string) { pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{


} kube::etcd::install() { <del> ( local os local arch os=$(kube::util::host_os) arch=$(kube::util::host_arch) cd "${KUBE_ROOT}/third_party" || return 1 if [[ $(readlink etcd) == etcd-v${ETCD_VERSION}-${os}-* ]]; then kube::log::info "etcd v${ETCD_VERSION} already installed. To use:" kube::log::info "export PATH="$(pwd)/etcd:${PATH}"" return  #already installed fi if [[ ${os} == "darwin" ]]; then download_file="etcd-v${ETCD_VERSION}-${os}-${arch}.zip" url="https://github.com/etcd-io/etcd/releases/download/v${ETCD_VERSION}/${download_file}" kube::util::download_file "${url}" "${download_file}" unzip -o "${download_file}" ln -fns "etcd-v${ETCD_VERSION}-${os}-${arch}" etcd rm "${download_file}" elif [[ ${os} == "linux" ]]; then url="https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-${os}-${arch}.tar.gz" download_file="etcd-v${ETCD_VERSION}-${os}-${arch}.tar.gz" kube::util::download_file "${url}" "${download_file}" tar xzf "${download_file}" ln -fns "etcd-v${ETCD_VERSION}-${os}-${arch}" etcd rm "${download_file}" else kube::log::info "${os} is NOT supported." fi kube::log::info "etcd v${ETCD_VERSION} installed. To use:" </del> <ins> local os local arch os=$(kube::util::host_os) arch=$(kube::util::host_arch) cd "${KUBE_ROOT}/third_party" || return 1 if [[ $(readlink etcd) == etcd-v${ETCD_VERSION}-${os}-* ]]; then kube::log::info "etcd v${ETCD_VERSION} already installed. To use:" </ins> kube::log::info "export PATH="$(pwd)/etcd:${PATH}"" <del> ) </del> <ins> # export into current process PATH="$(pwd)/etcd:${PATH}" export PATH return  #already installed fi if [[ ${os} == "darwin" ]]; then download_file="etcd-v${ETCD_VERSION}-${os}-${arch}.zip" url="https://github.com/etcd-io/etcd/releases/download/v${ETCD_VERSION}/${download_file}" kube::util::download_file "${url}" "${download_file}" unzip -o "${download_file}" ln -fns "etcd-v${ETCD_VERSION}-${os}-${arch}" etcd rm "${download_file}" elif [[ ${os} == "linux" ]]; then url="https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-${os}-${arch}.tar.gz" download_file="etcd-v${ETCD_VERSION}-${os}-${arch}.tar.gz" kube::util::download_file "${url}" "${download_file}" tar xzf "${download_file}" ln -fns "etcd-v${ETCD_VERSION}-${os}-${arch}" etcd rm "${download_file}" else kube::log::info "${os} is NOT supported." fi kube::log::info "etcd v${ETCD_VERSION} installed. To use:" kube::log::info "export PATH="$(pwd)/etcd:${PATH}"" # export into current process PATH="$(pwd)/etcd:${PATH}" export PATH </ins> }


kube::util::require-jq kube::golang::setup_env <ins> kube::etcd::install </ins> make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver


trap cleanup EXIT SIGINT <del> kube::golang::setup_env </del> TMP_DIR=${TMP_DIR:-$(kube::realpath "$(mktemp -d -t "$(basename "$0").XXXXXX")")} ETCD_HOST=${ETCD_HOST:-127.0.0.1} ETCD_PORT=${ETCD_PORT:-2379}


if err != nil { assert.NoErrorf(t, err, "unable to create temp file") } stdoutBuf := &bytes.Buffer{} stderrBuf := &bytes.Buffer{} containerID := "fake-container-id"


ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Start to follow the container's log. <ins> fileName := file.Name() </ins> go func(ctx context.Context) { podLogOptions := v1.PodLogOptions{ Follow: true, } opts := NewLogOptions(&podLogOptions, time.Now()) <del> ReadLogs(ctx, file.Name(), containerID, opts, fakeRuntimeService, stdoutBuf, stderrBuf) </del> <ins> _ = ReadLogs(ctx, fileName, containerID, opts, fakeRuntimeService, stdoutBuf, stderrBuf) </ins> }(ctx) // log in stdout


// Write 10 lines to log file. // Let ReadLogs start. time.Sleep(50 * time.Millisecond) for line := 0; line < 10; line++ { // Write the first three lines to log file now := time.Now().Format(types.RFC3339NanoLenient)


v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <ins> utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/features" </ins> kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod"


}) ginkgo.It("The containers terminated by OOM killer should have the reason set to OOMKilled", func() { <ins> cfg, configErr := getCurrentKubeletConfig(context.TODO()) framework.ExpectNoError(configErr) if utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) { // If Swap is enabled, we should test OOM with LimitedSwap. // UnlimitedSwap allows for workloads to use unbounded swap which // makes testing OOM challenging. // We are not able to change the default for these conformance tests, // so we will skip these tests if swap is enabled. if cfg.MemorySwap.SwapBehavior == "" || cfg.MemorySwap.SwapBehavior == "UnlimitedSwap" { ginkgo.Skip("OOMKiller should not run with UnlimitedSwap") } } </ins> ginkgo.By("Waiting for the pod to be failed") err := e2epod.WaitForPodTerminatedInNamespace(context.TODO(), f.ClientSet, testCase.podSpec.Name, "", f.Namespace.Name) framework.ExpectNoError(err, "Failed waiting for pod to terminate, %s/%s", f.Namespace.Name, testCase.podSpec.Name)


package storage import ( <del> utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/features" </del> <ins> "context" </ins> "testing" <ins> "time" </ins> <del> "context" </del> "github.com/google/go-cmp/cmp" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource"


genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing" "k8s.io/apiserver/pkg/registry/rest" etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" <ins> utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" </ins> api "k8s.io/kubernetes/pkg/apis/core" <ins> "k8s.io/kubernetes/pkg/features" </ins> "k8s.io/kubernetes/pkg/registry/core/persistentvolume" "k8s.io/kubernetes/pkg/registry/registrytest" )


t.Errorf("unexpected error: %v", err) } <ins> // We need to set custom timestamp which is not the same as the one on existing PV // - doing so will prevent timestamp update on phase change and custom one is used instead. pvStartTimestamp = &metav1.Time{Time: pvStartTimestamp.Time.Add(time.Second)} </ins> pvIn := &api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, Status: api.PersistentVolumeStatus{ <del> Phase: api.VolumeBound, // Set the same timestamp as original PV so this won't get updated on phase change breaking DeepEqual() later in test. </del> <ins> Phase:                   api.VolumeBound, </ins> LastPhaseTransitionTime: pvStartTimestamp, }, }


"k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" <ins> "k8s.io/component-base/metrics/legacyregistry" </ins> "k8s.io/component-base/metrics/testutil" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/features"


ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) metrics.RegisterMetrics() <ins> defer legacyregistry.Reset() </ins> // Create initial state var svc2 *v1.Service


if fp.needFullSync { t.Fatalf("Proxier unexpectedly already needs a full sync?") } <del> prFailures, err := testutil.GetCounterMetricValue(metrics.IptablesPartialRestoreFailuresTotal) </del> <ins> partialRestoreFailures, err := testutil.GetCounterMetricValue(metrics.IptablesPartialRestoreFailuresTotal) </ins> if err != nil { t.Fatalf("Could not get partial restore failures metric: %v", err) } <del> if prFailures != 0.0 { </del> <ins> if partialRestoreFailures != 0.0 { </ins> t.Errorf("Already did a partial resync? Something failed earlier!") }


if !fp.needFullSync { t.Errorf("Proxier did not fail on previous partial resync?") } <del> updatedPRFailures, err := testutil.GetCounterMetricValue(metrics.IptablesPartialRestoreFailuresTotal) </del> <ins> updatedPartialRestoreFailures, err := testutil.GetCounterMetricValue(metrics.IptablesPartialRestoreFailuresTotal) </ins> if err != nil { t.Errorf("Could not get partial restore failures metric: %v", err) } <del> if updatedPRFailures != prFailures+1.0 { t.Errorf("Partial restore failures metric was not incremented after failed partial resync (expected %.02f, got %.02f)", prFailures+1.0, updatedPRFailures) </del> <ins> if updatedPartialRestoreFailures != partialRestoreFailures+1.0 { t.Errorf("Partial restore failures metric was not incremented after failed partial resync (expected %.02f, got %.02f)", partialRestoreFailures+1.0, updatedPartialRestoreFailures) </ins> } // On retry we should do a full resync, which should succeed (and delete svc4)


return false, fmt.Errorf("mountVolume.NodeExpandVolume get PVC failed : %v", err) } <del> if volumeToMount.VolumeSpec.ReadOnly { simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MountVolume.NodeExpandVolume failed", "requested read-only file system") klog.Warningf(detailedMsg) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg) og.recorder.Eventf(pvc, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg) return true, nil } </del> pvcStatusCap := pvc.Status.Capacity[v1.ResourceStorage] pvSpecCap := pv.Spec.Capacity[v1.ResourceStorage] if pvcStatusCap.Cmp(pvSpecCap) < 0 { <ins> if volumeToMount.VolumeSpec.ReadOnly { simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MountVolume.NodeExpandVolume failed", "requested read-only file system") klog.Warningf(detailedMsg) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg) og.recorder.Eventf(pvc, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg) return true, nil } </ins> rsOpts.NewSize = pvSpecCap rsOpts.OldSize = pvcStatusCap resizeOp := nodeResizeOperationOpts{


} switch { <del> case o.Watch || o.WatchOnly: </del> <ins> case o.Watch: </ins> if len(o.SortBy) > 0 { <del> fmt.Fprintf(o.IOStreams.ErrOut, "warning: --watch or --watch-only requested, --sort-by will be ignoredn") </del> <ins> fmt.Fprintf(o.IOStreams.ErrOut, "warning: --watch requested, --sort-by will be ignored for watch events receivedn") } case o.WatchOnly: if len(o.SortBy) > 0 { fmt.Fprintf(o.IOStreams.ErrOut, "warning: --watch-only requested, --sort-by will be ignoredn") </ins> } default: if len(args) == 0 && cmdutil.IsFilenameSliceEmpty(o.Filenames, o.Kustomize) {


scheduledJobTimeout = 5 * time.Minute ) <del> var _ = framework.KubeDescribe("[Feature:ScheduledJob]", func() { </del> <ins> var _ = framework.KubeDescribe("ScheduledJob", func() { </ins> options := framework.FrameworkOptions{ ClientQPS:    20, ClientBurst:  50,


nodeAllocatableMilliCPU2, nodeAvailableMilliCPU2 := getNodeAllocatableAndAvailableMilliCPUValues(&node) framework.Logf("TEST2: Node '%s': NodeAllocatable MilliCPUs = %dm. MilliCPUs currently available to allocate = %dm.", node.Name, nodeAllocatableMilliCPU2, nodeAvailableMilliCPU2) <del> testPod3CPUQuantity := resource.NewMilliQuantity(nodeAvailableMilliCPU2+testPod1CPUQuantity.MilliValue()/2, resource.DecimalSI) </del> <ins> testPod3CPUQuantity := resource.NewMilliQuantity(nodeAvailableMilliCPU2+testPod1CPUQuantity.MilliValue()/4, resource.DecimalSI) </ins> testPod1CPUQuantityResized := resource.NewMilliQuantity(testPod1CPUQuantity.MilliValue()/3, resource.DecimalSI) framework.Logf("TEST2: testPod1 MilliCPUs after resize '%dm'", testPod1CPUQuantityResized.MilliValue())


<ins> # See the OWNERS docs at https://go.k8s.io/owners approvers: - aojea - danwinship reviewers: - aojea - danwinship </ins>


<ins> kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:network-policies namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile rules: - apiGroups: [""] resources: - pods - nodes - namespaces verbs: - get - watch - list # Watch for changes to Kubernetes NetworkPolicies. - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: - watch - list --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: kube-network-policies labels: addonmanager.kubernetes.io/mode: Reconcile roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:network-policies subjects: - kind: ServiceAccount name: kube-network-policies namespace: kube-system --- apiVersion: v1 kind: ServiceAccount metadata: name: kube-network-policies namespace: kube-system labels: k8s-app: kube-network-policies kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile </ins>  No newline at end of file


<ins> --- apiVersion: apps/v1 kind: DaemonSet metadata: name: kube-network-policies namespace: kube-system labels: tier: node app: kube-network-policies k8s-app: kube-network-policies addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: app: kube-network-policies template: metadata: labels: tier: node app: kube-network-policies k8s-app: kube-network-policies spec: hostNetwork: true tolerations: - operator: Exists effect: NoSchedule serviceAccountName: kube-network-policies containers: - name: kube-network-policies image: registry.k8s.io/networking/kube-network-policies:v0.1.0 command: - /bin/sh - -c - /bin/netpol -v 4 1>>/var/log/kube-network-policies.log 2>&1 resources: requests: cpu: "100m" memory: "50Mi" securityContext: privileged: true volumeMounts: - mountPath: /var/log name: varlog readOnly: false - mountPath: /lib/modules name: lib-modules readOnly: true volumes: - name: varlog hostPath: path: /var/log - name: lib-modules hostPath: path: /lib/modules </ins>


local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" sed -i -e "s@__CALICO_CNI_DIR__@/home/kubernetes/bin@g" "${ds_file}" fi <ins> if [[ "${NETWORK_POLICY_PROVIDER:-}" == "kube-network-policies" ]]; then setup-addon-manifests "addons" "kube-network-policies" fi </ins> if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" fi


} EOF if [[ "${KUBERNETES_MASTER:-}" != "true" ]]; then <del> if [[ "${NETWORK_POLICY_PROVIDER:-"none"}" != "none" || "${ENABLE_NETD:-}" == "true" ]]; then # Use Kubernetes cni daemonset on node if network policy provider is specified </del> <ins> if [[ "${NETWORK_POLICY_PROVIDER:-"none"}" == "calico" || "${ENABLE_NETD:-}" == "true" ]]; then # Use Kubernetes cni daemonset on node if network policy provider calico is specified </ins> # or netd is enabled. cni_template_path="" fi


readonly gcloud_supported_providers="gce gke" readonly master_logfiles="kube-apiserver.log kube-apiserver-audit.log kube-scheduler.log kube-controller-manager.log cloud-controller-manager.log etcd.log etcd-events.log glbc.log cluster-autoscaler.log kube-addon-manager.log konnectivity-server.log fluentd.log kubelet.cov" <del> readonly node_logfiles="kube-proxy.log containers/konnectivity-agent-*.log fluentd.log node-problem-detector.log kubelet.cov" </del> <ins> readonly node_logfiles="kube-proxy.log containers/konnectivity-agent-*.log fluentd.log node-problem-detector.log kubelet.cov kube-network-policies.log" </ins> readonly node_systemd_services="node-problem-detector" readonly hollow_node_logfiles="kubelet-hollow-node-*.log kubeproxy-hollow-node-*.log npd-hollow-node-*.log" readonly aws_logfiles="cloud-init-output.log"


- endpoint: createAuthenticationV1SelfSubjectReview reason: Cluster providers are allowed to choose to not serve this API, and the whoami command handles unavailability gracefully. link: https://github.com/kubernetes/enhancements/blob/master/keps/sig-auth/3325-self-subject-attributes-review-api/README.md#ga <ins> - endpoint: getInternalApiserverAPIGroup reason: The endpoint is part of a group that has no stable endpoints yet, reporting as a stable endpoint in apisnoop. link: https://github.com/kubernetes/issues/124248 - endpoint: getResourceAPIGroup reason: The endpoint is part of a group that has no stable endpoints yet, reporting as a stable endpoint in apisnoop. link: https://github.com/kubernetes/issues/124248 - endpoint: getStoragemigrationAPIGroup reason: The endpoint is part of a group that has no stable endpoints yet, reporting as a stable endpoint in apisnoop. link: https://github.com/kubernetes/issues/124248 </ins>


- deleteCoreV1Node - deleteStorageV1CollectionCSINode - deleteStorageV1CSINode <del> - getInternalApiserverAPIGroup - getResourceAPIGroup - getStoragemigrationAPIGroup </del> - listStorageV1CSINode - patchStorageV1CSINode - patchStorageV1VolumeAttachmentStatus


// Introduce some small jittering to ensure that over time the requests won't start // accumulating at approximately the same time from the set of nodes due to priority and // fairness effect. <del> go wait.JitterUntil(kl.syncNodeStatus, kl.nodeStatusUpdateFrequency, 0.04, true, wait.NeverStop) </del> <ins> go func() { // Call updateRuntimeUp once before syncNodeStatus to make sure kubelet had already checked runtime state // otherwise when restart kubelet, syncNodeStatus will report node notReady in first report period kl.updateRuntimeUp() wait.JitterUntil(kl.syncNodeStatus, kl.nodeStatusUpdateFrequency, 0.04, true, wait.NeverStop) }() </ins> go kl.fastStatusUpdateOnce() // start syncing lease


# Check to see if I can access the URL /logs/ kubectl auth can-i get /logs/ <ins> # Check to see if I can approve certificates.k8s.io kubectl auth can-i approve certificates.k8s.io </ins> # List all allowed actions in namespace "foo" kubectl auth can-i --list --namespace=foo`) <del> resourceVerbs       = sets.NewString("get", "list", "watch", "create", "update", "patch", "delete", "deletecollection", "use", "bind", "impersonate", "*") </del> <ins> resourceVerbs       = sets.NewString("get", "list", "watch", "create", "update", "patch", "delete", "deletecollection", "use", "bind", "impersonate", "*", "approve") </ins> nonResourceURLVerbs = sets.NewString("get", "put", "post", "head", "options", "delete", "patch", "*") // holds all the server-supported resources that cannot be discovered by clients. i.e. users and groups for the impersonate verb nonStandardResourceNames = sets.NewString("users", "groups")


default: errString := "you must specify two arguments: verb resource or verb resource/resourceName." usageString := "See 'kubectl auth can-i -h' for help and examples." <del> return errors.New(fmt.Sprintf("%sn%s", errString, usageString)) </del> <ins> return fmt.Errorf("%sn%s", errString, usageString) </ins> } }