// // If the webhook is running within the cluster, then you should use `service`. // <del> // If there is only one port open for the service, that port will be // used. If there are multiple ports open, port 443 will be used if it // is open, otherwise it is an error. </del> <ins> // Port 443 will be used if it is open, otherwise it is an error. </ins> // // +optional optional ServiceReference service = 1;


// // If the webhook is running within the cluster, then you should use `service`. // <del> // If there is only one port open for the service, that port will be // used. If there are multiple ports open, port 443 will be used if it // is open, otherwise it is an error. </del> <ins> // Port 443 will be used if it is open, otherwise it is an error. </ins> // // +optional Service *ServiceReference `json:"service" protobuf:"bytes,1,opt,name=service"`


var map_WebhookClientConfig = map[string]string{ "":         "WebhookClientConfig contains the information to make a TLS connection with the webhook", "url":      "`url` gives the location of the webhook, in standard URL form (`[scheme://]host:port/path`). Exactly one of `url` or `service` must be specified.nnThe `host` should not refer to a service running in the cluster; use the `service` field instead. The host might be resolved via external DNS in some apiservers (e.g., `kube-apiserver` cannot resolve in-cluster DNS as that would be a layering violation). `host` may also be an IP address.nnPlease note that using `localhost` or `127.0.0.1` as a `host` is risky unless you take great care to run this webhook on all hosts which run an apiserver which might need to make calls to this webhook. Such installs are likely to be non-portable, i.e., not easy to turn up in a new cluster.nnThe scheme must be "https"; the URL must begin with "https://".nnA path is optional, and if present may be any string permissible in a URL. You may use the path to pass an arbitrary string to the webhook, for example, a cluster identifier.nnAttempting to use a user or basic auth e.g. "user:password@" is not allowed. Fragments ("#...") and query parameters ("?...") are not allowed, either.", <del> "service":  "`service` is a reference to the service for this webhook. Either `service` or `url` must be specified.nnIf the webhook is running within the cluster, then you should use `service`.nnIf there is only one port open for the service, that port will be used. If there are multiple ports open, port 443 will be used if it is open, otherwise it is an error.", </del> <ins> "service":  "`service` is a reference to the service for this webhook. Either `service` or `url` must be specified.nnIf the webhook is running within the cluster, then you should use `service`.nnPort 443 will be used if it is open, otherwise it is an error.", </ins> "caBundle": "`caBundle` is a PEM encoded CA bundle which will be used to validate the webhook's server certificate. Required.", }


} func TestPerformance(t *testing.T) { <ins> if testing.Short() { // TODO (#61854) find why flakiness is caused by etcd connectivity before enabling always t.Skip("Skipping because we want to run short tests") } </ins> apiURL, masterShutdown := util.StartApiserver() defer masterShutdown()


createTagFactor       = 2.0 createTagSteps        = 9 <ins> // encryptedCheck* is configuration of poll for created volume to check // it has not been silently removed by AWS. // On a random AWS account (shared among several developers) it took 4s on // average. encryptedCheckInterval = 1 * time.Second encryptedCheckTimeout  = 30 * time.Second </ins> // Number of node names that can be added to a filter. The AWS limit is 200 // but we are using a lower limit on purpose filterNodeLimit = 150


request.VolumeType = aws.String(createType) request.Encrypted = aws.Bool(volumeOptions.Encrypted) if len(volumeOptions.KmsKeyId) > 0 { <del> if missing, err := c.checkEncryptionKey(volumeOptions.KmsKeyId); err != nil { if missing { // KSM key is missing, provisioning would fail return "", err } // Log checkEncryptionKey error and try provisioning anyway. glog.Warningf("Cannot check KSM key %s: %v", volumeOptions.KmsKeyId, err) } </del> request.KmsKeyId = aws.String(volumeOptions.KmsKeyId) request.Encrypted = aws.Bool(true) }


return "", fmt.Errorf("error tagging volume %s: %q", volumeName, err) } <ins> // AWS has a bad habbit of reporting success when creating a volume with // encryption keys that either don't exists or have wrong permissions. // Such volume lives for couple of seconds and then it's silently deleted // by AWS. There is no other check to ensure that given KMS key is correct, // because Kubernetes may have limited permissions to the key. if len(volumeOptions.KmsKeyId) > 0 { err := c.waitUntilVolumeAvailable(volumeName) if err != nil { if isAWSErrorVolumeNotFound(err) { err = fmt.Errorf("failed to create encrypted volume: the volume disappeared after creation, most likely due to inaccessible KMS encryption key") } return "", err } } </ins> return volumeName, nil } <del> // checkEncryptionKey tests that given encryption key exists. func (c *Cloud) checkEncryptionKey(keyId string) (missing bool, err error) { input := &kms.DescribeKeyInput{ KeyId: aws.String(keyId), } _, err = c.kms.DescribeKey(input) if err == nil { return false, nil </del> <ins> func (c *Cloud) waitUntilVolumeAvailable(volumeName KubernetesVolumeID) error { disk, err := newAWSDisk(c, volumeName) if err != nil { // Unreachable code return err </ins> } <del> if awsError, ok := err.(awserr.Error); ok { if awsError.Code() == "NotFoundException" { return true, fmt.Errorf("KMS key %s not found: %q", keyId, err) </del> <ins> err = wait.Poll(encryptedCheckInterval, encryptedCheckTimeout, func() (done bool, err error) { vol, err := disk.describeVolume() if err != nil { return true, err </ins> } <del> } return false, fmt.Errorf("Error checking KSM key %s: %q", keyId, err) </del> <ins> if vol.State != nil { switch *vol.State { case "available": // The volume is Available, it won't be deleted now. return true, nil case "creating": return false, nil default: return true, fmt.Errorf("unexpected State of newly created AWS EBS volume %s: %q", volumeName, *vol.State) } } return false, nil }) return err </ins> } // DeleteDisk implements Volumes.DeleteDisk


testDynamicProvisioning(test, c, claim, nil) }) }) <ins> Describe("Invalid AWS KMS key", func() { It("should report an error and create no PV", func() { framework.SkipUnlessProviderIs("aws") test := storageClassTest{ name:        "AWS EBS with invalid KMS key", provisioner: "kubernetes.io/aws-ebs", claimSize:   "2Gi", parameters:  map[string]string{"kmsKeyId": "arn:aws:kms:us-east-1:123456789012:key/55555555-5555-5555-5555-555555555555"}, } By("creating a StorageClass") suffix := fmt.Sprintf("invalid-aws") class := newStorageClass(test, ns, suffix) class, err := c.StorageV1().StorageClasses().Create(class) Expect(err).NotTo(HaveOccurred()) defer func() { framework.Logf("deleting storage class %s", class.Name) framework.ExpectNoError(c.StorageV1().StorageClasses().Delete(class.Name, nil)) }() By("creating a claim object with a suffix for gluster dynamic provisioner") claim := newClaim(test, ns, suffix) claim.Spec.StorageClassName = &class.Name claim, err = c.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim) Expect(err).NotTo(HaveOccurred()) defer func() { framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name) err = c.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil) if err != nil && !apierrs.IsNotFound(err) { framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err) } }() // Watch events until the message about invalid key appears err = wait.Poll(time.Second, framework.ClaimProvisionTimeout, func() (bool, error) { events, err := c.CoreV1().Events(claim.Namespace).List(metav1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) for _, event := range events.Items { if strings.Contains(event.Message, "failed to create encrypted volume: the volume disappeared after creation, most likely due to inaccessible KMS encryption key") { return true, nil } } return false, nil }) Expect(err).NotTo(HaveOccurred()) }) }) </ins> }) func getDefaultStorageClassName(c clientset.Interface) string {


if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then MASTER_KUBELET_TEST_ARGS=" --experimental-kernel-memcg-notification=true" fi <del> APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-} --vmodule=httplog=3 --runtime-config=extensions/v1beta1 ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT}" </del> <ins> APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-} --vmodule=httplog=3 --runtime-config=extensions/v1beta1,scheduling.k8s.io/v1alpha1 ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT}" </ins> CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_ARGS:-} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE}" SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}" KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"


fi if [[ -z "${KUBE_ADMISSION_CONTROL:-}" ]]; then <del> ADMISSION_CONTROL="Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection" </del> <ins> ADMISSION_CONTROL="Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection" </ins> if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then ADMISSION_CONTROL="${ADMISSION_CONTROL},PodSecurityPolicy" fi


KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}" <del> CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota}" </del> <ins> CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota}" </ins> # Master components' test arguments. <del> APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --vmodule=httplog=3 --runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS}" </del> <ins> APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --vmodule=httplog=3 --runtime-config=extensions/v1beta1,scheduling.k8s.io/v1alpha1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS}" </ins> CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}" SCHEDULER_TEST_ARGS="${KUBEMARK_SCHEDULER_TEST_ARGS:-} ${SCHEDULER_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}"


func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { var err error <del> lun, err := strconv.Atoi(devicePath) </del> <ins> volumeSource, err := getVolumeSource(spec) </ins> if err != nil { <del> return "", fmt.Errorf("azureDisk - Wait for attach expect device path as a lun number, instead got: %s (%v)", devicePath, err) </del> <ins> return "", err </ins> } <del> volumeSource, err := getVolumeSource(spec) </del> <ins> diskController, err := getDiskController(a.plugin.host) </ins> if err != nil { return "", err } <ins> nodeName := types.NodeName(a.plugin.host.GetHostName()) diskName := volumeSource.DiskName glog.V(5).Infof("azureDisk - WaitForAttach: begin to GetDiskLun by diskName(%s), DataDiskURI(%s), nodeName(%s), devicePath(%s)", diskName, volumeSource.DataDiskURI, nodeName, devicePath) lun, err := diskController.GetDiskLun(diskName, volumeSource.DataDiskURI, nodeName) if err != nil { return "", err } glog.V(5).Infof("azureDisk - WaitForAttach: GetDiskLun succeeded, got lun(%v)", lun) </ins> exec := a.plugin.host.GetExec(a.plugin.GetPluginName()) io := &osIOHandler{} scsiHostRescan(io, exec) <del> diskName := volumeSource.DiskName nodeName := a.plugin.host.GetHostName() </del> newDevicePath := "" err = wait.Poll(1*time.Second, timeout, func() (bool, error) { <del> if newDevicePath, err = findDiskByLun(lun, io, exec); err != nil { </del> <ins> if newDevicePath, err = findDiskByLun(int(lun), io, exec); err != nil { </ins> return false, fmt.Errorf("azureDisk - WaitForAttach ticker failed node (%s) disk (%s) lun(%v) err(%s)", nodeName, diskName, lun, err) }


} if !initializing { restarts = 0 <ins> hasRunning := false </ins> for i := len(pod.Status.ContainerStatuses) - 1; i >= 0; i-- { container := pod.Status.ContainerStatuses[i]


reason = fmt.Sprintf("ExitCode:%d", container.State.Terminated.ExitCode) } } else if container.Ready && container.State.Running != nil { <ins> hasRunning = true </ins> readyContainers++ } } <ins> // change pod status back to "Running" if there is at least one container still reporting as "Running" status if reason == "Completed" && hasRunning { reason = "Running" } </ins> } if pod.DeletionTimestamp != nil && pod.Status.Reason == node.NodeUnreachablePodReason {


}, []metav1beta1.TableRow{{Cells: []interface{}{"test5", "1/2", "podReason", 6, "<unknown>"}}}, }, <ins> { // Test pod has 2 containers, one is running and the other is completed. api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "test6"}, Spec:       api.PodSpec{Containers: make([]api.Container, 2)}, Status: api.PodStatus{ Phase:  "Running", Reason: "", ContainerStatuses: []api.ContainerStatus{ {Ready: true, RestartCount: 3, State: api.ContainerState{Terminated: &api.ContainerStateTerminated{Reason: "Completed", ExitCode: 0}}}, {Ready: true, RestartCount: 3, State: api.ContainerState{Running: &api.ContainerStateRunning{}}}, }, }, }, []metav1beta1.TableRow{{Cells: []interface{}{"test6", "1/2", "Running", 6, "<unknown>"}}}, }, </ins> } for i, test := range tests {


cinderVolumePluginName = "kubernetes.io/cinder" ) <ins> func getPath(uid types.UID, volName string, host volume.VolumeHost) string { return host.GetPodVolumeDir(uid, kstrings.EscapeQualifiedNameForDisk(cinderVolumePluginName), volName) } </ins> func (plugin *cinderPlugin) Init(host volume.VolumeHost) error { plugin.host = host plugin.volumeLocks = keymutex.NewKeyMutex()


return &cinderVolumeMounter{ cinderVolume: &cinderVolume{ <del> podUID:  podUID, volName: spec.Name(), pdName:  pdName, mounter: mounter, manager: manager, plugin:  plugin, </del> <ins> podUID:          podUID, volName:         spec.Name(), pdName:          pdName, mounter:         mounter, manager:         manager, plugin:          plugin, MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)), </ins> }, fsType:             fsType, readOnly:           readOnly,


func (plugin *cinderPlugin) newUnmounterInternal(volName string, podUID types.UID, manager cdManager, mounter mount.Interface) (volume.Unmounter, error) { return &cinderVolumeUnmounter{ &cinderVolume{ <del> podUID:  podUID, volName: volName, manager: manager, mounter: mounter, plugin:  plugin, </del> <ins> podUID:          podUID, volName:         volName, manager:         manager, mounter:         mounter, plugin:          plugin, MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)), </ins> }}, nil }


// diskMounter provides the interface that is used to mount the actual block device. blockDeviceMounter mount.Interface plugin             *cinderPlugin <del> volume.MetricsNil </del> <ins> volume.MetricsProvider </ins> } func (b *cinderVolumeMounter) GetAttributes() volume.Attributes {


} func (cd *cinderVolume) GetPath() string { <del> name := cinderVolumePluginName return cd.plugin.host.GetPodVolumeDir(cd.podUID, kstrings.EscapeQualifiedNameForDisk(name), cd.volName) </del> <ins> return getPath(cd.podUID, cd.volName, cd.plugin.host) </ins> } type cinderVolumeUnmounter struct {


var _ volume.Deleter = &cinderVolumeDeleter{} func (r *cinderVolumeDeleter) GetPath() string { <del> name := cinderVolumePluginName return r.plugin.host.GetPodVolumeDir(r.podUID, kstrings.EscapeQualifiedNameForDisk(name), r.volName) </del> <ins> return getPath(r.podUID, r.volName, r.plugin.host) </ins> } func (r *cinderVolumeDeleter) Delete() error {


SchedulerName:                  config.SchedulerName, Client:                         client, InformerFactory:                informers.NewSharedInformerFactory(client, 0), <del> PodInformer:                    factory.NewPodInformer(client, 0, config.SchedulerName), </del> <ins> PodInformer:                    factory.NewPodInformer(client, 0), </ins> AlgorithmSource:                config.AlgorithmSource, HardPodAffinitySymmetricWeight: config.HardPodAffinitySymmetricWeight, EventClient:                    eventClient,


FilterFunc: func(obj interface{}) bool { switch t := obj.(type) { case *v1.Pod: <del> return unassignedNonTerminatedPod(t) </del> <ins> return unassignedNonTerminatedPod(t) && responsibleForPod(t, schedulerName) </ins> case cache.DeletedFinalStateUnknown: if pod, ok := t.Obj.(*v1.Pod); ok { <del> return unassignedNonTerminatedPod(pod) </del> <ins> return unassignedNonTerminatedPod(pod) && responsibleForPod(pod, schedulerName) </ins> } runtime.HandleError(fmt.Errorf("unable to convert object %T to *v1.Pod in %T", obj, c)) return false


return true } <ins> // responsibleForPod returns true if the pod has asked to be scheduled by the given scheduler. func responsibleForPod(pod *v1.Pod, schedulerName string) bool { return schedulerName == pod.Spec.SchedulerName } </ins> // assignedPodLister filters the pods returned from a PodLister to // only include those that have a node name set. type assignedPodLister struct {


} // NewPodInformer creates a shared index informer that returns only non-terminal pods. <del> func NewPodInformer(client clientset.Interface, resyncPeriod time.Duration, schedulerName string) coreinformers.PodInformer { </del> <ins> func NewPodInformer(client clientset.Interface, resyncPeriod time.Duration) coreinformers.PodInformer { </ins> selector := fields.ParseSelectorOrDie( <del> "spec.schedulerName=" + schedulerName + ",status.phase!=" + string(v1.PodSucceeded) + </del> <ins> "status.phase!=" + string(v1.PodSucceeded) + </ins> ",status.phase!=" + string(v1.PodFailed)) lw := cache.NewListWatchFromClient(client.CoreV1().RESTClient(), string(v1.ResourcePods), metav1.NamespaceAll, selector) return &podInformer{


HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight, Client:          clientSet, InformerFactory: informerFactory, <del> PodInformer:     factory.NewPodInformer(clientSet, 0, v1.DefaultSchedulerName), </del> <ins> PodInformer:     factory.NewPodInformer(clientSet, 0), </ins> EventClient:     clientSet.CoreV1(), Recorder:        eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: v1.DefaultSchedulerName}), Broadcaster:     eventBroadcaster,


// 5. create and start a scheduler with name "foo-scheduler" clientSet2 := clientset.NewForConfigOrDie(&restclient.Config{Host: context.httpServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Groups[v1.GroupName].GroupVersion()}}) informerFactory2 := informers.NewSharedInformerFactory(context.clientSet, 0) <del> podInformer2 := factory.NewPodInformer(context.clientSet, 0, fooScheduler) </del> <ins> podInformer2 := factory.NewPodInformer(context.clientSet, 0) </ins> schedulerConfigFactory2 := createConfiguratorWithPodInformer(fooScheduler, clientSet2, podInformer2, informerFactory2) schedulerConfig2, err := schedulerConfigFactory2.Create()


t.Errorf("No PDB was deleted from the cache: %v", err) } } <ins> // TestSchedulerInformers tests that scheduler receives informer events and updates its cache when // pods are scheduled by other schedulers. func TestSchedulerInformers(t *testing.T) { // Initialize scheduler. context := initTest(t, "scheduler-informer") defer cleanupTest(t, context) cs := context.clientSet defaultPodRes := &v1.ResourceRequirements{Requests: v1.ResourceList{ v1.ResourceCPU:    *resource.NewMilliQuantity(200, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity(200, resource.BinarySI)}, } defaultNodeRes := &v1.ResourceList{ v1.ResourcePods:   *resource.NewQuantity(32, resource.DecimalSI), v1.ResourceCPU:    *resource.NewMilliQuantity(500, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity(500, resource.BinarySI), } type nodeConfig struct { name string res  *v1.ResourceList } tests := []struct { description         string nodes               []*nodeConfig existingPods        []*v1.Pod pod                 *v1.Pod preemptedPodIndexes map[int]struct{} }{ { description: "Pod cannot be scheduled when node is occupied by pods scheduled by other schedulers", nodes:       []*nodeConfig{{name: "node-1", res: defaultNodeRes}}, existingPods: []*v1.Pod{ initPausePod(context.clientSet, &pausePodConfig{ Name:          "pod1", Namespace:     context.ns.Name, Resources:     defaultPodRes, Labels:        map[string]string{"foo": "bar"}, NodeName:      "node-1", SchedulerName: "foo-scheduler", }), initPausePod(context.clientSet, &pausePodConfig{ Name:          "pod2", Namespace:     context.ns.Name, Resources:     defaultPodRes, Labels:        map[string]string{"foo": "bar"}, NodeName:      "node-1", SchedulerName: "bar-scheduler", }), }, pod: initPausePod(cs, &pausePodConfig{ Name:      "unschedulable-pod", Namespace: context.ns.Name, Resources: defaultPodRes, }), preemptedPodIndexes: map[int]struct{}{2: {}}, }, } for _, test := range tests { for _, nodeConf := range test.nodes { _, err := createNode(cs, nodeConf.name, nodeConf.res) if err != nil { t.Fatalf("Error creating node %v: %v", nodeConf.name, err) } } pods := make([]*v1.Pod, len(test.existingPods)) var err error // Create and run existingPods. for i, p := range test.existingPods { if pods[i], err = runPausePod(cs, p); err != nil { t.Fatalf("Test [%v]: Error running pause pod: %v", test.description, err) } } // Create the new "pod". unschedulable, err := createPausePod(cs, test.pod) if err != nil { t.Errorf("Error while creating new pod: %v", err) } if err := waitForPodUnschedulable(cs, unschedulable); err != nil { t.Errorf("Pod %v got scheduled: %v", unschedulable.Name, err) } // Cleanup pods = append(pods, unschedulable) cleanupPods(cs, t, pods) cs.PolicyV1beta1().PodDisruptionBudgets(context.ns.Name).DeleteCollection(nil, metav1.ListOptions{}) cs.CoreV1().Nodes().DeleteCollection(nil, metav1.ListOptions{}) } } </ins>


// create independent pod informer if required if setPodInformer { <del> podInformer = factory.NewPodInformer(context.clientSet, 12*time.Hour, v1.DefaultSchedulerName) </del> <ins> podInformer = factory.NewPodInformer(context.clientSet, 12*time.Hour) </ins> } else { podInformer = context.informerFactory.Core().V1().Pods() }


import ( "bufio" <del> //"fmt" </del> "net" "net/http" "regexp" "strconv" "strings" <ins> "sync" </ins> "time" utilnet "k8s.io/apimachinery/pkg/util/net"


"github.com/prometheus/client_golang/prometheus" ) <ins> // resettableCollector is the interface implemented by prometheus.MetricVec // that can be used by Prometheus to collect metrics and reset their values. type resettableCollector interface { prometheus.Collector Reset() } </ins> var ( // TODO(a-robinson): Add unit tests for the handling of these metrics once // the upstream library supports it.


[]string{"requestKind"}, ) kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl.exe))`) <ins> metrics = []resettableCollector{ requestCounter, longRunningRequestGauge, requestLatencies, requestLatenciesSummary, responseSizes, DroppedRequests, currentInflightRequests, } </ins> ) const (


MutatingKind = "mutating" ) <del> func init() { // Register all metrics. prometheus.MustRegister(requestCounter) prometheus.MustRegister(longRunningRequestGauge) prometheus.MustRegister(requestLatencies) prometheus.MustRegister(requestLatenciesSummary) prometheus.MustRegister(responseSizes) prometheus.MustRegister(DroppedRequests) prometheus.MustRegister(currentInflightRequests) </del> <ins> var registerMetrics sync.Once // Register all metrics. func Register() { registerMetrics.Do(func() { for _, metric := range metrics { prometheus.MustRegister(metric) } }) } // Reset all metrics. func Reset() { for _, metric := range metrics { metric.Reset() } </ins> } func UpdateInflightRequestMetrics(nonmutating, mutating int) {


} } <del> func Reset() { requestCounter.Reset() requestLatencies.Reset() requestLatenciesSummary.Reset() responseSizes.Reset() } </del> // InstrumentRouteFunc works like Prometheus' InstrumentHandlerFunc but wraps // the go-restful RouteFunction instead of a HandlerFunc plus some Kubernetes endpoint specific information. func InstrumentRouteFunc(verb, resource, subresource, scope string, routeFunc restful.RouteFunction) restful.RouteFunction {


// Install adds the DefaultMetrics handler func (m DefaultMetrics) Install(c *mux.PathRecorderMux) { <ins> register() </ins> c.Handle("/metrics", prometheus.Handler()) }


// Install adds the MetricsWithReset handler func (m MetricsWithReset) Install(c *mux.PathRecorderMux) { <ins> register() </ins> defaultMetricsHandler := prometheus.Handler().ServeHTTP c.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { if req.Method == "DELETE" {


defaultMetricsHandler(w, req) }) } <ins> // register apiserver and etcd metrics func register() { apimetrics.Register() etcdmetrics.Register() } </ins>


cache utilcache.Cache } <del> func init() { metrics.Register() } </del> // Implements storage.Interface. func (h *etcdHelper) Versioner() storage.Versioner { return h.versioner


// did we find it? if newDevicePath != "" { <del> // the current sequence k8s uses for unformated disk (check-disk, mount, fail, mkfs.extX) hangs on // Azure Managed disk scsi interface. this is a hack and will be replaced once we identify and solve // the root case on Azure. formatIfNotFormatted(newDevicePath, *volumeSource.FSType, exec) </del> return true, nil }


} return "", err } <del> func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) { notFormatted, err := diskLooksUnformatted(disk, exec) if err == nil && notFormatted { args := []string{disk} // Disk is unformatted so format it. // Use 'ext4' as the default if len(fstype) == 0 { fstype = "ext4" } if fstype == "ext4" || fstype == "ext3" { args = []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", disk} } glog.Infof("azureDisk - Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", disk, fstype, args) _, err := exec.Run("mkfs."+fstype, args...) if err == nil { // the disk has been formatted successfully try to mount it again. glog.Infof("azureDisk - Disk successfully formatted with 'mkfs.%s %v'", fstype, args) } else { glog.Warningf("azureDisk - Error formatting volume with 'mkfs.%s %v': %v", fstype, args, err) } } else { if err != nil { glog.Warningf("azureDisk - Failed to check if the disk %s formatted with error %s, will attach anyway", disk, err) } else { glog.Infof("azureDisk - Disk %s already formatted, will not format", disk) } } } func diskLooksUnformatted(disk string, exec mount.Exec) (bool, error) { args := []string{"-nd", "-o", "FSTYPE", disk} glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args) dataOut, err := exec.Run("lsblk", args...) if err != nil { glog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) return false, err } output := libstrings.TrimSpace(string(dataOut)) return output == "", nil } </del>


func findDiskByLun(lun int, io ioHandler, exec mount.Exec) (string, error) { return "", nil } <del> func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) { } </del>


awaiting <-chan interface{} finished *time.Time lock     sync.Mutex <del> notify   chan struct{} </del> <ins> notify   chan bool </ins> } // Operations tracks all the ongoing operations.


op := &Operation{ ID:       strconv.FormatInt(id, 10), awaiting: from, <del> notify:   make(chan struct{}), </del> <ins> notify:   make(chan bool, 1), </ins> } go op.wait() go ops.insert(op)


op.result = result finished := time.Now() op.finished = &finished <del> close(op.notify) </del> <ins> op.notify <- true </ins> } // WaitFor waits for the specified duration, or until the operation finishes,


select { case <-time.After(timeout): case <-op.notify: <ins> // Re-send on this channel in case there are others // waiting for notification. op.notify <- true </ins> } }


if accountName == "" || accountKey == "" { return "", "", fmt.Errorf("Invalid %v/%v, couldn't extract azurestorageaccountname or azurestorageaccountkey", nameSpace, secretName) } <ins> accountName = strings.TrimSpace(accountName) </ins> return accountName, accountKey, nil }


"//vendor/k8s.io/client-go/discovery:go_default_library", ], ) <ins> go_test( name = "go_default_test", srcs = ["status_strategy_test.go"], embed = [":go_default_library"], deps = ["//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library"], ) </ins>


} func (a statusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { <ins> // update is only allowed to set status </ins> newCustomResourceObject := obj.(*unstructured.Unstructured) <del> oldCustomResourceObject := old.(*unstructured.Unstructured) </del> newCustomResource := newCustomResourceObject.UnstructuredContent() <del> oldCustomResource := oldCustomResourceObject.UnstructuredContent() // update is not allowed to set spec and metadata _, ok1 := newCustomResource["spec"] _, ok2 := oldCustomResource["spec"] switch { case ok2: newCustomResource["spec"] = oldCustomResource["spec"] case ok1: delete(newCustomResource, "spec") } </del> <ins> status, ok := newCustomResource["status"] </ins> <del> newCustomResourceObject.SetAnnotations(oldCustomResourceObject.GetAnnotations()) newCustomResourceObject.SetFinalizers(oldCustomResourceObject.GetFinalizers()) newCustomResourceObject.SetGeneration(oldCustomResourceObject.GetGeneration()) newCustomResourceObject.SetLabels(oldCustomResourceObject.GetLabels()) newCustomResourceObject.SetOwnerReferences(oldCustomResourceObject.GetOwnerReferences()) newCustomResourceObject.SetSelfLink(oldCustomResourceObject.GetSelfLink()) </del> <ins> // copy old object into new object oldCustomResourceObject := old.(*unstructured.Unstructured) // overridding the resourceVersion in metadata is safe here, we have already checked that // new object and old object have the same resourceVersion. *newCustomResourceObject = *oldCustomResourceObject.DeepCopy() // set status newCustomResource = newCustomResourceObject.UnstructuredContent() if ok { newCustomResource["status"] = status } else { delete(newCustomResource, "status") } </ins> } // ValidateUpdate is the default update validation for an end user updating status.


<ins> /* Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package customresource import ( "context" "reflect" "testing" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) func TestPrepareForUpdate(t *testing.T) { strategy := statusStrategy{} tcs := []struct { old      *unstructured.Unstructured obj      *unstructured.Unstructured expected *unstructured.Unstructured }{ { // changes to spec are ignored old: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec": "old", }, }, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec": "new", }, }, expected: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec": "old", }, }, }, { // changes to other places are also ignored old: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec": "old", }, }, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "new": "new", }, }, expected: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec": "old", }, }, }, { // delete status old: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec":   "old", "status": "old", }, }, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec": "old", }, }, expected: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec": "old", }, }, }, { // update status old: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec":   "old", "status": "old", }, }, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "status": "new", }, }, expected: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec":   "old", "status": "new", }, }, }, { // update status and other parts old: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec":   "old", "status": "old", }, }, obj: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec":   "new", "new":    "new", "status": "new", }, }, expected: &unstructured.Unstructured{ Object: map[string]interface{}{ "spec":   "old", "status": "new", }, }, }, } for index, tc := range tcs { strategy.PrepareForUpdate(context.TODO(), tc.obj, tc.old) if !reflect.DeepEqual(tc.obj, tc.expected) { t.Errorf("test %d failed: expected: %v, got %v", index, tc.expected, tc.obj) } } } </ins>


var IP string // TODO: Remove this when sandbox is available on windows <ins> // Currently windows supports both sandbox and non-sandbox cases. </ins> // This is a workaround for windows, where sandbox is not in use, and pod IP is determined through containers belonging to the Pod. <del> if IP = ds.determinePodIPBySandboxID(podSandboxID); IP == "" { </del> <ins> if IP = ds.determinePodIPBySandboxID(podSandboxID, r); IP == "" { </ins> IP = ds.getIP(podSandboxID, r) }


return nil } <del> func (ds *dockerService) determinePodIPBySandboxID(uid string) string { </del> <ins> func (ds *dockerService) determinePodIPBySandboxID(uid string, sandbox *dockertypes.ContainerJSON) string { </ins> return "" }


return nil } <del> func (ds *dockerService) determinePodIPBySandboxID(uid string) string { </del> <ins> func (ds *dockerService) determinePodIPBySandboxID(uid string, sandbox *dockertypes.ContainerJSON) string { </ins> glog.Warningf("determinePodIPBySandboxID is unsupported in this build") return "" }


return nil } <del> func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) string { </del> <ins> func (ds *dockerService) determinePodIPBySandboxID(sandboxID string, sandbox *dockertypes.ContainerJSON) string { // Versions and feature support // ============================ // Windows version >= Windows Server, Version 1709, Supports both sandbox and non-sandbox case // Windows version == Windows Server 2016 Support only non-sandbox case // Windows version < Windows Server 2016 is Not Supported // Sandbox support in Windows mandates CNI Plugin. // Presence of CONTAINER_NETWORK flag is considered as non-Sandbox cases here // Hyper-V isolated containers are also considered as non-Sandbox cases // Todo: Add a kernel version check for more validation // Hyper-V only supports one container per Pod yet and the container will have a different // IP address from sandbox. Retrieve the IP from the containers as this is a non-Sandbox case. // TODO(feiskyer): remove this workaround after Hyper-V supports multiple containers per Pod. if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" && sandbox.HostConfig.Isolation != kubeletapis.HypervIsolationValue { // Sandbox case, fetch the IP from the sandbox container. return ds.getIP(sandboxID, sandbox) } // Non-Sandbox case, fetch the IP from the containers within the Pod. </ins> opts := dockertypes.ContainerListOptions{ All:     true, Filters: dockerfilters.NewArgs(),


continue } <del> // Versions and feature support // ============================ // Windows version == Windows Server, Version 1709,, Supports both sandbox and non-sandbox case // Windows version == Windows Server 2016   Support only non-sandbox case // Windows version < Windows Server 2016 is Not Supported // Sandbox support in Windows mandates CNI Plugin. // Presence of CONTAINER_NETWORK flag is considered as non-Sandbox cases here // Todo: Add a kernel version check for more validation if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" { // On Windows, every container that is created in a Sandbox, needs to invoke CNI plugin again for adding the Network, // with the shared container name as NetNS info, // This is passed down to the platform to replicate some necessary information to the new container // // This place is chosen as a hack for now, since ds.getIP would end up calling CNI's addToNetwork // That is why addToNetwork is required to be idempotent // Instead of relying on this call, an explicit call to addToNetwork should be // done immediately after ContainerCreation, in case of Windows only. TBD Issue # to handle this if r.HostConfig.Isolation == kubeletapis.HypervIsolationValue { // Hyper-V only supports one container per Pod yet and the container will have a different // IP address from sandbox. Return the first non-sandbox container IP as POD IP. // TODO(feiskyer): remove this workaround after Hyper-V supports multiple containers per Pod. if containerIP := ds.getIP(c.ID, r); containerIP != "" { return containerIP } } else { // Do not return any IP, so that we would continue and get the IP of the Sandbox ds.getIP(sandboxID, r) } } else { // ds.getIP will call the CNI plugin to fetch the IP if containerIP := ds.getIP(c.ID, r); containerIP != "" { return containerIP } </del> <ins> if containerIP := ds.getIP(c.ID, r); containerIP != "" { return containerIP </ins> } }


// by REST storage - these typically indicate programmer // error by not using pkg/api/errors, or unexpected failure // cases. <del> runtime.HandleError(fmt.Errorf("apiserver received an error that is not an metav1.Status: %v", err)) </del> <ins> runtime.HandleError(fmt.Errorf("apiserver received an error that is not an metav1.Status: %#+v", err)) </ins> return &metav1.Status{ TypeMeta: metav1.TypeMeta{ Kind:       "Status",


}, { "ImportPath": "github.com/evanphx/json-patch", <del> "Rev": "ed7cfbae1fffc071f71e068df27bf4f0521402d8" </del> <ins> "Rev": "94e38aa1586e8a6c8a75770bddf5ff84c48a106b" </ins> }, { "ImportPath": "github.com/exponent-io/jsonpath",


}, { "ImportPath": "github.com/evanphx/json-patch", <del> "Rev": "ed7cfbae1fffc071f71e068df27bf4f0521402d8" </del> <ins> "Rev": "94e38aa1586e8a6c8a75770bddf5ff84c48a106b" </ins> }, { "ImportPath": "github.com/ghodss/yaml",


<del> ## JSON-Patch </del> <ins> # JSON-Patch `jsonpatch` is a library which provides functionallity for both applying [RFC6902 JSON patches](http://tools.ietf.org/html/rfc6902) against documents, as well as for calculating & applying [RFC7396 JSON merge patches](https://tools.ietf.org/html/rfc7396). </ins> <del> Provides the ability to modify and test a JSON according to a [RFC6902 JSON patch](http://tools.ietf.org/html/rfc6902) and [RFC7396 JSON Merge Patch](https://tools.ietf.org/html/rfc7396). </del> <ins> [![GoDoc](https://godoc.org/github.com/evanphx/json-patch?status.svg)](http://godoc.org/github.com/evanphx/json-patch) [![Build Status](https://travis-ci.org/evanphx/json-patch.svg?branch=master)](https://travis-ci.org/evanphx/json-patch) [![Report Card](https://goreportcard.com/badge/github.com/evanphx/json-patch)](https://goreportcard.com/report/github.com/evanphx/json-patch) </ins> <del> *Version*: **1.0** </del> <ins> # Get It! </ins> <del> [![GoDoc](https://godoc.org/github.com/evanphx/json-patch?status.svg)](http://godoc.org/github.com/evanphx/json-patch) </del> <ins> **Latest and greatest**: ```bash go get -u github.com/evanphx/json-patch ``` </ins> <del> [![Build Status](https://travis-ci.org/evanphx/json-patch.svg?branch=master)](https://travis-ci.org/evanphx/json-patch) </del> <ins> **Stable Versions**: * Version 3: `go get -u gopkg.in/evanphx/json-patch.v3` (previous versions below `v3` are unavailable) # Use It! * [Create and apply a merge patch](#create-and-apply-a-merge-patch) * [Create and apply a JSON Patch](#create-and-apply-a-json-patch) * [Comparing JSON documents](#comparing-json-documents) * [Combine merge patches](#combine-merge-patches) ## Create and apply a merge patch Given both an original JSON document and a modified JSON document, you can create a [Merge Patch](https://tools.ietf.org/html/rfc7396) document. It can describe the changes needed to convert from the original to the modified JSON document. Once you have a merge patch, you can apply it to other JSON documents using the `jsonpatch.MergePatch(document, patch)` function. ```go package main import ( "fmt" jsonpatch "github.com/evanphx/json-patch" ) func main() { // Let's create a merge patch from these two documents... original := []byte(`{"name": "John", "age": 24, "height": 3.21}`) target := []byte(`{"name": "Jane", "age": 24}`) patch, err := jsonpatch.CreateMergePatch(original, target) if err != nil { panic(err) } // Now lets apply the patch against a different JSON document... alternative := []byte(`{"name": "Tina", "age": 28, "height": 3.75}`) modifiedAlternative, err := jsonpatch.MergePatch(alternative, patch) fmt.Printf("patch document:   %sn", patch) fmt.Printf("updated alternative doc: %sn", modifiedAlternative) } ``` When ran, you get the following output: ```bash $ go run main.go patch document:   {"height":null,"name":"Jane"} updated tina doc: {"age":28,"name":"Jane"} ``` ## Create and apply a JSON Patch You can create patch objects using `DecodePatch([]byte)`, which can then be applied against JSON documents. The following is an example of creating a patch from two operations, and applying it against a JSON document. ```go package main import ( "fmt" jsonpatch "github.com/evanphx/json-patch" ) func main() { original := []byte(`{"name": "John", "age": 24, "height": 3.21}`) patchJSON := []byte(`[ {"op": "replace", "path": "/name", "value": "Jane"}, {"op": "remove", "path": "/height"} ]`) patch, err := jsonpatch.DecodePatch(patchJSON) if err != nil { panic(err) } modified, err := patch.Apply(original) if err != nil { panic(err) } fmt.Printf("Original document: %sn", original) fmt.Printf("Modified document: %sn", modified) } ``` When ran, you get the following output: ```bash $ go run main.go Original document: {"name": "John", "age": 24, "height": 3.21} Modified document: {"age":24,"name":"Jane"} ``` ## Comparing JSON documents Due to potential whitespace and ordering differences, one cannot simply compare JSON strings or byte-arrays directly. As such, you can instead use `jsonpatch.Equal(document1, document2)` to determine if two JSON documents are _structurally_ equal. This ignores whitespace differences, and key-value ordering. ```go package main import ( "fmt" jsonpatch "github.com/evanphx/json-patch" ) func main() { original := []byte(`{"name": "John", "age": 24, "height": 3.21}`) similar := []byte(` { "age": 24, "height": 3.21, "name": "John" } `) different := []byte(`{"name": "Jane", "age": 20, "height": 3.37}`) if jsonpatch.Equal(original, similar) { fmt.Println(`"original" is structurally equal to "similar"`) } if !jsonpatch.Equal(original, different) { fmt.Println(`"original" is _not_ structurally equal to "similar"`) } } ``` When ran, you get the following output: ```bash $ go run main.go "original" is structurally equal to "similar" "original" is _not_ structurally equal to "similar" ``` ## Combine merge patches Given two JSON merge patch documents, it is possible to combine them into a single merge patch which can describe both set of changes. The resulting merge patch can be used such that applying it results in a document structurally similar as merging each merge patch to the document in succession. ```go package main import ( "fmt" jsonpatch "github.com/evanphx/json-patch" ) func main() { original := []byte(`{"name": "John", "age": 24, "height": 3.21}`) nameAndHeight := []byte(`{"height":null,"name":"Jane"}`) ageAndEyes := []byte(`{"age":4.23,"eyes":"blue"}`) // Let's combine these merge patch documents... combinedPatch, err := jsonpatch.MergeMergePatches(nameAndHeight, ageAndEyes) if err != nil { panic(err) } // Apply each patch individual against the original document withoutCombinedPatch, err := jsonpatch.MergePatch(original, nameAndHeight) if err != nil { panic(err) } withoutCombinedPatch, err = jsonpatch.MergePatch(withoutCombinedPatch, ageAndEyes) if err != nil { panic(err) } // Apply the combined patch against the original document withCombinedPatch, err := jsonpatch.MergePatch(original, combinedPatch) if err != nil { panic(err) } // Do both result in the same thing? They should! if jsonpatch.Equal(withCombinedPatch, withoutCombinedPatch) { fmt.Println("Both JSON documents are structurally the same!") } fmt.Printf("combined merge patch: %s", combinedPatch) } ``` When ran, you get the following output: ```bash $ go run main.go Both JSON documents are structurally the same! combined merge patch: {"age":4.23,"eyes":"blue","height":null,"name":"Jane"} ``` # CLI for comparing JSON documents You can install the commandline program `json-patch`. This program can take multiple JSON patch documents as arguments, and fed a JSON document from `stdin`. It will apply the patch(es) against the document and output the modified doc. **patch.1.json** ```json [ {"op": "replace", "path": "/name", "value": "Jane"}, {"op": "remove", "path": "/height"} ] ``` **patch.2.json** ```json [ {"op": "add", "path": "/address", "value": "123 Main St"}, {"op": "replace", "path": "/age", "value": "21"} ] ``` </ins> <del> ### API Usage </del> <ins> **document.json** ```json { "name": "John", "age": 24, "height": 3.21 } ``` </ins> <del> * Given a `[]byte`, obtain a Patch object </del> <ins> You can then run: </ins> <del> `obj, err := jsonpatch.DecodePatch(patch)` </del> <ins> ```bash $ go install github.com/evanphx/json-patch/cmd/json-patch $ cat document.json | json-patch -p patch.1.json -p patch.2.json {"address":"123 Main St","age":"21","name":"Jane"} ``` </ins> <del> * Apply the patch and get a new document back </del> <ins> # Help It! Contributions are welcomed! Leave [an issue](https://github.com/evanphx/json-patch/issues) or [create a PR](https://github.com/evanphx/json-patch/compare). </ins> <del> `out, err := obj.Apply(doc)` * Create a JSON Merge Patch document based on two json documents (a to b): </del> <ins> Before creating a pull request, we'd ask that you make sure tests are passing and that you have added new tests when applicable. </ins> <del> `mergeDoc, err := jsonpatch.CreateMergePatch(a, b)` * Bonus API: compare documents for structural equality </del> <ins> Contributors can run tests using: </ins> <del> `jsonpatch.Equal(doca, docb)` </del> <ins> ```bash go test -cover ./... ``` Builds for pull requests are tested automatically using [TravisCI](https://travis-ci.org/evanphx/json-patch). </ins>


package jsonpatch import ( <ins> "bytes" </ins> "encoding/json" "fmt" "reflect"


var errBadJSONDoc = fmt.Errorf("Invalid JSON Document") var errBadJSONPatch = fmt.Errorf("Invalid JSON Patch") <ins> var errBadMergeTypes = fmt.Errorf("Mismatched JSON Documents") </ins> // MergeMergePatches merges two merge patches together, such that // applying this resulting merged merge patch to a document yields the same


return json.Marshal(doc) } <del> // CreateMergePatch creates a merge patch as specified in http://tools.ietf.org/html/draft-ietf-appsawg-json-merge-patch-07 // // 'a' is original, 'b' is the modified document. Both are to be given as json encoded content. // The function will return a mergeable json document with differences from a to b. // // An error will be returned if any of the two documents are invalid. func CreateMergePatch(a, b []byte) ([]byte, error) { aI := map[string]interface{}{} bI := map[string]interface{}{} err := json.Unmarshal(a, &aI) </del> <ins> // resemblesJSONArray indicates whether the byte-slice "appears" to be // a JSON array or not. // False-positives are possible, as this function does not check the internal // structure of the array. It only checks that the outer syntax is present and // correct. func resemblesJSONArray(input []byte) bool { input = bytes.TrimSpace(input) hasPrefix := bytes.HasPrefix(input, []byte("[")) hasSuffix := bytes.HasSuffix(input, []byte("]")) return hasPrefix && hasSuffix } // CreateMergePatch will return a merge patch document capable of converting // the original document(s) to the modified document(s). // The parameters can be bytes of either two JSON Documents, or two arrays of // JSON documents. // The merge patch returned follows the specification defined at http://tools.ietf.org/html/draft-ietf-appsawg-json-merge-patch-07 func CreateMergePatch(originalJSON, modifiedJSON []byte) ([]byte, error) { originalResemblesArray := resemblesJSONArray(originalJSON) modifiedResemblesArray := resemblesJSONArray(modifiedJSON) // Do both byte-slices seem like JSON arrays? if originalResemblesArray && modifiedResemblesArray { return createArrayMergePatch(originalJSON, modifiedJSON) } // Are both byte-slices are not arrays? Then they are likely JSON objects... if !originalResemblesArray && !modifiedResemblesArray { return createObjectMergePatch(originalJSON, modifiedJSON) } // None of the above? Then return an error because of mismatched types. return nil, errBadMergeTypes } // createObjectMergePatch will return a merge-patch document capable of // converting the original document to the modified document. func createObjectMergePatch(originalJSON, modifiedJSON []byte) ([]byte, error) { originalDoc := map[string]interface{}{} modifiedDoc := map[string]interface{}{} err := json.Unmarshal(originalJSON, &originalDoc) </ins> if err != nil { return nil, errBadJSONDoc } <del> err = json.Unmarshal(b, &bI) </del> <ins> err = json.Unmarshal(modifiedJSON, &modifiedDoc) </ins> if err != nil { return nil, errBadJSONDoc } <del> dest, err := getDiff(aI, bI) </del> <ins> dest, err := getDiff(originalDoc, modifiedDoc) </ins> if err != nil { return nil, err } return json.Marshal(dest) } <ins> // createArrayMergePatch will return an array of merge-patch documents capable // of converting the original document to the modified document for each // pair of JSON documents provided in the arrays. // Arrays of mismatched sizes will result in an error. func createArrayMergePatch(originalJSON, modifiedJSON []byte) ([]byte, error) { originalDocs := []json.RawMessage{} modifiedDocs := []json.RawMessage{} err := json.Unmarshal(originalJSON, &originalDocs) if err != nil { return nil, errBadJSONDoc } err = json.Unmarshal(modifiedJSON, &modifiedDocs) if err != nil { return nil, errBadJSONDoc } total := len(originalDocs) if len(modifiedDocs) != total { return nil, errBadJSONDoc } result := []json.RawMessage{} for i := 0; i < len(originalDocs); i++ { original := originalDocs[i] modified := modifiedDocs[i] patch, err := createObjectMergePatch(original, modified) if err != nil { return nil, err } result = append(result, json.RawMessage(patch)) } return json.Marshal(result) } </ins> // Returns true if the array matches (must be json types). // As is idiomatic for go, an empty array is not the same as a nil array. func matchesArray(a, b []interface{}) bool {


} idx = len(ary) - idx } <ins> if idx < 0 || idx >= len(ary) || idx > len(cur) { return fmt.Errorf("Unable to access invalid index: %d", idx) } </ins> copy(ary[0:idx], cur[0:idx]) ary[idx] = val copy(ary[idx+1:], cur[idx:])


return fmt.Errorf("couldn't get key for object %#v: %v", ds, err) } <ins> // If the DaemonSet is being deleted (either by foreground deletion or // orphan deletion), we cannot be sure if the DaemonSet history objects // it owned still exist -- those history objects can either be deleted // or orphaned. Garbage collector doesn't guarantee that it will delete // DaemonSet pods before deleting DaemonSet history objects, because // DaemonSet history doesn't own DaemonSet pods. We cannot reliably // calculate the status of a DaemonSet being deleted. Therefore, return // here without updating status for the DaemonSet being deleted. if ds.DeletionTimestamp != nil { return nil } </ins> // Construct histories of the DaemonSet, and get the hash of current history cur, old, err := dsc.constructHistory(ds) if err != nil {


} hash := cur.Labels[apps.DefaultDaemonSetUniqueLabelKey] <del> if ds.DeletionTimestamp != nil || !dsc.expectations.SatisfiedExpectations(dsKey) { </del> <ins> if !dsc.expectations.SatisfiedExpectations(dsKey) { </ins> // Only update status. return dsc.updateDaemonSetStatus(ds, hash) }


"//vendor/github.com/renstrom/dedent:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", <del> "//vendor/k8s.io/api/apps/v1:go_default_library", </del> "//vendor/k8s.io/api/autoscaling/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/policy/v1beta1:go_default_library",


"github.com/golang/glog" "github.com/spf13/cobra" <del> appsv1 "k8s.io/api/apps/v1" </del> "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <del> "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" </del> "k8s.io/client-go/dynamic" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"


} func (o *DeleteOptions) deleteResource(info *resource.Info, deleteOptions *metav1.DeleteOptions) error { <del> // TODO: this should be removed as soon as DaemonSet controller properly handles object deletion // see https://github.com/kubernetes/kubernetes/issues/64313 for details mapping := info.ResourceMapping() if mapping.Resource.GroupResource() == (schema.GroupResource{Group: "extensions", Resource: "daemonsets"}) || mapping.Resource.GroupResource() == (schema.GroupResource{Group: "apps", Resource: "daemonsets"}) { if err := updateDaemonSet(info.Namespace, info.Name, o.DynamicClient); err != nil { return err } } </del> if err := resource.NewHelper(info.Client, info.Mapping).DeleteWithOptions(info.Namespace, info.Name, deleteOptions); err != nil { return cmdutil.AddSourceToErr("deleting", info.Source, err) }


return nil } <del> func updateDaemonSet(namespace, name string, dynamicClient dynamic.Interface) error { dsClient := dynamicClient.Resource(schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"}).Namespace(namespace) obj, err := dsClient.Get(name, metav1.GetOptions{}) if err != nil { return err } ds := &appsv1.DaemonSet{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, ds); err != nil { return err } // We set the nodeSelector to a random label. This label is nearly guaranteed // to not be set on any node so the DameonSetController will start deleting // daemon pods. Once it's done deleting the daemon pods, it's safe to delete // the DaemonSet. ds.Spec.Template.Spec.NodeSelector = map[string]string{ string(uuid.NewUUID()): string(uuid.NewUUID()), } // force update to avoid version conflict ds.ResourceVersion = "" out, err := runtime.DefaultUnstructuredConverter.ToUnstructured(ds) if err != nil { return err } if _, err = dsClient.Update(&unstructured.Unstructured{Object: out}); err != nil { return err } // Wait for the daemon set controller to kill all the daemon pods. if err := wait.Poll(1*time.Second, 5*time.Minute, func() (bool, error) { updatedObj, err := dsClient.Get(name, metav1.GetOptions{}) if err != nil { return false, nil } updatedDS := &appsv1.DaemonSet{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(updatedObj.Object, ds); err != nil { return false, nil } return updatedDS.Status.CurrentNumberScheduled+updatedDS.Status.NumberMisscheduled == 0, nil }); err != nil { return err } return nil } </del> // deletion printing is special because we do not have an object to print. // This mirrors name printer behavior func (o *DeleteOptions) PrintObj(info *resource.Info) {


deleteFlags.AddFlags(cmd) <del> cmd.Flags().Bool("wait", true, `If true, wait for resources to be gone before returning.  This waits for finalizers.`) </del> cmdutil.AddIncludeUninitializedFlag(cmd) return cmd }


} if o.GracePeriod == 0 && !o.ForceDeletion { // To preserve backwards compatibility, but prevent accidental data loss, we convert --grace-period=0 <del> // into --grace-period=1 and wait until the object is successfully deleted. Users may provide --force // to bypass this wait. o.WaitForDeletion = true </del> <ins> // into --grace-period=1. Users may provide --force to bypass this conversion. </ins> o.GracePeriod = 1 } <del> if b, err := cmd.Flags().GetBool("wait"); err == nil { o.WaitForDeletion = b } </del> includeUninitialized := cmdutil.ShouldIncludeUninitialized(cmd, false) r := f.NewBuilder().


return fmt.Errorf("cannot set --all and --field-selector at the same time") } <ins> if o.GracePeriod == 0 && !o.ForceDeletion && !o.WaitForDeletion { // With the explicit --wait flag we need extra validation for backward compatibility return fmt.Errorf("--grace-period=0 must have either --force specified, or --wait to be set to true") } </ins> switch { case o.GracePeriod == 0 && o.ForceDeletion: fmt.Fprintf(o.ErrOut, "warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.n")


IgnoreNotFound *bool Now            *bool Timeout        *time.Duration <ins> Wait           *bool </ins> Output         *string }


if f.Timeout != nil { options.Timeout = *f.Timeout } <ins> if f.Wait != nil { options.WaitForDeletion = *f.Wait } </ins> return options }


if f.IgnoreNotFound != nil { cmd.Flags().BoolVar(f.IgnoreNotFound, "ignore-not-found", *f.IgnoreNotFound, "Treat "resource not found" as a successful delete. Defaults to "true" when --all is specified.") } <ins> if f.Wait != nil { cmd.Flags().BoolVar(f.Wait, "wait", *f.Wait, "If true, wait for resources to be gone before returning. This waits for finalizers.") } </ins> if f.Output != nil { cmd.Flags().StringVarP(f.Output, "output", "o", *f.Output, "Output mode. Use "-o name" for shorter output (resource/name).") } } // NewDeleteCommandFlags provides default flags and values for use with the "delete" command


labelSelector := "" fieldSelector := "" timeout := time.Duration(0) <ins> wait := true </ins> filenames := []string{} recursive := false


IgnoreNotFound: &ignoreNotFound, Now:            &now, Timeout:        &timeout, <ins> Wait:           &wait, </ins> Output:         &output, } }


force := false timeout := time.Duration(0) <ins> wait := false </ins> filenames := []string{} recursive := false


// add non-defaults Force:   &force, Timeout: &timeout, <ins> Wait:    &wait, </ins> } }


effectiveTimeout = 168 * time.Hour } waitOptions := kubectlwait.WaitOptions{ <del> ResourceFinder: genericclioptions.ResourceFinderForResult(o.Result), </del> <ins> ResourceFinder: genericclioptions.ResourceFinderForResult(r), </ins> DynamicClient:  o.DynamicClient, Timeout:        effectiveTimeout,


operator: Exists - key: {{ .MasterTaintKey }} effect: NoSchedule <del> # TODO: Remove this affinity field as soon as we are using manifest lists affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: beta.kubernetes.io/arch operator: In values: - {{ .Arch }} </del> <ins> nodeSelector: beta.kubernetes.io/arch: {{ .Arch }} </ins> ` // KubeDNSService is the kube-dns Service manifest


importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy", deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", <ins> "//cmd/kubeadm/app/constants:go_default_library", </ins> "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",


readOnly: true hostNetwork: true serviceAccountName: kube-proxy <del> tolerations: - operator: Exists </del> volumes: - name: kube-proxy configMap:


- name: lib-modules hostPath: path: /lib/modules <ins> tolerations: - key: CriticalAddonsOnly operator: Exists - key: {{ .MasterTaintKey }} effect: NoSchedule nodeSelector: beta.kubernetes.io/arch: {{ .Arch }} </ins> ` )


clientset "k8s.io/client-go/kubernetes" clientsetscheme "k8s.io/client-go/kubernetes/scheme" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" <ins> kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" </ins> kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" kubeproxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"


if err != nil { return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err) } <del> proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ ImageRepository, Arch, Version, ImageOverride string }{ </del> <ins> proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ ImageRepository, Arch, Version, ImageOverride, MasterTaintKey string }{ </ins> ImageRepository: cfg.GetControlPlaneImageRepository(), Arch:            runtime.GOARCH, Version:         kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion), ImageOverride:   cfg.UnifiedControlPlaneImage, <ins> MasterTaintKey:  kubeadmconstants.LabelNodeRoleMaster, </ins> }) if err != nil { return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err)


spec = specHandler.DefaultSpec() } <del> // Pass all parameters as volume labels for Portworx server-side processing. spec.VolumeLabels = p.options.Parameters </del> <ins> // Pass all parameters as volume labels for Portworx server-side processing if len(p.options.Parameters) > 0 { spec.VolumeLabels = p.options.Parameters } else { spec.VolumeLabels = make(map[string]string, 0) } </ins> // Update the requested size in the spec spec.Size = uint64(requestGiB * volutil.GIB)


// HTTP Handler interface func (server *ApiServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { <ins> defer func() { if x := recover(); x != nil { w.WriteHeader(http.StatusInternalServerError) fmt.Fprint(w, "apiserver panic. Look in log for details.") log.Printf("ApiServer panic'd: %#vn", x) } }() </ins> log.Printf("%s %s", req.Method, req.RequestURI) url, err := url.ParseRequestURI(req.RequestURI) if err != nil {


func (storage *ControllerRegistryStorage) Get(id string) (interface{}, error) { controller, err := storage.registry.GetController(id) <ins> if err != nil { return nil, err } </ins> controller.Kind = "cluster#replicationController" return controller, err }


func (sr *ServiceRegistryStorage) List(*url.URL) (interface{}, error) { list, err := sr.registry.ListServices() <ins> if err != nil { return nil, err } </ins> list.Kind = "cluster#serviceList" return list, err } func (sr *ServiceRegistryStorage) Get(id string) (interface{}, error) { service, err := sr.registry.GetService(id) <ins> if err != nil { return nil, err } </ins> service.Kind = "cluster#service" return service, err }


fi } <ins> # A helper function to set up a custom yaml for a k8s addon. # # $1: addon category under /etc/kubernetes # $2: manifest source dir # $3: manifest file # $4: custom yaml function setup-addon-custom-yaml { local -r manifest_path="/etc/kubernetes/$1/$2/$3" local -r custom_yaml="$4" if [ -n "${custom_yaml:-}" ]; then # Replace with custom manifest. cat > "${manifest_path}" <<EOF $custom_yaml EOF fi } </ins> # Prepares the manifests of k8s addons, and starts the addon manager. # Vars assumed: #   CLUSTER_NAME


if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" <ins> setup-addon-custom-yaml "addons" "calico-policy-controller" "calico-node-daemonset.yaml" "${CUSTOM_CALICO_NODE_DAEMONSET_YAML:-}" setup-addon-custom-yaml "addons" "calico-policy-controller" "typha-deployment.yaml" "${CUSTOM_TYPHA_DEPLOYMENT_YAML:-}" </ins> # Configure Calico CNI directory. local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" sed -i -e "s@__CALICO_CNI_DIR__@/home/kubernetes/bin@g" "${ds_file}"


} // add service Cluster IP:Port to kubeServiceAccess ip set for the purpose of solving hairpin. // proxier.kubeServiceAccessSet.activeEntries.Insert(entry.String()) <del> // Install masquerade rules if 'masqueradeAll' or 'clusterCIDR' is specified. if proxier.masqueradeAll || len(proxier.clusterCIDR) > 0 { if valid := proxier.ipsetList[kubeClusterIPSet].validateEntry(entry); !valid { glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeClusterIPSet].Name)) continue } proxier.ipsetList[kubeClusterIPSet].activeEntries.Insert(entry.String()) </del> <ins> if valid := proxier.ipsetList[kubeClusterIPSet].validateEntry(entry); !valid { glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.ipsetList[kubeClusterIPSet].Name)) continue </ins> } <ins> proxier.ipsetList[kubeClusterIPSet].activeEntries.Insert(entry.String()) </ins> // ipvs call serv := &utilipvs.VirtualServer{ Address:   svcInfo.ClusterIP,


// KindFor fulfills meta.RESTMapper func (e shortcutExpander) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) { <del> return e.RESTMapper.KindFor(e.expandResourceShortcut(resource)) </del> <ins> // expandResourceShortcut works with current API resources as read from discovery cache. // In case of new CRDs this means we potentially don't have current state of discovery. // In the current wiring in k8s.io/cli-runtime/pkg/genericclioptions/config_flags.go#toRESTMapper, // we are using DeferredDiscoveryRESTMapper which on KindFor failure will clear the // cache and fetch all data from a cluster (see vendor/k8s.io/client-go/restmapper/discovery.go#KindFor). // Thus another call to expandResourceShortcut, after a NoMatchError should successfully // read Kind to the user or an error. gvk, err := e.RESTMapper.KindFor(e.expandResourceShortcut(resource)) if meta.IsNoMatchError(err) { return e.RESTMapper.KindFor(e.expandResourceShortcut(resource)) } return gvk, err </ins> } // KindsFor fulfills meta.RESTMapper


"testing" openapi_v2 "github.com/google/gnostic/openapiv2" <ins> "github.com/google/go-cmp/cmp" </ins> "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta"


} } <ins> func TestKindForWithNewCRDs(t *testing.T) { tests := map[string]struct { in       schema.GroupVersionResource expected schema.GroupVersionKind srvRes   []*metav1.APIResourceList }{ "": { in:       schema.GroupVersionResource{Group: "a", Version: "", Resource: "sc"}, expected: schema.GroupVersionKind{Group: "a", Version: "v1", Kind: "StorageClass"}, srvRes: []*metav1.APIResourceList{ { GroupVersion: "a/v1", APIResources: []metav1.APIResource{ { Name:       "storageclasses", ShortNames: []string{"sc"}, Kind:       "StorageClass", }, }, }, }, }, } for name, test := range tests { t.Run(name, func(t *testing.T) { invalidateCalled := false fakeDiscovery := &fakeDiscoveryClient{} fakeDiscovery.serverResourcesHandler = func() ([]*metav1.APIResourceList, error) { if invalidateCalled { return test.srvRes, nil } return []*metav1.APIResourceList{}, nil } fakeCachedDiscovery := &fakeCachedDiscoveryClient{DiscoveryInterface: fakeDiscovery} fakeCachedDiscovery.invalidateHandler = func() { invalidateCalled = true } fakeCachedDiscovery.freshHandler = func() bool { return invalidateCalled } // in real world the discovery client is fronted with a cache which // will answer the initial request, only failure to match will trigger // the cache invalidation and live discovery call delegate := NewDeferredDiscoveryRESTMapper(fakeCachedDiscovery) mapper := NewShortcutExpander(delegate, fakeCachedDiscovery) gvk, err := mapper.KindFor(test.in) if err != nil { t.Errorf("unexpected error: %v", err) } if diff := cmp.Equal(gvk, test.expected); !diff { t.Errorf("unexpected data returned %#v, expected %#v", gvk, test.expected) } }) } } </ins> type fakeRESTMapper struct { kindForInput schema.GroupVersionResource }


func (c *fakeDiscoveryClient) OpenAPIV3() openapi.Client { panic("implement me") } <ins> type fakeCachedDiscoveryClient struct { discovery.DiscoveryInterface freshHandler      func() bool invalidateHandler func() } var _ discovery.CachedDiscoveryInterface = &fakeCachedDiscoveryClient{} func (c *fakeCachedDiscoveryClient) Fresh() bool { if c.freshHandler != nil { return c.freshHandler() } return true } func (c *fakeCachedDiscoveryClient) Invalidate() { if c.invalidateHandler != nil { c.invalidateHandler() } } </ins>


"//staging/src/k8s.io/api/policy/v1beta1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", <del> "//staging/src/k8s.io/api/scheduling/v1alpha1:go_default_library", </del> <ins> "//staging/src/k8s.io/api/scheduling/v1beta1:go_default_library", </ins> "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",


"//staging/src/k8s.io/api/policy/v1beta1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", <del> "//staging/src/k8s.io/api/scheduling/v1alpha1:go_default_library", </del> <ins> "//staging/src/k8s.io/api/scheduling/v1beta1:go_default_library", </ins> "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",


import ( "fmt" <del> scheduling "k8s.io/api/scheduling/v1alpha1" </del> <ins> scheduling "k8s.io/api/scheduling/v1beta1" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" )


package kubectl import ( <del> scheduling "k8s.io/api/scheduling/v1alpha1" </del> <ins> scheduling "k8s.io/api/scheduling/v1beta1" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "reflect"


} }, ContainerGCPeriod, wait.NeverStop) <del> stopChan := make(chan struct{}) defer close(stopChan) </del> // when the high threshold is set to 100, stub the image GC manager if kl.kubeletConfiguration.ImageGCHighThresholdPercent == 100 { glog.V(2).Infof("ImageGCHighThresholdPercent is set 100, Disable image GC") <del> go func() { stopChan <- struct{}{} }() </del> <ins> return </ins> } prevImageGCFailed := false


glog.V(vLevel).Infof("Image garbage collection succeeded") } <del> }, ImageGCPeriod, stopChan) </del> <ins> }, ImageGCPeriod, wait.NeverStop) </ins> } // initializeModules will initialize internal modules that do not require the container runtime to be up.


return allErrs } <del> // validateTopologySelectorLabelRequirement tests that the specified TopologySelectorLabelRequirement fields has valid data func validateTopologySelectorLabelRequirement(rq core.TopologySelectorLabelRequirement, fldPath *field.Path) field.ErrorList { </del> <ins> // validateTopologySelectorLabelRequirement tests that the specified TopologySelectorLabelRequirement fields has valid data, // and constructs a set containing all of its Values. func validateTopologySelectorLabelRequirement(rq core.TopologySelectorLabelRequirement, fldPath *field.Path) (sets.String, field.ErrorList) { </ins> allErrs := field.ErrorList{} <ins> valueSet := make(sets.String) valuesPath := fldPath.Child("values") </ins> if len(rq.Values) == 0 { <del> allErrs = append(allErrs, field.Forbidden(fldPath.Child("values"), "must specify as least one value")) </del> <ins> allErrs = append(allErrs, field.Required(valuesPath, "")) </ins> } <ins> // Validate set property of Values field for i, value := range rq.Values { if valueSet.Has(value) { allErrs = append(allErrs, field.Duplicate(valuesPath.Index(i), value)) } valueSet.Insert(value) } </ins> allErrs = append(allErrs, unversionedvalidation.ValidateLabelName(rq.Key, fldPath.Child("key"))...) <del> return allErrs </del> <ins> return valueSet, allErrs </ins> } <del> // ValidateTopologySelectorTerm tests that the specified topology selector term has valid data func ValidateTopologySelectorTerm(term core.TopologySelectorTerm, fldPath *field.Path) field.ErrorList { </del> <ins> // ValidateTopologySelectorTerm tests that the specified topology selector term has valid data, // and constructs a map representing the term in raw form. func ValidateTopologySelectorTerm(term core.TopologySelectorTerm, fldPath *field.Path) (map[string]sets.String, field.ErrorList) { </ins> allErrs := field.ErrorList{} <ins> exprMap := make(map[string]sets.String) exprPath := fldPath.Child("matchLabelExpressions") </ins> if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) { <ins> // Allow empty MatchLabelExpressions, in case this field becomes optional in the future. </ins> for i, req := range term.MatchLabelExpressions { <del> allErrs = append(allErrs, validateTopologySelectorLabelRequirement(req, fldPath.Child("matchLabelExpressions").Index(i))...) </del> <ins> idxPath := exprPath.Index(i) valueSet, exprErrs := validateTopologySelectorLabelRequirement(req, idxPath) allErrs = append(allErrs, exprErrs...) // Validate no duplicate keys exist. if _, exists := exprMap[req.Key]; exists { allErrs = append(allErrs, field.Duplicate(idxPath.Child("key"), req.Key)) } exprMap[req.Key] = valueSet </ins> } } else if len(term.MatchLabelExpressions) != 0 { allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate DynamicProvisioningScheduling")) } <del> return allErrs </del> <ins> return exprMap, allErrs </ins> } // ValidateAvoidPodsInNodeAnnotations tests that the serialized AvoidPods in Node.Annotations has valid data


importpath = "k8s.io/kubernetes/pkg/apis/storage/validation", deps = [ "//pkg/apis/core:go_default_library", <ins> "//pkg/apis/core/helper:go_default_library", </ins> "//pkg/apis/core/validation:go_default_library", "//pkg/apis/storage:go_default_library", "//pkg/features:go_default_library",


"k8s.io/apimachinery/pkg/util/validation/field" utilfeature "k8s.io/apiserver/pkg/util/feature" api "k8s.io/kubernetes/pkg/apis/core" <ins> "k8s.io/kubernetes/pkg/apis/core/helper" </ins> apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" "k8s.io/kubernetes/pkg/apis/storage" "k8s.io/kubernetes/pkg/features"


allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate DynamicProvisioningScheduling")) } <ins> rawTopologies := make([]map[string]sets.String, len(topologies)) </ins> for i, term := range topologies { <del> allErrs = append(allErrs, apivalidation.ValidateTopologySelectorTerm(term, fldPath.Index(i))...) </del> <ins> idxPath := fldPath.Index(i) exprMap, termErrs := apivalidation.ValidateTopologySelectorTerm(term, fldPath.Index(i)) allErrs = append(allErrs, termErrs...) // TODO (verult) consider improving runtime for _, t := range rawTopologies { if helper.Semantic.DeepEqual(exprMap, t) { allErrs = append(allErrs, field.Duplicate(idxPath.Child("matchLabelExpressions"), "")) } } rawTopologies = append(rawTopologies, exprMap) </ins> } return allErrs


}, } <ins> topologyDupValues := []api.TopologySelectorTerm{ { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "kubernetes.io/hostname", Values: []string{"node1", "node1"}, }, }, }, } topologyMultiValues := []api.TopologySelectorTerm{ { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "kubernetes.io/hostname", Values: []string{"node1", "node2"}, }, }, }, } topologyEmptyMatchLabelExpressions := []api.TopologySelectorTerm{ { MatchLabelExpressions: nil, }, } topologyDupKeys := []api.TopologySelectorTerm{ { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "kubernetes.io/hostname", Values: []string{"node1"}, }, { Key:    "kubernetes.io/hostname", Values: []string{"node2"}, }, }, }, } topologyMultiTerm := []api.TopologySelectorTerm{ { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "kubernetes.io/hostname", Values: []string{"node1"}, }, }, }, { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "kubernetes.io/hostname", Values: []string{"node2"}, }, }, }, } topologyDupTermsIdentical := []api.TopologySelectorTerm{ { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "failure-domain.beta.kubernetes.io/zone", Values: []string{"zone1"}, }, { Key:    "kubernetes.io/hostname", Values: []string{"node1"}, }, }, }, { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "failure-domain.beta.kubernetes.io/zone", Values: []string{"zone1"}, }, { Key:    "kubernetes.io/hostname", Values: []string{"node1"}, }, }, }, } topologyExprsOneSameOneDiff := []api.TopologySelectorTerm{ { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "failure-domain.beta.kubernetes.io/zone", Values: []string{"zone1"}, }, { Key:    "kubernetes.io/hostname", Values: []string{"node1"}, }, }, }, { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "failure-domain.beta.kubernetes.io/zone", Values: []string{"zone1"}, }, { Key:    "kubernetes.io/hostname", Values: []string{"node2"}, }, }, }, } topologyValuesOneSameOneDiff := []api.TopologySelectorTerm{ { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "kubernetes.io/hostname", Values: []string{"node1", "node2"}, }, }, }, { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "kubernetes.io/hostname", Values: []string{"node1", "node3"}, }, }, }, } topologyDupTermsDiffExprOrder := []api.TopologySelectorTerm{ { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "kubernetes.io/hostname", Values: []string{"node1"}, }, { Key:    "failure-domain.beta.kubernetes.io/zone", Values: []string{"zone1"}, }, }, }, { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "failure-domain.beta.kubernetes.io/zone", Values: []string{"zone1"}, }, { Key:    "kubernetes.io/hostname", Values: []string{"node1"}, }, }, }, } topologyDupTermsDiffValueOrder := []api.TopologySelectorTerm{ { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "failure-domain.beta.kubernetes.io/zone", Values: []string{"zone1", "zone2"}, }, }, }, { MatchLabelExpressions: []api.TopologySelectorLabelRequirement{ { Key:    "failure-domain.beta.kubernetes.io/zone", Values: []string{"zone2", "zone1"}, }, }, }, } </ins> cases := map[string]bindingTest{ "no topology": { class:         makeClass(nil, nil),


class:         makeClass(nil, topologyLackOfValues), shouldSucceed: false, }, <ins> "duplicate TopologySelectorRequirement values": { class:         makeClass(nil, topologyDupValues), shouldSucceed: false, }, "multiple TopologySelectorRequirement values": { class:         makeClass(nil, topologyMultiValues), shouldSucceed: true, }, "empty MatchLabelExpressions": { class:         makeClass(nil, topologyEmptyMatchLabelExpressions), shouldSucceed: false, }, "duplicate MatchLabelExpression keys": { class:         makeClass(nil, topologyDupKeys), shouldSucceed: false, }, "duplicate MatchLabelExpression keys but across separate terms": { class:         makeClass(nil, topologyMultiTerm), shouldSucceed: true, }, "duplicate AllowedTopologies terms - identical": { class:         makeClass(nil, topologyDupTermsIdentical), shouldSucceed: false, }, "two AllowedTopologies terms, with a pair of the same MatchLabelExpressions and a pair of different ones": { class:         makeClass(nil, topologyExprsOneSameOneDiff), shouldSucceed: true, }, "two AllowedTopologies terms, with a pair of the same Values and a pair of different ones": { class:         makeClass(nil, topologyValuesOneSameOneDiff), shouldSucceed: true, }, "duplicate AllowedTopologies terms - different MatchLabelExpressions order": { class:         makeClass(nil, topologyDupTermsDiffExprOrder), shouldSucceed: false, }, "duplicate AllowedTopologies terms - different TopologySelectorRequirement values order": { class:         makeClass(nil, topologyDupTermsDiffValueOrder), shouldSucceed: false, }, } // Disable VolumeScheduling so nil VolumeBindingMode doesn't fail to validate. err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false") if err != nil { t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err) </ins> } // TODO: remove when feature gate not required <del> err := utilfeature.DefaultFeatureGate.Set("DynamicProvisioningScheduling=true") </del> <ins> err = utilfeature.DefaultFeatureGate.Set("DynamicProvisioningScheduling=true") </ins> if err != nil { t.Fatalf("Failed to enable feature gate for DynamicProvisioningScheduling: %v", err) }


readonly ENABLE_LEGACY_ABAC=false readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty <del> readonly LOG_OWNER_USER=$(whoami) </del> <ins> readonly LOG_OWNER_USER=$(id -un) </ins> readonly LOG_OWNER_GROUP=$(id -gn) ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}} ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}


<ins> apiVersion: v1 kind: Service metadata: name: a spec: selector: app: test clusterIP: None ports: - port: 80 </ins>


<ins> apiVersion: v1 kind: Service metadata: name: a spec: selector: app: test clusterIP: 10.0.0.12 ports: - port: 80 </ins>


} patchBytes, patchObject, err = p.patchSimple(current, modified, source, namespace, name, errOut) } <del> if err != nil && errors.IsConflict(err) && p.force { </del> <ins> if err != nil && (errors.IsConflict(err) || errors.IsInvalid(err)) && p.force { </ins> patchBytes, patchObject, err = p.deleteAndCreate(current, modified, namespace, name) } return patchBytes, patchObject, err


# cleanup kubectl delete svc prune-svc 2>&1 "${kube_flags[@]}" <ins> ## kubectl apply -f some.yml --force # Pre-condition: no service exists kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" '' # apply service a kubectl apply -f hack/testdata/service-revision1.yaml "${kube_flags[@]}" # check right service exists kube::test::get_object_assert 'services a' "{{${id_field}}}" 'a' # change immutable field and apply service a output_message=$(! kubectl apply -f hack/testdata/service-revision2.yaml 2>&1 "${kube_flags[@]}") kube::test::if_has_string "${output_message}" 'field is immutable' # apply --force to recreate resources for immutable fields kubectl apply -f hack/testdata/service-revision2.yaml --force "${kube_flags[@]}" # check immutable field exists kube::test::get_object_assert 'services a' "{{.spec.clusterIP}}" '10.0.0.12' # cleanup kubectl delete -f hack/testdata/service-revision2.yaml "${kube_flags[@]}" </ins> set +o nounset set +o errexit }


}, { "ImportPath": "github.com/json-iterator/go", <del> "Comment": "1.1.3-22-gf2b4162afba355", "Rev": "f2b4162afba35581b6d4a50d3b8f34e33c144682" </del> <ins> "Comment": "1.1.4", "Rev": "ab8a2e0c74be9d3be70b3184d9acc634935ded82" </ins> }, { "ImportPath": "github.com/jteeuwen/go-bindata",


}, { "ImportPath": "github.com/json-iterator/go", <del> "Rev": "f2b4162afba35581b6d4a50d3b8f34e33c144682" </del> <ins> "Rev": "ab8a2e0c74be9d3be70b3184d9acc634935ded82" </ins> }, { "ImportPath": "github.com/mailru/easyjson/buffer",


}, { "ImportPath": "github.com/json-iterator/go", <del> "Rev": "f2b4162afba35581b6d4a50d3b8f34e33c144682" </del> <ins> "Rev": "ab8a2e0c74be9d3be70b3184d9acc634935ded82" </ins> }, { "ImportPath": "github.com/modern-go/concurrent",


[[projects]] <del> name = "github.com/json-iterator/go" packages = ["."] revision = "ca39e5af3ece67bbcda3d0f4f56a8e24d9f2dad4" version = "1.1.3" [[projects]] </del> name = "github.com/modern-go/concurrent" packages = ["."] revision = "e0a39a4cb4216ea8db28e22a69f4ec25610d513a"


[[projects]] name = "github.com/modern-go/reflect2" packages = ["."] <del> revision = "1df9eeb2bb81f327b96228865c5687bc2194af3f" version = "1.0.0" </del> <ins> revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" version = "1.0.1" </ins> [solve-meta] analyzer-name = "dep" analyzer-version = 1 <del> inputs-digest = "56a0b9e9e61d2bc8af5e1b68537401b7f4d60805eda3d107058f3171aa5cf793" </del> <ins> inputs-digest = "ea54a775e5a354cb015502d2e7aa4b74230fc77e894f34a838b268c25ec8eeb8" </ins> solver-name = "gps-cdcl" solver-version = 1


[[constraint]] name = "github.com/modern-go/reflect2" <del> version = "1.0.0" </del> <ins> version = "1.0.1" </ins>


srcs = [ "cert_key.go", "client_ca.go", <del> "dynamicfile_content.go", </del> <ins> "dynamic_cafile_content.go", "dynamic_serving_content.go", </ins> "named_certificates.go", "static_content.go", "tlsconfig.go",


<ins> /* Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dynamiccertificates import ( "bytes" "crypto/x509" "fmt" "io/ioutil" "sync/atomic" "time" "k8s.io/client-go/util/cert" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" "k8s.io/klog" ) // FileRefreshDuration is exposed so that integration tests can crank up the reload speed. var FileRefreshDuration = 1 * time.Minute // Listener is an interface to use to notify interested parties of a change. type Listener interface { // Enqueue should be called when an input may have changed Enqueue() } // Notifier is a way to add listeners type Notifier interface { // AddListener is adds a listener to be notified of potential input changes AddListener(listener Listener) } // ControllerRunner is a generic interface for starting a controller type ControllerRunner interface { // RunOnce runs the sync loop a single time.  This useful for synchronous priming RunOnce() error // Run should be called a go .Run Run(workers int, stopCh <-chan struct{}) } // DynamicFileCAContent provies a CAContentProvider that can dynamically react to new file content // It also fulfills the authenticator interface to provide verifyoptions type DynamicFileCAContent struct { name string // filename is the name the file to read. filename string // caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file caBundle atomic.Value listeners []Listener // queue only ever has one item, but it has nice error handling backoff/retry semantics queue workqueue.RateLimitingInterface } var _ Notifier = &DynamicFileCAContent{} var _ CAContentProvider = &DynamicFileCAContent{} var _ ControllerRunner = &DynamicFileCAContent{} type caBundleAndVerifier struct { caBundle      []byte verifyOptions x509.VerifyOptions } // NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) { if len(filename) == 0 { return nil, fmt.Errorf("missing filename for ca bundle") } name := fmt.Sprintf("%s::%s", purpose, filename) ret := &DynamicFileCAContent{ name:     name, filename: filename, queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)), } if err := ret.loadCABundle(); err != nil { return nil, err } return ret, nil } // AddListener adds a listener to be notified when the CA content changes. func (c *DynamicFileCAContent) AddListener(listener Listener) { c.listeners = append(c.listeners, listener) } // loadCABundle determines the next set of content for the file. func (c *DynamicFileCAContent) loadCABundle() error { caBundle, err := ioutil.ReadFile(c.filename) if err != nil { return err } if len(caBundle) == 0 { return fmt.Errorf("missing content for CA bundle %q", c.Name()) } // check to see if we have a change. If the values are the same, do nothing. if !c.hasCAChanged(caBundle) { return nil } caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), caBundle) if err != nil { return err } c.caBundle.Store(caBundleAndVerifier) for _, listener := range c.listeners { listener.Enqueue() } return nil } // hasCAChanged returns true if the caBundle is different than the current. func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool { uncastExisting := c.caBundle.Load() if uncastExisting == nil { return true } // check to see if we have a change. If the values are the same, do nothing. existing, ok := uncastExisting.(*caBundleAndVerifier) if !ok { return true } if !bytes.Equal(existing.caBundle, caBundle) { return true } return false } // RunOnce runs a single sync loop func (c *DynamicFileCAContent) RunOnce() error { return c.loadCABundle() } // Run starts the kube-apiserver and blocks until stopCh is closed. func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() klog.Infof("Starting %s", c.name) defer klog.Infof("Shutting down %s", c.name) // doesn't matter what workers say, only start one. go wait.Until(c.runWorker, time.Second, stopCh) // start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly. _ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) { c.queue.Add(workItemKey) return false, nil }, stopCh) // TODO this can be wired to an fsnotifier as well. <-stopCh } func (c *DynamicFileCAContent) runWorker() { for c.processNextWorkItem() { } } func (c *DynamicFileCAContent) processNextWorkItem() bool { dsKey, quit := c.queue.Get() if quit { return false } defer c.queue.Done(dsKey) err := c.loadCABundle() if err == nil { c.queue.Forget(dsKey) return true } utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err)) c.queue.AddRateLimited(dsKey) return true } // Name is just an identifier func (c *DynamicFileCAContent) Name() string { return c.name } // CurrentCABundleContent provides ca bundle byte content func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) { return c.caBundle.Load().(*caBundleAndVerifier).caBundle } // VerifyOptions provides verifyoptions compatible with authenticators func (c *DynamicFileCAContent) VerifyOptions() x509.VerifyOptions { return c.caBundle.Load().(*caBundleAndVerifier).verifyOptions } // newVerifyOptions creates a new verification func from a file.  It reads the content and then fails. // It will return a nil function if you pass an empty CA file. func newCABundleAndVerifier(name string, caBundle []byte) (*caBundleAndVerifier, error) { if len(caBundle) == 0 { return nil, fmt.Errorf("missing content for CA bundle %q", name) } // Wrap with an x509 verifier var err error verifyOptions := defaultVerifyOptions() verifyOptions.Roots, err = cert.NewPoolFromBytes(caBundle) if err != nil { return nil, fmt.Errorf("error loading CA bundle for %q: %v", name, err) } return &caBundleAndVerifier{ caBundle:      caBundle, verifyOptions: verifyOptions, }, nil } // defaultVerifyOptions returns VerifyOptions that use the system root certificates, current time, // and requires certificates to be valid for client auth (x509.ExtKeyUsageClientAuth) func defaultVerifyOptions() x509.VerifyOptions { return x509.VerifyOptions{ KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, } } </ins>


<ins> /* Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dynamiccertificates import ( "crypto/tls" "fmt" "io/ioutil" "sync/atomic" "time" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" "k8s.io/klog" ) // DynamicFileServingContent provides a CertKeyContentProvider that can dynamically react to new file content type DynamicFileServingContent struct { name string // certFile is the name of the certificate file to read. certFile string // keyFile is the name of the key file to read. keyFile string // servingCert is a certKeyContent that contains the last read, non-zero length content of the key and cert servingCert atomic.Value listeners []Listener // queue only ever has one item, but it has nice error handling backoff/retry semantics queue workqueue.RateLimitingInterface } var _ Notifier = &DynamicFileServingContent{} var _ CertKeyContentProvider = &DynamicFileServingContent{} var _ ControllerRunner = &DynamicFileServingContent{} // NewDynamicServingContentFromFiles returns a dynamic CertKeyContentProvider based on a cert and key filename func NewDynamicServingContentFromFiles(purpose, certFile, keyFile string) (*DynamicFileServingContent, error) { if len(certFile) == 0 || len(keyFile) == 0 { return nil, fmt.Errorf("missing filename for serving cert") } name := fmt.Sprintf("%s::%s::%s", purpose, certFile, keyFile) ret := &DynamicFileServingContent{ name:     name, certFile: certFile, keyFile:  keyFile, queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)), } if err := ret.loadServingCert(); err != nil { return nil, err } return ret, nil } // AddListener adds a listener to be notified when the serving cert content changes. func (c *DynamicFileServingContent) AddListener(listener Listener) { c.listeners = append(c.listeners, listener) } // loadServingCert determines the next set of content for the file. func (c *DynamicFileServingContent) loadServingCert() error { cert, err := ioutil.ReadFile(c.certFile) if err != nil { return err } key, err := ioutil.ReadFile(c.keyFile) if err != nil { return err } if len(cert) == 0 || len(key) == 0 { return fmt.Errorf("missing content for serving cert %q", c.Name()) } // Ensure that the key matches the cert and both are valid _, err = tls.X509KeyPair(cert, key) if err != nil { return err } newCertKey := &certKeyContent{ cert: cert, key:  key, } // check to see if we have a change. If the values are the same, do nothing. existing, ok := c.servingCert.Load().(*certKeyContent) if ok && existing != nil && existing.Equal(newCertKey) { return nil } c.servingCert.Store(newCertKey) for _, listener := range c.listeners { listener.Enqueue() } return nil } // RunOnce runs a single sync loop func (c *DynamicFileServingContent) RunOnce() error { return c.loadServingCert() } // Run starts the controller and blocks until stopCh is closed. func (c *DynamicFileServingContent) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() klog.Infof("Starting %s", c.name) defer klog.Infof("Shutting down %s", c.name) // doesn't matter what workers say, only start one. go wait.Until(c.runWorker, time.Second, stopCh) // start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly. _ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) { c.queue.Add(workItemKey) return false, nil }, stopCh) // TODO this can be wired to an fsnotifier as well. <-stopCh } func (c *DynamicFileServingContent) runWorker() { for c.processNextWorkItem() { } } func (c *DynamicFileServingContent) processNextWorkItem() bool { dsKey, quit := c.queue.Get() if quit { return false } defer c.queue.Done(dsKey) err := c.loadServingCert() if err == nil { c.queue.Forget(dsKey) return true } utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err)) c.queue.AddRateLimited(dsKey) return true } // Name is just an identifier func (c *DynamicFileServingContent) Name() string { return c.name } // CurrentCertKeyContent provides serving cert byte content func (c *DynamicFileServingContent) CurrentCertKeyContent() ([]byte, []byte) { certKeyContent := c.servingCert.Load().(*certKeyContent) return certKeyContent.cert, certKeyContent.key } </ins>