func (c *cacheWatcher) add(event *watchCacheEvent, budget *timeBudget) { // Try to send the event immediately, without blocking. select { <del> case c.input <- event: </del> <ins> case c.input <- *event: </ins> return default: }


defer timerPool.Put(t) select { <del> case c.input <- event: </del> <ins> case c.input <- *event: </ins> stopped := t.Stop() if !stopped { // Consume triggered (but not yet received) timer event


} // only send events newer than resourceVersion if event.ResourceVersion > resourceVersion { <del> c.sendWatchCacheEvent(event) </del> <ins> c.sendWatchCacheEvent(&event) </ins> } } }


return utilerrors.NewAggregate(errList) } <del> // Find process(es) with a specified name (exact match) </del> <ins> // Find process(es) with a specified name (regexp match) </ins> // and return their pid(s) func PidOf(name string) ([]int, error) { if len(name) == 0 {


exit 1 fi <ins> echo "$(date +'%Y-%m-%d %H:%M:%S') Detecting if migration is needed" </ins> if [ "${TARGET_STORAGE}" != "etcd2" -a "${TARGET_STORAGE}" != "etcd3" ]; then echo "Not supported version of storage: ${TARGET_STORAGE}" exit 1


CURRENT_VERSION=${step} echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}" fi <del> if [ "$(echo ${CURRENT_VERSION} | cut -c1-2)" = "3." -a "${CURRENT_STORAGE}" = "etcd2" -a "${TARGET_STORAGE}" = "etcd3" ]; then </del> <ins> if [ "$(echo ${CURRENT_VERSION} | cut -c1-2)" = "3." -a "${CURRENT_VERSION}" = "${step}" -a "${CURRENT_STORAGE}" = "etcd2" -a "${TARGET_STORAGE}" = "etcd3" ]; then </ins> # If it is the first 3.x release in the list and we are migrating # also from 'etcd2' to 'etcd3', do the migration now. echo "Performing etcd2 -> etcd3 migration"


CURRENT_VERSION="2.3.7" echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}" fi <ins> echo "$(date +'%Y-%m-%d %H:%M:%S') Migration finished" </ins>


} } <del> func (s *StatefulSetTester) waitForStatus(ss *apps.StatefulSet, expectedReplicas int32) { </del> <ins> // WaitForStatus waits for the ss.Status.Replicas to be equal to expectedReplicas func (s *StatefulSetTester) WaitForStatus(ss *apps.StatefulSet, expectedReplicas int32) { </ins> Logf("Waiting for statefulset status.replicas updated to %d", expectedReplicas) ns, name := ss.Namespace, ss.Name


if err != nil { return false, err } <ins> if *ssGet.Status.ObservedGeneration < ss.Generation { return false, nil } </ins> if ssGet.Status.Replicas != expectedReplicas { Logf("Waiting for stateful set status to become %d, currently %d", expectedReplicas, ssGet.Status.Replicas) return false, nil


if err := sst.Scale(&ss, 0); err != nil { errList = append(errList, fmt.Sprintf("%v", err)) } <del> sst.waitForStatus(&ss, 0) </del> <ins> sst.WaitForStatus(&ss, 0) </ins> Logf("Deleting statefulset %v", ss.Name) if err := c.Apps().StatefulSets(ss.Namespace).Delete(ss.Name, nil); err != nil { errList = append(errList, fmt.Sprintf("%v", err))


By("Before scale up finished setting 2nd pod to be not ready by breaking readiness probe") sst.BreakProbe(ss, testProbe) <ins> sst.WaitForStatus(ss, 0) </ins> sst.WaitForRunningAndNotReady(2, ss) By("Continue scale operation after the 2nd pod, and scaling down to 1 replica")


By("Confirming that stateful set scale up will halt with unhealthy stateful pod") sst.BreakProbe(ss, testProbe) sst.WaitForRunningAndNotReady(*ss.Spec.Replicas, ss) <ins> sst.WaitForStatus(ss, 0) </ins> sst.UpdateReplicas(ss, 3) sst.ConfirmStatefulPodCount(1, ss, 10*time.Second)


Expect(err).NotTo(HaveOccurred()) sst.BreakProbe(ss, testProbe) <ins> sst.WaitForStatus(ss, 0) </ins> sst.WaitForRunningAndNotReady(3, ss) sst.UpdateReplicas(ss, 0) sst.ConfirmStatefulPodCount(3, ss, 10*time.Second)


}) // Rc <del> output := framework.RunKubectlOrDie("describe", "rc", "redis-master", nsFlag) </del> requiredStrings := [][]string{ {"Name:", "redis-master"}, {"Namespace:", ns},


{"Pod Template:"}, {"Image:", redisImage}, {"Events:"}} <del> checkOutput(output, requiredStrings) </del> <ins> checkKubectlOutputWithRetry(requiredStrings, "describe", "rc", "redis-master", nsFlag) </ins> // Service <del> output = framework.RunKubectlOrDie("describe", "service", "redis-master", nsFlag) </del> <ins> output := framework.RunKubectlOrDie("describe", "service", "redis-master", nsFlag) </ins> requiredStrings = [][]string{ {"Name:", "redis-master"}, {"Namespace:", ns},


}) // Checks whether the output split by line contains the required elements. <del> func checkOutput(output string, required [][]string) { </del> <ins> func checkOutputReturnError(output string, required [][]string) error { </ins> outputLines := strings.Split(output, "n") currentLine := 0 for _, requirement := range required {


currentLine++ } if currentLine == len(outputLines) { <del> framework.Failf("Failed to find %s in %s", requirement[0], output) </del> <ins> return fmt.Errorf("failed to find %s in %s", requirement[0], output) </ins> } for _, item := range requirement[1:] { if !strings.Contains(outputLines[currentLine], item) { <del> framework.Failf("Failed to find %s in %s", item, outputLines[currentLine]) </del> <ins> return fmt.Errorf("failed to find %s in %s", item, outputLines[currentLine]) </ins> } } } <ins> return nil } func checkOutput(output string, required [][]string) { err := checkOutputReturnError(output, required) if err != nil { framework.Failf("%v", err) } } func checkKubectlOutputWithRetry(required [][]string, args ...string) { var pollErr error wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { output := framework.RunKubectlOrDie(args...) err := checkOutputReturnError(output, required) if err != nil { pollErr = err return false, nil } pollErr = nil return true, nil }) if pollErr != nil { framework.Failf("%v", pollErr) } return </ins> } func getAPIVersions(apiEndpoint string) (*metav1.APIVersions, error) {


} // Error caught by validation _, maxUnavailable, _ := ResolveFenceposts(deployment.Spec.Strategy.RollingUpdate.MaxSurge, deployment.Spec.Strategy.RollingUpdate.MaxUnavailable, *(deployment.Spec.Replicas)) <ins> if maxUnavailable > *deployment.Spec.Replicas { return *deployment.Spec.Replicas } </ins> return maxUnavailable }


} } } <ins> func TestMaxUnavailable(t *testing.T) { deployment := func(replicas int32, maxUnavailable intstr.IntOrString) extensions.Deployment { return extensions.Deployment{ Spec: extensions.DeploymentSpec{ Replicas: func(i int32) *int32 { return &i }(replicas), Strategy: extensions.DeploymentStrategy{ RollingUpdate: &extensions.RollingUpdateDeployment{ MaxSurge:       func(i int) *intstr.IntOrString { x := intstr.FromInt(i); return &x }(int(1)), MaxUnavailable: &maxUnavailable, }, Type: extensions.RollingUpdateDeploymentStrategyType, }, }, } } tests := []struct { name       string deployment extensions.Deployment expected   int32 }{ { name:       "maxUnavailable less than replicas", deployment: deployment(10, intstr.FromInt(5)), expected:   int32(5), }, { name:       "maxUnavailable equal replicas", deployment: deployment(10, intstr.FromInt(10)), expected:   int32(10), }, { name:       "maxUnavailable greater than replicas", deployment: deployment(5, intstr.FromInt(10)), expected:   int32(5), }, { name:       "maxUnavailable with replicas is 0", deployment: deployment(0, intstr.FromInt(10)), expected:   int32(0), }, { name: "maxUnavailable with Recreate deployment strategy", deployment: extensions.Deployment{ Spec: extensions.DeploymentSpec{ Strategy: extensions.DeploymentStrategy{ Type: extensions.RecreateDeploymentStrategyType, }, }, }, expected: int32(0), }, { name:       "maxUnavailable less than replicas with percents", deployment: deployment(10, intstr.FromString("50%")), expected:   int32(5), }, { name:       "maxUnavailable equal replicas with percents", deployment: deployment(10, intstr.FromString("100%")), expected:   int32(10), }, { name:       "maxUnavailable greater than replicas with percents", deployment: deployment(5, intstr.FromString("100%")), expected:   int32(5), }, } for _, test := range tests { t.Log(test.name) maxUnavailable := MaxUnavailable(test.deployment) if test.expected != maxUnavailable { t.Fatalf("expected:%v, got:%v", test.expected, maxUnavailable) } } } </ins>


package handlers import ( <del> "encoding/json" </del> "fmt" "k8s.io/apimachinery/pkg/conversion/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" <ins> "k8s.io/apimachinery/pkg/util/json" </ins> "k8s.io/apimachinery/pkg/util/strategicpatch" "github.com/evanphx/json-patch"


} <ins> func TestNumberConversion(t *testing.T) { codec := api.Codecs.LegacyCodec(schema.GroupVersion{Version: "v1"}) currentVersionedObject := &v1.Service{ TypeMeta:   metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "test-service"}, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ { Port:     80, Protocol: "TCP", NodePort: 31678, }, }, }, } versionedObjToUpdate := &v1.Service{} versionedObj := &v1.Service{} patchJS := []byte(`{"spec":{"ports":[{"port":80,"nodePort":31789}]}}`) _, _, err := strategicPatchObject(codec, currentVersionedObject, patchJS, versionedObjToUpdate, versionedObj) if err != nil { t.Fatal(err) } ports := versionedObjToUpdate.Spec.Ports if len(ports) != 1 || ports[0].Port != 80 || ports[0].NodePort != 31789 { t.Fatal(errors.New("Ports failed to merge because of number conversion issue")) } } </ins> func TestPatchResourceWithVersionConflict(t *testing.T) { namespace := "bar" name := "foo"


"//vendor:k8s.io/apimachinery/pkg/runtime/serializer/streaming", "//vendor:k8s.io/apimachinery/pkg/types", "//vendor:k8s.io/apimachinery/pkg/util/httpstream", <ins> "//vendor:k8s.io/apimachinery/pkg/util/json", </ins> "//vendor:k8s.io/apimachinery/pkg/util/mergepatch", "//vendor:k8s.io/apimachinery/pkg/util/net", "//vendor:k8s.io/apimachinery/pkg/util/runtime",


// Returns selectors of services, RCs and RSs matching the given pod. func getSelectors(pod *v1.Pod, sl algorithm.ServiceLister, cl algorithm.ControllerLister, rsl algorithm.ReplicaSetLister, ssl algorithm.StatefulSetLister) []labels.Selector { <del> selectors := make([]labels.Selector, 0, 3) </del> <ins> var selectors []labels.Selector </ins> if services, err := sl.GetPodServices(pod); err == nil { for _, service := range services { selectors = append(selectors, labels.SelectorFromSet(service.Spec.Selector))


return antiAffinity.CalculateAntiAffinityPriority } <ins> // Classifies nodes into ones with labels and without labels. func (s *ServiceAntiAffinity) getNodeClassificationByLabels(nodes []*v1.Node) (map[string]string, []string) { labeledNodes := map[string]string{} nonLabeledNodes := []string{} for _, node := range nodes { if labels.Set(node.Labels).Has(s.label) { label := labels.Set(node.Labels).Get(s.label) labeledNodes[node.Name] = label } else { nonLabeledNodes = append(nonLabeledNodes, node.Name) } } return labeledNodes, nonLabeledNodes } </ins> // CalculateAntiAffinityPriority spreads pods by minimizing the number of pods belonging to the same service // on machines with the same value for a particular label. // The label to be considered is provided to the struct (ServiceAntiAffinity).


} // separate out the nodes that have the label from the ones that don't <del> otherNodes := []string{} labeledNodes := map[string]string{} for _, node := range nodes { if labels.Set(node.Labels).Has(s.label) { label := labels.Set(node.Labels).Get(s.label) labeledNodes[node.Name] = label } else { otherNodes = append(otherNodes, node.Name) } } </del> <ins> labeledNodes, nonLabeledNodes := s.getNodeClassificationByLabels(nodes) </ins> podCounts := map[string]int{} for _, pod := range nsServicePods { label, exists := labeledNodes[pod.Spec.NodeName]


} podCounts[label]++ } numServicePods := len(nsServicePods) result := []schedulerapi.HostPriority{} //score int - scale of 0-maxPriority


result = append(result, schedulerapi.HostPriority{Host: node, Score: int(fScore)}) } // add the open nodes with a score of 0 <del> for _, node := range otherNodes { </del> <ins> for _, node := range nonLabeledNodes { </ins> result = append(result, schedulerapi.HostPriority{Host: node, Score: 0}) } return result, nil }


} } <ins> func TestGetNodeClassificationByLabels(t *testing.T) { const machine01 = "machine01" const machine02 = "machine02" const zoneA = "zoneA" zone1 := map[string]string{ "zone": zoneA, } labeledNodes := map[string]map[string]string{ machine01: zone1, } expectedNonLabeledNodes := []string{machine02} serviceAffinity := ServiceAntiAffinity{label: "zone"} newLabeledNodes, noNonLabeledNodes := serviceAffinity.getNodeClassificationByLabels(makeLabeledNodeList(labeledNodes)) noLabeledNodes, newnonLabeledNodes := serviceAffinity.getNodeClassificationByLabels(makeNodeList(expectedNonLabeledNodes)) label, _ := newLabeledNodes[machine01] if label != zoneA && len(noNonLabeledNodes) != 0 { t.Errorf("Expected only labeled node with label zoneA and no noNonLabeledNodes") } if len(noLabeledNodes) != 0 && newnonLabeledNodes[0] != machine02 { t.Errorf("Expected only non labled nodes") } } </ins> func makeLabeledNodeList(nodeMap map[string]map[string]string) []*v1.Node { nodes := make([]*v1.Node, 0, len(nodeMap)) for nodeName, labels := range nodeMap {


openssh-client  nfs-common  socat  <ins> udev  </ins> util-linux COPY cni-bin/bin /opt/cni/bin


REGISTRY?=staging-k8s.gcr.io IMAGE?=debian-hyperkube-base <del> TAG=0.9 </del> <ins> TAG=0.10 </ins> ARCH?=amd64 CACHEBUST?=1


docker_pull( name = "debian-hyperkube-base-amd64", <del> digest = "sha256:d83594ecd85345144584523e7fa5388467edf5d2dfa30d0a1bcbf184cddf4a7b", </del> <ins> digest = "sha256:cc782ed16599000ca4c85d47ec6264753747ae1e77520894dca84b104a7621e2", </ins> registry = "k8s.gcr.io", repository = "debian-hyperkube-base-amd64", <del> tag = "0.9",  # ignored, but kept here for documentation </del> <ins> tag = "0.10",  # ignored, but kept here for documentation </ins> ) docker_pull(


OUT_DIR?=_output HYPERKUBE_BIN?=$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/hyperkube <del> BASEIMAGE=k8s.gcr.io/debian-hyperkube-base-$(ARCH):0.8 </del> <ins> BASEIMAGE=k8s.gcr.io/debian-hyperkube-base-$(ARCH):0.10 </ins> TEMP_DIR:=$(shell mktemp -d -t hyperkubeXXXXXX) all: build


fi # Network plugin if [[ -n "${NETWORK_PROVIDER:-}" ]]; then <del> if [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then flags+=" --cni-bin-dir=/opt/kubernetes/bin" else flags+=" --network-plugin-dir=/opt/kubernetes/bin" fi </del> <ins> flags+=" --cni-bin-dir=/opt/kubernetes/bin" </ins> flags+=" --network-plugin=${NETWORK_PROVIDER}" fi if [[ -n "${NON_MASQUERADE_CIDR:-}" ]]; then


fi # Network plugin if [[ -n "${NETWORK_PROVIDER:-}" || -n "${NETWORK_POLICY_PROVIDER:-}" ]]; then <del> if [[ "${NETWORK_PROVIDER:-}" == "cni" || "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then flags+=" --cni-bin-dir=/home/kubernetes/bin" else flags+=" --network-plugin-dir=/home/kubernetes/bin" fi </del> <ins> flags+=" --cni-bin-dir=/home/kubernetes/bin" </ins> if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then # Calico uses CNI always. flags+=" --network-plugin=cni"


{% if pillar.get('network_provider', '').lower() == 'opencontrail' %} {% set network_plugin = "--network-plugin=opencontrail" %} {% elif pillar.get('network_provider', '').lower() == 'cni' %} <del> {% set network_plugin = "--network-plugin=cni --network-plugin-dir=/etc/cni/net.d/" %} </del> <ins> {% set network_plugin = "--network-plugin=cni --cni-bin-dir=/etc/cni/net.d/" %} </ins> {%elif pillar.get('network_policy_provider', '').lower() == 'calico' and grains['roles'][0] != 'kubernetes-master' -%} <del> {% set network_plugin = "--network-plugin=cni --network-plugin-dir=/etc/cni/net.d/ --cni-bin-dir=/home/kubernetes/bin/" %} </del> <ins> {% set network_plugin = "--network-plugin=cni --cni-conf-dir=/etc/cni/net.d/ --cni-bin-dir=/home/kubernetes/bin/" %} </ins> {% elif pillar.get('network_provider', '').lower() == 'kubenet' %} {% set network_plugin = "--network-plugin=kubenet" -%} {% endif -%}


// Network plugin settings. Shared by both docker and rkt. fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, "<Warning: Alpha feature> The name of the network plugin to be invoked for various events in kubelet/pod lifecycle") <ins> //TODO(#46410): Remove the network-plugin-dir flag. </ins> fs.StringVar(&s.NetworkPluginDir, "network-plugin-dir", s.NetworkPluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for network plugins or CNI config") <ins> fs.MarkDeprecated("network-plugin-dir", "Use --cni-bin-dir instead. This flag will be removed in a future version.") </ins> fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, "<Warning: Alpha feature> The full path of the directory in which to search for CNI config files. Default: /etc/cni/net.d") fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, "<Warning: Alpha feature> The full path of the directory in which to search for CNI plugin binaries. Default: /opt/cni/bin") fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, "<Warning: Alpha feature> The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU.")


KUBELET_FLAGS=${KUBELET_FLAGS:-""} # Name of the network plugin, eg: "kubenet" NET_PLUGIN=${NET_PLUGIN:-""} <del> # Place the binaries required by NET_PLUGIN in this directory, eg: "/home/kubernetes/bin". NET_PLUGIN_DIR=${NET_PLUGIN_DIR:-""} </del> <ins> # Place the config files and binaries required by NET_PLUGIN in these directory, # eg: "/etc/cni/net.d" for config files, and "/opt/cni/bin" for binaries. CNI_CONF_DIR=${CNI_CONF_DIR:-""} CNI_BIN_DIR=${CNI_BIN_DIR:-""} </ins> SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/24} FIRST_SERVICE_CLUSTER_IP=${FIRST_SERVICE_CLUSTER_IP:-10.0.0.1} # if enabled, must set CGROUP_ROOT


auth_args="${auth_args} --client-ca-file=${CLIENT_CA_FILE}" fi <del> net_plugin_dir_args="" if [[ -n "${NET_PLUGIN_DIR}" ]]; then net_plugin_dir_args="--network-plugin-dir=${NET_PLUGIN_DIR}" </del> <ins> cni_conf_dir_args="" if [[ -n "${CNI_CONF_DIR}" ]]; then cni_conf_dir_args="--cni-conf-dir=${CNI_CONF_DIR}" fi cni_bin_dir_args="" if [[ -n "${CNI_BIN_DIR}" ]]; then cni_bin_dir_args="--cni-bin-dir=${CNI_BIN_DIR}" </ins> fi container_runtime_endpoint_args=""


--pod-manifest-path="${POD_MANIFEST_PATH}"  ${auth_args}  ${dns_args}  <del> ${net_plugin_dir_args}  </del> <ins> ${cni_conf_dir_args}  ${cni_bin_dir_args}  </ins> ${net_plugin_args}  ${container_runtime_endpoint_args}  ${image_service_endpoint_args}


# Do not use any network plugin by default. User could override the flags with # test_args. <del> test_args='--kubelet-flags="--network-plugin= --network-plugin-dir=" '$test_args </del> <ins> test_args='--kubelet-flags="--network-plugin= --cni-bin-dir=" '$test_args </ins> # Runtime flags test_args='--kubelet-flags="--container-runtime='$runtime'" '$test_args


# plugin by default. NETWORK_PLUGIN=${NETWORK_PLUGIN:-""} <del> # NETWORK_PLUGIN_PATH is the path to network plugin binary. NETWORK_PLUGIN_PATH=${NETWORK_PLUGIN_PATH:-""} </del> <ins> # CNI_CONF_DIR is the path to network plugin binaries. CNI_CONF_DIR=${CNI_CONF_DIR:-""} # CNI_BIN_DIR is the path to network plugin config files. CNI_BIN_DIR=${CNI_BIN_DIR:-""} </ins> # start_kubelet starts kubelet and redirect kubelet log to $LOG_DIR/kubelet.log. kubelet_log=kubelet.log


--system-cgroups=/system  --cgroup-root=/  --network-plugin=$NETWORK_PLUGIN  <del> --network-plugin-dir=$NETWORK_PLUGIN_PATH  </del> <ins> --cni-conf-dir=$CNI_CONF_DIR  --cni-bin-dir=$CNI_BIN_DIR  </ins> --v=$log_level  --logtostderr


clusters = f.GetRegisteredClusters() ns = f.FederationNamespace.Name // create backend service <del> service = createLBServiceOrFail(f.FederationClientset, ns, FederatedIngressServiceName) </del> <ins> service = createLBServiceOrFail(f.FederationClientset, ns, FederatedIngressServiceName, clusters) </ins> // create the TLS secret secret = createTLSSecretOrFail(f.FederationClientset, ns, FederatedIngressTLSSecretName) // wait for services objects sync


backendPods = createBackendPodsOrFail(clusters, nsName, FederatedServicePodName) <del> service = createLBServiceOrFail(f.FederationClientset, nsName, FederatedServiceName) </del> <ins> service = createLBServiceOrFail(f.FederationClientset, nsName, FederatedServiceName, clusters) </ins> obj, err := scheme.Scheme.DeepCopy(service) // Cloning shouldn't fail. On the off-chance it does, we // should shallow copy service to serviceShard before


return clientset.CoreV1().Services(namespace).Create(service) } <del> func createLBService(clientset *fedclientset.Clientset, namespace, name string) (*v1.Service, error) { </del> <ins> func createLBService(clientset *fedclientset.Clientset, namespace, name string, clusters fedframework.ClusterSlice) (*v1.Service, error) { </ins> if clientset == nil || len(namespace) == 0 { return nil, fmt.Errorf("Internal error: invalid parameters passed to createService: clientset: %v, namespace: %v", clientset, namespace) }


// Tests can be run in parallel, so we need a different nodePort for // each test. <del> // We add 1 to FederatedSvcNodePortLast because IntnRange's range end // is not inclusive. nodePort := int32(rand.IntnRange(FederatedSvcNodePortFirst, FederatedSvcNodePortLast+1)) </del> <ins> // we add in a array all the "available" ports availablePorts := make([]int32, FederatedSvcNodePortLast-FederatedSvcNodePortFirst) for i := range availablePorts { availablePorts[i] = int32(FederatedSvcNodePortFirst + i) } var err error var service *v1.Service retry := 10 // the function should retry the service creation on different port only 10 time. // until the availablePort list is not empty, lets try to create the service for len(availablePorts) > 0 && retry > 0 { // select the Id of an available port i := rand.Intn(len(availablePorts)) By(fmt.Sprintf("try creating federated service %q in namespace %q with nodePort %d", name, namespace, availablePorts[i])) service, err = createServiceWithNodePort(clientset, namespace, name, availablePorts[i]) if err == nil { // check if service have been created properly in all clusters. // if the service is not present in one of the clusters, we should cleanup all services if err = checkServicesCreation(namespace, name, clusters); err == nil { // everything was created properly so returns the federated service. return service, nil } } // in case of error, cleanup everything if service != nil { if err = deleteService(clientset, namespace, name, nil); err != nil { framework.ExpectNoError(err, "Deleting service %q after a partial createService() error", service.Name) return nil, err } cleanupServiceShardsAndProviderResources(namespace, service, clusters) } // creation failed, lets try with another port // first remove from the availablePorts the port with which the creation failed availablePorts = append(availablePorts[:i], availablePorts[i+1:]...) retry-- } return nil, err } func createServiceWithNodePort(clientset *fedclientset.Clientset, namespace, name string, nodePort int32) (*v1.Service, error) { </ins> service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name:      name,


return clientset.CoreV1().Services(namespace).Create(service) } <ins> // checkServicesCreation checks if the service have been created successfuly in all the clusters. // if the service is not present in at least one of the clusters, this function returns an error. func checkServicesCreation(namespace, serviceName string, clusters fedframework.ClusterSlice) error { framework.Logf("check if service %q have been created in %d clusters", serviceName, len(clusters)) for _, cluster := range clusters { name := cluster.Name err := wait.PollImmediate(framework.Poll, fedframework.FederatedDefaultTestTimeout, func() (bool, error) { var err error _, err = cluster.Clientset.CoreV1().Services(namespace).Get(serviceName, metav1.GetOptions{}) if err != nil && !errors.IsNotFound(err) { // Get failed with an error, try again. framework.Logf("Failed to find service %q in namespace %q, in cluster %q: %v. Trying again in %s", serviceName, namespace, name, err, framework.Poll) return false, err } else if errors.IsNotFound(err) { framework.Logf("Service %q in namespace %q in cluster %q not found. Trying again in %s", serviceName, namespace, name, framework.Poll) return false, nil } By(fmt.Sprintf("Service %q in namespace %q in cluster %q found", serviceName, namespace, name)) return true, nil }) if err != nil { return err } } return nil } </ins> func createServiceOrFail(clientset *fedclientset.Clientset, namespace, name string) *v1.Service { service, err := createService(clientset, namespace, name) framework.ExpectNoError(err, "Creating service %q in namespace %q", service.Name, namespace)


return service } <del> func createLBServiceOrFail(clientset *fedclientset.Clientset, namespace, name string) *v1.Service { service, err := createLBService(clientset, namespace, name) </del> <ins> func createLBServiceOrFail(clientset *fedclientset.Clientset, namespace, name string, clusters fedframework.ClusterSlice) *v1.Service { service, err := createLBService(clientset, namespace, name, clusters) </ins> framework.ExpectNoError(err, "Creating service %q in namespace %q", service.Name, namespace) By(fmt.Sprintf("Successfully created federated service (type: load balancer) %q in namespace %q", name, namespace)) return service


Fail(fmt.Sprintf("Internal error: invalid parameters passed to deleteServiceOrFail: clientset: %v, namespace: %v, service: %v", clientset, namespace, serviceName)) } framework.Logf("Deleting service %q in namespace %v", serviceName, namespace) <ins> err := deleteService(clientset, namespace, serviceName, orphanDependents) if err != nil { framework.ExpectNoError(err, "Error deleting service %q from namespace %q", serviceName, namespace) } } func deleteService(clientset *fedclientset.Clientset, namespace string, serviceName string, orphanDependents *bool) error { </ins> err := clientset.CoreV1().Services(namespace).Delete(serviceName, &metav1.DeleteOptions{OrphanDependents: orphanDependents}) <del> framework.ExpectNoError(err, "Error deleting service %q from namespace %q", serviceName, namespace) </del> <ins> if err != nil { return err } </ins> // Wait for the service to be deleted. err = wait.Poll(5*time.Second, fedframework.FederatedDefaultTestTimeout, func() (bool, error) { _, err := clientset.Core().Services(namespace).Get(serviceName, metav1.GetOptions{})


} return false, err }) <del> if err != nil { framework.DescribeSvc(namespace) framework.Failf("Error in deleting service %s: %v", serviceName, err) } </del> <ins> return err </ins> } func cleanupServiceShardsAndProviderResources(namespace string, service *v1.Service, clusters fedframework.ClusterSlice) {


if [ "${DRY_RUN}" = false ]; then ls "${CLIENT_REPO}" | { grep -v '_tmp' || true; } | xargs rm -rf mv "${CLIENT_REPO_TEMP}"/* "${CLIENT_REPO}" <ins> git checkout HEAD -- $(find "${CLIENT_REPO}" -name BUILD) </ins> fi


var ( // TODO: Deprecate gitMajor and gitMinor, use only gitVersion instead. gitMajor     string = "0"              // major version, always numeric <del> gitMinor     string = "8.2+"           // minor version, numeric possibly followed by "+" gitVersion   string = "v0.8.2-dev"     // version from git, output of $(git describe) </del> <ins> gitMinor     string = "8.3"            // minor version, numeric possibly followed by "+" gitVersion   string = "v0.8.3"         // version from git, output of $(git describe) </ins> gitCommit    string = ""               // sha1 from git, output of $(git rev-parse HEAD) gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty" )


package spdy import ( <ins> "bufio" </ins> "fmt" <ins> "io" "net" </ins> "net/http" "strings" <ins> "sync/atomic" </ins> "k8s.io/apimachinery/pkg/util/httpstream" "k8s.io/apimachinery/pkg/util/runtime"


type responseUpgrader struct { } <ins> // connWrapper is used to wrap a hijacked connection and its bufio.Reader. All // calls will be handled directly by the underlying net.Conn with the exception // of Read and Close calls, which will consider data in the bufio.Reader. This // ensures that data already inside the used bufio.Reader instance is also // read. type connWrapper struct { net.Conn closed    int32 bufReader *bufio.Reader } func (w *connWrapper) Read(b []byte) (n int, err error) { if atomic.LoadInt32(&w.closed) == 1 { return 0, io.EOF } return w.bufReader.Read(b) } func (w *connWrapper) Close() error { err := w.Conn.Close() atomic.StoreInt32(&w.closed, 1) return err } </ins> // NewResponseUpgrader returns a new httpstream.ResponseUpgrader that is // capable of upgrading HTTP responses using SPDY/3.1 via the // spdystream package.


w.Header().Add(httpstream.HeaderUpgrade, HeaderSpdy31) w.WriteHeader(http.StatusSwitchingProtocols) <del> conn, _, err := hijacker.Hijack() </del> <ins> conn, bufrw, err := hijacker.Hijack() </ins> if err != nil { runtime.HandleError(fmt.Errorf("unable to upgrade: error hijacking response: %v", err)) return nil } <del> spdyConn, err := NewServerConnection(conn, newStreamHandler) </del> <ins> connWithBuf := &connWrapper{Conn: conn, bufReader: bufrw.Reader} spdyConn, err := NewServerConnection(connWithBuf, newStreamHandler) </ins> if err != nil { runtime.HandleError(fmt.Errorf("unable to upgrade: error creating SPDY server connection: %v", err)) return nil


} var left, right interface{} <del> if len(lefts) != 1 { </del> <ins> switch { case len(lefts) == 0: continue case len(lefts) > 1: </ins> return input, fmt.Errorf("can only compare one element at a time") } left = lefts[0].Interface()


if err != nil { return input, err } <del> if len(rights) != 1 { </del> <ins> switch { case len(rights) == 0: continue case len(rights) > 1: </ins> return input, fmt.Errorf("can only compare one element at a time") } right = rights[0].Interface()


) type jsonpathTest struct { <del> name     string template string input    interface{} expect   string </del> <ins> name        string template    string input       interface{} expect      string expectError bool </ins> } func testJSONPath(tests []jsonpathTest, allowMissingKeys bool, t *testing.T) {


} buf := new(bytes.Buffer) err = j.Execute(buf, test.input) <del> if err != nil { </del> <ins> if test.expectError { if test.expectError && err == nil { t.Errorf("in %s, expected execute error", test.name) } continue } else if err != nil { </ins> t.Errorf("in %s, execute error %v", test.name, err) } out := buf.String()


} storeTests := []jsonpathTest{ <del> {"plain", "hello jsonpath", nil, "hello jsonpath"}, {"recursive", "{..}", []int{1, 2, 3}, "[1 2 3]"}, {"filter", "{[?(@<5)]}", []int{2, 6, 3, 7}, "2 3"}, {"quote", `{"{"}`, nil, "{"}, {"union", "{[1,3,4]}", []int{0, 1, 2, 3, 4}, "1 3 4"}, {"array", "{[0:2]}", []string{"Monday", "Tudesday"}, "Monday Tudesday"}, {"variable", "hello {.Name}", storeData, "hello jsonpath"}, {"dict/", "{$.Labels.web/html}", storeData, "15"}, {"dict/", "{$.Employees.jason}", storeData, "manager"}, {"dict/", "{$.Employees.dan}", storeData, "clerk"}, {"dict-", "{.Labels.k8s-app}", storeData, "20"}, {"nest", "{.Bicycle[*].Color}", storeData, "red green"}, {"allarray", "{.Book[*].Author}", storeData, "Nigel Rees Evelyn Waugh Herman Melville"}, {"allfileds", "{.Bicycle.*}", storeData, "{red 19.95 true} {green 20.01 false}"}, {"recurfileds", "{..Price}", storeData, "8.95 12.99 8.99 19.95 20.01"}, </del> <ins> {"plain", "hello jsonpath", nil, "hello jsonpath", false}, {"recursive", "{..}", []int{1, 2, 3}, "[1 2 3]", false}, {"filter", "{[?(@<5)]}", []int{2, 6, 3, 7}, "2 3", false}, {"quote", `{"{"}`, nil, "{", false}, {"union", "{[1,3,4]}", []int{0, 1, 2, 3, 4}, "1 3 4", false}, {"array", "{[0:2]}", []string{"Monday", "Tudesday"}, "Monday Tudesday", false}, {"variable", "hello {.Name}", storeData, "hello jsonpath", false}, {"dict/", "{$.Labels.web/html}", storeData, "15", false}, {"dict/", "{$.Employees.jason}", storeData, "manager", false}, {"dict/", "{$.Employees.dan}", storeData, "clerk", false}, {"dict-", "{.Labels.k8s-app}", storeData, "20", false}, {"nest", "{.Bicycle[*].Color}", storeData, "red green", false}, {"allarray", "{.Book[*].Author}", storeData, "Nigel Rees Evelyn Waugh Herman Melville", false}, {"allfileds", "{.Bicycle.*}", storeData, "{red 19.95 true} {green 20.01 false}", false}, {"recurfileds", "{..Price}", storeData, "8.95 12.99 8.99 19.95 20.01", false}, </ins> {"lastarray", "{.Book[-1:]}", storeData, <del> "{Category: fiction, Author: Herman Melville, Title: Moby Dick, Price: 8.99}"}, </del> <ins> "{Category: fiction, Author: Herman Melville, Title: Moby Dick, Price: 8.99}", false}, </ins> {"recurarray", "{..Book[2]}", storeData, <del> "{Category: fiction, Author: Herman Melville, Title: Moby Dick, Price: 8.99}"}, {"bool", "{.Bicycle[?(@.IsNew==true)]}", storeData, "{red 19.95 true}"}, </del> <ins> "{Category: fiction, Author: Herman Melville, Title: Moby Dick, Price: 8.99}", false}, {"bool", "{.Bicycle[?(@.IsNew==true)]}", storeData, "{red 19.95 true}", false}, </ins> } testJSONPath(storeTests, false, t) missingKeyTests := []jsonpathTest{ <del> {"nonexistent field", "{.hello}", storeData, ""}, </del> <ins> {"nonexistent field", "{.hello}", storeData, "", false}, </ins> } testJSONPath(missingKeyTests, true, t) failStoreTests := []jsonpathTest{ <del> {"invalid identifier", "{hello}", storeData, "unrecognized identifier hello"}, {"nonexistent field", "{.hello}", storeData, "hello is not found"}, {"invalid array", "{.Labels[0]}", storeData, "map[string]int is not array or slice"}, {"invalid filter operator", "{.Book[?(@.Price<>10)]}", storeData, "unrecognized filter operator <>"}, {"redundent end", "{range .Labels.*}{@}{end}{end}", storeData, "not in range, nothing to end"}, </del> <ins> {"invalid identifier", "{hello}", storeData, "unrecognized identifier hello", false}, {"nonexistent field", "{.hello}", storeData, "hello is not found", false}, {"invalid array", "{.Labels[0]}", storeData, "map[string]int is not array or slice", false}, {"invalid filter operator", "{.Book[?(@.Price<>10)]}", storeData, "unrecognized filter operator <>", false}, {"redundent end", "{range .Labels.*}{@}{end}{end}", storeData, "not in range, nothing to end", false}, </ins> } testFailJSONPath(failStoreTests, t) }


t.Error(err) } pointsTests := []jsonpathTest{ <del> {"exists filter", "{[?(@.z)].id}", pointsData, "i2 i5"}, {"bracket key", "{[0]['id']}", pointsData, "i1"}, </del> <ins> {"exists filter", "{[?(@.z)].id}", pointsData, "i2 i5", false}, {"bracket key", "{[0]['id']}", pointsData, "i1", false}, </ins> } testJSONPath(pointsTests, false, t) }


} nodesTests := []jsonpathTest{ <del> {"range item", `{range .items[*]}{.metadata.name}, {end}{.kind}`, nodesData, "127.0.0.1, 127.0.0.2, List"}, {"range item with quote", `{range .items[*]}{.metadata.name}{"t"}{end}`, nodesData, "127.0.0.1t127.0.0.2t"}, </del> <ins> {"range item", `{range .items[*]}{.metadata.name}, {end}{.kind}`, nodesData, "127.0.0.1, 127.0.0.2, List", false}, {"range item with quote", `{range .items[*]}{.metadata.name}{"t"}{end}`, nodesData, "127.0.0.1t127.0.0.2t", false}, </ins> {"range addresss", `{.items[*].status.addresses[*].address}`, nodesData, <del> "127.0.0.1 127.0.0.2 127.0.0.3"}, </del> <ins> "127.0.0.1 127.0.0.2 127.0.0.3", false}, </ins> {"double range", `{range .items[*]}{range .status.addresses[*]}{.address}, {end}{end}`, nodesData, <del> "127.0.0.1, 127.0.0.2, 127.0.0.3, "}, {"item name", `{.items[*].metadata.name}`, nodesData, "127.0.0.1 127.0.0.2"}, </del> <ins> "127.0.0.1, 127.0.0.2, 127.0.0.3, ", false}, {"item name", `{.items[*].metadata.name}`, nodesData, "127.0.0.1 127.0.0.2", false}, </ins> {"union nodes capacity", `{.items[*]['metadata.name', 'status.capacity']}`, nodesData, <del> "127.0.0.1 127.0.0.2 map[cpu:4] map[cpu:8]"}, </del> <ins> "127.0.0.1 127.0.0.2 map[cpu:4] map[cpu:8]", false}, </ins> {"range nodes capacity", `{range .items[*]}[{.metadata.name}, {.status.capacity}] {end}`, nodesData, <del> "[127.0.0.1, map[cpu:4]] [127.0.0.2, map[cpu:8]] "}, {"user password", `{.users[?(@.name=="e2e")].user.password}`, &nodesData, "secret"}, {"hostname", `{.items[0].metadata.labels.kubernetes.io/hostname}`, &nodesData, "127.0.0.1"}, {"hostname filter", `{.items[?(@.metadata.labels.kubernetes.io/hostname=="127.0.0.1")].kind}`, &nodesData, "None"}, {"bool item", `{.items[?(@..ready==true)].metadata.name}`, &nodesData, "127.0.0.1"}, </del> <ins> "[127.0.0.1, map[cpu:4]] [127.0.0.2, map[cpu:8]] ", false}, {"user password", `{.users[?(@.name=="e2e")].user.password}`, &nodesData, "secret", false}, {"hostname", `{.items[0].metadata.labels.kubernetes.io/hostname}`, &nodesData, "127.0.0.1", false}, {"hostname filter", `{.items[?(@.metadata.labels.kubernetes.io/hostname=="127.0.0.1")].kind}`, &nodesData, "None", false}, {"bool item", `{.items[?(@..ready==true)].metadata.name}`, &nodesData, "127.0.0.1", false}, </ins> } testJSONPath(nodesTests, false, t) randomPrintOrderTests := []jsonpathTest{ <del> {"recursive name", "{..name}", nodesData, `127.0.0.1 127.0.0.2 myself e2e`}, </del> <ins> {"recursive name", "{..name}", nodesData, `127.0.0.1 127.0.0.2 myself e2e`, false}, </ins> } testJSONPathSortOutput(randomPrintOrderTests, t) } <ins> func TestFilterPartialMatchesSometimesMissingAnnotations(t *testing.T) { // for https://issues.k8s.io/45546 var input = []byte(`{ "kind": "List", "items": [ { "kind": "Pod", "metadata": { "name": "pod1", "annotations": { "color": "blue" } } }, { "kind": "Pod", "metadata": { "name": "pod2" } }, { "kind": "Pod", "metadata": { "name": "pod3", "annotations": { "color": "green" } } }, { "kind": "Pod", "metadata": { "name": "pod4", "annotations": { "color": "blue" } } } ] }`) var data interface{} err := json.Unmarshal(input, &data) if err != nil { t.Fatal(err) } testJSONPath( []jsonpathTest{ { "filter, should only match a subset, some items don't have annotations, tolerate missing items", `{.items[?(@.metadata.annotations.color=="blue")].metadata.name}`, data, "pod1 pod4", false, // expect no error }, }, true, // allow missing keys t, ) testJSONPath( []jsonpathTest{ { "filter, should only match a subset, some items don't have annotations, error on missing items", `{.items[?(@.metadata.annotations.color=="blue")].metadata.name}`, data, "", true, // expect an error }, }, false, // don't allow missing keys t, ) } </ins>


"integration", ], deps = [ <ins> "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", </ins> "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",


"testing" "time" <ins> "k8s.io/apimachinery/pkg/api/errors" </ins> "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"


if e, a := createdNoxuInstance, gottenNoxuInstance2; !reflect.DeepEqual(e, a) { t.Errorf("expected %v, got %v", e, a) } <ins> } func TestDeRegistrationAndReRegistration(t *testing.T) { stopCh, apiExtensionClient, clientPool, err := testserver.StartDefaultServer() if err != nil { t.Fatal(err) } defer close(stopCh) noxuDefinition := testserver.NewNoxuCustomResourceDefinition() ns := "not-the-default" sameInstanceName := "foo" func() { noxuVersionClient, err := testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, clientPool) if err != nil { t.Fatal(err) } noxuNamespacedResourceClient := NewNamespacedCustomResourceClient(ns, noxuVersionClient, noxuDefinition) if _, err := instantiateCustomResource(t, testserver.NewNoxuInstance(ns, sameInstanceName), noxuNamespacedResourceClient, noxuDefinition); err != nil { t.Fatal(err) } // Remove sameInstanceName since at the moment there's no finalizers. // TODO: as soon finalizers will be implemented Delete can be removed. if err := noxuNamespacedResourceClient.Delete(sameInstanceName, nil); err != nil { t.Fatal(err) } if err := testserver.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil { t.Fatal(err) } if _, err := testserver.GetCustomResourceDefinition(noxuDefinition, apiExtensionClient); err == nil || !errors.IsNotFound(err) { t.Fatalf("expected a NotFound error, got:%v", err) } if _, err = noxuNamespacedResourceClient.List(metav1.ListOptions{}); err == nil || !errors.IsNotFound(err) { t.Fatalf("expected a NotFound error, got:%v", err) } if _, err = noxuNamespacedResourceClient.Get("foo"); err == nil || !errors.IsNotFound(err) { t.Fatalf("expected a NotFound error, got:%v", err) } }() func() { if _, err := testserver.GetCustomResourceDefinition(noxuDefinition, apiExtensionClient); err == nil || !errors.IsNotFound(err) { t.Fatalf("expected a NotFound error, got:%v", err) } noxuVersionClient, err := testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, clientPool) if err != nil { t.Fatal(err) } noxuNamespacedResourceClient := NewNamespacedCustomResourceClient(ns, noxuVersionClient, noxuDefinition) initialList, err := noxuNamespacedResourceClient.List(metav1.ListOptions{}) if err != nil { t.Fatal(err) } if _, err = noxuNamespacedResourceClient.Get(sameInstanceName); err == nil || !errors.IsNotFound(err) { t.Fatalf("expected a NotFound error, got:%v", err) } if e, a := 0, len(initialList.(*unstructured.UnstructuredList).Items); e != a { t.Fatalf("expected %v, got %v", e, a) } createdNoxuInstance, err := instantiateCustomResource(t, testserver.NewNoxuInstance(ns, sameInstanceName), noxuNamespacedResourceClient, noxuDefinition) if err != nil { t.Fatal(err) } gottenNoxuInstance, err := noxuNamespacedResourceClient.Get(sameInstanceName) if err != nil { t.Fatal(err) } if e, a := createdNoxuInstance, gottenNoxuInstance; !reflect.DeepEqual(e, a) { t.Fatalf("expected %v, got %v", e, a) } listWithItem, err := noxuNamespacedResourceClient.List(metav1.ListOptions{}) if err != nil { t.Fatal(err) } if e, a := 1, len(listWithItem.(*unstructured.UnstructuredList).Items); e != a { t.Fatalf("expected %v, got %v", e, a) } if e, a := *createdNoxuInstance, listWithItem.(*unstructured.UnstructuredList).Items[0]; !reflect.DeepEqual(e, a) { t.Fatalf("expected %v, got %v", e, a) } if err := noxuNamespacedResourceClient.Delete(sameInstanceName, nil); err != nil { t.Fatal(err) } if _, err = noxuNamespacedResourceClient.Get(sameInstanceName); err == nil || !errors.IsNotFound(err) { t.Fatalf("expected a NotFound error, got:%v", err) } listWithoutItem, err := noxuNamespacedResourceClient.List(metav1.ListOptions{}) if err != nil { t.Fatal(err) } if e, a := 0, len(listWithoutItem.(*unstructured.UnstructuredList).Items); e != a { t.Fatalf("expected %v, got %v", e, a) } }() </ins> }


tags = ["automanaged"], deps = [ "//vendor/github.com/pborman/uuid:go_default_library", <ins> "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", </ins> "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",


import ( "time" <ins> "k8s.io/apimachinery/pkg/api/errors" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema"


} return dynamicClient, nil } <ins> func DeleteCustomResourceDefinition(customResource *apiextensionsv1alpha1.CustomResourceDefinition, apiExtensionsClient clientset.Interface) error { if err := apiExtensionsClient.Apiextensions().CustomResourceDefinitions().Delete(customResource.Name, nil); err != nil { return err } err := wait.PollImmediate(30*time.Millisecond, 30*time.Second, func() (bool, error) { if _, err := apiExtensionsClient.Discovery().ServerResourcesForGroupVersion(customResource.Spec.Group + "/" + customResource.Spec.Version); err != nil { if errors.IsNotFound(err) { return true, nil } return false, err } return false, nil }) return err } func GetCustomResourceDefinition(customResource *apiextensionsv1alpha1.CustomResourceDefinition, apiExtensionsClient clientset.Interface) (*apiextensionsv1alpha1.CustomResourceDefinition, error) { return apiExtensionsClient.Apiextensions().CustomResourceDefinitions().Get(customResource.Name, metav1.GetOptions{}) } </ins>


library = ":go_default_library", tags = ["automanaged"], deps = [ <ins> "//pkg/cloudprovider/providers/azure:go_default_library", "//pkg/cloudprovider/providers/fake:go_default_library", </ins> "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/testing:go_default_library",


"k8s.io/kubernetes/pkg/volume" "github.com/golang/glog" <ins> "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" </ins> "k8s.io/kubernetes/pkg/volume/util" ) <del> // This is the primary entrypoint for volume plugins. </del> <ins> // ProbeVolumePlugins is the primary endpoint for volume plugins </ins> func ProbeVolumePlugins() []volume.VolumePlugin { return []volume.VolumePlugin{&azureFilePlugin{nil}} }


if accountName, accountKey, err = b.util.GetAzureCredentials(b.plugin.host, b.pod.Namespace, b.secretName); err != nil { return err } <del> os.MkdirAll(dir, 0750) source := fmt.Sprintf("//%s.file.core.windows.net/%s", accountName, b.shareName) </del> <ins> os.MkdirAll(dir, 0700) source := fmt.Sprintf("//%s.file.%s/%s", accountName, getStorageEndpointSuffix(b.plugin.host.GetCloudProvider()), b.shareName) </ins> // parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/ <del> options := []string{fmt.Sprintf("vers=3.0,username=%s,password=%s,dir_mode=0777,file_mode=0777", accountName, accountKey)} </del> <ins> options := []string{fmt.Sprintf("vers=3.0,username=%s,password=%s,dir_mode=0700,file_mode=0700", accountName, accountKey)} </ins> if b.readOnly { options = append(options, "ro") }


return nil, false, fmt.Errorf("Spec does not reference an AzureFile volume type") } <ins> func getAzureCloud(cloudProvider cloudprovider.Interface) (*azure.Cloud, error) { azure, ok := cloudProvider.(*azure.Cloud) if !ok || azure == nil { return nil, fmt.Errorf("Failed to get Azure Cloud Provider. GetCloudProvider returned %v instead", cloudProvider) } return azure, nil } func getStorageEndpointSuffix(cloudprovider cloudprovider.Interface) string { const publicCloudStorageEndpointSuffix = "core.windows.net" azure, err := getAzureCloud(cloudprovider) if err != nil { glog.Warningf("No Azure cloud provider found. Using the Azure public cloud endpoint: %s", publicCloudStorageEndpointSuffix) return publicCloudStorageEndpointSuffix } return azure.Environment.StorageEndpointSuffix } </ins>


"io/ioutil" "os" "path" <ins> "strings" </ins> "testing" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" <ins> "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" fakecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/fake" </ins> "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing"


return false } <del> func TestPlugin(t *testing.T) { </del> <ins> func getAzureTestCloud(t *testing.T) *azure.Cloud { config := `{ "aadClientId": "--aad-client-id--", "aadClientSecret": "--aad-client-secret--" }` configReader := strings.NewReader(config) cloud, err := azure.NewCloud(configReader) if err != nil { t.Error(err) } azureCloud, ok := cloud.(*azure.Cloud) if !ok { t.Error("NewCloud returned incorrect type") } return azureCloud } func getTestTempDir(t *testing.T) string { </ins> tmpDir, err := ioutil.TempDir(os.TempDir(), "azurefileTest") if err != nil { t.Fatalf("can't make a temp dir: %v", err) } <ins> return tmpDir } func TestPluginAzureCloudProvider(t *testing.T) { tmpDir := getTestTempDir(t) defer os.RemoveAll(tmpDir) testPlugin(t, tmpDir, volumetest.NewFakeVolumeHostWithCloudProvider(tmpDir, nil, nil, getAzureTestCloud(t))) } func TestPluginWithoutCloudProvider(t *testing.T) { tmpDir := getTestTempDir(t) defer os.RemoveAll(tmpDir) testPlugin(t, tmpDir, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) } func TestPluginWithOtherCloudProvider(t *testing.T) { tmpDir := getTestTempDir(t) </ins> defer os.RemoveAll(tmpDir) <ins> cloud := &fakecloud.FakeCloud{} testPlugin(t, tmpDir, volumetest.NewFakeVolumeHostWithCloudProvider(tmpDir, nil, nil, cloud)) } func testPlugin(t *testing.T, tmpDir string, volumeHost volume.VolumeHost) { </ins> plugMgr := volume.VolumePluginMgr{} <del> plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) </del> <ins> plugMgr.InitPlugins(ProbeVolumePlugins(), volumeHost) </ins> plug, err := plugMgr.FindPluginByName("kubernetes.io/azure-file") if err != nil {


} func NewFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin) *fakeVolumeHost { <del> host := &fakeVolumeHost{rootDir: rootDir, kubeClient: kubeClient, cloud: nil} </del> <ins> return newFakeVolumeHost(rootDir, kubeClient, plugins, nil) } func NewFakeVolumeHostWithCloudProvider(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, cloud cloudprovider.Interface) *fakeVolumeHost { return newFakeVolumeHost(rootDir, kubeClient, plugins, cloud) } func newFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, cloud cloudprovider.Interface) *fakeVolumeHost { host := &fakeVolumeHost{rootDir: rootDir, kubeClient: kubeClient, cloud: cloud} </ins> host.mounter = &mount.FakeMounter{} host.writer = &io.StdWriter{} host.pluginMgr.InitPlugins(plugins, host)


} // SetRef stores a reference to a pod's container, associating it with the given container ID. <ins> // TODO: move this to client-go v1.ObjectReference </ins> func (c *RefManager) SetRef(id ContainerID, ref *v1.ObjectReference) { c.Lock() defer c.Unlock()


} // GetRef returns the container reference of the given ID, or (nil, false) if none is stored. <ins> // TODO: move this to client-go v1.ObjectReference </ins> func (c *RefManager) GetRef(id ContainerID) (ref *v1.ObjectReference, ok bool) { c.RLock() defer c.RUnlock()


glog.Warningf("No ref for container %q", containerID) return } <del> m.recorder.Event(ref, eventType, reason, message) </del> <ins> m.recorder.Event(events.ToObjectReference(ref), eventType, reason, message) </ins> } // executePreStopHook runs the pre-stop lifecycle hooks if applicable and returns the duration it takes.


if err != nil { glog.V(1).Infof("%s probe for %q errored: %v", probeType, ctrName, err) if hasRef { <del> pb.recorder.Eventf(ref, v1.EventTypeWarning, events.ContainerUnhealthy, "%s probe errored: %v", probeType, err) </del> <ins> pb.recorder.Eventf(events.ToObjectReference(ref), v1.EventTypeWarning, events.ContainerUnhealthy, "%s probe errored: %v", probeType, err) </ins> } } else { // result != probe.Success glog.V(1).Infof("%s probe for %q failed (%v): %s", probeType, ctrName, result, output) if hasRef { <del> pb.recorder.Eventf(ref, v1.EventTypeWarning, events.ContainerUnhealthy, "%s probe failed: %s", probeType, output) </del> <ins> pb.recorder.Eventf(events.ToObjectReference(ref), v1.EventTypeWarning, events.ContainerUnhealthy, "%s probe failed: %s", probeType, output) </ins> } } return results.Failure, err


uuid := utilstrings.ShortenString(id.uuid, 8) switch reason { case "Created": <del> r.recorder.Eventf(ref, v1.EventTypeNormal, events.CreatedContainer, "Created with rkt id %v", uuid) </del> <ins> r.recorder.Eventf(events.ToObjectReference(ref), v1.EventTypeNormal, events.CreatedContainer, "Created with rkt id %v", uuid) </ins> case "Started": <del> r.recorder.Eventf(ref, v1.EventTypeNormal, events.StartedContainer, "Started with rkt id %v", uuid) </del> <ins> r.recorder.Eventf(events.ToObjectReference(ref), v1.EventTypeNormal, events.StartedContainer, "Started with rkt id %v", uuid) </ins> case "Failed": <del> r.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedToStartContainer, "Failed to start with rkt id %v with error %v", uuid, failure) </del> <ins> r.recorder.Eventf(events.ToObjectReference(ref), v1.EventTypeWarning, events.FailedToStartContainer, "Failed to start with rkt id %v with error %v", uuid, failure) </ins> case "Killing": <del> r.recorder.Eventf(ref, v1.EventTypeNormal, events.KillingContainer, "Killing with rkt id %v", uuid) </del> <ins> r.recorder.Eventf(events.ToObjectReference(ref), v1.EventTypeNormal, events.KillingContainer, "Killing with rkt id %v", uuid) </ins> default: glog.Errorf("rkt: Unexpected event %q", reason) }


if len(namespace) == 0 { namespace = metav1.NamespaceDefault } <del> selfLink = fmt.Sprintf("/api/"+api.Registry.GroupOrDie(api.GroupName).GroupVersion.Version+"/pods/namespaces/%s/%s", name, namespace) </del> <ins> selfLink = fmt.Sprintf("/api/"+api.Registry.GroupOrDie(api.GroupName).GroupVersion.Version+"/namespaces/%s/pods/%s", namespace, name) </ins> return selfLink }


} } } <ins> func TestGetSelfLink(t *testing.T) { var testCases = []struct { desc             string name             string namespace        string expectedSelfLink string }{ { desc:             "No namespace specified", name:             "foo", namespace:        "", expectedSelfLink: "/api/v1/namespaces/default/pods/foo", }, { desc:             "Namespace specified", name:             "foo", namespace:        "bar", expectedSelfLink: "/api/v1/namespaces/bar/pods/foo", }, } for _, testCase := range testCases { selfLink := getSelfLink(testCase.name, testCase.namespace) if testCase.expectedSelfLink != selfLink { t.Errorf("%s: getSelfLink error, expected: %s, got: %s", testCase.desc, testCase.expectedSelfLink, selfLink) } } } </ins>


"//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library", "//pkg/kubeapiserver/admission:go_default_library", <ins> "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", </ins> "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", ],


"fmt" "io" <ins> "k8s.io/apimachinery/pkg/api/errors" </ins> "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/admission" "k8s.io/kubernetes/pkg/api"


return nil case admission.Delete: <del> // get the existing pod </del> <ins> // get the existing pod from the server cache </ins> existingPod, err := c.podsGetter.Pods(a.GetNamespace()).Get(a.GetName(), v1.GetOptions{ResourceVersion: "0"}) <ins> if errors.IsNotFound(err) { // wasn't found in the server cache, do a live lookup before forbidding existingPod, err = c.podsGetter.Pods(a.GetNamespace()).Get(a.GetName(), v1.GetOptions{}) } </ins> if err != nil { return admission.NewForbidden(a, err) }


tolerations: - operator: "Exists" effect: "NoExecute" <ins> - key: "CriticalAddonsOnly" operator: "Exists" </ins>


if kind.Version == runtime.APIVersionInternal { continue } <del> if kind == api.Unversioned.WithKind("Status") { </del> <ins> if kind == metav1.Unversioned.WithKind("Status") { </ins> // this is added below as unversioned continue } metaOnlyObject := gvkToMetadataOnlyObject(kind) scheme.AddKnownTypeWithName(kind, metaOnlyObject) } <del> scheme.AddUnversionedTypes(api.Unversioned, &metav1.Status{}) </del> <ins> scheme.AddUnversionedTypes(metav1.Unversioned, &metav1.Status{}) </ins> return serializer.NewCodecFactory(scheme) }


// SchemeGroupVersion is group version used to register these objects var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1"} <ins> // Unversioned is group version for unversioned API objects // TODO: this should be v1 probably var Unversioned = schema.GroupVersion{Group: "", Version: "v1"} </ins> // WatchEventKind is name reserved for serializing watch events. const WatchEventKind = "WatchEvent"


Convert_versioned_Event_to_versioned_InternalEvent, ) <ins> // Register Unversioned types under their own special group scheme.AddUnversionedTypes(Unversioned, &Status{}, &APIVersions{}, &APIGroupList{}, &APIGroup{}, &APIResourceList{}, ) </ins> // register manually. This usually goes through the SchemeBuilder, which we cannot use here. scheme.AddGeneratedDeepCopyFuncs(GetGeneratedDeepCopyFuncs()...) AddConversionFuncs(scheme)


t := reflect.TypeOf(obj).Elem() gvk := version.WithKind(t.Name()) s.unversionedTypes[t] = gvk <del> if _, ok := s.unversionedKinds[gvk.Kind]; ok { panic(fmt.Sprintf("%v has already been registered as unversioned kind %q - kind name must be unique", reflect.TypeOf(t), gvk.Kind)) </del> <ins> if old, ok := s.unversionedKinds[gvk.Kind]; ok && t != old { panic(fmt.Sprintf("%v.%v has already been registered as unversioned kind %q - kind name must be unique", old.PkgPath(), old.Name(), gvk)) </ins> } s.unversionedKinds[gvk.Kind] = t }


} s.AddUnversionedTypes(gv, &InternalSimple{}) <ins> s.AddUnversionedTypes(gv, &InternalSimple{}) </ins> if len(s.KnownTypes(gv)) != 1 { t.Errorf("expected only one %v type after double registration with custom name", gv) }


} } <ins> // EmbeddableTypeMeta passes GetObjectKind to the type which embeds it. type EmbeddableTypeMeta runtime.TypeMeta func (tm *EmbeddableTypeMeta) GetObjectKind() schema.ObjectKind { return (*runtime.TypeMeta)(tm) } </ins> func TestConflictingAddKnownTypes(t *testing.T) { s := runtime.NewScheme() gv := schema.GroupVersion{Group: "foo", Version: "v1"}


panicked <- true } }() s.AddUnversionedTypes(gv, &InternalSimple{}) <ins> // redefine InternalSimple with the same name, but obviously as a different type type InternalSimple struct { EmbeddableTypeMeta `json:",inline"` TestString         string `json:"testString"` } </ins> s.AddUnversionedTypes(gv, &InternalSimple{}) panicked <- false }()


// SchemeGroupVersion is group version used to register these objects var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal} <del> // Unversioned is group version for unversioned API objects // TODO: this should be v1 probably var Unversioned = schema.GroupVersion{Group: "", Version: "v1"} </del> // ParameterCodec handles versioning of objects that are converted to query parameters. var ParameterCodec = runtime.NewParameterCodec(Scheme)


&ConfigMapList{}, ) <del> // Register Unversioned types under their own special group scheme.AddUnversionedTypes(Unversioned, &metav1.Status{}, &metav1.APIVersions{}, &metav1.APIGroupList{}, &metav1.APIGroup{}, &metav1.APIResourceList{}, ) </del> return nil }


} } <ins> func TestStatus(t *testing.T) { _, s, closeFn := framework.RunAMaster(nil) defer closeFn() u := s.URL + "/apis/batch/v1/namespaces/default/jobs/foo" resp, err := http.Get(u) if err != nil { t.Fatalf("unexpected error getting %s: %v", u, err) } if resp.StatusCode != http.StatusNotFound { t.Fatalf("got status %v instead of 404", resp.StatusCode) } defer resp.Body.Close() data, _ := ioutil.ReadAll(resp.Body) decodedData := map[string]interface{}{} if err := json.Unmarshal(data, &decodedData); err != nil { t.Logf("body: %s", string(data)) t.Fatalf("got error decoding data: %v", err) } t.Logf("body: %s", string(data)) if got, expected := decodedData["apiVersion"], "v1"; got != expected { t.Errorf("unexpected apiVersion %q, expected %q", got, expected) } if got, expected := decodedData["kind"], "Status"; got != expected { t.Errorf("unexpected kind %q, expected %q", got, expected) } if got, expected := decodedData["status"], "Failure"; got != expected { t.Errorf("unexpected status %q, expected %q", got, expected) } if got, expected := decodedData["code"], float64(404); got != expected { t.Errorf("unexpected code %v, expected %v", got, expected) } } </ins> func TestWatchSucceedsWithoutArgs(t *testing.T) { _, s, closeFn := framework.RunAMaster(nil) defer closeFn()


PodWaiting PodStatus = "Waiting" // PodRunning means that the pod is up and running. PodRunning PodStatus = "Running" <del> // PodTerminated means that the pod has stopped. </del> <ins> // PodTerminated means that the pod has stopped with error(s) </ins> PodTerminated PodStatus = "Terminated" // PodUnknown means that we failed to obtain info about the pod. PodUnknown PodStatus = "Unknown" <ins> // PodSucceeded means that the pod has stopped without error(s) PodSucceeded PodStatus = "Succeeded" </ins> ) type ContainerStateWaiting struct {


case newer.PodRunning: *out = PodRunning case newer.PodSucceeded: <del> *out = PodTerminated </del> <ins> *out = PodSucceeded </ins> case newer.PodFailed: *out = PodTerminated case newer.PodUnknown:


case PodTerminated: // Older API versions did not contain enough info to map to PodSucceeded *out = newer.PodFailed <ins> case PodSucceeded: *out = newer.PodSucceeded </ins> case PodUnknown: *out = newer.PodUnknown default:


PodTerminated PodStatus = "Terminated" // PodUnknown means that we failed to obtain info about the pod. PodUnknown PodStatus = "Unknown" <ins> // PodSucceeded means that the pod has stopped without error(s) PodSucceeded PodStatus = "Succeeded" </ins> ) type ContainerStateWaiting struct {


package e2e import ( <ins> "io/ioutil" "net/http" </ins> "strings" "time"


"k8s.io/kubernetes/pkg/api/v1" extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" "k8s.io/kubernetes/test/e2e/framework" <del> "k8s.io/kubernetes/test/e2e/generated" </del> . "github.com/onsi/ginkgo" . "github.com/onsi/gomega"


// Nvidia driver installation can take upwards of 5 minutes. driverInstallTimeout = 10 * time.Minute // Nvidia COS driver installer daemonset. <del> cosNvidiaDriverInstallerPath = "cluster/gce/gci/nvidia-gpus/cos-installer-daemonset.yaml" </del> <ins> cosNvidiaDriverInstallerUrl = "https://raw.githubusercontent.com/ContainerEngine/accelerators/stable/cos-nvidia-gpu-installer/daemonset.yaml" </ins> ) func makeCudaAdditionTestPod() *v1.Pod {


// GPU drivers might have already been installed. if !areGPUsAvailableOnAllSchedulableNodes(f) { // Install Nvidia Drivers. <del> ds := dsFromManifest(cosNvidiaDriverInstallerPath) </del> <ins> ds := dsFromManifest(cosNvidiaDriverInstallerUrl) </ins> ds.Namespace = f.Namespace.Name _, err := f.ClientSet.Extensions().DaemonSets(f.Namespace.Name).Create(ds) framework.ExpectNoError(err, "failed to create daemonset")


} // dsFromManifest reads a .json/yaml file and returns the daemonset in it. <del> func dsFromManifest(fileName string) *extensions.DaemonSet { </del> <ins> func dsFromManifest(url string) *extensions.DaemonSet { </ins> var controller extensions.DaemonSet <del> framework.Logf("Parsing ds from %v", fileName) data := generated.ReadOrDie(fileName) </del> <ins> framework.Logf("Parsing ds from %v", url) var response *http.Response var err error for i := 1; i <= 5; i++ { response, err = http.Get(url) if err == nil && response.StatusCode == 200 { break } time.Sleep(time.Duration(i) * time.Second) } Expect(err).NotTo(HaveOccurred()) Expect(response.StatusCode).To(Equal(200)) defer response.Body.Close() data, err := ioutil.ReadAll(response.Body) Expect(err).NotTo(HaveOccurred()) </ins> json, err := utilyaml.ToJSON(data) Expect(err).NotTo(HaveOccurred())


runcmd: - modprobe configs <del> - docker run -v /dev:/dev -v /home/kubernetes/bin/nvidia:/rootfs/nvidia -v /etc/os-release:/rootfs/etc/os-release -v /proc/sysrq-trigger:/sysrq -e LAKITU_KERNEL_SHA1=26481563cb3788ad254c2bf2126b843c161c7e48 -e BASE_DIR=/rootfs/nvidia --privileged gcr.io/google_containers/cos-nvidia-driver-install@sha256:ad83ede6e0c6d768bf7cf69a7dec972aa5e8f88778142ca46afd3286ad58cfc8 </del> <ins> - docker run -v /dev:/dev -v /home/kubernetes/bin/nvidia:/rootfs/nvidia -v /etc/os-release:/rootfs/etc/os-release -v /proc/sysrq-trigger:/sysrq -e BASE_DIR=/rootfs/nvidia --privileged gcr.io/google_containers/cos-nvidia-driver-install@sha256:cb55c7971c337fece62f2bfe858662522a01e43ac9984a2dd1dd5c71487d225c </ins> - mount /tmp /tmp -o remount,exec,suid - usermod -a -G docker jenkins - mkdir -p /var/lib/kubelet


image: e2e-node-containervm-v20161208-image # docker 1.11.2 project: kubernetes-node-e2e-images gci: <del> image_regex: cos-beta-59-9460-20-0 # docker 1.11.2 </del> <ins> image_regex: cos-stable-59-9460-60-0 # docker 1.11.2 </ins> project: cos-cloud metadata: "user-data<test/e2e_node/jenkins/gci-init-gpu.yaml,gci-update-strategy=update_disabled" resources: accelerators: - type: nvidia-tesla-k80 count: 2


// absGidMin <= defGidMin <= defGidMax <= absGidMax absoluteGidMin          = 2000 absoluteGidMax          = math.MaxInt32 <del> heketiAnn               = "heketi-dynamic-provisioner" glusterTypeAnn          = "gluster.org/type" glusterDescAnn          = "Gluster: Dynamically provisioned PV" </del> linuxGlusterMountBinary = "mount.glusterfs" autoUnmountBinaryVer    = "3.11" )


if kubeClient == nil { return nil, fmt.Errorf("glusterfs: failed to get kube client to initialize mounter") } ep, err := kubeClient.Core().Endpoints(ns).Get(epName, metav1.GetOptions{}) <del> if err != nil && errors.IsNotFound(err) { claim := spec.PersistentVolume.Spec.ClaimRef.Name checkEpName := dynamicEpSvcPrefix + claim if epName != checkEpName { return nil, fmt.Errorf("failed to get endpoint %s, error %v", epName, err) } glog.Errorf("glusterfs: failed to get endpoint %s[%v]", epName, err) if spec != nil && spec.PersistentVolume.Annotations[volumehelper.VolumeDynamicallyCreatedByKey] == heketiAnn { class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume) if err != nil { return nil, fmt.Errorf("glusterfs: failed to get storageclass, error: %v", err) } cfg, err := parseClassParameters(class.Parameters, plugin.host.GetKubeClient()) if err != nil { return nil, fmt.Errorf("glusterfs: failed to parse parameters, error: %v", err) } scConfig := *cfg cli := gcli.NewClient(scConfig.url, scConfig.user, scConfig.secretValue) if cli == nil { return nil, fmt.Errorf("glusterfs: failed to create heketi client, error: %v", err) } volumeID := dstrings.TrimPrefix(source.Path, volPrefix) volInfo, err := cli.VolumeInfo(volumeID) if err != nil { return nil, fmt.Errorf("glusterfs: failed to get volume info, error: %v", err) } endpointIPs, err := getClusterNodes(cli, volInfo.Cluster) if err != nil { return nil, fmt.Errorf("glusterfs: failed to get cluster nodes, error: %v", err) } // Give an attempt to recreate endpoint/service. _, _, err = plugin.createEndpointService(ns, epName, endpointIPs, claim) if err != nil && !errors.IsAlreadyExists(err) { glog.Errorf("glusterfs: failed to recreate endpoint/service, error: %v", err) return nil, fmt.Errorf("failed to recreate endpoint/service, error: %v", err) } glog.V(3).Infof("glusterfs: endpoint/service [%v] successfully recreated ", epName) } else { return nil, err } </del> <ins> if err != nil { glog.Errorf("glusterfs: failed to get endpoints %s[%v]", epName, err) return nil, err </ins> } <ins> glog.V(1).Infof("glusterfs: endpoints %v", ep) </ins> return plugin.newMounterInternal(spec, ep, pod, plugin.host.GetMounter(), exec.New()) }


return newGidTable, nil } <del> //createEndpointService create an endpoint and service in provided namespace. func (plugin *glusterfsPlugin) createEndpointService(namespace string, epServiceName string, hostips []string, pvcname string) (endpoint *v1.Endpoints, service *v1.Service, err error) { addrlist := make([]v1.EndpointAddress, len(hostips)) for i, v := range hostips { addrlist[i].IP = v } endpoint = &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name:      epServiceName, Labels: map[string]string{ "gluster.kubernetes.io/provisioned-for-pvc": pvcname, }, }, Subsets: []v1.EndpointSubset{{ Addresses: addrlist, Ports:     []v1.EndpointPort{{Port: 1, Protocol: "TCP"}}, }}, } kubeClient := plugin.host.GetKubeClient() if kubeClient == nil { return nil, nil, fmt.Errorf("glusterfs: failed to get kube client when creating endpoint service") } _, err = kubeClient.Core().Endpoints(namespace).Create(endpoint) if err != nil && errors.IsAlreadyExists(err) { glog.V(1).Infof("glusterfs: endpoint [%s] already exist in namespace [%s]", endpoint, namespace) err = nil } if err != nil { glog.Errorf("glusterfs: failed to create endpoint: %v", err) return nil, nil, fmt.Errorf("error creating endpoint: %v", err) } service = &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name:      epServiceName, Namespace: namespace, Labels: map[string]string{ "gluster.kubernetes.io/provisioned-for-pvc": pvcname, }, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ {Protocol: "TCP", Port: 1}}}} _, err = kubeClient.Core().Services(namespace).Create(service) if err != nil && errors.IsAlreadyExists(err) { glog.V(1).Infof("glusterfs: service [%s] already exist in namespace [%s]", service, namespace) err = nil } if err != nil { glog.Errorf("glusterfs: failed to create service: %v", err) return nil, nil, fmt.Errorf("error creating service: %v", err) } return endpoint, service, nil } // deleteEndpointService delete the endpoint and service from the provided namespace. func (plugin *glusterfsPlugin) deleteEndpointService(namespace string, epServiceName string) (err error) { kubeClient := plugin.host.GetKubeClient() if kubeClient == nil { return fmt.Errorf("glusterfs: failed to get kube client when deleting endpoint service") } err = kubeClient.Core().Services(namespace).Delete(epServiceName, nil) if err != nil { glog.Errorf("glusterfs: error deleting service %s/%s: %v", namespace, epServiceName, err) return fmt.Errorf("error deleting service %s/%s: %v", namespace, epServiceName, err) } glog.V(1).Infof("glusterfs: service/endpoint %s/%s deleted successfully", namespace, epServiceName) return nil } </del> func (d *glusterfsVolumeDeleter) getGid() (int, bool, error) { gidStr, ok := d.spec.Annotations[volumehelper.VolumeGidAnnotationKey]


dynamicEndpoint = pvSpec.Glusterfs.EndpointsName } glog.V(3).Infof("glusterfs: dynamic namespace and endpoint : [%v/%v]", dynamicNamespace, dynamicEndpoint) <del> err = d.plugin.deleteEndpointService(dynamicNamespace, dynamicEndpoint) </del> <ins> err = d.deleteEndpointService(dynamicNamespace, dynamicEndpoint) </ins> if err != nil { glog.Errorf("glusterfs: error when deleting endpoint/service :%v", err) } else {


gidStr := strconv.FormatInt(int64(gid), 10) pv.Annotations = map[string]string{ <del> volumehelper.VolumeGidAnnotationKey:        gidStr, volumehelper.VolumeDynamicallyCreatedByKey: heketiAnn, glusterTypeAnn:                             "file", "Description":                              glusterDescAnn, v1.MountOptionAnnotation:                   "auto_unmount", </del> <ins> volumehelper.VolumeGidAnnotationKey: gidStr, "kubernetes.io/createdby":           "heketi-dynamic-provisioner", "gluster.org/type":                  "file", "Description":                       "Gluster: Dynamically provisioned PV", v1.MountOptionAnnotation:            "auto_unmount", </ins> } pv.Spec.Capacity = v1.ResourceList{


return pv, nil } <ins> func (p *glusterfsVolumeProvisioner) GetClusterNodes(cli *gcli.Client, cluster string) (dynamicHostIps []string, err error) { clusterinfo, err := cli.ClusterInfo(cluster) if err != nil { glog.Errorf("glusterfs: failed to get cluster details: %v", err) return nil, fmt.Errorf("failed to get cluster details: %v", err) } // For the dynamically provisioned volume, we gather the list of node IPs // of the cluster on which provisioned volume belongs to, as there can be multiple // clusters. for _, node := range clusterinfo.Nodes { nodei, err := cli.NodeInfo(string(node)) if err != nil { glog.Errorf("glusterfs: failed to get hostip: %v", err) return nil, fmt.Errorf("failed to get hostip: %v", err) } ipaddr := dstrings.Join(nodei.NodeAddRequest.Hostnames.Storage, "") dynamicHostIps = append(dynamicHostIps, ipaddr) } glog.V(3).Infof("glusterfs: hostlist :%v", dynamicHostIps) if len(dynamicHostIps) == 0 { glog.Errorf("glusterfs: no hosts found: %v", err) return nil, fmt.Errorf("no hosts found: %v", err) } return dynamicHostIps, nil } </ins> func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsVolumeSource, size int, err error) { var clusterIDs []string capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]


return nil, 0, fmt.Errorf("error creating volume %v", err) } glog.V(1).Infof("glusterfs: volume with size: %d and name: %s created", volume.Size, volume.Name) <del> dynamicHostIps, err := getClusterNodes(cli, volume.Cluster) </del> <ins> dynamicHostIps, err := p.GetClusterNodes(cli, volume.Cluster) </ins> if err != nil { glog.Errorf("glusterfs: error [%v] when getting cluster nodes for volume %s", err, volume) return nil, 0, fmt.Errorf("error [%v] when getting cluster nodes for volume %s", err, volume)


// of volume creation. epServiceName := dynamicEpSvcPrefix + p.options.PVC.Name epNamespace := p.options.PVC.Namespace <del> endpoint, service, err := p.plugin.createEndpointService(epNamespace, epServiceName, dynamicHostIps, p.options.PVC.Name) </del> <ins> endpoint, service, err := p.createEndpointService(epNamespace, epServiceName, dynamicHostIps, p.options.PVC.Name) </ins> if err != nil { glog.Errorf("glusterfs: failed to create endpoint/service: %v", err) deleteErr := cli.VolumeDelete(volume.Id)


}, sz, nil } <ins> func (p *glusterfsVolumeProvisioner) createEndpointService(namespace string, epServiceName string, hostips []string, pvcname string) (endpoint *v1.Endpoints, service *v1.Service, err error) { addrlist := make([]v1.EndpointAddress, len(hostips)) for i, v := range hostips { addrlist[i].IP = v } endpoint = &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name:      epServiceName, Labels: map[string]string{ "gluster.kubernetes.io/provisioned-for-pvc": pvcname, }, }, Subsets: []v1.EndpointSubset{{ Addresses: addrlist, Ports:     []v1.EndpointPort{{Port: 1, Protocol: "TCP"}}, }}, } kubeClient := p.plugin.host.GetKubeClient() if kubeClient == nil { return nil, nil, fmt.Errorf("glusterfs: failed to get kube client when creating endpoint service") } _, err = kubeClient.Core().Endpoints(namespace).Create(endpoint) if err != nil && errors.IsAlreadyExists(err) { glog.V(1).Infof("glusterfs: endpoint [%s] already exist in namespace [%s]", endpoint, namespace) err = nil } if err != nil { glog.Errorf("glusterfs: failed to create endpoint: %v", err) return nil, nil, fmt.Errorf("error creating endpoint: %v", err) } service = &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name:      epServiceName, Namespace: namespace, Labels: map[string]string{ "gluster.kubernetes.io/provisioned-for-pvc": pvcname, }, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ {Protocol: "TCP", Port: 1}}}} _, err = kubeClient.Core().Services(namespace).Create(service) if err != nil && errors.IsAlreadyExists(err) { glog.V(1).Infof("glusterfs: service [%s] already exist in namespace [%s]", service, namespace) err = nil } if err != nil { glog.Errorf("glusterfs: failed to create service: %v", err) return nil, nil, fmt.Errorf("error creating service: %v", err) } return endpoint, service, nil } func (d *glusterfsVolumeDeleter) deleteEndpointService(namespace string, epServiceName string) (err error) { kubeClient := d.plugin.host.GetKubeClient() if kubeClient == nil { return fmt.Errorf("glusterfs: failed to get kube client when deleting endpoint service") } err = kubeClient.Core().Services(namespace).Delete(epServiceName, nil) if err != nil { glog.Errorf("glusterfs: error deleting service %s/%s: %v", namespace, epServiceName, err) return fmt.Errorf("error deleting service %s/%s: %v", namespace, epServiceName, err) } glog.V(1).Infof("glusterfs: service/endpoint %s/%s deleted successfully", namespace, epServiceName) return nil } </ins> // parseSecret finds a given Secret instance and reads user password from it. func parseSecret(namespace, secretName string, kubeClient clientset.Interface) (string, error) { secretMap, err := volutil.GetSecretForPV(namespace, secretName, glusterfsPluginName, kubeClient)


return &cfg, nil } <del> // getClusterNodes() returns the cluster nodes of a given cluster func getClusterNodes(cli *gcli.Client, cluster string) (dynamicHostIps []string, err error) { clusterinfo, err := cli.ClusterInfo(cluster) if err != nil { glog.Errorf("glusterfs: failed to get cluster details: %v", err) return nil, fmt.Errorf("failed to get cluster details: %v", err) } // For the dynamically provisioned volume, we gather the list of node IPs // of the cluster on which provisioned volume belongs to, as there can be multiple // clusters. for _, node := range clusterinfo.Nodes { nodei, err := cli.NodeInfo(string(node)) if err != nil { glog.Errorf("glusterfs: failed to get hostip: %v", err) return nil, fmt.Errorf("failed to get hostip: %v", err) } ipaddr := dstrings.Join(nodei.NodeAddRequest.Hostnames.Storage, "") dynamicHostIps = append(dynamicHostIps, ipaddr) } glog.V(3).Infof("glusterfs: hostlist :%v", dynamicHostIps) if len(dynamicHostIps) == 0 { glog.Errorf("glusterfs: no hosts found: %v", err) return nil, fmt.Errorf("no hosts found: %v", err) } return dynamicHostIps, nil } </del>


DNS_SERVER_IP=${KUBE_DNS_SERVER_IP:-10.0.0.10} DNS_DOMAIN=${KUBE_DNS_NAME:-"cluster.local"} KUBECTL=${KUBECTL:-cluster/kubectl.sh} <del> WAIT_FOR_URL_API_SERVER=${WAIT_FOR_URL_API_SERVER:-10} </del> <ins> WAIT_FOR_URL_API_SERVER=${WAIT_FOR_URL_API_SERVER:-20} </ins> ENABLE_DAEMON=${ENABLE_DAEMON:-false} HOSTNAME_OVERRIDE=${HOSTNAME_OVERRIDE:-"127.0.0.1"} CLOUD_PROVIDER=${CLOUD_PROVIDER:-""}


# this uses the API port because if you don't have any authenticator, you can't seem to use the secure port at all. # this matches what happened with the combination in 1.4. # TODO change this conditionally based on whether API_PORT is on or off <del> kube::util::wait_for_url "http://${API_HOST_IP}:${API_SECURE_PORT}/healthz" "apiserver: " 1 ${WAIT_FOR_URL_API_SERVER}  </del> <ins> kube::util::wait_for_url "https://${API_HOST_IP}:${API_SECURE_PORT}/healthz" "apiserver: " 1 ${WAIT_FOR_URL_API_SERVER}  </ins> || { echo "check apiserver logs: ${APISERVER_LOG}" ; exit 1 ; } # Create kubeconfigs for all components, using client certs


apiVersion: apps/v1beta1 kind: Deployment metadata: <del> name: event-exporter-v0.1.0 </del> <ins> name: event-exporter-v0.1.4 </ins> namespace: kube-system labels: k8s-app: event-exporter


containers: # TODO: Add resources in 1.8 - name: event-exporter <del> image: gcr.io/google-containers/event-exporter:v0.1.0-r2 </del> <ins> image: gcr.io/google-containers/event-exporter:v0.1.4 </ins> command: - '/event-exporter' - name: prometheus-to-sd-exporter


} } <del> // For a given node checks its conditions and tries to update it. Returns grace period to which given node // is entitled, state of current and last observed Ready Condition, and an error if it occurred. </del> <ins> // tryUpdateNodeStatus checks a given node's conditions and tries to update it. Returns grace period to // which given node is entitled, state of current and last observed Ready Condition, and an error if it occurred. </ins> func (nc *NodeController) tryUpdateNodeStatus(node *v1.Node) (time.Duration, v1.NodeCondition, *v1.NodeCondition, error) { var err error var gracePeriod time.Duration


// otherwise we leave it as it is. if savedCondition.LastTransitionTime != observedCondition.LastTransitionTime { glog.V(3).Infof("ReadyCondition for Node %s transitioned from %v to %v", node.Name, savedCondition.Status, observedCondition) transitionTime = nc.now() } else { transitionTime = savedNodeStatus.readyTransitionTimestamp


} // remaining node conditions should also be set to Unknown <del> remainingNodeConditionTypes := []v1.NodeConditionType{v1.NodeOutOfDisk, v1.NodeMemoryPressure, v1.NodeDiskPressure} </del> <ins> remainingNodeConditionTypes := []v1.NodeConditionType{ v1.NodeOutOfDisk, v1.NodeMemoryPressure, v1.NodeDiskPressure, // We don't change 'NodeInodePressure' condition, as it'll be removed in future. // v1.NodeInodePressure, // We don't change 'NodeNetworkUnavailable' condition, as it's managed on a control plane level. // v1.NodeNetworkUnavailable, } </ins> nowTimestamp := nc.now() for _, nodeConditionType := range remainingNodeConditionTypes { _, currentCondition := nodeutil.GetNodeCondition(&node.Status, nodeConditionType)


return 0 } <del> // This function is expected to get a slice of NodeReadyConditions for all Nodes in a given zone. </del> <ins> // ComputeZoneState returns a slice of NodeReadyConditions for all Nodes in a given zone. </ins> // The zone is considered: // - fullyDisrupted if there're no Ready Nodes, // - partiallyDisrupted if at least than nc.unhealthyZoneThreshold percent of Nodes are not Ready,


"math" "time" <del> meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" </del> "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/instrumentation"


nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items maxPodCount := 10 <del> jobDuration := 1 * time.Hour </del> <ins> jobDuration := 30 * time.Minute </ins> linesPerPodPerSecond := 100 <del> testDuration := 21 * time.Hour </del> <ins> // TODO(crassirostris): Increase to 21 hrs testDuration := 3 * time.Hour </ins> ingestionTimeout := testDuration + 30*time.Minute allowedRestarts := int(math.Ceil(float64(testDuration) / float64(time.Hour) * maxAllowedRestartsPerHour))


podRunCount := maxPodCount*(int(testDuration/jobDuration)-1) + 1 linesPerPod := linesPerPodPerSecond * int(jobDuration.Seconds()) <ins> // pods is a flat array of all pods to be run and to expect in Stackdriver. </ins> pods := []*loggingPod{} <ins> // podsByRun is a two-dimensional array of pods, first dimension is the run // index, the second dimension is the node index. Since we want to create // an equal load on all nodes, for the same run we have one pod per node. podsByRun := [][]*loggingPod{} </ins> for runIdx := 0; runIdx < podRunCount; runIdx++ { <ins> podsInRun := []*loggingPod{} </ins> for nodeIdx, node := range nodes { podName := fmt.Sprintf("job-logs-generator-%d-%d-%d-%d", maxPodCount, linesPerPod, runIdx, nodeIdx) <del> pods = append(pods, newLoggingPod(podName, node.Name, linesPerPod, jobDuration)) </del> <ins> pod := newLoggingPod(podName, node.Name, linesPerPod, jobDuration) pods = append(pods, pod) podsInRun = append(podsInRun, pod) </ins> } <ins> podsByRun = append(podsByRun, podsInRun) </ins> } By("Running short-living pods") go func() { <del> for _, pod := range pods { pod.Start(f) </del> <ins> for runIdx := 0; runIdx < podRunCount; runIdx++ { // Starting one pod on each node. for _, pod := range podsByRun[runIdx] { pod.Start(f) } </ins> time.Sleep(podRunDelay) <del> defer f.PodClient().Delete(pod.Name, &meta_v1.DeleteOptions{}) </del> } // Waiting until the last pod has completed time.Sleep(jobDuration - podRunDelay + lastPodIngestionSlack)


return nil } <del> list, err := c.lister.PodPresets(pod.GetNamespace()).List(labels.Everything()) </del> <ins> list, err := c.lister.PodPresets(a.GetNamespace()).List(labels.Everything()) </ins> // Ignore if exclusion annotation is present if podAnnotations := pod.GetAnnotations(); podAnnotations != nil {


} } <ins> func TestAdmitEmptyPodNamespace(t *testing.T) { containerName := "container" pod := &api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "mypod", Labels: map[string]string{ "security": "S2", }, }, Spec: api.PodSpec{ Containers: []api.Container{ { Name: containerName, Env:  []api.EnvVar{{Name: "abc", Value: "value2"}, {Name: "ABCD", Value: "value3"}}, }, }, }, } pip := &settings.PodPreset{ ObjectMeta: v1.ObjectMeta{ Name:      "hello", Namespace: "different", // (pod will be submitted to namespace 'namespace') }, Spec: settings.PodPresetSpec{ Selector: v1.LabelSelector{ MatchExpressions: []v1.LabelSelectorRequirement{ { Key:      "security", Operator: v1.LabelSelectorOpIn, Values:   []string{"S2"}, }, }, }, Volumes: []api.Volume{{Name: "vol", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}}}, Env:     []api.EnvVar{{Name: "abcd", Value: "value"}, {Name: "ABC", Value: "value"}}, EnvFrom: []api.EnvFromSource{ { ConfigMapRef: &api.ConfigMapEnvSource{ LocalObjectReference: api.LocalObjectReference{Name: "abc"}, }, }, { Prefix: "pre_", ConfigMapRef: &api.ConfigMapEnvSource{ LocalObjectReference: api.LocalObjectReference{Name: "abc"}, }, }, }, }, } originalPod, err := api.Scheme.Copy(pod) if err != nil { t.Fatal(err) } err = admitPod(pod, pip) if err != nil { t.Fatal(err) } // verify PodSpec has not been mutated if !reflect.DeepEqual(pod, originalPod) { t.Fatalf("Expected pod spec of '%v' to be unchanged", pod.Name) } } </ins> func admitPod(pod *api.Pod, pip *settings.PodPreset) error { informerFactory := informers.NewSharedInformerFactory(nil, controller.NoResyncPeriodFunc()) store := informerFactory.Settings().InternalVersion().PodPresets().Informer().GetStore()


"spec.nodeName", "spec.restartPolicy", "spec.serviceAccountName", <ins> "spec.schedulerName", </ins> "status.phase", "status.hostIP", "status.podIP":


// amount of allocations needed to create the fields.Set. If you add any // field here or the number of object-meta related fields changes, this should // be adjusted. <del> podSpecificFieldsSet := make(fields.Set, 6) </del> <ins> podSpecificFieldsSet := make(fields.Set, 7) </ins> podSpecificFieldsSet["spec.nodeName"] = pod.Spec.NodeName podSpecificFieldsSet["spec.restartPolicy"] = string(pod.Spec.RestartPolicy) <ins> podSpecificFieldsSet["spec.schedulerName"] = string(pod.Spec.SchedulerName) </ins> podSpecificFieldsSet["status.phase"] = string(pod.Status.Phase) podSpecificFieldsSet["status.podIP"] = string(pod.Status.PodIP) return generic.AddObjectMetaFieldsSet(podSpecificFieldsSet, &pod.ObjectMeta, true)


}, { in: &api.Pod{ <ins> Spec: api.PodSpec{SchedulerName: "scheduler1"}, }, fieldSelector: fields.ParseSelectorOrDie("spec.schedulerName=scheduler1"), expectMatch:   true, }, { in: &api.Pod{ Spec: api.PodSpec{SchedulerName: "scheduler1"}, }, fieldSelector: fields.ParseSelectorOrDie("spec.schedulerName=scheduler2"), expectMatch:   false, }, { in: &api.Pod{ </ins> Status: api.PodStatus{Phase: api.PodRunning}, }, fieldSelector: fields.ParseSelectorOrDie("status.phase=Running"),


return zones, nil } <ins> // ListInstanceNames returns a string of instance names seperated by spaces. func (gce *GCECloud) ListInstanceNames(project, zone string) (string, error) { res, err := gce.service.Instances.List(project, zone).Fields("items(name)").Do() if err != nil { return "", err } var output string for _, item := range res.Items { output += item.Name + " " } return output, nil } // DeleteInstance deletes an instance specified by project, zone, and name func (gce *GCECloud) DeleteInstance(project, zone, name string) (*compute.Operation, error) { return gce.service.Instances.Delete(project, zone, name).Do() } </ins> // Implementation of Instances.CurrentNodeName func (gce *GCECloud) CurrentNodeName(hostname string) (types.NodeName, error) { return types.NodeName(hostname), nil


import ( "fmt" mathrand "math/rand" <del> "os/exec" </del> "strings" "time"


// Verify that disk shows up in node 0's volumeInUse list framework.ExpectNoError(waitForPDInVolumesInUse(nodeClient, diskName, host0Name, nodeStatusTimeout, true /* should exist*/)) <del> output, err := exec.Command("gcloud", "compute", "instances", "list", "--project="+framework.TestContext.CloudConfig.ProjectID).CombinedOutput() </del> <ins> gceCloud, err := framework.GetGCECloud() framework.ExpectNoError(err, fmt.Sprintf("Unable to create gcloud client err=%v", err)) output, err := gceCloud.ListInstanceNames(framework.TestContext.CloudConfig.ProjectID, framework.TestContext.CloudConfig.Zone) </ins> framework.ExpectNoError(err, fmt.Sprintf("Unable to get list of node instances err=%v output=%s", err, output)) Expect(true, strings.Contains(string(output), string(host0Name))) By("deleting host0") <ins> resp, err := gceCloud.DeleteInstance(framework.TestContext.CloudConfig.ProjectID, framework.TestContext.CloudConfig.Zone, string(host0Name)) framework.ExpectNoError(err, fmt.Sprintf("Failed to delete host0pod: err=%v response=%#v", err, resp)) </ins> <del> output, err = exec.Command("gcloud", "compute", "instances", "delete", string(host0Name), "--project="+framework.TestContext.CloudConfig.ProjectID, "--zone="+framework.TestContext.CloudConfig.Zone).CombinedOutput() framework.ExpectNoError(err, fmt.Sprintf("Failed to delete host0pod: err=%v output=%s", err, output)) output, err = exec.Command("gcloud", "compute", "instances", "list", "--project="+framework.TestContext.CloudConfig.ProjectID).CombinedOutput() </del> <ins> output, err = gceCloud.ListInstanceNames(framework.TestContext.CloudConfig.ProjectID, framework.TestContext.CloudConfig.Zone) </ins> framework.ExpectNoError(err, fmt.Sprintf("Unable to get list of node instances err=%v output=%s", err, output)) Expect(false, strings.Contains(string(output), string(host0Name)))


//NewRandomNameCustomResourceDefinition generates a CRD with random name to avoid name conflict in e2e tests func NewRandomNameCustomResourceDefinition(scope apiextensionsv1beta1.ResourceScope) *apiextensionsv1beta1.CustomResourceDefinition { <del> gName := names.SimpleNameGenerator.GenerateName("foo") </del> <ins> // ensure the singular doesn't end in an s for now gName := names.SimpleNameGenerator.GenerateName("foo") + "a" </ins> return &apiextensionsv1beta1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: gName + "s.mygroup.example.com"}, Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{


glog.V(4).Infof("Disk %s is already attached to instance %s", volumeID, instanceID) return volume.ID, nil } <del> glog.V(2).Infof("Disk %s is attached to a different instance (%s), detaching", volumeID, volume.AttachedServerId) err = os.DetachDisk(volume.AttachedServerId, volumeID) if err != nil { return "", err } </del> <ins> errmsg := fmt.Sprintf("Disk %s is attached to a different instance (%s)", volumeID, volume.AttachedServerId) glog.V(2).Infof(errmsg) return "", errors.New(errmsg) </ins> } startTime := time.Now()


go_library( name = "go_default_library", srcs = [ <ins> "conversion.go", </ins> "deepcopy.go", "doc.go", "generated.pb.go",


<ins> /* Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1alpha1 import "k8s.io/apimachinery/pkg/conversion" // Convert_Slice_string_To_v1alpha1_IncludeObjectPolicy allows converting a URL query parameter value func Convert_Slice_string_To_v1alpha1_IncludeObjectPolicy(input *[]string, out *IncludeObjectPolicy, s conversion.Scope) error { if len(*input) > 0 { *out = IncludeObjectPolicy((*input)[0]) } return nil } </ins>


&PartialObjectMetadataList{}, ) <ins> if err := scheme.AddConversionFuncs( Convert_Slice_string_To_v1alpha1_IncludeObjectPolicy, ); err != nil { panic(err) } </ins> // register manually. This usually goes through the SchemeBuilder, which we cannot use here. //scheme.AddGeneratedDeepCopyFuncs(GetGeneratedDeepCopyFuncs()...) }


}, }, }, <ins> { accept: runtime.ContentTypeJSON + ";as=Table;v=v1alpha1;g=meta.k8s.io", params: url.Values{"includeObject": []string{"Metadata"}}, expected: &metav1alpha1.Table{ TypeMeta: metav1.TypeMeta{Kind: "Table", APIVersion: "meta.k8s.io/v1alpha1"}, ColumnDefinitions: []metav1alpha1.TableColumnDefinition{ {Name: "Name", Type: "string", Description: metaDoc["name"]}, {Name: "Created At", Type: "date", Description: metaDoc["creationTimestamp"]}, }, Rows: []metav1alpha1.TableRow{ {Cells: []interface{}{"foo1", now.Time.UTC().Format(time.RFC3339)}, Object: runtime.RawExtension{Raw: encodedBody}}, }, }, }, </ins> } for i, test := range tests { u, err := url.Parse(server.URL + "/" + prefix + "/" + testGroupVersion.Group + "/" + testGroupVersion.Version + "/namespaces/default/simple/id")


} if test.statusCode != 0 { if resp.StatusCode != test.statusCode { <del> t.Errorf("%d: unexpected response: %#v", resp) </del> <ins> t.Errorf("%d: unexpected response: %#v", i, resp) </ins> } continue } if resp.StatusCode != http.StatusOK { <del> t.Errorf("%d: unexpected response: %#v", resp) </del> <ins> t.Errorf("%d: unexpected response: %#v", i, resp) </ins> } var itemOut metav1alpha1.Table <del> if _, err = extractBody(resp, &itemOut); err != nil { </del> <ins> body, err := extractBody(resp, &itemOut) if err != nil { </ins> t.Fatal(err) } if !reflect.DeepEqual(test.expected, &itemOut) { <ins> t.Log(body) </ins> t.Errorf("%d: did not match: %s", i, diff.ObjectReflectDiff(test.expected, &itemOut)) } }


t.Errorf("unexpected error: %v", err) } if len(times) != 0 { <del> t.Errorf("expected 0 start times, got: , got: %v", times) </del> <ins> t.Errorf("expected 0 start times, got: %v", times) </ins> } } {


t.Errorf("unexpected error: %v", err) } if len(times) != 1 { <del> t.Errorf("expected 2 start times, got: , got: %v", times) </del> <ins> t.Errorf("expected 1 start times, got: %v", times) </ins> } else if !times[0].Equal(T2) { t.Errorf("expected: %v, got: %v", T1, times[0]) }


t.Errorf("unexpected error: %v", err) } if len(times) != 2 { <del> t.Errorf("expected 2 start times, got: , got: %v", times) </del> <ins> t.Errorf("expected 2 start times, got: %v", times) </ins> } else { if !times[0].Equal(T1) { t.Errorf("expected: %v, got: %v", T1, times[0])


set = obj.(*apps.StatefulSet) } if set.Status.Replicas != 3 { <del> t.Error("Falied to scale statefulset to 3 replicas") </del> <ins> t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) </ins> } }


set = obj.(*apps.StatefulSet) } if set.Status.Replicas != 3 { <del> t.Error("Falied to scale statefulset to 3 replicas") </del> <ins> t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) </ins> } *set.Spec.Replicas = 0 if err := scaleDownStatefulSetController(set, ssc, spc); err != nil {


set = obj.(*apps.StatefulSet) } if set.Status.Replicas != 3 { <del> t.Error("Falied to scale statefulset to 3 replicas") </del> <ins> t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) </ins> } pods, err := spc.addTerminatingPod(set, 3) if err != nil {


set = obj.(*apps.StatefulSet) } if set.Status.Replicas != 0 { <del> t.Error("Falied to scale statefulset to 3 replicas") </del> <ins> t.Errorf("set.Status.Replicas = %v; want 0", set.Status.Replicas) </ins> } }


set = obj.(*apps.StatefulSet) } if set.Status.Replicas != 3 { <del> t.Error("Falied to scale statefulset to 3 replicas") </del> <ins> t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) </ins> } *set.Spec.Replicas = 5 fakeResourceVersion(set)


} func sleepUpTo(d time.Duration) { <del> time.Sleep(time.Duration(rand.Int63n(d.Nanoseconds()))) </del> <ins> if d.Nanoseconds() > 0 { time.Sleep(time.Duration(rand.Int63n(d.Nanoseconds()))) } </ins> } func createAllResources(configs []testutils.RunObjectConfig, creatingTime time.Duration) {


dockercontainer "github.com/docker/docker/api/types/container" dockerfilters "github.com/docker/docker/api/types/filters" "github.com/golang/glog" <del> "k8s.io/api/core/v1" </del> runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" )


return 0 } <del> func (ds *dockerService) getSecurityOpts(containerName string, sandboxConfig *runtimeapi.PodSandboxConfig, separator rune) ([]string, error) { hasSeccompSetting := false annotations := sandboxConfig.GetAnnotations() if _, ok := annotations[v1.SeccompContainerAnnotationKeyPrefix+containerName]; !ok { _, hasSeccompSetting = annotations[v1.SeccompPodAnnotationKey] } else { hasSeccompSetting = true </del> <ins> func (ds *dockerService) getSecurityOpts(seccompProfile string, separator rune) ([]string, error) { if seccompProfile != "" { glog.Warningf("seccomp annotations are not supported on windows") </ins> } <del> if hasSeccompSetting { glog.Warningf("seccomp annotations found, but it is not supported on windows") } </del> return nil, nil }


existed = true } <ins> // If address exists, get it by IP, because name might be different. // This can specifically happen if the IP was changed from ephemeral to static, // which results in a new name for the IP. if existingIP != "" { addr, err := s.GetRegionAddressByIP(region, existingIP) if err != nil { return "", false, fmt.Errorf("error getting static IP address: %v", err) } return addr.Address, existed, nil } // Otherwise, get address by name </ins> addr, err := s.GetRegionAddress(name, region) if err != nil { return "", false, fmt.Errorf("error getting static IP address: %v", err)


if err != nil || !existed || ip != ipPrime { t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, gce, ipName, serviceName, gce.region, ip, ipPrime, existed, err, ip) } <ins> // Ensure call with different name ipName = "another-name-for-static-ip" ipPrime, existed, err = ensureStaticIP(gce, ipName, serviceName, gce.region, ip, cloud.NetworkTierDefault) if err != nil || !existed || ip != ipPrime { t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, gce, ipName, serviceName, gce.region, ip, ipPrime, existed, err, ip) } </ins> } func TestEnsureStaticIPWithTier(t *testing.T) {


--network ${NETWORK}  --region ${REGION}  --range ${NODE_IP_RANGE}  <del> --secondary-range "pods-default=${CLUSTER_IP_RANGE}" </del> <ins> --secondary-range "pods-default=${CLUSTER_IP_RANGE}"  --secondary-range "services-default=${SERVICE_CLUSTER_IP_RANGE}" </ins> echo "Created subnetwork ${IP_ALIAS_SUBNETWORK}" else if ! echo ${subnet} | grep --quiet secondaryIpRanges ${subnet}; then


exit 1 fi fi <del> # Services subnetwork. local subnet=$(gcloud beta compute networks subnets describe  --project "${PROJECT}"  --region ${REGION}  ${SERVICE_CLUSTER_IP_SUBNETWORK} 2>/dev/null) if [[ -z ${subnet} ]]; then if [[ ${SERVICE_CLUSTER_IP_SUBNETWORK} != ${INSTANCE_PREFIX}-subnet-services ]]; then echo "${color_red}Subnetwork ${NETWORK}:${SERVICE_CLUSTER_IP_SUBNETWORK} does not exist${color_norm}" exit 1 fi echo "Creating subnet for reserving service cluster IPs ${NETWORK}:${SERVICE_CLUSTER_IP_SUBNETWORK}" gcloud beta compute networks subnets create  ${SERVICE_CLUSTER_IP_SUBNETWORK}  --description "Automatically generated subnet for ${INSTANCE_PREFIX} cluster. This will be removed on cluster teardown."  --project "${PROJECT}"  --network ${NETWORK}  --region ${REGION}  --range ${SERVICE_CLUSTER_IP_RANGE} echo "Created subnetwork ${SERVICE_CLUSTER_IP_SUBNETWORK}" else echo "Subnet ${SERVICE_CLUSTER_IP_SUBNETWORK} already exists" fi </del> } function delete-firewall-rules() {


${IP_ALIAS_SUBNETWORK} fi fi <del> if [[ ${SERVICE_CLUSTER_IP_SUBNETWORK} == ${INSTANCE_PREFIX}-subnet-services ]]; then echo "Removing auto-created subnet ${NETWORK}:${SERVICE_CLUSTER_IP_SUBNETWORK}" if [[ -n $(gcloud beta compute networks subnets describe  --project "${PROJECT}"  --region ${REGION}  ${SERVICE_CLUSTER_IP_SUBNETWORK} 2>/dev/null) ]]; then gcloud --quiet beta compute networks subnets delete  --project "${PROJECT}"  --region ${REGION}  ${SERVICE_CLUSTER_IP_SUBNETWORK} fi fi </del> } # Generates SSL certificates for etcd cluster. Uses cfssl program.