Question:
The Kubelet REST end-point has turned into a complicated switch-case code block that is harder to maintain overtime. The Kubelet should move to a http.ServeMux pattern and register handler functions to a pattern.
I am working on a PR to submit in next couple of days that will make this change, so feel free to assign to me if possible.
Tagging because I asked for this in one of his PRs
This one is also under my radar. If no one had time, feel free to assign it to me too.
I have a branch in progress. Feel free to offer feedback on it when I submit. Sent from my iPhone

ServeLogs(w http.ResponseWriter, req *http.Request) } <ins> // NewServer initializes and configures a kubelet.Server object to handle HTTP requests func NewServer(host HostInterface, updates chan<- interface{}) Server { server := Server{ host:    host, updates: updates, mux:     http.NewServeMux(), } server.InstallDefaultHandlers() return server } // InstallDefaultHandlers registers the set of supported HTTP request patterns with the mux func (s *Server) InstallDefaultHandlers() { s.mux.HandleFunc("/healthz", s.handleHealth) s.mux.HandleFunc("/container", s.handleContainer) s.mux.HandleFunc("/containers", s.handleContainers) s.mux.HandleFunc("/podInfo", s.handlePodInfo) s.mux.HandleFunc("/stats/", s.handleStats) s.mux.HandleFunc("/logs/", s.handleLogs) s.mux.HandleFunc("/spec/", s.handleSpec) } // error serializes an error object into an HTTP response </ins> func (s *Server) error(w http.ResponseWriter, err error) { http.Error(w, fmt.Sprintf("Internal Error: %v", err), http.StatusInternalServerError) } <del> func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { defer httplog.MakeLogged(req, &w).StacktraceWhen( httplog.StatusIsNot( http.StatusOK, http.StatusNotFound, ), ).Log() </del> <ins> // handleHealth handles health checking requests against the Kubelet func (s *Server) handleHealth(w http.ResponseWriter, req *http.Request) { } // handleContainer handles container requests against the Kubelet func (s *Server) handleContainer(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() data, err := ioutil.ReadAll(req.Body) if err != nil { s.error(w, err) return } // This is to provide backward compatibility. It only supports a single manifest var pod Pod err = yaml.Unmarshal(data, &pod.Manifest) if err != nil { s.error(w, err) return } //TODO: sha1 of manifest? pod.Name = "1" s.updates <- PodUpdate{[]Pod{pod}, SET} } // handleContainers handles containers requests against the Kubelet func (s *Server) handleContainers(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() data, err := ioutil.ReadAll(req.Body) if err != nil { s.error(w, err) return } var manifests []api.ContainerManifest err = yaml.Unmarshal(data, &manifests) if err != nil { s.error(w, err) return } pods := make([]Pod, len(manifests)) for i := range manifests { pods[i].Name = fmt.Sprintf("%d", i+1) pods[i].Manifest = manifests[i] } s.updates <- PodUpdate{pods, SET} } // handlePodInfo handles podInfo requests against the Kubelet func (s *Server) handlePodInfo(w http.ResponseWriter, req *http.Request) { </ins> u, err := url.ParseRequestURI(req.RequestURI) if err != nil { s.error(w, err) return } <del> // TODO: use an http.ServeMux instead of a switch. switch { case u.Path == "/container" || u.Path == "/containers": defer req.Body.Close() data, err := ioutil.ReadAll(req.Body) if err != nil { s.error(w, err) return } if u.Path == "/container" { // This is to provide backward compatibility. It only supports a single manifest var pod Pod err = yaml.Unmarshal(data, &pod.Manifest) if err != nil { s.error(w, err) return } //TODO: sha1 of manifest? pod.Name = "1" s.updates <- PodUpdate{[]Pod{pod}, SET} } else if u.Path == "/containers" { var manifests []api.ContainerManifest err = yaml.Unmarshal(data, &manifests) if err != nil { s.error(w, err) return } pods := make([]Pod, len(manifests)) for i := range manifests { pods[i].Name = fmt.Sprintf("%d", i+1) pods[i].Manifest = manifests[i] } s.updates <- PodUpdate{pods, SET} } case u.Path == "/podInfo": podID := u.Query().Get("podID") if len(podID) == 0 { w.WriteHeader(http.StatusBadRequest) http.Error(w, "Missing 'podID=' query entry.", http.StatusBadRequest) return } // TODO: backwards compatibility with existing API, needs API change podFullName := GetPodFullName(&Pod{Name: podID, Namespace: "etcd"}) info, err := s.host.GetPodInfo(podFullName) if err == ErrNoContainersInPod { http.Error(w, "Pod does not exist", http.StatusNotFound) return } if err != nil { s.error(w, err) return } data, err := json.Marshal(info) if err != nil { s.error(w, err) return } w.WriteHeader(http.StatusOK) w.Header().Add("Content-type", "application/json") w.Write(data) case strings.HasPrefix(u.Path, "/stats"): s.serveStats(w, req) case strings.HasPrefix(u.Path, "/spec"): info, err := s.host.GetMachineInfo() if err != nil { s.error(w, err) return } data, err := json.Marshal(info) if err != nil { s.error(w, err) return } w.Header().Add("Content-type", "application/json") w.Write(data) case strings.HasPrefix(u.Path, "/logs/"): s.host.ServeLogs(w, req) default: if s.handler != nil { s.handler.ServeHTTP(w, req) } </del> <ins> podID := u.Query().Get("podID") if len(podID) == 0 { w.WriteHeader(http.StatusBadRequest) http.Error(w, "Missing 'podID=' query entry.", http.StatusBadRequest) return } // TODO: backwards compatibility with existing API, needs API change podFullName := GetPodFullName(&Pod{Name: podID, Namespace: "etcd"}) info, err := s.host.GetPodInfo(podFullName) if err == ErrNoContainersInPod { http.Error(w, "Pod does not exist", http.StatusNotFound) return </ins> } <ins> if err != nil { s.error(w, err) return } data, err := json.Marshal(info) if err != nil { s.error(w, err) return } w.WriteHeader(http.StatusOK) w.Header().Add("Content-type", "application/json") w.Write(data) } // handleStats handles stats requests against the Kubelet func (s *Server) handleStats(w http.ResponseWriter, req *http.Request) { s.serveStats(w, req) } // handleLogs handles logs requests against the Kubelet func (s *Server) handleLogs(w http.ResponseWriter, req *http.Request) { s.host.ServeLogs(w, req) } // handleSpec handles spec requests against the Kubelet func (s *Server) handleSpec(w http.ResponseWriter, req *http.Request) { info, err := s.host.GetMachineInfo() if err != nil { s.error(w, err) return } data, err := json.Marshal(info) if err != nil { s.error(w, err) return } w.Header().Add("Content-type", "application/json") w.Write(data) } // ServeHTTP responds to HTTP requests on the Kubelet func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { defer httplog.MakeLogged(req, &w).StacktraceWhen( httplog.StatusIsNot( http.StatusOK, http.StatusNotFound, ), ).Log() s.mux.ServeHTTP(w, req) </ins> } <ins> // serveStats implements stats logic </ins> func (s *Server) serveStats(w http.ResponseWriter, req *http.Request) { // /stats/<podfullname>/<containerName> components := strings.Split(strings.TrimPrefix(path.Clean(req.URL.Path), "/"), "/")


Question:
The Kubelet REST end-point has turned into a complicated switch-case code block that is harder to maintain overtime. The Kubelet should move to a http.ServeMux pattern and register handler functions to a pattern.
I am working on a PR to submit in next couple of days that will make this change, so feel free to assign to me if possible.
Tagging because I asked for this in one of his PRs
This one is also under my radar. If no one had time, feel free to assign it to me too.
I have a branch in progress. Feel free to offer feedback on it when I submit. Sent from my iPhone

} fw.updateReader = startReading(fw.updateChan) fw.fakeKubelet = &fakeKubelet{} <del> fw.serverUnderTest = &Server{ host:    fw.fakeKubelet, updates: fw.updateChan, } </del> <ins> server := NewServer(fw.fakeKubelet, fw.updateChan) fw.serverUnderTest = &server </ins> fw.testHTTPServer = httptest.NewServer(fw.serverUnderTest) return fw }


Question:
In the guestbook example, the redis slaves are instructed to connect to the redis-master using the proxy provided by the minion on which they reside. The minion is designated by its hostname "kubernetes-minion-N". The container cannot resolve this name to an IP address and so slave connections fail. 1) create vagrant cluster 2) create redis-master pod 3) create redis-master service 4) create redis-slave replicationController Verify redis-slave pod locations: list pods ID Image(s) Host Labels Status redis-master-2 dockerfile/redis kubernetes-minion-1/10.245.2.2 name=redis-master Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-1/10.245.2.2 name=redisslave,replicationController=redisSlaveController Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-3/10.245.2.4 name=redisslave,replicationController=redisSlaveController Running Log onto a minion containing a slave (note: vagrant hostnames are minion-N, not kubernetes-minion-N) Find the slave container docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES brendanburns/redis-slave:latest /bin/sh -c About an hour ago Up About an hour k8s--slave.---3839-11e4-a922- docker logs | tail [7] 09 Sep 17:27:08.428  Connecting to MASTER kubernetes-minion-1: [7] 09 Sep 17:27:08.430 # Unable to connect to MASTER: No such file or directory [7] 09 Sep 17:27:09.435  Connecting to MASTER kubernetes-minion-1: sudo nsenter -m -u -n -i -p -t $(docker inspect --format '{{ .State.Pid }}' ) /bin/bash [ root ]$ ping kubernetes-minion-1 ping: unknown host kubernetes-minion-1 [ root ]$ cat /etc/hosts 244.1.4 -3839-11e4-a922- 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters [ root ]$ cat nameserver 10.0.2.3 cat /proc/1/environ HOME=/rootPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binHOSTNAME=-3839-11e4-a922-0800279696e1REDISMASTERSERVICEPORT=10000REDISMASTERPORT=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCP=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCPPROTO=tcpREDISMASTERPORT6379TCPPORT=10000REDISMASTERPORT6379TCPADDR=kubernetes-minion-1SERVICE_HOST=kubernetes-
Tim - are you the right guy to handle this? I think we need to resolve the name of the service host and only give IPs to the minion in env variables. We can't assume the containers have the same DNS set up as the host system.
To be clear -- this example is for the checked in Vagrant stuff? If that is the case would be the guy? I'm trying to catch up on IRC discussions from earlier today.
I can look at this, but there are two possible solutions. The first solution is in the vagrant cluster (or any other local vm setup) we could run a dns name server on the kubernetes-master that can resolve each minion by host name. We would then pass a -dns option to the docker daemon to specify that name server to use in each container. The second solution is that we pass IP address instead of host name and make no assumption or override on the containers specified name server. Is there any obvious negative to this solution? Has much thought been done on the expected Nameserver used by containers spawned by our service? - if you can weigh in, I would be happy to resolve using either approach. My preference is solution two.
So, the vagrant case is a little special as there is no sane DNS set up by default. Another suggestion (3) would be to revert the stuff and just devolve to using ip addresses directly. I would do (3) before (2) before (1). But I'm cool with any of them. As for the nameserver of the containers that are spawned -- docker currently just takes the of the host and copies that into the container. It doesn't do that for . that is the core of the issue here. If/when we do offer some level of split horizon DNS (I'd love to) we can start exposing an extra layer of DNS on top of whatever is configured for the host. That is part of the "ip per service portal" stuff that is being talked about a lot right now.
If I do (3) there is a piece of code in pod storage that mangles my hostname where I return 10.245.x.x and it asks the cloud provider to resolve the IP address for "10". Since other providers appear to also be returning IP as host, I will look to change this so kubernetes makes no modifications to hostnames that it's given. Look for PR tomorrow.
you're right I'd missed that point. This really only applies to the Vagrant cluster and so a fix to the vagrant setup is sufficient. I also agree with your priorities. Injecting a line into /etc/hosts (if that's reasonable) is the simplest solution. Either reverting or allowing a switch to pass the minion public IP rather than hostname is next. Adding a DNS service to the vagrant master and forcing DNS resolution to the master host is most complicated and least preferable. Thanks both of you for helping me think this through better. Mark
I don't grok what the difference between 2 and 3 is? I sent a PR to docker a while back () to insert arbitrary lines to /etc/hosts. I still think it is a reasonable thing to want to do.
- difference between (2) and (3) is that in the past, the vagrant cloud provider was returning the IP address as the literal hostname for the minion. I had changed this after noticing that it had some ugly side-effects in the CLI, where a call to list pods would report pod ip addresses as IP / nil. The issue (which I now believe is a bug that I want to change independent of what we do here) is that the code at this line mangles the hostname returned by the cloudprovider: This makes using the IP address as a hostname (10.245.2.2) to have the ugly side-effect of the code then asking the cloudprovider for the IP address of hostname "10", and then the cloudprovider having no idea how to proceed. Independent of what is done in this issue, I want to submit a PR tomorrow to stop that code from manipulating the reported hostname that was provided by the cloudprovider before asking that same cloudprovider to return the IP address using a potentially different input value. so with that backlog (3) means revert vagrant cloudprovider to use IPAddress as hostname, which means when we pass in what we believe is hostname in this environment, it will just work because it is in-fact the IPAddress. (2) means do not pass in hostname but pass in IPAddress by changing more core k8s code. all that being said, there is an option (4) that I am intrigued by, - after following your PR through its final conclusion, it looks like as of docker 1.2.0 you can change the /etc/hosts file of a running container. this introduces an option where the kubelet injects the /etc/hosts file of minion into the running container so it has the /etc/hosts values of the minion. I would need to play with that support a little more to know pros/cons more. I think longer term, we need a plan on what nameserver our containers are expecting to reference, and if they are inherently different in some way. I have run into a number of times the issue where doing things like running docker in docker causes the nested container to default to the nameserver 8.8.8.8, which happens to be blocked on the Red Hat network ;-) For now to unblock I will probably pursue (3) and fix the hostname bug referenced here at the same time. It does make the CLI uglier when working with this environment, but not a huge deal, at least it will function.
Thanks both for the complete explanation of what's happening and for the suggested fixes. I think in the long run using hostnames in SERVICEHOST and ensuriing either through IP injection into container /etc/hosts or by providing DNS are the best options. I'm not sure if there might still be cases where resolution of SERVICEHOST would still be best done by presenting the IP address or not.

MASTER_NAME="${INSTANCE_PREFIX}-master" MASTER_TAG="${INSTANCE_PREFIX}-master" MINION_TAG="${INSTANCE_PREFIX}-minion" <del> MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) </del> <ins> # Unable to use hostnames yet because DNS is not in cluster, so we revert external look-up name to use the minion IP #MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) </ins> # IP LOCATIONS FOR INTERACTING WITH THE MINIONS MINION_IP_BASE="10.245.2." for (( i=0; i <${NUM_MINIONS}; i++)) do KUBE_MINION_IP_ADDRESSES[$i]="${MINION_IP_BASE}$[$i+2]" MINION_IP[$i]="${MINION_IP_BASE}$[$i+2]" <ins> MINION_NAMES[$i]="${MINION_IP[$i]}" </ins> VAGRANT_MINION_NAMES[$i]="minion-$[$i+1]" done


Question:
In the guestbook example, the redis slaves are instructed to connect to the redis-master using the proxy provided by the minion on which they reside. The minion is designated by its hostname "kubernetes-minion-N". The container cannot resolve this name to an IP address and so slave connections fail. 1) create vagrant cluster 2) create redis-master pod 3) create redis-master service 4) create redis-slave replicationController Verify redis-slave pod locations: list pods ID Image(s) Host Labels Status redis-master-2 dockerfile/redis kubernetes-minion-1/10.245.2.2 name=redis-master Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-1/10.245.2.2 name=redisslave,replicationController=redisSlaveController Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-3/10.245.2.4 name=redisslave,replicationController=redisSlaveController Running Log onto a minion containing a slave (note: vagrant hostnames are minion-N, not kubernetes-minion-N) Find the slave container docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES brendanburns/redis-slave:latest /bin/sh -c About an hour ago Up About an hour k8s--slave.---3839-11e4-a922- docker logs | tail [7] 09 Sep 17:27:08.428  Connecting to MASTER kubernetes-minion-1: [7] 09 Sep 17:27:08.430 # Unable to connect to MASTER: No such file or directory [7] 09 Sep 17:27:09.435  Connecting to MASTER kubernetes-minion-1: sudo nsenter -m -u -n -i -p -t $(docker inspect --format '{{ .State.Pid }}' ) /bin/bash [ root ]$ ping kubernetes-minion-1 ping: unknown host kubernetes-minion-1 [ root ]$ cat /etc/hosts 244.1.4 -3839-11e4-a922- 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters [ root ]$ cat nameserver 10.0.2.3 cat /proc/1/environ HOME=/rootPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binHOSTNAME=-3839-11e4-a922-0800279696e1REDISMASTERSERVICEPORT=10000REDISMASTERPORT=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCP=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCPPROTO=tcpREDISMASTERPORT6379TCPPORT=10000REDISMASTERPORT6379TCPADDR=kubernetes-minion-1SERVICE_HOST=kubernetes-
Tim - are you the right guy to handle this? I think we need to resolve the name of the service host and only give IPs to the minion in env variables. We can't assume the containers have the same DNS set up as the host system.
To be clear -- this example is for the checked in Vagrant stuff? If that is the case would be the guy? I'm trying to catch up on IRC discussions from earlier today.
I can look at this, but there are two possible solutions. The first solution is in the vagrant cluster (or any other local vm setup) we could run a dns name server on the kubernetes-master that can resolve each minion by host name. We would then pass a -dns option to the docker daemon to specify that name server to use in each container. The second solution is that we pass IP address instead of host name and make no assumption or override on the containers specified name server. Is there any obvious negative to this solution? Has much thought been done on the expected Nameserver used by containers spawned by our service? - if you can weigh in, I would be happy to resolve using either approach. My preference is solution two.
So, the vagrant case is a little special as there is no sane DNS set up by default. Another suggestion (3) would be to revert the stuff and just devolve to using ip addresses directly. I would do (3) before (2) before (1). But I'm cool with any of them. As for the nameserver of the containers that are spawned -- docker currently just takes the of the host and copies that into the container. It doesn't do that for . that is the core of the issue here. If/when we do offer some level of split horizon DNS (I'd love to) we can start exposing an extra layer of DNS on top of whatever is configured for the host. That is part of the "ip per service portal" stuff that is being talked about a lot right now.
If I do (3) there is a piece of code in pod storage that mangles my hostname where I return 10.245.x.x and it asks the cloud provider to resolve the IP address for "10". Since other providers appear to also be returning IP as host, I will look to change this so kubernetes makes no modifications to hostnames that it's given. Look for PR tomorrow.
you're right I'd missed that point. This really only applies to the Vagrant cluster and so a fix to the vagrant setup is sufficient. I also agree with your priorities. Injecting a line into /etc/hosts (if that's reasonable) is the simplest solution. Either reverting or allowing a switch to pass the minion public IP rather than hostname is next. Adding a DNS service to the vagrant master and forcing DNS resolution to the master host is most complicated and least preferable. Thanks both of you for helping me think this through better. Mark
I don't grok what the difference between 2 and 3 is? I sent a PR to docker a while back () to insert arbitrary lines to /etc/hosts. I still think it is a reasonable thing to want to do.
- difference between (2) and (3) is that in the past, the vagrant cloud provider was returning the IP address as the literal hostname for the minion. I had changed this after noticing that it had some ugly side-effects in the CLI, where a call to list pods would report pod ip addresses as IP / nil. The issue (which I now believe is a bug that I want to change independent of what we do here) is that the code at this line mangles the hostname returned by the cloudprovider: This makes using the IP address as a hostname (10.245.2.2) to have the ugly side-effect of the code then asking the cloudprovider for the IP address of hostname "10", and then the cloudprovider having no idea how to proceed. Independent of what is done in this issue, I want to submit a PR tomorrow to stop that code from manipulating the reported hostname that was provided by the cloudprovider before asking that same cloudprovider to return the IP address using a potentially different input value. so with that backlog (3) means revert vagrant cloudprovider to use IPAddress as hostname, which means when we pass in what we believe is hostname in this environment, it will just work because it is in-fact the IPAddress. (2) means do not pass in hostname but pass in IPAddress by changing more core k8s code. all that being said, there is an option (4) that I am intrigued by, - after following your PR through its final conclusion, it looks like as of docker 1.2.0 you can change the /etc/hosts file of a running container. this introduces an option where the kubelet injects the /etc/hosts file of minion into the running container so it has the /etc/hosts values of the minion. I would need to play with that support a little more to know pros/cons more. I think longer term, we need a plan on what nameserver our containers are expecting to reference, and if they are inherently different in some way. I have run into a number of times the issue where doing things like running docker in docker causes the nested container to default to the nameserver 8.8.8.8, which happens to be blocked on the Red Hat network ;-) For now to unblock I will probably pursue (3) and fix the hostname bug referenced here at the same time. It does make the CLI uglier when working with this environment, but not a huge deal, at least it will function.
Thanks both for the complete explanation of what's happening and for the suggested fixes. I think in the long run using hostnames in SERVICEHOST and ensuriing either through IP injection into container /etc/hosts or by providing DNS are the best options. I'm not sure if there might still be cases where resolution of SERVICEHOST would still be best done by presenting the IP address or not.

roles: - kubernetes-pool cbr-cidr: $MINION_IP_RANGE <ins> minion_ip: $MINION_IP </ins> EOF # we will run provision to update code each time we test, so we do not want to do salt install each time


Question:
In the guestbook example, the redis slaves are instructed to connect to the redis-master using the proxy provided by the minion on which they reside. The minion is designated by its hostname "kubernetes-minion-N". The container cannot resolve this name to an IP address and so slave connections fail. 1) create vagrant cluster 2) create redis-master pod 3) create redis-master service 4) create redis-slave replicationController Verify redis-slave pod locations: list pods ID Image(s) Host Labels Status redis-master-2 dockerfile/redis kubernetes-minion-1/10.245.2.2 name=redis-master Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-1/10.245.2.2 name=redisslave,replicationController=redisSlaveController Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-3/10.245.2.4 name=redisslave,replicationController=redisSlaveController Running Log onto a minion containing a slave (note: vagrant hostnames are minion-N, not kubernetes-minion-N) Find the slave container docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES brendanburns/redis-slave:latest /bin/sh -c About an hour ago Up About an hour k8s--slave.---3839-11e4-a922- docker logs | tail [7] 09 Sep 17:27:08.428  Connecting to MASTER kubernetes-minion-1: [7] 09 Sep 17:27:08.430 # Unable to connect to MASTER: No such file or directory [7] 09 Sep 17:27:09.435  Connecting to MASTER kubernetes-minion-1: sudo nsenter -m -u -n -i -p -t $(docker inspect --format '{{ .State.Pid }}' ) /bin/bash [ root ]$ ping kubernetes-minion-1 ping: unknown host kubernetes-minion-1 [ root ]$ cat /etc/hosts 244.1.4 -3839-11e4-a922- 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters [ root ]$ cat nameserver 10.0.2.3 cat /proc/1/environ HOME=/rootPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binHOSTNAME=-3839-11e4-a922-0800279696e1REDISMASTERSERVICEPORT=10000REDISMASTERPORT=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCP=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCPPROTO=tcpREDISMASTERPORT6379TCPPORT=10000REDISMASTERPORT6379TCPADDR=kubernetes-minion-1SERVICE_HOST=kubernetes-
Tim - are you the right guy to handle this? I think we need to resolve the name of the service host and only give IPs to the minion in env variables. We can't assume the containers have the same DNS set up as the host system.
To be clear -- this example is for the checked in Vagrant stuff? If that is the case would be the guy? I'm trying to catch up on IRC discussions from earlier today.
I can look at this, but there are two possible solutions. The first solution is in the vagrant cluster (or any other local vm setup) we could run a dns name server on the kubernetes-master that can resolve each minion by host name. We would then pass a -dns option to the docker daemon to specify that name server to use in each container. The second solution is that we pass IP address instead of host name and make no assumption or override on the containers specified name server. Is there any obvious negative to this solution? Has much thought been done on the expected Nameserver used by containers spawned by our service? - if you can weigh in, I would be happy to resolve using either approach. My preference is solution two.
So, the vagrant case is a little special as there is no sane DNS set up by default. Another suggestion (3) would be to revert the stuff and just devolve to using ip addresses directly. I would do (3) before (2) before (1). But I'm cool with any of them. As for the nameserver of the containers that are spawned -- docker currently just takes the of the host and copies that into the container. It doesn't do that for . that is the core of the issue here. If/when we do offer some level of split horizon DNS (I'd love to) we can start exposing an extra layer of DNS on top of whatever is configured for the host. That is part of the "ip per service portal" stuff that is being talked about a lot right now.
If I do (3) there is a piece of code in pod storage that mangles my hostname where I return 10.245.x.x and it asks the cloud provider to resolve the IP address for "10". Since other providers appear to also be returning IP as host, I will look to change this so kubernetes makes no modifications to hostnames that it's given. Look for PR tomorrow.
you're right I'd missed that point. This really only applies to the Vagrant cluster and so a fix to the vagrant setup is sufficient. I also agree with your priorities. Injecting a line into /etc/hosts (if that's reasonable) is the simplest solution. Either reverting or allowing a switch to pass the minion public IP rather than hostname is next. Adding a DNS service to the vagrant master and forcing DNS resolution to the master host is most complicated and least preferable. Thanks both of you for helping me think this through better. Mark
I don't grok what the difference between 2 and 3 is? I sent a PR to docker a while back () to insert arbitrary lines to /etc/hosts. I still think it is a reasonable thing to want to do.
- difference between (2) and (3) is that in the past, the vagrant cloud provider was returning the IP address as the literal hostname for the minion. I had changed this after noticing that it had some ugly side-effects in the CLI, where a call to list pods would report pod ip addresses as IP / nil. The issue (which I now believe is a bug that I want to change independent of what we do here) is that the code at this line mangles the hostname returned by the cloudprovider: This makes using the IP address as a hostname (10.245.2.2) to have the ugly side-effect of the code then asking the cloudprovider for the IP address of hostname "10", and then the cloudprovider having no idea how to proceed. Independent of what is done in this issue, I want to submit a PR tomorrow to stop that code from manipulating the reported hostname that was provided by the cloudprovider before asking that same cloudprovider to return the IP address using a potentially different input value. so with that backlog (3) means revert vagrant cloudprovider to use IPAddress as hostname, which means when we pass in what we believe is hostname in this environment, it will just work because it is in-fact the IPAddress. (2) means do not pass in hostname but pass in IPAddress by changing more core k8s code. all that being said, there is an option (4) that I am intrigued by, - after following your PR through its final conclusion, it looks like as of docker 1.2.0 you can change the /etc/hosts file of a running container. this introduces an option where the kubelet injects the /etc/hosts file of minion into the running container so it has the /etc/hosts values of the minion. I would need to play with that support a little more to know pros/cons more. I think longer term, we need a plan on what nameserver our containers are expecting to reference, and if they are inherently different in some way. I have run into a number of times the issue where doing things like running docker in docker causes the nested container to default to the nameserver 8.8.8.8, which happens to be blocked on the Red Hat network ;-) For now to unblock I will probably pursue (3) and fix the hostname bug referenced here at the same time. It does make the CLI uglier when working with this environment, but not a huge deal, at least it will function.
Thanks both for the complete explanation of what's happening and for the suggested fixes. I think in the long run using hostnames in SERVICEHOST and ensuriing either through IP injection into container /etc/hosts or by providing DNS are the best options. I'm not sure if there might still be cases where resolution of SERVICEHOST would still be best done by presenting the IP address or not.

} filteredMinions := v.saltMinionsByRole(minions, "kubernetes-pool") for _, minion := range filteredMinions { <del> fmt.Println("Minion: ", minion.Host, " , ", instance, " IP: ", minion.IP) if minion.Host == instance { </del> <ins> // Due to vagrant not running with a dedicated DNS setup, we return the IP address of a minion as its hostname at this time if minion.IP == instance { </ins> return net.ParseIP(minion.IP), nil } }


Question:
In the guestbook example, the redis slaves are instructed to connect to the redis-master using the proxy provided by the minion on which they reside. The minion is designated by its hostname "kubernetes-minion-N". The container cannot resolve this name to an IP address and so slave connections fail. 1) create vagrant cluster 2) create redis-master pod 3) create redis-master service 4) create redis-slave replicationController Verify redis-slave pod locations: list pods ID Image(s) Host Labels Status redis-master-2 dockerfile/redis kubernetes-minion-1/10.245.2.2 name=redis-master Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-1/10.245.2.2 name=redisslave,replicationController=redisSlaveController Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-3/10.245.2.4 name=redisslave,replicationController=redisSlaveController Running Log onto a minion containing a slave (note: vagrant hostnames are minion-N, not kubernetes-minion-N) Find the slave container docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES brendanburns/redis-slave:latest /bin/sh -c About an hour ago Up About an hour k8s--slave.---3839-11e4-a922- docker logs | tail [7] 09 Sep 17:27:08.428  Connecting to MASTER kubernetes-minion-1: [7] 09 Sep 17:27:08.430 # Unable to connect to MASTER: No such file or directory [7] 09 Sep 17:27:09.435  Connecting to MASTER kubernetes-minion-1: sudo nsenter -m -u -n -i -p -t $(docker inspect --format '{{ .State.Pid }}' ) /bin/bash [ root ]$ ping kubernetes-minion-1 ping: unknown host kubernetes-minion-1 [ root ]$ cat /etc/hosts 244.1.4 -3839-11e4-a922- 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters [ root ]$ cat nameserver 10.0.2.3 cat /proc/1/environ HOME=/rootPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binHOSTNAME=-3839-11e4-a922-0800279696e1REDISMASTERSERVICEPORT=10000REDISMASTERPORT=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCP=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCPPROTO=tcpREDISMASTERPORT6379TCPPORT=10000REDISMASTERPORT6379TCPADDR=kubernetes-minion-1SERVICE_HOST=kubernetes-
Tim - are you the right guy to handle this? I think we need to resolve the name of the service host and only give IPs to the minion in env variables. We can't assume the containers have the same DNS set up as the host system.
To be clear -- this example is for the checked in Vagrant stuff? If that is the case would be the guy? I'm trying to catch up on IRC discussions from earlier today.
I can look at this, but there are two possible solutions. The first solution is in the vagrant cluster (or any other local vm setup) we could run a dns name server on the kubernetes-master that can resolve each minion by host name. We would then pass a -dns option to the docker daemon to specify that name server to use in each container. The second solution is that we pass IP address instead of host name and make no assumption or override on the containers specified name server. Is there any obvious negative to this solution? Has much thought been done on the expected Nameserver used by containers spawned by our service? - if you can weigh in, I would be happy to resolve using either approach. My preference is solution two.
So, the vagrant case is a little special as there is no sane DNS set up by default. Another suggestion (3) would be to revert the stuff and just devolve to using ip addresses directly. I would do (3) before (2) before (1). But I'm cool with any of them. As for the nameserver of the containers that are spawned -- docker currently just takes the of the host and copies that into the container. It doesn't do that for . that is the core of the issue here. If/when we do offer some level of split horizon DNS (I'd love to) we can start exposing an extra layer of DNS on top of whatever is configured for the host. That is part of the "ip per service portal" stuff that is being talked about a lot right now.
If I do (3) there is a piece of code in pod storage that mangles my hostname where I return 10.245.x.x and it asks the cloud provider to resolve the IP address for "10". Since other providers appear to also be returning IP as host, I will look to change this so kubernetes makes no modifications to hostnames that it's given. Look for PR tomorrow.
you're right I'd missed that point. This really only applies to the Vagrant cluster and so a fix to the vagrant setup is sufficient. I also agree with your priorities. Injecting a line into /etc/hosts (if that's reasonable) is the simplest solution. Either reverting or allowing a switch to pass the minion public IP rather than hostname is next. Adding a DNS service to the vagrant master and forcing DNS resolution to the master host is most complicated and least preferable. Thanks both of you for helping me think this through better. Mark
I don't grok what the difference between 2 and 3 is? I sent a PR to docker a while back () to insert arbitrary lines to /etc/hosts. I still think it is a reasonable thing to want to do.
- difference between (2) and (3) is that in the past, the vagrant cloud provider was returning the IP address as the literal hostname for the minion. I had changed this after noticing that it had some ugly side-effects in the CLI, where a call to list pods would report pod ip addresses as IP / nil. The issue (which I now believe is a bug that I want to change independent of what we do here) is that the code at this line mangles the hostname returned by the cloudprovider: This makes using the IP address as a hostname (10.245.2.2) to have the ugly side-effect of the code then asking the cloudprovider for the IP address of hostname "10", and then the cloudprovider having no idea how to proceed. Independent of what is done in this issue, I want to submit a PR tomorrow to stop that code from manipulating the reported hostname that was provided by the cloudprovider before asking that same cloudprovider to return the IP address using a potentially different input value. so with that backlog (3) means revert vagrant cloudprovider to use IPAddress as hostname, which means when we pass in what we believe is hostname in this environment, it will just work because it is in-fact the IPAddress. (2) means do not pass in hostname but pass in IPAddress by changing more core k8s code. all that being said, there is an option (4) that I am intrigued by, - after following your PR through its final conclusion, it looks like as of docker 1.2.0 you can change the /etc/hosts file of a running container. this introduces an option where the kubelet injects the /etc/hosts file of minion into the running container so it has the /etc/hosts values of the minion. I would need to play with that support a little more to know pros/cons more. I think longer term, we need a plan on what nameserver our containers are expecting to reference, and if they are inherently different in some way. I have run into a number of times the issue where doing things like running docker in docker causes the nested container to default to the nameserver 8.8.8.8, which happens to be blocked on the Red Hat network ;-) For now to unblock I will probably pursue (3) and fix the hostname bug referenced here at the same time. It does make the CLI uglier when working with this environment, but not a huge deal, at least it will function.
Thanks both for the complete explanation of what's happening and for the suggested fixes. I think in the long run using hostnames in SERVICEHOST and ensuriing either through IP injection into container /etc/hosts or by providing DNS are the best options. I'm not sure if there might still be cases where resolution of SERVICEHOST would still be best done by presenting the IP address or not.

filteredMinions := v.saltMinionsByRole(minions, "kubernetes-pool") var instances []string for _, instance := range filteredMinions { <del> instances = append(instances, instance.Host) </del> <ins> // With no dedicated DNS setup in cluster, IP address is used as hostname instances = append(instances, instance.IP) </ins> } return instances, nil


Question:
In the guestbook example, the redis slaves are instructed to connect to the redis-master using the proxy provided by the minion on which they reside. The minion is designated by its hostname "kubernetes-minion-N". The container cannot resolve this name to an IP address and so slave connections fail. 1) create vagrant cluster 2) create redis-master pod 3) create redis-master service 4) create redis-slave replicationController Verify redis-slave pod locations: list pods ID Image(s) Host Labels Status redis-master-2 dockerfile/redis kubernetes-minion-1/10.245.2.2 name=redis-master Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-1/10.245.2.2 name=redisslave,replicationController=redisSlaveController Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-3/10.245.2.4 name=redisslave,replicationController=redisSlaveController Running Log onto a minion containing a slave (note: vagrant hostnames are minion-N, not kubernetes-minion-N) Find the slave container docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES brendanburns/redis-slave:latest /bin/sh -c About an hour ago Up About an hour k8s--slave.---3839-11e4-a922- docker logs | tail [7] 09 Sep 17:27:08.428  Connecting to MASTER kubernetes-minion-1: [7] 09 Sep 17:27:08.430 # Unable to connect to MASTER: No such file or directory [7] 09 Sep 17:27:09.435  Connecting to MASTER kubernetes-minion-1: sudo nsenter -m -u -n -i -p -t $(docker inspect --format '{{ .State.Pid }}' ) /bin/bash [ root ]$ ping kubernetes-minion-1 ping: unknown host kubernetes-minion-1 [ root ]$ cat /etc/hosts 244.1.4 -3839-11e4-a922- 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters [ root ]$ cat nameserver 10.0.2.3 cat /proc/1/environ HOME=/rootPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binHOSTNAME=-3839-11e4-a922-0800279696e1REDISMASTERSERVICEPORT=10000REDISMASTERPORT=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCP=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCPPROTO=tcpREDISMASTERPORT6379TCPPORT=10000REDISMASTERPORT6379TCPADDR=kubernetes-minion-1SERVICE_HOST=kubernetes-
Tim - are you the right guy to handle this? I think we need to resolve the name of the service host and only give IPs to the minion in env variables. We can't assume the containers have the same DNS set up as the host system.
To be clear -- this example is for the checked in Vagrant stuff? If that is the case would be the guy? I'm trying to catch up on IRC discussions from earlier today.
I can look at this, but there are two possible solutions. The first solution is in the vagrant cluster (or any other local vm setup) we could run a dns name server on the kubernetes-master that can resolve each minion by host name. We would then pass a -dns option to the docker daemon to specify that name server to use in each container. The second solution is that we pass IP address instead of host name and make no assumption or override on the containers specified name server. Is there any obvious negative to this solution? Has much thought been done on the expected Nameserver used by containers spawned by our service? - if you can weigh in, I would be happy to resolve using either approach. My preference is solution two.
So, the vagrant case is a little special as there is no sane DNS set up by default. Another suggestion (3) would be to revert the stuff and just devolve to using ip addresses directly. I would do (3) before (2) before (1). But I'm cool with any of them. As for the nameserver of the containers that are spawned -- docker currently just takes the of the host and copies that into the container. It doesn't do that for . that is the core of the issue here. If/when we do offer some level of split horizon DNS (I'd love to) we can start exposing an extra layer of DNS on top of whatever is configured for the host. That is part of the "ip per service portal" stuff that is being talked about a lot right now.
If I do (3) there is a piece of code in pod storage that mangles my hostname where I return 10.245.x.x and it asks the cloud provider to resolve the IP address for "10". Since other providers appear to also be returning IP as host, I will look to change this so kubernetes makes no modifications to hostnames that it's given. Look for PR tomorrow.
you're right I'd missed that point. This really only applies to the Vagrant cluster and so a fix to the vagrant setup is sufficient. I also agree with your priorities. Injecting a line into /etc/hosts (if that's reasonable) is the simplest solution. Either reverting or allowing a switch to pass the minion public IP rather than hostname is next. Adding a DNS service to the vagrant master and forcing DNS resolution to the master host is most complicated and least preferable. Thanks both of you for helping me think this through better. Mark
I don't grok what the difference between 2 and 3 is? I sent a PR to docker a while back () to insert arbitrary lines to /etc/hosts. I still think it is a reasonable thing to want to do.
- difference between (2) and (3) is that in the past, the vagrant cloud provider was returning the IP address as the literal hostname for the minion. I had changed this after noticing that it had some ugly side-effects in the CLI, where a call to list pods would report pod ip addresses as IP / nil. The issue (which I now believe is a bug that I want to change independent of what we do here) is that the code at this line mangles the hostname returned by the cloudprovider: This makes using the IP address as a hostname (10.245.2.2) to have the ugly side-effect of the code then asking the cloudprovider for the IP address of hostname "10", and then the cloudprovider having no idea how to proceed. Independent of what is done in this issue, I want to submit a PR tomorrow to stop that code from manipulating the reported hostname that was provided by the cloudprovider before asking that same cloudprovider to return the IP address using a potentially different input value. so with that backlog (3) means revert vagrant cloudprovider to use IPAddress as hostname, which means when we pass in what we believe is hostname in this environment, it will just work because it is in-fact the IPAddress. (2) means do not pass in hostname but pass in IPAddress by changing more core k8s code. all that being said, there is an option (4) that I am intrigued by, - after following your PR through its final conclusion, it looks like as of docker 1.2.0 you can change the /etc/hosts file of a running container. this introduces an option where the kubelet injects the /etc/hosts file of minion into the running container so it has the /etc/hosts values of the minion. I would need to play with that support a little more to know pros/cons more. I think longer term, we need a plan on what nameserver our containers are expecting to reference, and if they are inherently different in some way. I have run into a number of times the issue where doing things like running docker in docker causes the nested container to default to the nameserver 8.8.8.8, which happens to be blocked on the Red Hat network ;-) For now to unblock I will probably pursue (3) and fix the hostname bug referenced here at the same time. It does make the CLI uglier when working with this environment, but not a huge deal, at least it will function.
Thanks both for the complete explanation of what's happening and for the suggested fixes. I think in the long run using hostnames in SERVICEHOST and ensuriing either through IP injection into container /etc/hosts or by providing DNS are the best options. I'm not sure if there might still be cases where resolution of SERVICEHOST would still be best done by presenting the IP address or not.

t.Fatalf("Incorrect number of instances returned") } <del> if instances[0] != "kubernetes-minion-1" { </del> <ins> // no DNS in vagrant cluster, so we return IP as hostname expectedInstanceHost := "10.245.2.2" expectedInstanceIP := "10.245.2.2" if instances[0] != expectedInstanceHost { </ins> t.Fatalf("Invalid instance returned") }


Question:
In the guestbook example, the redis slaves are instructed to connect to the redis-master using the proxy provided by the minion on which they reside. The minion is designated by its hostname "kubernetes-minion-N". The container cannot resolve this name to an IP address and so slave connections fail. 1) create vagrant cluster 2) create redis-master pod 3) create redis-master service 4) create redis-slave replicationController Verify redis-slave pod locations: list pods ID Image(s) Host Labels Status redis-master-2 dockerfile/redis kubernetes-minion-1/10.245.2.2 name=redis-master Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-1/10.245.2.2 name=redisslave,replicationController=redisSlaveController Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-3/10.245.2.4 name=redisslave,replicationController=redisSlaveController Running Log onto a minion containing a slave (note: vagrant hostnames are minion-N, not kubernetes-minion-N) Find the slave container docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES brendanburns/redis-slave:latest /bin/sh -c About an hour ago Up About an hour k8s--slave.---3839-11e4-a922- docker logs | tail [7] 09 Sep 17:27:08.428  Connecting to MASTER kubernetes-minion-1: [7] 09 Sep 17:27:08.430 # Unable to connect to MASTER: No such file or directory [7] 09 Sep 17:27:09.435  Connecting to MASTER kubernetes-minion-1: sudo nsenter -m -u -n -i -p -t $(docker inspect --format '{{ .State.Pid }}' ) /bin/bash [ root ]$ ping kubernetes-minion-1 ping: unknown host kubernetes-minion-1 [ root ]$ cat /etc/hosts 244.1.4 -3839-11e4-a922- 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters [ root ]$ cat nameserver 10.0.2.3 cat /proc/1/environ HOME=/rootPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binHOSTNAME=-3839-11e4-a922-0800279696e1REDISMASTERSERVICEPORT=10000REDISMASTERPORT=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCP=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCPPROTO=tcpREDISMASTERPORT6379TCPPORT=10000REDISMASTERPORT6379TCPADDR=kubernetes-minion-1SERVICE_HOST=kubernetes-
Tim - are you the right guy to handle this? I think we need to resolve the name of the service host and only give IPs to the minion in env variables. We can't assume the containers have the same DNS set up as the host system.
To be clear -- this example is for the checked in Vagrant stuff? If that is the case would be the guy? I'm trying to catch up on IRC discussions from earlier today.
I can look at this, but there are two possible solutions. The first solution is in the vagrant cluster (or any other local vm setup) we could run a dns name server on the kubernetes-master that can resolve each minion by host name. We would then pass a -dns option to the docker daemon to specify that name server to use in each container. The second solution is that we pass IP address instead of host name and make no assumption or override on the containers specified name server. Is there any obvious negative to this solution? Has much thought been done on the expected Nameserver used by containers spawned by our service? - if you can weigh in, I would be happy to resolve using either approach. My preference is solution two.
So, the vagrant case is a little special as there is no sane DNS set up by default. Another suggestion (3) would be to revert the stuff and just devolve to using ip addresses directly. I would do (3) before (2) before (1). But I'm cool with any of them. As for the nameserver of the containers that are spawned -- docker currently just takes the of the host and copies that into the container. It doesn't do that for . that is the core of the issue here. If/when we do offer some level of split horizon DNS (I'd love to) we can start exposing an extra layer of DNS on top of whatever is configured for the host. That is part of the "ip per service portal" stuff that is being talked about a lot right now.
If I do (3) there is a piece of code in pod storage that mangles my hostname where I return 10.245.x.x and it asks the cloud provider to resolve the IP address for "10". Since other providers appear to also be returning IP as host, I will look to change this so kubernetes makes no modifications to hostnames that it's given. Look for PR tomorrow.
you're right I'd missed that point. This really only applies to the Vagrant cluster and so a fix to the vagrant setup is sufficient. I also agree with your priorities. Injecting a line into /etc/hosts (if that's reasonable) is the simplest solution. Either reverting or allowing a switch to pass the minion public IP rather than hostname is next. Adding a DNS service to the vagrant master and forcing DNS resolution to the master host is most complicated and least preferable. Thanks both of you for helping me think this through better. Mark
I don't grok what the difference between 2 and 3 is? I sent a PR to docker a while back () to insert arbitrary lines to /etc/hosts. I still think it is a reasonable thing to want to do.
- difference between (2) and (3) is that in the past, the vagrant cloud provider was returning the IP address as the literal hostname for the minion. I had changed this after noticing that it had some ugly side-effects in the CLI, where a call to list pods would report pod ip addresses as IP / nil. The issue (which I now believe is a bug that I want to change independent of what we do here) is that the code at this line mangles the hostname returned by the cloudprovider: This makes using the IP address as a hostname (10.245.2.2) to have the ugly side-effect of the code then asking the cloudprovider for the IP address of hostname "10", and then the cloudprovider having no idea how to proceed. Independent of what is done in this issue, I want to submit a PR tomorrow to stop that code from manipulating the reported hostname that was provided by the cloudprovider before asking that same cloudprovider to return the IP address using a potentially different input value. so with that backlog (3) means revert vagrant cloudprovider to use IPAddress as hostname, which means when we pass in what we believe is hostname in this environment, it will just work because it is in-fact the IPAddress. (2) means do not pass in hostname but pass in IPAddress by changing more core k8s code. all that being said, there is an option (4) that I am intrigued by, - after following your PR through its final conclusion, it looks like as of docker 1.2.0 you can change the /etc/hosts file of a running container. this introduces an option where the kubelet injects the /etc/hosts file of minion into the running container so it has the /etc/hosts values of the minion. I would need to play with that support a little more to know pros/cons more. I think longer term, we need a plan on what nameserver our containers are expecting to reference, and if they are inherently different in some way. I have run into a number of times the issue where doing things like running docker in docker causes the nested container to default to the nameserver 8.8.8.8, which happens to be blocked on the Red Hat network ;-) For now to unblock I will probably pursue (3) and fix the hostname bug referenced here at the same time. It does make the CLI uglier when working with this environment, but not a huge deal, at least it will function.
Thanks both for the complete explanation of what's happening and for the suggested fixes. I think in the long run using hostnames in SERVICEHOST and ensuriing either through IP injection into container /etc/hosts or by providing DNS are the best options. I'm not sure if there might still be cases where resolution of SERVICEHOST would still be best done by presenting the IP address or not.

t.Fatalf("Unexpected error, should have returned a valid IP address: %s", err) } <del> if ip.String() != "10.245.2.2" { </del> <ins> if ip.String() != expectedInstanceIP { </ins> t.Fatalf("Invalid IP address returned") } }


Question:
In the guestbook example, the redis slaves are instructed to connect to the redis-master using the proxy provided by the minion on which they reside. The minion is designated by its hostname "kubernetes-minion-N". The container cannot resolve this name to an IP address and so slave connections fail. 1) create vagrant cluster 2) create redis-master pod 3) create redis-master service 4) create redis-slave replicationController Verify redis-slave pod locations: list pods ID Image(s) Host Labels Status redis-master-2 dockerfile/redis kubernetes-minion-1/10.245.2.2 name=redis-master Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-1/10.245.2.2 name=redisslave,replicationController=redisSlaveController Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-3/10.245.2.4 name=redisslave,replicationController=redisSlaveController Running Log onto a minion containing a slave (note: vagrant hostnames are minion-N, not kubernetes-minion-N) Find the slave container docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES brendanburns/redis-slave:latest /bin/sh -c About an hour ago Up About an hour k8s--slave.---3839-11e4-a922- docker logs | tail [7] 09 Sep 17:27:08.428  Connecting to MASTER kubernetes-minion-1: [7] 09 Sep 17:27:08.430 # Unable to connect to MASTER: No such file or directory [7] 09 Sep 17:27:09.435  Connecting to MASTER kubernetes-minion-1: sudo nsenter -m -u -n -i -p -t $(docker inspect --format '{{ .State.Pid }}' ) /bin/bash [ root ]$ ping kubernetes-minion-1 ping: unknown host kubernetes-minion-1 [ root ]$ cat /etc/hosts 244.1.4 -3839-11e4-a922- 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters [ root ]$ cat nameserver 10.0.2.3 cat /proc/1/environ HOME=/rootPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binHOSTNAME=-3839-11e4-a922-0800279696e1REDISMASTERSERVICEPORT=10000REDISMASTERPORT=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCP=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCPPROTO=tcpREDISMASTERPORT6379TCPPORT=10000REDISMASTERPORT6379TCPADDR=kubernetes-minion-1SERVICE_HOST=kubernetes-
Tim - are you the right guy to handle this? I think we need to resolve the name of the service host and only give IPs to the minion in env variables. We can't assume the containers have the same DNS set up as the host system.
To be clear -- this example is for the checked in Vagrant stuff? If that is the case would be the guy? I'm trying to catch up on IRC discussions from earlier today.
I can look at this, but there are two possible solutions. The first solution is in the vagrant cluster (or any other local vm setup) we could run a dns name server on the kubernetes-master that can resolve each minion by host name. We would then pass a -dns option to the docker daemon to specify that name server to use in each container. The second solution is that we pass IP address instead of host name and make no assumption or override on the containers specified name server. Is there any obvious negative to this solution? Has much thought been done on the expected Nameserver used by containers spawned by our service? - if you can weigh in, I would be happy to resolve using either approach. My preference is solution two.
So, the vagrant case is a little special as there is no sane DNS set up by default. Another suggestion (3) would be to revert the stuff and just devolve to using ip addresses directly. I would do (3) before (2) before (1). But I'm cool with any of them. As for the nameserver of the containers that are spawned -- docker currently just takes the of the host and copies that into the container. It doesn't do that for . that is the core of the issue here. If/when we do offer some level of split horizon DNS (I'd love to) we can start exposing an extra layer of DNS on top of whatever is configured for the host. That is part of the "ip per service portal" stuff that is being talked about a lot right now.
If I do (3) there is a piece of code in pod storage that mangles my hostname where I return 10.245.x.x and it asks the cloud provider to resolve the IP address for "10". Since other providers appear to also be returning IP as host, I will look to change this so kubernetes makes no modifications to hostnames that it's given. Look for PR tomorrow.
you're right I'd missed that point. This really only applies to the Vagrant cluster and so a fix to the vagrant setup is sufficient. I also agree with your priorities. Injecting a line into /etc/hosts (if that's reasonable) is the simplest solution. Either reverting or allowing a switch to pass the minion public IP rather than hostname is next. Adding a DNS service to the vagrant master and forcing DNS resolution to the master host is most complicated and least preferable. Thanks both of you for helping me think this through better. Mark
I don't grok what the difference between 2 and 3 is? I sent a PR to docker a while back () to insert arbitrary lines to /etc/hosts. I still think it is a reasonable thing to want to do.
- difference between (2) and (3) is that in the past, the vagrant cloud provider was returning the IP address as the literal hostname for the minion. I had changed this after noticing that it had some ugly side-effects in the CLI, where a call to list pods would report pod ip addresses as IP / nil. The issue (which I now believe is a bug that I want to change independent of what we do here) is that the code at this line mangles the hostname returned by the cloudprovider: This makes using the IP address as a hostname (10.245.2.2) to have the ugly side-effect of the code then asking the cloudprovider for the IP address of hostname "10", and then the cloudprovider having no idea how to proceed. Independent of what is done in this issue, I want to submit a PR tomorrow to stop that code from manipulating the reported hostname that was provided by the cloudprovider before asking that same cloudprovider to return the IP address using a potentially different input value. so with that backlog (3) means revert vagrant cloudprovider to use IPAddress as hostname, which means when we pass in what we believe is hostname in this environment, it will just work because it is in-fact the IPAddress. (2) means do not pass in hostname but pass in IPAddress by changing more core k8s code. all that being said, there is an option (4) that I am intrigued by, - after following your PR through its final conclusion, it looks like as of docker 1.2.0 you can change the /etc/hosts file of a running container. this introduces an option where the kubelet injects the /etc/hosts file of minion into the running container so it has the /etc/hosts values of the minion. I would need to play with that support a little more to know pros/cons more. I think longer term, we need a plan on what nameserver our containers are expecting to reference, and if they are inherently different in some way. I have run into a number of times the issue where doing things like running docker in docker causes the nested container to default to the nameserver 8.8.8.8, which happens to be blocked on the Red Hat network ;-) For now to unblock I will probably pursue (3) and fix the hostname bug referenced here at the same time. It does make the CLI uglier when working with this environment, but not a huge deal, at least it will function.
Thanks both for the complete explanation of what's happening and for the suggested fixes. I think in the long run using hostnames in SERVICEHOST and ensuriing either through IP injection into container /etc/hosts or by providing DNS are the best options. I'm not sure if there might still be cases where resolution of SERVICEHOST would still be best done by presenting the IP address or not.

import ( "fmt" <del> "strings" </del> "sync" "time"


Question:
In the guestbook example, the redis slaves are instructed to connect to the redis-master using the proxy provided by the minion on which they reside. The minion is designated by its hostname "kubernetes-minion-N". The container cannot resolve this name to an IP address and so slave connections fail. 1) create vagrant cluster 2) create redis-master pod 3) create redis-master service 4) create redis-slave replicationController Verify redis-slave pod locations: list pods ID Image(s) Host Labels Status redis-master-2 dockerfile/redis kubernetes-minion-1/10.245.2.2 name=redis-master Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-1/10.245.2.2 name=redisslave,replicationController=redisSlaveController Running -3839-11e4-a922- brendanburns/redis-slave kubernetes-minion-3/10.245.2.4 name=redisslave,replicationController=redisSlaveController Running Log onto a minion containing a slave (note: vagrant hostnames are minion-N, not kubernetes-minion-N) Find the slave container docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES brendanburns/redis-slave:latest /bin/sh -c About an hour ago Up About an hour k8s--slave.---3839-11e4-a922- docker logs | tail [7] 09 Sep 17:27:08.428  Connecting to MASTER kubernetes-minion-1: [7] 09 Sep 17:27:08.430 # Unable to connect to MASTER: No such file or directory [7] 09 Sep 17:27:09.435  Connecting to MASTER kubernetes-minion-1: sudo nsenter -m -u -n -i -p -t $(docker inspect --format '{{ .State.Pid }}' ) /bin/bash [ root ]$ ping kubernetes-minion-1 ping: unknown host kubernetes-minion-1 [ root ]$ cat /etc/hosts 244.1.4 -3839-11e4-a922- 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters [ root ]$ cat nameserver 10.0.2.3 cat /proc/1/environ HOME=/rootPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/binHOSTNAME=-3839-11e4-a922-0800279696e1REDISMASTERSERVICEPORT=10000REDISMASTERPORT=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCP=tcp://kubernetes-minion-1:10000REDISMASTERPORT6379TCPPROTO=tcpREDISMASTERPORT6379TCPPORT=10000REDISMASTERPORT6379TCPADDR=kubernetes-minion-1SERVICE_HOST=kubernetes-
Tim - are you the right guy to handle this? I think we need to resolve the name of the service host and only give IPs to the minion in env variables. We can't assume the containers have the same DNS set up as the host system.
To be clear -- this example is for the checked in Vagrant stuff? If that is the case would be the guy? I'm trying to catch up on IRC discussions from earlier today.
I can look at this, but there are two possible solutions. The first solution is in the vagrant cluster (or any other local vm setup) we could run a dns name server on the kubernetes-master that can resolve each minion by host name. We would then pass a -dns option to the docker daemon to specify that name server to use in each container. The second solution is that we pass IP address instead of host name and make no assumption or override on the containers specified name server. Is there any obvious negative to this solution? Has much thought been done on the expected Nameserver used by containers spawned by our service? - if you can weigh in, I would be happy to resolve using either approach. My preference is solution two.
So, the vagrant case is a little special as there is no sane DNS set up by default. Another suggestion (3) would be to revert the stuff and just devolve to using ip addresses directly. I would do (3) before (2) before (1). But I'm cool with any of them. As for the nameserver of the containers that are spawned -- docker currently just takes the of the host and copies that into the container. It doesn't do that for . that is the core of the issue here. If/when we do offer some level of split horizon DNS (I'd love to) we can start exposing an extra layer of DNS on top of whatever is configured for the host. That is part of the "ip per service portal" stuff that is being talked about a lot right now.
If I do (3) there is a piece of code in pod storage that mangles my hostname where I return 10.245.x.x and it asks the cloud provider to resolve the IP address for "10". Since other providers appear to also be returning IP as host, I will look to change this so kubernetes makes no modifications to hostnames that it's given. Look for PR tomorrow.
you're right I'd missed that point. This really only applies to the Vagrant cluster and so a fix to the vagrant setup is sufficient. I also agree with your priorities. Injecting a line into /etc/hosts (if that's reasonable) is the simplest solution. Either reverting or allowing a switch to pass the minion public IP rather than hostname is next. Adding a DNS service to the vagrant master and forcing DNS resolution to the master host is most complicated and least preferable. Thanks both of you for helping me think this through better. Mark
I don't grok what the difference between 2 and 3 is? I sent a PR to docker a while back () to insert arbitrary lines to /etc/hosts. I still think it is a reasonable thing to want to do.
- difference between (2) and (3) is that in the past, the vagrant cloud provider was returning the IP address as the literal hostname for the minion. I had changed this after noticing that it had some ugly side-effects in the CLI, where a call to list pods would report pod ip addresses as IP / nil. The issue (which I now believe is a bug that I want to change independent of what we do here) is that the code at this line mangles the hostname returned by the cloudprovider: This makes using the IP address as a hostname (10.245.2.2) to have the ugly side-effect of the code then asking the cloudprovider for the IP address of hostname "10", and then the cloudprovider having no idea how to proceed. Independent of what is done in this issue, I want to submit a PR tomorrow to stop that code from manipulating the reported hostname that was provided by the cloudprovider before asking that same cloudprovider to return the IP address using a potentially different input value. so with that backlog (3) means revert vagrant cloudprovider to use IPAddress as hostname, which means when we pass in what we believe is hostname in this environment, it will just work because it is in-fact the IPAddress. (2) means do not pass in hostname but pass in IPAddress by changing more core k8s code. all that being said, there is an option (4) that I am intrigued by, - after following your PR through its final conclusion, it looks like as of docker 1.2.0 you can change the /etc/hosts file of a running container. this introduces an option where the kubelet injects the /etc/hosts file of minion into the running container so it has the /etc/hosts values of the minion. I would need to play with that support a little more to know pros/cons more. I think longer term, we need a plan on what nameserver our containers are expecting to reference, and if they are inherently different in some way. I have run into a number of times the issue where doing things like running docker in docker causes the nested container to default to the nameserver 8.8.8.8, which happens to be blocked on the Red Hat network ;-) For now to unblock I will probably pursue (3) and fix the hostname bug referenced here at the same time. It does make the CLI uglier when working with this environment, but not a huge deal, at least it will function.
Thanks both for the complete explanation of what's happening and for the suggested fixes. I think in the long run using hostnames in SERVICEHOST and ensuriing either through IP injection into container /etc/hosts or by providing DNS are the best options. I'm not sure if there might still be cases where resolution of SERVICEHOST would still be best done by presenting the IP address or not.

if instances == nil || !ok { return "" } <del> ix := strings.Index(host, ".") if ix != -1 { host = host[:ix] } </del> addr, err := instances.IPAddress(host) if err != nil { glog.Errorf("Error getting instance IP: %#v", err)


Question:
Kubernetes use kubernetes/pause image to create network container. The problem is that whatever signals the pause container received, it will exit with code 2. Instead, with a normal exit, it should exit with code 0; while it exit with port binding issue, it should exit with code -1.
The port-binding issues will occur before the pause image gets invoked, but the pause image itself should exit 0.
I filed to docker to capture port-binding issue there. Port-binding is not my concern since we validate the port to avoid the conflict anyway. The concern I am having is PreStart we plan to introduce between docker container creation and start ().
There seems to be another issue - quick testing shows that if there is a conflict with a port allocated OUTSIDE of docker (e.g. the kube-proxy), Docker will accept the container and run it, but the host port doesn't work at all, and there's no way to tell except trawling logs. , Dawn Chen wrote:
Is there anything we could do to help go through, to eliminate the need for the network container?
Good catch. But following your steps above, my quick testing shows that docker container is running, but doesn't work at all. Also daemon logs doesn't have any error information. Will file another issue to docker: 1) # netstat -lnt Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State tcp 0 0 0.0.0.0:22 0.0.0.0: LISTEN tcp6 0 0 :::4194 ::: LISTEN tcp6 0 0 ::: ::: LISTEN tcp6 0 0 :::8080 ::: LISTEN tcp6 0 0 :::22 :::* LISTEN 2) # docker run -p :80 -d dockerimages/apache 6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11 3) # docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES dockerimages/apache:latest "apache2 -DFOREGROUN 6 seconds ago Up 5 seconds 0.0.0.0:-80/tcp jollynewton 4) # docker logs AH00558: apache2: Could not reliably determine the server's fully qualified domain name, using 10.244.3.11. Set the 'ServerName' directive globally to suppress this message AH00558: apache2: Could not reliably determine the server's fully qualified domain name, using 10.244.3.11. Set the 'ServerName' directive globally to suppress this message [Tue Sep 30 00:03:16. 2014] [core:warn] [pid 1:tid ] AH00098: pid file overwritten -- Unclean shutdown of previous Apache run? [Tue Sep 30 00:03:16. 2014] [mpmevent:notice] [pid 1:tid ] AH00489: Apache/2.4.7 (Ubuntu) configured -- resuming normal operations [Tue Sep 30 00:03:16. 2014] [core:notice] [pid 1:tid ] AH00094: Command line: 'apache2 -D FOREGROUND' 5) # cat | grep [] +job log(create, 6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11, dockerimages/apache:latest) [] -job log(create, 6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11, dockerimages/apache:latest) = OK (0) [info] POST /v1.14/containers/6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11/start [] +job start(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) [] +job allocateinterface(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) [] -job allocateinterface(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) = OK (0) [] +job allocateport(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) [] -job allocateport(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) = OK (0) [] +job log(start, 6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11, dockerimages/apache:latest) [] -job log(start, 6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11, dockerimages/apache:latest) = OK (0) [info] GET /containers/6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11/json [] +job containerinspect(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) [] -job start(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) = OK (0) [] -job containerinspect(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) = OK (0) [info] GET /containers/6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11/json [] +job containerinspect(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) [] -job containerinspect(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) = OK (0) [info] GET /v1.14/containers//json [] +job containerinspect() [] -job containerinspect() = OK (0) [info] GET /containers//logs?stderr=1&stdout=1&tail=all [] +job containerinspect() [] -job containerinspect() = OK (0) [] +job logs() [] -job logs() = OK (0)
I agreed we should help docker/docker go through, so that network container should be gone forever which makes our system, especially kubelet much simple. But that doesn't help with docker/docker and a new issue found by following.
have you tried this on docker master because it should have been fixed by , just want to prevent duplicate issues
I just rebuild docker from HEAD, and tested it with above steps I posted. The problem is fixed: 37a039245cd9faba6ae900cc463313eae77c9c9dce589616ec992a7e8eecb458 2014/09/30 16:29:20 Error response from daemon: Cannot start container 37a039245cd9faba6ae900cc463313eae77c9c9dce589616ec992a7e8eecb458: Bind for 0.0.0.0:8080 failed: port is already allocated But another issue I filed (docker/docker) is still exist. Thanks!
great, just wanted to make sure! and yes I can see if want to change the existing functionality for docker/docker

package main <del> import "syscall" </del> <ins> import ( "os" "os/signal" "syscall" ) </ins> func main() { <del> // Halts execution, waiting on signal. syscall.Pause() </del> <ins> c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM) // Block until a signal is received. <-c </ins> }


Question:
Kubernetes use kubernetes/pause image to create network container. The problem is that whatever signals the pause container received, it will exit with code 2. Instead, with a normal exit, it should exit with code 0; while it exit with port binding issue, it should exit with code -1.
The port-binding issues will occur before the pause image gets invoked, but the pause image itself should exit 0.
I filed to docker to capture port-binding issue there. Port-binding is not my concern since we validate the port to avoid the conflict anyway. The concern I am having is PreStart we plan to introduce between docker container creation and start ().
There seems to be another issue - quick testing shows that if there is a conflict with a port allocated OUTSIDE of docker (e.g. the kube-proxy), Docker will accept the container and run it, but the host port doesn't work at all, and there's no way to tell except trawling logs. , Dawn Chen wrote:
Is there anything we could do to help go through, to eliminate the need for the network container?
Good catch. But following your steps above, my quick testing shows that docker container is running, but doesn't work at all. Also daemon logs doesn't have any error information. Will file another issue to docker: 1) # netstat -lnt Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State tcp 0 0 0.0.0.0:22 0.0.0.0: LISTEN tcp6 0 0 :::4194 ::: LISTEN tcp6 0 0 ::: ::: LISTEN tcp6 0 0 :::8080 ::: LISTEN tcp6 0 0 :::22 :::* LISTEN 2) # docker run -p :80 -d dockerimages/apache 6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11 3) # docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES dockerimages/apache:latest "apache2 -DFOREGROUN 6 seconds ago Up 5 seconds 0.0.0.0:-80/tcp jollynewton 4) # docker logs AH00558: apache2: Could not reliably determine the server's fully qualified domain name, using 10.244.3.11. Set the 'ServerName' directive globally to suppress this message AH00558: apache2: Could not reliably determine the server's fully qualified domain name, using 10.244.3.11. Set the 'ServerName' directive globally to suppress this message [Tue Sep 30 00:03:16. 2014] [core:warn] [pid 1:tid ] AH00098: pid file overwritten -- Unclean shutdown of previous Apache run? [Tue Sep 30 00:03:16. 2014] [mpmevent:notice] [pid 1:tid ] AH00489: Apache/2.4.7 (Ubuntu) configured -- resuming normal operations [Tue Sep 30 00:03:16. 2014] [core:notice] [pid 1:tid ] AH00094: Command line: 'apache2 -D FOREGROUND' 5) # cat | grep [] +job log(create, 6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11, dockerimages/apache:latest) [] -job log(create, 6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11, dockerimages/apache:latest) = OK (0) [info] POST /v1.14/containers/6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11/start [] +job start(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) [] +job allocateinterface(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) [] -job allocateinterface(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) = OK (0) [] +job allocateport(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) [] -job allocateport(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) = OK (0) [] +job log(start, 6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11, dockerimages/apache:latest) [] -job log(start, 6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11, dockerimages/apache:latest) = OK (0) [info] GET /containers/6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11/json [] +job containerinspect(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) [] -job start(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) = OK (0) [] -job containerinspect(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) = OK (0) [info] GET /containers/6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11/json [] +job containerinspect(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) [] -job containerinspect(6f7309782be63f95dbc3f87bded69637e3299cd255668f1f171dc5eef28f5a11) = OK (0) [info] GET /v1.14/containers//json [] +job containerinspect() [] -job containerinspect() = OK (0) [info] GET /containers//logs?stderr=1&stdout=1&tail=all [] +job containerinspect() [] -job containerinspect() = OK (0) [] +job logs() [] -job logs() = OK (0)
I agreed we should help docker/docker go through, so that network container should be gone forever which makes our system, especially kubelet much simple. But that doesn't help with docker/docker and a new issue found by following.
have you tried this on docker master because it should have been fixed by , just want to prevent duplicate issues
I just rebuild docker from HEAD, and tested it with above steps I posted. The problem is fixed: 37a039245cd9faba6ae900cc463313eae77c9c9dce589616ec992a7e8eecb458 2014/09/30 16:29:20 Error response from daemon: Cannot start container 37a039245cd9faba6ae900cc463313eae77c9c9dce589616ec992a7e8eecb458: Bind for 0.0.0.0:8080 failed: port is already allocated But another issue I filed (docker/docker) is still exist. Thanks!
great, just wanted to make sure! and yes I can see if want to change the existing functionality for docker/docker

set -e set -x <ins> # Build the binary. </ins> go build --ldflags '-extldflags "-static" -s' pause.go <ins> # Run goupx to shrink binary size. go get github.com/pwaller/goupx goupx pause </ins>


Question: It's not well tested nor widely used; so it should either be removed or have tests and uses .

) var ( <del> configFile     = flag.String("configfile", "/tmp/proxy_config", "Configuration file for the proxy") </del> etcdServerList util.StringList etcdConfigFile = flag.String("etcd_config", "", "The config file for the etcd client. Mutually exclusive with -etcd_servers") bindAddress    = util.IP(net.ParseIP("0.0.0.0"))


Question: It's not well tested nor widely used; so it should either be removed or have tests and uses .

} } <del> // And create a configuration source that reads from a local file config.NewConfigSourceFile(*configFile, serviceConfig.Channel("file"), endpointsConfig.Channel("file")) glog.Infof("Using configuration file %s", *configFile) </del> loadBalancer := proxy.NewLoadBalancerRR() proxier := proxy.NewProxier(loadBalancer, net.IP(bindAddress)) // Wire proxier to handle changes to services


Question: It's not well tested nor widely used; so it should either be removed or have tests and uses .

**-bindaddress**="0.0.0.0" The address for the proxy server to serve on (set to 0.0.0.0 or "" for all interfaces) <del> **-configfile**="/tmp/proxy_config" Configuration file for the proxy </del> **-etcd_servers**=[] List of etcd servers to watch (http://ip:port), comma separated (optional)


Question: It's not well tested nor widely used; so it should either be removed or have tests and uses .

The address for the proxy server to serve on (set to 0.0.0.0 or "" for all interfaces) .PP <del> fB-configfilefP="/tmp/proxy_config" Configuration file for the proxy .PP </del> fB-etcd_serversfP=[] List of etcd servers to watch ( [la]http://ip:port[ra]), comma separated (optional)


Question: It's not well tested nor widely used; so it should either be removed or have tests and uses .

<del> /* Copyright 2014 Google Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Reads the configuration from the file. Example file for two services [nodejs & mysql] //{"Services": [ //   { //      "Name":"nodejs", //      "Port":10000, //      "Endpoints":["10.240.180.168:8000", "10.240.254.199:8000", "10.240.62.150:8000"] //   }, //   { //      "Name":"mysql", //      "Port":10001, //      "Endpoints":["10.240.180.168:9000", "10.240.254.199:9000", "10.240.62.150:9000"] //   } //] //} package config import ( "bytes" "encoding/json" "fmt" "io/ioutil" "reflect" "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/golang/glog" ) // serviceConfig is a deserialized form of the config file format which ConfigSourceFile accepts. // TODO: this is apparently untested; is it used? type serviceConfig struct { Services []struct { Name      string   `json: "name"` Port      int      `json: "port"` Endpoints []string `json: "endpoints"` } `json:"service"` } // ConfigSourceFile periodically reads service configurations in JSON from a file, and sends the services and endpoints defined in the file to the specified channels. type ConfigSourceFile struct { serviceChannel   chan ServiceUpdate endpointsChannel chan EndpointsUpdate filename         string } // NewConfigSourceFile creates a new ConfigSourceFile and let it immediately runs the created ConfigSourceFile in a goroutine. func NewConfigSourceFile(filename string, serviceChannel chan ServiceUpdate, endpointsChannel chan EndpointsUpdate) ConfigSourceFile { config := ConfigSourceFile{ filename:         filename, serviceChannel:   serviceChannel, endpointsChannel: endpointsChannel, } go config.Run() return config } // Run begins watching the config file. func (s ConfigSourceFile) Run() { glog.V(1).Infof("Watching file %s", s.filename) var lastData []byte var lastServices []api.Service var lastEndpoints []api.Endpoints sleep := 5 * time.Second // Used to avoid spamming the error log file, makes error logging edge triggered. hadSuccess := true for { data, err := ioutil.ReadFile(s.filename) if err != nil { msg := fmt.Sprintf("Couldn't read file: %s : %v", s.filename, err) if hadSuccess { glog.Error(msg) } else { glog.V(1).Info(msg) } hadSuccess = false time.Sleep(sleep) continue } hadSuccess = true if bytes.Equal(lastData, data) { time.Sleep(sleep) continue } lastData = data config := &serviceConfig{} if err = json.Unmarshal(data, config); err != nil { glog.Errorf("Couldn't unmarshal configuration from file : %s %v", data, err) continue } // Ok, we have a valid configuration, send to channel for // rejiggering. newServices := make([]api.Service, len(config.Services)) newEndpoints := make([]api.Endpoints, len(config.Services)) for i, service := range config.Services { newServices[i] = api.Service{TypeMeta: api.TypeMeta{ID: service.Name}, Port: service.Port} newEndpoints[i] = api.Endpoints{TypeMeta: api.TypeMeta{ID: service.Name}, Endpoints: service.Endpoints} } if !reflect.DeepEqual(lastServices, newServices) { serviceUpdate := ServiceUpdate{Op: SET, Services: newServices} s.serviceChannel <- serviceUpdate lastServices = newServices } if !reflect.DeepEqual(lastEndpoints, newEndpoints) { endpointsUpdate := EndpointsUpdate{Op: SET, Endpoints: newEndpoints} s.endpointsChannel <- endpointsUpdate lastEndpoints = newEndpoints } time.Sleep(sleep) } } </del>


Question:
Right now, we have a mixture of user documentation and developer documentation in the top-level directory. We should make it easier to find a just user documentation. Step 1 would be to move developer documentation into a subdir like docs/development or something like that. User docs would stay in the top-level docs dir.
move to

## Protocols for Collaborative Development <del> Please read [this doc](docs/collab.md) for information on how we're running development for the project. </del> <ins> Please read [this doc](docs/devel/collab.md) for information on how we're running development for the project. </ins> ## Adding dependencies


Question:
Right now, we have a mixture of user documentation and developer documentation in the top-level directory. We should make it easier to find a just user documentation. Step 1 would be to move developer documentation into a subdir like docs/development or something like that. User docs would stay in the top-level docs dir.
move to

* [Example of dynamic updates](examples/update-demo/README.md) * [Cluster monitoring with heapster and cAdvisor](https://github.com/GoogleCloudPlatform/heapster) * [Community projects](https://github.com/GoogleCloudPlatform/kubernetes/wiki/Kubernetes-Community) <del> * [Development guide](docs/development.md) </del> <ins> * [Development guide](docs/devel/development.md) </ins> Or fork and start hacking!


Question:
Right now, we have a mixture of user documentation and developer documentation in the top-level directory. We should make it easier to find a just user documentation. Step 1 would be to move developer documentation into a subdir like docs/development or something like that. User docs would stay in the top-level docs dir.
move to

<ins> # On Collaborative Development Kubernetes is open source, but many of the people working on it do so as their day job.  In order to avoid forcing people to be "at work" effectively 24/7, we want to establish some semi-formal protocols around development.  Hopefully these rules make things go more smoothly.  If you find that this is not the case, please complain loudly. ## Patches welcome First and foremost: as a potential contributor, your changes and ideas are welcome at any hour of the day or night, weekdays, weekends, and holidays.  Please do not ever hesitate to ask a question or send a PR. ## Timezones and calendars For the time being, most of the people working on this project are in the US and on Pacific time.  Any times mentioned henceforth will refer to this timezone.  Any references to "work days" will refer to the US calendar. ## Code reviews All changes must be code reviewed.  For non-maintainers this is obvious, since you can't commit anyway.  But even for maintainers, we want all changes to get at least one review, preferably from someone who knows the areas the change touches.  For non-trivial changes we may want two reviewers.  The primary reviewer will make this decision and nominate a second reviewer, if needed.  Except for trivial changes, PRs should sit for at least a 2 hours to allow for wider review. Most PRs will find reviewers organically.  If a maintainer intends to be the primary reviewer of a PR they should set themselves as the assignee on GitHub and say so in a reply to the PR.  Only the primary reviewer of a change should actually do the merge, except in rare cases (e.g. they are unavailable in a reasonable timeframe). If a PR has gone 2 work days without an owner emerging, please poke the PR thread and ask for a reviewer to be assigned. Except for rare cases, such as trivial changes (e.g. typos, comments) or emergencies (e.g. broken builds), maintainers should not merge their own changes. Expect reviewers to request that you avoid [common go style mistakes](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) in your PRs. ## Assigned reviews Maintainers can assign reviews to other maintainers, when appropriate. The assignee becomes the shepherd for that PR and is responsible for merging the PR once they are satisfied with it or else closing it.  The assignee might request reviews from non-maintainers. ## Merge hours Maintainers will do merges between the hours of 7:00 am Monday and 7:00 pm (19:00h) Friday.  PRs that arrive over the weekend or on holidays will only be merged if there is a very good reason for it and if the code review requirements have been met. There may be discussion an even approvals granted outside of the above hours, but merges will generally be deferred. ## Holds Any maintainer or core contributor who wants to review a PR but does not have time immediately may put a hold on a PR simply by saying so on the PR discussion and offering an ETA measured in single-digit days at most.  Any PR that has a hold shall not be merged until the person who requested the hold acks the review, withdraws their hold, or is overruled by a preponderance of maintainers. </ins>