@@ -1,6 +1,6 @@
 /* execute_cmd.c -- Execute a COMMAND structure. */

-/* Copyright (C) 1987-2015 Free Software Foundation, Inc.
+/* Copyright (C) 1987-2016 Free Software Foundation, Inc.

    This file is part of GNU Bash, the Bourne Again SHell.

@@ -836,7 +836,9 @@ execute_command_internal (command, asynchronous, pipe_in, pipe_out,
 #if !defined (JOB_CONTROL)
 	      /* Do not wait for asynchronous processes started from
 		 startup files. */
-	    if (last_made_pid != last_asynchronous_pid)
+	    if (last_made_pid != NO_PID && last_made_pid != last_asynchronous_pid)
+#else
+	    if (last_made_pid != NO_PID)
 #endif
 	    /* When executing a shell function that executes other
 	       commands, this causes the last simple command in
@@ -2118,6 +2120,7 @@ coproc_fdrestore (cp)
 void
 coproc_pidchk (pid, status)
      pid_t pid;
+     int status;
 {
   struct coproc *cp;

@@ -2149,18 +2152,48 @@ coproc_setvars (cp)
   SHELL_VAR *v;
   char *namevar, *t;
   int l;
+  WORD_DESC w;
 #if defined (ARRAY_VARS)
   arrayind_t ind;
 #endif

   if (cp->c_name == 0)
     return;

+  /* We could do more here but right now we only check the name, warn if it's
+     not a valid identifier, and refuse to create variables with invalid names
+     if a coproc with such a name is supplied. */
+  w.word = cp->c_name;
+  w.flags = 0;
+  if (check_identifier (&w, 1) == 0)
+    return;
+
   l = strlen (cp->c_name);
   namevar = xmalloc (l + 16);

 #if defined (ARRAY_VARS)
   v = find_variable (cp->c_name);
+
+  /* This is the same code as in find_or_make_array_variable */
+  if (v == 0)
+    {
+      v = find_variable_nameref_for_create (cp->c_name, 1);
+      if (v == INVALID_NAMEREF_VALUE)
+	return;
+      if (v && nameref_p (v))
+	{
+	  free (cp->c_name);
+	  cp->c_name = savestring (nameref_cell (v));
+	  v = make_new_array_variable (cp->c_name);
+	}
+    }
+
+  if (v && (readonly_p (v) || noassign_p (v)))
+    {
+      if (readonly_p (v))
+	err_readonly (cp->c_name);
+      return;
+    }
   if (v == 0)
     v = make_new_array_variable (cp->c_name);
   if (array_p (v) == 0)
@@ -2208,10 +2241,10 @@ coproc_unsetvars (cp)
   namevar = xmalloc (l + 16);

   sprintf (namevar, "%s_PID", cp->c_name);
-  unbind_variable (namevar);
+  unbind_variable_noref (namevar);

 #if defined (ARRAY_VARS)
-  unbind_variable (cp->c_name);
+  check_unbind_variable (cp->c_name);
 #else
   sprintf (namevar, "%s_READ", cp->c_name);
   unbind_variable (namevar);
@@ -2237,7 +2270,7 @@ execute_coproc (command, pipe_in, pipe_out, fds_to_close)
   /* XXX -- can be removed after changes to handle multiple coprocs */
 #if !MULTIPLE_COPROCS
   if (sh_coproc.c_pid != NO_PID)
-    internal_warning ("execute_coproc: coproc [%d:%s] still exists", sh_coproc.c_pid, sh_coproc.c_name);
+    internal_warning (_("execute_coproc: coproc [%d:%s] still exists"), sh_coproc.c_pid, sh_coproc.c_name);
   coproc_init (&sh_coproc);
 #endif

@@ -2468,7 +2501,7 @@ execute_pipeline (command, asynchronous, pipe_in, pipe_out, fds_to_close)
           lstdin = wait_for (lastpid);
         }
       else
-        lstdin = wait_for_single_pid (lastpid);		/* checks bgpids list */
+        lstdin = wait_for_single_pid (lastpid, 0);		/* checks bgpids list */
 #else
       lstdin = wait_for (lastpid);
 #endif
@@ -2732,17 +2765,24 @@ execute_for_command (for_command)

       this_command_name = (char *)NULL;
       /* XXX - special ksh93 for command index variable handling */
-      v = find_variable_last_nameref (identifier);
+      v = find_variable_last_nameref (identifier, 1);
       if (v && nameref_p (v))
-        {
-          v = bind_variable_value (v, list->word->word, 0);
-        }
+	{
+	  if (valid_nameref_value (list->word->word, 1) == 0)
+	    {
+	      sh_invalidid (list->word->word);
+	      v = 0;
+	    }
+	  else
+	    v = bind_variable_value (v, list->word->word, 0);
+	}
       else
-        v = bind_variable (identifier, list->word->word, 0);
-      if (readonly_p (v) || noassign_p (v))
+	v = bind_variable (identifier, list->word->word, 0);
+
+      if (v == 0 || readonly_p (v) || noassign_p (v))
 	{
 	  line_number = save_line_number;
-	  if (readonly_p (v) && interactive_shell == 0 && posixly_correct)
+	  if (v && readonly_p (v) && interactive_shell == 0 && posixly_correct)
 	    {
 	      last_command_exit_value = EXECUTION_FAILURE;
 	      jump_to_top_level (FORCE_EOF);
@@ -2755,6 +2795,10 @@ execute_for_command (for_command)
 	      return (EXECUTION_FAILURE);
 	    }
 	}
+
+      if (ifsname (identifier))
+	setifs (v);
+
       retval = execute_command (for_command->action);
       REAP ();
       QUIT;
@@ -3232,9 +3276,9 @@ execute_select_command (select_command)
 	}

       v = bind_variable (identifier, selection, 0);
-      if (readonly_p (v) || noassign_p (v))
+      if (v == 0 || readonly_p (v) || noassign_p (v))
 	{
-	  if (readonly_p (v) && interactive_shell == 0 && posixly_correct)
+	  if (v && readonly_p (v) && interactive_shell == 0 && posixly_correct)
 	    {
 	      last_command_exit_value = EXECUTION_FAILURE;
 	      jump_to_top_level (FORCE_EOF);
@@ -3765,7 +3809,8 @@ bind_lastarg (arg)
   if (arg == 0)
     arg = "";
   var = bind_variable ("_", arg, 0);
-  VUNSETATTR (var, att_exported);
+  if (var)
+    VUNSETATTR (var, att_exported);
 }

 /* Execute a null command.  Fork a subshell if the command uses pipes or is
@@ -5474,6 +5519,7 @@ shell_execve (command, args, env)
       if (check_binary_file (sample, sample_len))
 	{
 	  internal_error (_("%s: cannot execute binary file: %s"), command, strerror (i));
+	  errno = i;
 	  return (EX_BINARY_FILE);
 	}
     }


@@ -230,7 +230,7 @@ extern int set_tty_state __P((void));
 extern int job_exit_status __P((int));
 extern int job_exit_signal __P((int));

-extern int wait_for_single_pid __P((pid_t));
+extern int wait_for_single_pid __P((pid_t, int));
 extern void wait_for_background_pids __P((void));
 extern int wait_for __P((pid_t));
 extern int wait_for_job __P((int));


@@ -1,7 +1,7 @@
 /* rltty.c -- functions to prepare and restore the terminal for readline's
    use. */

-/* Copyright (C) 1992-2015 Free Software Foundation, Inc.
+/* Copyright (C) 1992-2016 Free Software Foundation, Inc.

    This file is part of the GNU Readline Library (Readline), a library
    for reading lines of text with interactive input and history editing.
@@ -714,6 +714,19 @@ rl_deprep_terminal ()
   _rl_release_sigint ();
 }
 #endif /* !NO_TTY_DRIVER */
+
+/* Set readline's idea of whether or not it is echoing output to the terminal,
+   returning the old value. */
+int
+rl_tty_set_echoing (u)
+     int u;
+{
+  int o;
+
+  o = _rl_echoing_p;
+  _rl_echoing_p = u;
+  return o;
+}

 /* **************************************************************** */
 /*								    */
@@ -876,6 +889,11 @@ _rl_bind_tty_special_chars (kmap, ttybuff)
 #  endif /* VLNEXT && TERMIOS_TTY_DRIVER */

 #  if defined (VWERASE) && defined (TERMIOS_TTY_DRIVER)
+#    if defined (VI_MODE)
+  if (rl_editing_mode == vi_mode)
+    SET_SPECIAL (VWERASE, rl_vi_unix_word_rubout);
+  else
+#    endif
   SET_SPECIAL (VWERASE, rl_unix_word_rubout);
 #  endif /* VWERASE && TERMIOS_TTY_DRIVER */
 }


@@ -1,6 +1,6 @@
 /* patchlevel.h -- current bash patch level */

-/* Copyright (C) 2001-2012 Free Software Foundation, Inc.
+/* Copyright (C) 2001-2016 Free Software Foundation, Inc.

    This file is part of GNU Bash, the Bourne Again SHell.


Prevent CRLF injections described in CVE-2019-12387

Author: markrwilliams

Reviewers: glyph

Fixes: ticket:9647

Twisted's HTTP client APIs were vulnerable to maliciously constructed
HTTP methods, hosts, and/or paths, URI components such as paths and
query parameters.  These vulnerabilities were beyond the header name
and value injection vulnerabilities addressed in:

https://twistedmatrix.com/trac/ticket/9420
https://github.com/twisted/twisted/pull/999/

The following client APIs will raise a ValueError if given a method,
host, or URI that includes newlines or other disallowed characters:

- twisted.web.client.Agent.request
- twisted.web.client.ProxyAgent.request
- twisted.web.client.Request.__init__
- twisted.web.client.Request.writeTo

ProxyAgent is patched separately from Agent because unlike other
agents (e.g. CookieAgent) it is not implemented as an Agent wrapper.

Request.__init__ checks its method and URI so that errors occur closer
to their originating input.  Request.method and Request.uri are both
public APIs, however, so Request.writeTo (via Request._writeHeaders)
also checks the validity of both before writing anything to the wire.

Additionally, the following deprecated client APIs have also been
patched:

- twisted.web.client.HTTPPageGetter.__init__
- twisted.web.client.HTTPPageDownloader.__init__
- twisted.web.client.HTTPClientFactory.__init__
- twisted.web.client.HTTPClientFactory.setURL
- twisted.web.client.HTTPDownloader.__init__
- twisted.web.client.HTTPDownloader.setURL
- twisted.web.client.getPage
- twisted.web.client.downloadPage

These have been patched prior to their removal so that they won't be
vulnerable in the last Twisted release that includes them.  They
represent a best effort, because testing every combination of these
public APIs would require more code than deprecated APIs warrant.

In all cases URI components, including hostnames, are restricted to
the characters allowed in path components.  This mirrors the CPython
patch (for bpo-30458) that addresses equivalent vulnerabilities:

https://github.com/python/cpython/commit/bb8071a4cae5ab3fe321481dd3d73662ffb26052

HTTP methods, however, are checked against the set of characters
described in RFC-7230.

@@ -0,0 +1,168 @@
+"""
+Helpers for URI and method injection tests.
+
+@see: U{CVE-2019-12387}
+"""
+
+import string
+
+
+UNPRINTABLE_ASCII = (
+    frozenset(range(0, 128)) -
+    frozenset(bytearray(string.printable, 'ascii'))
+)
+
+NONASCII = frozenset(range(128, 256))
+
+
+
+class MethodInjectionTestsMixin(object):
+    """
+    A mixin that runs HTTP method injection tests.  Define
+    L{MethodInjectionTestsMixin.attemptRequestWithMaliciousMethod} in
+    a L{twisted.trial.unittest.SynchronousTestCase} subclass to test
+    how HTTP client code behaves when presented with malicious HTTP
+    methods.
+
+    @see: U{CVE-2019-12387}
+    """
+
+    def attemptRequestWithMaliciousMethod(self, method):
+        """
+        Attempt to send a request with the given method.  This should
+        synchronously raise a L{ValueError} if either is invalid.
+
+        @param method: the method (e.g. C{GET\x00})
+
+        @param uri: the URI
+
+        @type method:
+        """
+        raise NotImplementedError()
+
+
+    def test_methodWithCLRFRejected(self):
+        """
+        Issuing a request with a method that contains a carriage
+        return and line feed fails with a L{ValueError}.
+        """
+        with self.assertRaises(ValueError) as cm:
+            method = b"GET\r\nX-Injected-Header: value"
+            self.attemptRequestWithMaliciousMethod(method)
+        self.assertRegex(str(cm.exception), "^Invalid method")
+
+
+    def test_methodWithUnprintableASCIIRejected(self):
+        """
+        Issuing a request with a method that contains unprintable
+        ASCII characters fails with a L{ValueError}.
+        """
+        for c in UNPRINTABLE_ASCII:
+            method = b"GET%s" % (bytearray([c]),)
+            with self.assertRaises(ValueError) as cm:
+                self.attemptRequestWithMaliciousMethod(method)
+            self.assertRegex(str(cm.exception), "^Invalid method")
+
+
+    def test_methodWithNonASCIIRejected(self):
+        """
+        Issuing a request with a method that contains non-ASCII
+        characters fails with a L{ValueError}.
+        """
+        for c in NONASCII:
+            method = b"GET%s" % (bytearray([c]),)
+            with self.assertRaises(ValueError) as cm:
+                self.attemptRequestWithMaliciousMethod(method)
+            self.assertRegex(str(cm.exception), "^Invalid method")
+
+
+
+class URIInjectionTestsMixin(object):
+    """
+    A mixin that runs HTTP URI injection tests.  Define
+    L{MethodInjectionTestsMixin.attemptRequestWithMaliciousURI} in a
+    L{twisted.trial.unittest.SynchronousTestCase} subclass to test how
+    HTTP client code behaves when presented with malicious HTTP
+    URIs.
+    """
+
+    def attemptRequestWithMaliciousURI(self, method):
+        """
+        Attempt to send a request with the given URI.  This should
+        synchronously raise a L{ValueError} if either is invalid.
+
+        @param uri: the URI.
+
+        @type method:
+        """
+        raise NotImplementedError()
+
+
+    def test_hostWithCRLFRejected(self):
+        """
+        Issuing a request with a URI whose host contains a carriage
+        return and line feed fails with a L{ValueError}.
+        """
+        with self.assertRaises(ValueError) as cm:
+            uri = b"http://twisted\r\n.invalid/path"
+            self.attemptRequestWithMaliciousURI(uri)
+        self.assertRegex(str(cm.exception), "^Invalid URI")
+
+
+    def test_hostWithWithUnprintableASCIIRejected(self):
+        """
+        Issuing a request with a URI whose host contains unprintable
+        ASCII characters fails with a L{ValueError}.
+        """
+        for c in UNPRINTABLE_ASCII:
+            uri = b"http://twisted%s.invalid/OK" % (bytearray([c]),)
+            with self.assertRaises(ValueError) as cm:
+                self.attemptRequestWithMaliciousURI(uri)
+            self.assertRegex(str(cm.exception), "^Invalid URI")
+
+
+    def test_hostWithNonASCIIRejected(self):
+        """
+        Issuing a request with a URI whose host contains non-ASCII
+        characters fails with a L{ValueError}.
+        """
+        for c in NONASCII:
+            uri = b"http://twisted%s.invalid/OK" % (bytearray([c]),)
+            with self.assertRaises(ValueError) as cm:
+                self.attemptRequestWithMaliciousURI(uri)
+            self.assertRegex(str(cm.exception), "^Invalid URI")
+
+
+    def test_pathWithCRLFRejected(self):
+        """
+        Issuing a request with a URI whose path contains a carriage
+        return and line feed fails with a L{ValueError}.
+        """
+        with self.assertRaises(ValueError) as cm:
+            uri = b"http://twisted.invalid/\r\npath"
+            self.attemptRequestWithMaliciousURI(uri)
+        self.assertRegex(str(cm.exception), "^Invalid URI")
+
+
+    def test_pathWithWithUnprintableASCIIRejected(self):
+        """
+        Issuing a request with a URI whose path contains unprintable
+        ASCII characters fails with a L{ValueError}.
+        """
+        for c in UNPRINTABLE_ASCII:
+            uri = b"http://twisted.invalid/OK%s" % (bytearray([c]),)
+            with self.assertRaises(ValueError) as cm:
+                self.attemptRequestWithMaliciousURI(uri)
+            self.assertRegex(str(cm.exception), "^Invalid URI")
+
+
+    def test_pathWithNonASCIIRejected(self):
+        """
+        Issuing a request with a URI whose path contains non-ASCII
+        characters fails with a L{ValueError}.
+        """
+        for c in NONASCII:
+            uri = b"http://twisted.invalid/OK%s" % (bytearray([c]),)
+            with self.assertRaises(ValueError) as cm:
+                self.attemptRequestWithMaliciousURI(uri)
+            self.assertRegex(str(cm.exception), "^Invalid URI")


bpo-34155: Dont parse domains containing @ (GH-13079)

Before:

        >>> email.message_from_string('From: a@malicious.org@important.com', policy=email.policy.default)['from'].addresses
        (Address(display_name='', username='a', domain='malicious.org'),)

        >>> parseaddr('a@malicious.org@important.com')
        ('', 'a@malicious.org')

    After:

        >>> email.message_from_string('From: a@malicious.org@important.com', policy=email.policy.default)['from'].addresses
        (Address(display_name='', username='', domain=''),)

        >>> parseaddr('a@malicious.org@important.com')
        ('', 'a@')

https://bugs.python.org/issue34155

@@ -1587,6 +1587,8 @@ def get_domain(value):
         token, value = get_dot_atom(value)
     except errors.HeaderParseError:
         token, value = get_atom(value)
+    if value and value[0] == '@':
+        raise errors.HeaderParseError('Invalid Domain')
     if leader is not None:
         token[:0] = [leader]
     domain.append(token)


bpo-35766: Merge typed_ast back into CPython (GH-11645)

@@ -88,3 +88,7 @@
 #define encoding_decl 341
 #define yield_expr 342
 #define yield_arg 343
+#define func_body_suite 344
+#define func_type_input 345
+#define func_type 346
+#define typelist 347


@@ -890,6 +890,15 @@ def visitModule(self, mod):
     return obj2ast_object(obj, out, arena);
 }

+static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena)
+{
+    if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) {
+        PyErr_SetString(PyExc_TypeError, "AST string must be of type str");
+        return 1;
+    }
+    return obj2ast_object(obj, out, arena);
+}
+
 static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
 {
     int i;
@@ -993,6 +1002,8 @@ def visitModule(self, mod):
         self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1)
         self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1)
         self.emit("return NULL;", 2)
+        self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)', 1)
+        self.emit("return NULL;", 2)
         for dfn in mod.dfns:
             self.visit(dfn)
         self.emit("return m;", 1)
@@ -1176,18 +1187,19 @@ class PartingShots(StaticVisitor):
 }

 /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */
+/* and 3 for "func_type" */
 mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
 {
     mod_ty res;
     PyObject *req_type[3];
-    char *req_name[] = {"Module", "Expression", "Interactive"};
+    char *req_name[] = {"Module", "Expression", "Interactive", "FunctionType"};
     int isinstance;

     req_type[0] = (PyObject*)Module_type;
     req_type[1] = (PyObject*)Expression_type;
     req_type[2] = (PyObject*)Interactive_type;

-    assert(0 <= mode && mode <= 2);
+    assert(0 <= mode && mode <= 3);

     if (!init_types())
         return NULL;


@@ -158,6 +158,8 @@ static int PARSER_FLAGS(PyCompilerFlags *flags)
         parser_flags |= PyPARSE_IGNORE_COOKIE;
     if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL)
         parser_flags |= PyPARSE_BARRY_AS_BDFL;
+    if (flags->cf_flags & PyCF_TYPE_COMMENTS)
+        parser_flags |= PyPARSE_TYPE_COMMENTS;
     return parser_flags;
 }


Fully incorporate the code from Python 3.7.2 (#78)

This is a full port, following the recipe in update_process.md. I've also tried to keep the recipe up to date and improved the automation (see tools/script). I haven't cleaned up the commits. As of #77 there are a few tests that sanity-check this (though it's far from a full test suite), and they're run by Travis-CI and AppVeyor.

@@ -8,15 +8,15 @@ extern "C" {
 #endif

 typedef struct _node {
-    short		n_type;
-    char		*n_str;
-    int			n_lineno;
-    int			n_col_offset;
-    int			n_nchildren;
-    struct _node	*n_child;
+    short               n_type;
+    char                *n_str;
+    int                 n_lineno;
+    int                 n_col_offset;
+    int                 n_nchildren;
+    struct _node        *n_child;
 } node;

-extern node *Ta3Node_New(int type);
+extern node * Ta3Node_New(int type);
 extern int Ta3Node_AddChild(node *n, int type,
                                       char *str, int lineno, int col_offset);
 extern void Ta3Node_Free(node *n);
@@ -25,12 +25,12 @@ extern Py_ssize_t _Ta3Node_SizeOf(node *n);
 #endif

 /* Node access functions */
-#define NCH(n)		((n)->n_nchildren)
+#define NCH(n)          ((n)->n_nchildren)

-#define CHILD(n, i)	(&(n)->n_child[i])
-#define RCHILD(n, i)	(CHILD(n, NCH(n) + i))
-#define TYPE(n)		((n)->n_type)
-#define STR(n)		((n)->n_str)
+#define CHILD(n, i)     (&(n)->n_child[i])
+#define RCHILD(n, i)    (CHILD(n, NCH(n) + i))
+#define TYPE(n)         ((n)->n_type)
+#define STR(n)          ((n)->n_str)
 #define LINENO(n)       ((n)->n_lineno)

 /* Assert that the type of a node is what we expect */


@@ -1,113 +0,0 @@
-
-/* Computation of FIRST stets */
-
-#include "pgenheaders.h"
-#include "grammar.h"
-#include "token.h"
-
-extern int Py_DebugFlag;
-
-/* Forward */
-static void calcfirstset(grammar *, dfa *);
-
-void
-addfirstsets(grammar *g)
-{
-    int i;
-    dfa *d;
-
-    if (Py_DebugFlag)
-        printf("Adding FIRST sets ...\n");
-    for (i = 0; i < g->g_ndfas; i++) {
-        d = &g->g_dfa[i];
-        if (d->d_first == NULL)
-            calcfirstset(g, d);
-    }
-}
-
-static void
-calcfirstset(grammar *g, dfa *d)
-{
-    int i, j;
-    state *s;
-    arc *a;
-    int nsyms;
-    int *sym;
-    int nbits;
-    static bitset dummy;
-    bitset result;
-    int type;
-    dfa *d1;
-    label *l0;
-
-    if (Py_DebugFlag)
-        printf("Calculate FIRST set for '%s'\n", d->d_name);
-
-    if (dummy == NULL)
-        dummy = newbitset(1);
-    if (d->d_first == dummy) {
-        fprintf(stderr, "Left-recursion for '%s'\n", d->d_name);
-        return;
-    }
-    if (d->d_first != NULL) {
-        fprintf(stderr, "Re-calculating FIRST set for '%s' ???\n",
-            d->d_name);
-    }
-    d->d_first = dummy;
-
-    l0 = g->g_ll.ll_label;
-    nbits = g->g_ll.ll_nlabels;
-    result = newbitset(nbits);
-
-    sym = (int *)PyObject_MALLOC(sizeof(int));
-    if (sym == NULL)
-        Py_FatalError("no mem for new sym in calcfirstset");
-    nsyms = 1;
-    sym[0] = findlabel(&g->g_ll, d->d_type, (char *)NULL);
-
-    s = &d->d_state[d->d_initial];
-    for (i = 0; i < s->s_narcs; i++) {
-        a = &s->s_arc[i];
-        for (j = 0; j < nsyms; j++) {
-            if (sym[j] == a->a_lbl)
-                break;
-        }
-        if (j >= nsyms) { /* New label */
-            sym = (int *)PyObject_REALLOC(sym,
-                                    sizeof(int) * (nsyms + 1));
-            if (sym == NULL)
-                Py_FatalError(
-                    "no mem to resize sym in calcfirstset");
-            sym[nsyms++] = a->a_lbl;
-            type = l0[a->a_lbl].lb_type;
-            if (ISNONTERMINAL(type)) {
-                d1 = Ta3Grammar_FindDFA(g, type);
-                if (d1->d_first == dummy) {
-                    fprintf(stderr,
-                        "Left-recursion below '%s'\n",
-                        d->d_name);
-                }
-                else {
-                    if (d1->d_first == NULL)
-                        calcfirstset(g, d1);
-                    mergebitset(result,
-                                d1->d_first, nbits);
-                }
-            }
-            else if (ISTERMINAL(type)) {
-                addbit(result, a->a_lbl);
-            }
-        }
-    }
-    d->d_first = result;
-    if (Py_DebugFlag) {
-        printf("FIRST set for '%s': {", d->d_name);
-        for (i = 0; i < nbits; i++) {
-            if (testbit(result, i))
-                printf(" %s", Ta3Grammar_LabelRepr(&l0[i]));
-        }
-        printf(" }\n");
-    }
-
-    PyObject_FREE(sym);
-}


@@ -1,2 +0,0 @@
-#define PGEN
-#include "tokenizer.c"


@@ -22,6 +22,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
 #define PyCF_DONT_IMPLY_DEDENT 0x0200
 #define PyCF_ONLY_AST 0x0400
 #define PyCF_IGNORE_COOKIE 0x0800
+#define PyCF_TYPE_COMMENTS 0x1000

 #ifndef Py_LIMITED_API
 typedef struct {
@@ -85,10 +86,10 @@ PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, int optimize);

 #endif /* !Py_LIMITED_API */

-/* These definitions must match corresponding definitions in graminit.h.
-   There's code in compile.c that checks that they are the same. */
+/* These definitions must match corresponding definitions in graminit.h. */
 #define Py_single_input 256
 #define Py_file_input 257
 #define Py_eval_input 258
+#define Py_func_type_input 345

 #endif /* !Py_COMPILE_H */


@@ -663,6 +663,12 @@ validate_node(node *tree)
     for (pos = 0; pos < nch; ++pos) {
         node *ch = CHILD(tree, pos);
         int ch_type = TYPE(ch);
+        if (ch_type == suite && TYPE(tree) == funcdef) {
+            /* This is the opposite hack of what we do in parser.c
+               (search for func_body_suite), except we don't ever
+               support type comments here. */
+            ch_type = func_body_suite;
+        }
         for (arc = 0; arc < dfa_state->s_narcs; ++arc) {
             short a_label = dfa_state->s_arc[arc].a_lbl;
             assert(a_label < _PyParser_Grammar.g_ll.ll_nlabels);


@@ -12,58 +12,58 @@ extern "C" {
 /* A label of an arc */

 typedef struct {
-    int		 lb_type;
-    char	*lb_str;
+    int          lb_type;
+    char        *lb_str;
 } label;

-#define EMPTY 0		/* Label number 0 is by definition the empty label */
+#define EMPTY 0         /* Label number 0 is by definition the empty label */

 /* A list of labels */

 typedef struct {
-    int		 ll_nlabels;
-    label	*ll_label;
+    int          ll_nlabels;
+    label       *ll_label;
 } labellist;

 /* An arc from one state to another */

 typedef struct {
-    short	a_lbl;		/* Label of this arc */
-    short	a_arrow;	/* State where this arc goes to */
+    short       a_lbl;          /* Label of this arc */
+    short       a_arrow;        /* State where this arc goes to */
 } arc;

 /* A state in a DFA */

 typedef struct {
-    int		 s_narcs;
-    arc		*s_arc;		/* Array of arcs */
+    int          s_narcs;
+    arc         *s_arc;         /* Array of arcs */

     /* Optional accelerators */
-    int		 s_lower;	/* Lowest label index */
-    int		 s_upper;	/* Highest label index */
-    int		*s_accel;	/* Accelerator */
-    int		 s_accept;	/* Nonzero for accepting state */
+    int          s_lower;       /* Lowest label index */
+    int          s_upper;       /* Highest label index */
+    int         *s_accel;       /* Accelerator */
+    int          s_accept;      /* Nonzero for accepting state */
 } state;

 /* A DFA */

 typedef struct {
-    int		 d_type;	/* Non-terminal this represents */
-    char	*d_name;	/* For printing */
-    int		 d_initial;	/* Initial state */
-    int		 d_nstates;
-    state	*d_state;	/* Array of states */
-    bitset	 d_first;
+    int          d_type;        /* Non-terminal this represents */
+    char        *d_name;        /* For printing */
+    int          d_initial;     /* Initial state */
+    int          d_nstates;
+    state       *d_state;       /* Array of states */
+    bitset       d_first;
 } dfa;

 /* A grammar */

 typedef struct {
-    int		 g_ndfas;
-    dfa		*g_dfa;		/* Array of DFAs */
-    labellist	 g_ll;
-    int		 g_start;	/* Start symbol of the grammar */
-    int		 g_accel;	/* Set if accelerators present */
+    int          g_ndfas;
+    dfa         *g_dfa;         /* Array of DFAs */
+    labellist    g_ll;
+    int          g_start;       /* Start symbol of the grammar */
+    int          g_accel;       /* Set if accelerators present */
 } grammar;

 /* FUNCTIONS */


@@ -1,154 +0,0 @@
-/* Copyright (c) 2008-2009, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * ---
- * Author: Kostya Serebryany
- */
-
-#ifdef _MSC_VER
-# include <windows.h>
-#endif
-
-#ifdef __cplusplus
-# error "This file should be built as pure C to avoid name mangling"
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-
-#include "dynamic_annotations.h"
-
-/* Each function is empty and called (via a macro) only in debug mode.
-   The arguments are captured by dynamic tools at runtime. */
-
-#if DYNAMIC_ANNOTATIONS_ENABLED == 1
-
-void AnnotateRWLockCreate(const char *file, int line,
-                          const volatile void *lock){}
-void AnnotateRWLockDestroy(const char *file, int line,
-                           const volatile void *lock){}
-void AnnotateRWLockAcquired(const char *file, int line,
-                            const volatile void *lock, long is_w){}
-void AnnotateRWLockReleased(const char *file, int line,
-                            const volatile void *lock, long is_w){}
-void AnnotateBarrierInit(const char *file, int line,
-                         const volatile void *barrier, long count,
-                         long reinitialization_allowed) {}
-void AnnotateBarrierWaitBefore(const char *file, int line,
-                               const volatile void *barrier) {}
-void AnnotateBarrierWaitAfter(const char *file, int line,
-                              const volatile void *barrier) {}
-void AnnotateBarrierDestroy(const char *file, int line,
-                            const volatile void *barrier) {}
-
-void AnnotateCondVarWait(const char *file, int line,
-                         const volatile void *cv,
-                         const volatile void *lock){}
-void AnnotateCondVarSignal(const char *file, int line,
-                           const volatile void *cv){}
-void AnnotateCondVarSignalAll(const char *file, int line,
-                              const volatile void *cv){}
-void AnnotatePublishMemoryRange(const char *file, int line,
-                                const volatile void *address,
-                                long size){}
-void AnnotateUnpublishMemoryRange(const char *file, int line,
-                                  const volatile void *address,
-                                  long size){}
-void AnnotatePCQCreate(const char *file, int line,
-                       const volatile void *pcq){}
-void AnnotatePCQDestroy(const char *file, int line,
-                        const volatile void *pcq){}
-void AnnotatePCQPut(const char *file, int line,
-                    const volatile void *pcq){}
-void AnnotatePCQGet(const char *file, int line,
-                    const volatile void *pcq){}
-void AnnotateNewMemory(const char *file, int line,
-                       const volatile void *mem,
-                       long size){}
-void AnnotateExpectRace(const char *file, int line,
-                        const volatile void *mem,
-                        const char *description){}
-void AnnotateBenignRace(const char *file, int line,
-                        const volatile void *mem,
-                        const char *description){}
-void AnnotateBenignRaceSized(const char *file, int line,
-                             const volatile void *mem,
-                             long size,
-                             const char *description) {}
-void AnnotateMutexIsUsedAsCondVar(const char *file, int line,
-                                  const volatile void *mu){}
-void AnnotateTraceMemory(const char *file, int line,
-                         const volatile void *arg){}
-void AnnotateThreadName(const char *file, int line,
-                        const char *name){}
-void AnnotateIgnoreReadsBegin(const char *file, int line){}
-void AnnotateIgnoreReadsEnd(const char *file, int line){}
-void AnnotateIgnoreWritesBegin(const char *file, int line){}
-void AnnotateIgnoreWritesEnd(const char *file, int line){}
-void AnnotateIgnoreSyncBegin(const char *file, int line){}
-void AnnotateIgnoreSyncEnd(const char *file, int line){}
-void AnnotateEnableRaceDetection(const char *file, int line, int enable){}
-void AnnotateNoOp(const char *file, int line,
-                  const volatile void *arg){}
-void AnnotateFlushState(const char *file, int line){}
-
-static int GetRunningOnValgrind(void) {
-#ifdef RUNNING_ON_VALGRIND
-  if (RUNNING_ON_VALGRIND) return 1;
-#endif
-
-#ifndef _MSC_VER
-  char *running_on_valgrind_str = getenv("RUNNING_ON_VALGRIND");
-  if (running_on_valgrind_str) {
-    return strcmp(running_on_valgrind_str, "0") != 0;
-  }
-#else
-  /* Visual Studio issues warnings if we use getenv,
-   * so we use GetEnvironmentVariableA instead.
-   */
-  char value[100] = "1";
-  int res = GetEnvironmentVariableA("RUNNING_ON_VALGRIND",
-                                    value, sizeof(value));
-  /* value will remain "1" if res == 0 or res >= sizeof(value). The latter
-   * can happen only if the given value is long, in this case it can't be "0".
-   */
-  if (res > 0 && !strcmp(value, "0"))
-    return 1;
-#endif
-  return 0;
-}
-
-/* See the comments in dynamic_annotations.h */
-int RunningOnValgrind(void) {
-  static volatile int running_on_valgrind = -1;
-  /* C doesn't have thread-safe initialization of statics, and we
-     don't want to depend on pthread_once here, so hack it. */
-  int local_running_on_valgrind = running_on_valgrind;
-  if (local_running_on_valgrind == -1)
-    running_on_valgrind = local_running_on_valgrind = GetRunningOnValgrind();
-  return local_running_on_valgrind;
-}
-
-#endif  /* DYNAMIC_ANNOTATIONS_ENABLED == 1 */


CVE-2019-18933: Fix insecure account creation via social authentication.

A bug in Zulip's new user signup process meant that users who
registered their account using social authentication (e.g. GitHub or
Google SSO) in an organization that also allows password
authentication could have their personal API key stolen by an
unprivileged attacker, allowing nearly full access to the user's
account.

Zulip versions between 1.7.0 and 2.0.6 were affected.

This commit fixes the original bug and also contains a database
migration to fix any users with corrupt `password` fields in the
database as a result of the bug.

Out of an abundance of caution (and to protect the users of any
installations that delay applying this commit), the migration also
resets the API keys of any users where Zulip's logs cannot prove the
user's API key was not previously stolen via this bug.  Resetting
those API keys will be inconvenient for users:

* Users of the Zulip mobile and terminal apps whose API keys are reset
  will be logged out and need to login again.
* Users using their personal API keys for any other reason will need
  to re-fetch their personal API key.

We discovered this bug internally and don't believe it was disclosed
prior to our publishing it through this commit.  Because the algorithm
for determining which users might have been affected is very
conservative, many users who were never at risk will have their API
keys reset by this migration.

To avoid this on self-hosted installations that have always used
e.g. LDAP authentication, we skip resetting API keys on installations
that don't have password authentication enabled.  System
administrators on installations that used to have email authentication
enabled, but no longer do, should temporarily enable EmailAuthBackend
before applying this migration.

The migration also records which users had their passwords or API keys
reset in the usual RealmAuditLog table.

@@ -416,6 +416,7 @@
              'zerver/migrations/0060_move_avatars_to_be_uid_based.py',
              'zerver/migrations/0104_fix_unreads.py',
              'zerver/migrations/0206_stream_rendered_description.py',
+             'zerver/migrations/0209_user_profile_no_empty_password.py',
              'pgroonga/migrations/0002_html_escape_subject.py',
          ]),
          'description': "Don't import models or other code in migrations; see docs/subsystems/schema-migrations.md",


BUG: weave: fix issues with temporary directory usage

@@ -34,6 +34,7 @@

 import os
 import sys
+import stat
 import pickle
 import socket
 import tempfile
@@ -133,7 +134,7 @@ def is_writable(dir):

     # Do NOT use a hardcoded name here due to the danger from race conditions
     # on NFS when multiple processes are accessing the same base directory in
-    # parallel.  We use both hostname and pocess id for the prefix in an
+    # parallel.  We use both hostname and process id for the prefix in an
     # attempt to ensure that there can really be no name collisions (tempfile
     # appends 6 random chars to this prefix).
     prefix = 'dummy_%s_%s_' % (socket.gethostname(),os.getpid())
@@ -150,6 +151,88 @@ def whoami():
     """return a string identifying the user."""
     return os.environ.get("USER") or os.environ.get("USERNAME") or "unknown"

+
+def _create_dirs(path):
+    """ create provided path, ignore errors """
+    try:
+        os.makedirs(path, mode=0o700)
+    except OSError:
+        pass
+
+
+def default_dir_posix(tmp_dir=None):
+    """
+    Create or find default catalog store for posix systems
+
+    purpose of 'tmp_dir' is to enable way how to test this function easily
+    """
+    path_candidates = []
+    python_name = "python%d%d_compiled" % tuple(sys.version_info[:2])
+
+    if tmp_dir:
+        home_dir = tmp_dir
+    else:
+        home_dir = os.path.expanduser('~')
+    tmp_dir = tmp_dir or tempfile.gettempdir()
+
+    home_temp_dir_name = '.' + python_name
+    home_temp_dir = os.path.join(home_dir, home_temp_dir_name)
+    path_candidates.append(home_temp_dir)
+
+    temp_dir_name = repr(os.getuid()) + '_' + python_name
+    temp_dir_path = os.path.join(tmp_dir, temp_dir_name)
+    path_candidates.append(temp_dir_path)
+
+    for path in path_candidates:
+        _create_dirs(path)
+        if check_dir(path):
+            return path
+
+    # since we got here, both dirs are not useful
+    tmp_dir_path = find_valid_temp_dir(temp_dir_name, tmp_dir)
+    if not tmp_dir_path:
+        tmp_dir_path = create_temp_dir(temp_dir_name, tmp_dir=tmp_dir)
+    return tmp_dir_path
+
+
+def default_dir_win(tmp_dir=None):
+    """
+    Create or find default catalog store for Windows systems
+
+    purpose of 'tmp_dir' is to enable way how to test this function easily
+    """
+    def create_win_temp_dir(prefix, inner_dir=None, tmp_dir=None):
+        """
+        create temp dir starting with 'prefix' in 'tmp_dir' or
+        'tempfile.gettempdir'; if 'inner_dir' is specified, it should be
+        created inside
+        """
+        tmp_dir_path = find_valid_temp_dir(prefix, tmp_dir)
+        if tmp_dir_path:
+            if inner_dir:
+                tmp_dir_path = os.path.join(tmp_dir_path, inner_dir)
+                if not os.path.isdir(tmp_dir_path):
+                    os.mkdir(tmp_dir_path, 0o700)
+        else:
+            tmp_dir_path = create_temp_dir(prefix, inner_dir, tmp_dir)
+        return tmp_dir_path
+
+    python_name = "python%d%d_compiled" % tuple(sys.version_info[:2])
+    tmp_dir = tmp_dir or tempfile.gettempdir()
+
+    temp_dir_name = "%s" % whoami()
+    temp_root_dir = os.path.join(tmp_dir, temp_dir_name)
+    temp_dir_path = os.path.join(temp_root_dir, python_name)
+    _create_dirs(temp_dir_path)
+    if check_dir(temp_dir_path) and check_dir(temp_root_dir):
+        return temp_dir_path
+    else:
+        if check_dir(temp_root_dir):
+            return create_win_temp_dir(python_name, tmp_dir=temp_root_dir)
+        else:
+            return create_win_temp_dir(temp_dir_name, python_name, tmp_dir)
+
+
 def default_dir():
     """ Return a default location to store compiled files and catalogs.

@@ -164,61 +247,151 @@ def default_dir():
         in the user's home, /tmp/<uid>_pythonXX_compiled is used.  If it
         doesn't exist, it is created.  The directory is marked rwx------
         to try and keep people from being able to sneak a bad module
-        in on you.
-
+        in on you. If the directory already exists in /tmp/ and is not
+        secure, new one is created.
     """
-
     # Use a cached value for fast return if possible
-    if hasattr(default_dir,"cached_path") and \
-       os.path.exists(default_dir.cached_path) and \
-       os.access(default_dir.cached_path, os.W_OK):
+    if hasattr(default_dir, "cached_path") and \
+       check_dir(default_dir.cached_path):
         return default_dir.cached_path

-    python_name = "python%d%d_compiled" % tuple(sys.version_info[:2])
-    path_candidates = []
-    if sys.platform != 'win32':
-        try:
-            path_candidates.append(os.path.join(os.environ['HOME'],
-                                                '.' + python_name))
-        except KeyError:
-            pass
-
-        temp_dir = repr(os.getuid()) + '_' + python_name
-        path_candidates.append(os.path.join(tempfile.gettempdir(), temp_dir))
+    if sys.platform == 'win32':
+        path = default_dir_win()
     else:
-        path_candidates.append(os.path.join(tempfile.gettempdir(),
-                                           "%s" % whoami(), python_name))
-
-    writable = False
-    for path in path_candidates:
-        if not os.path.exists(path):
-            try:
-                os.makedirs(path, mode=0o700)
-            except OSError:
-                continue
-        if is_writable(path):
-            writable = True
-            break
-
-    if not writable:
-        print('warning: default directory is not write accessible.')
-        print('default:', path)
+        path = default_dir_posix()

     # Cache the default dir path so that this function returns quickly after
     # being called once (nothing in it should change after the first call)
     default_dir.cached_path = path

     return path

-def intermediate_dir():
-    """ Location in temp dir for storing .cpp and .o  files during
-        builds.
+
+def check_dir(im_dir):
     """
-    python_name = "python%d%d_intermediate" % tuple(sys.version_info[:2])
-    path = os.path.join(tempfile.gettempdir(),"%s"%whoami(),python_name)
-    if not os.path.exists(path):
-        os.makedirs(path, mode=0o700)
-    return path
+    Check if dir is safe; if it is, return True.
+    These checks make sense only on posix:
+     * directory has correct owner
+     * directory has correct permissions (0700)
+     * directory is not a symlink
+    """
+    def check_is_dir():
+        return os.path.isdir(im_dir)
+
+    def check_permissions():
+        """ If on posix, permissions should be 0700. """
+        writable = is_writable(im_dir)
+        if sys.platform != 'win32':
+            try:
+                im_dir_stat = os.stat(im_dir)
+            except OSError:
+                return False
+            writable &= stat.S_IMODE(im_dir_stat.st_mode) == 0o0700
+        return writable
+
+    def check_ownership():
+        """ Intermediate dir owner should be same as owner of process. """
+        if sys.platform != 'win32':
+            try:
+                im_dir_stat = os.stat(im_dir)
+            except OSError:
+                return False
+            proc_uid = os.getuid()
+            return proc_uid == im_dir_stat.st_uid
+        return True
+
+    def check_is_symlink():
+        """ Check if intermediate dir is symlink. """
+        try:
+            return not os.path.islink(im_dir)
+        except OSError:
+            return False
+
+    checks = [check_is_dir, check_permissions,
+              check_ownership, check_is_symlink]
+
+    for check in checks:
+        if not check():
+            return False
+
+    return True
+
+
+def create_temp_dir(prefix, inner_dir=None, tmp_dir=None):
+    """
+    Create intermediate dirs <tmp>/<prefix+random suffix>/<inner_dir>/
+
+    argument 'tmp_dir' is used in unit tests
+    """
+    if not tmp_dir:
+        tmp_dir_path = tempfile.mkdtemp(prefix=prefix)
+    else:
+        tmp_dir_path = tempfile.mkdtemp(prefix=prefix, dir=tmp_dir)
+    if inner_dir:
+        tmp_dir_path = os.path.join(tmp_dir_path, inner_dir)
+        os.mkdir(tmp_dir_path, 0o700)
+    return tmp_dir_path
+
+
+def intermediate_dir_prefix():
+    """ Prefix of root intermediate dir (<tmp>/<root_im_dir>). """
+    return "%s-%s-" % ("scipy", whoami())
+
+
+def find_temp_dir(prefix, tmp_dir=None):
+    """ Find temp dirs in 'tmp_dir' starting with 'prefix'"""
+    matches = []
+    tmp_dir = tmp_dir or tempfile.gettempdir()
+    for tmp_file in os.listdir(tmp_dir):
+        if tmp_file.startswith(prefix):
+            matches.append(os.path.join(tmp_dir, tmp_file))
+    return matches
+
+
+def find_valid_temp_dir(prefix, tmp_dir=None):
+    """
+    Try to look for existing temp dirs.
+    If there is one suitable found, return it, otherwise return None.
+    """
+    matches = find_temp_dir(prefix, tmp_dir)
+    for match in matches:
+        if check_dir(match):
+            # as soon as we find correct dir, we can stop searching
+            return match
+
+
+def py_intermediate_dir():
+    """
+    Name of intermediate dir for current python interpreter:
+    <temp dir>/<name>/pythonXY_intermediate/
+    """
+    name = "python%d%d_intermediate" % tuple(sys.version_info[:2])
+    return name
+
+
+def create_intermediate_dir(tmp_dir=None):
+    py_im_dir = py_intermediate_dir()
+    return create_temp_dir(intermediate_dir_prefix(), py_im_dir, tmp_dir)
+
+
+def intermediate_dir(tmp_dir=None):
+    """
+    Temporary directory for storing .cpp and .o files during builds.
+
+    First, try to find the dir and if it exists, verify it is safe.
+    Otherwise, create it.
+    """
+    im_dir = find_valid_temp_dir(intermediate_dir_prefix(), tmp_dir)
+    py_im_dir = py_intermediate_dir()
+    if im_dir is None:
+        py_im_dir = py_intermediate_dir()
+        im_dir = create_intermediate_dir(tmp_dir)
+    else:
+        im_dir = os.path.join(im_dir, py_im_dir)
+        if not os.path.isdir(im_dir):
+            os.mkdir(im_dir, 0o700)
+    return im_dir
+

 def default_temp_dir():
     path = os.path.join(default_dir(),'temp')


Remove support for non CRLF line endings

https://tools.ietf.org/html/rfc7230#section-3.5 says that servers MAY
implement their parsers to use only the LF as a delimeter between lines,
however if the frontend server does NOT do the same you can potentially
allow a single HTTP request to be treated differently by the two
servers.

This issue can be used to cause HTTP request smuggling or HTTP desync
which may lead to vulnerabilities.

To increase robustness Waitress will no longer allow bare LF for HTTP
messages/headers and chunked encoding and instead now enforces that the
line endings at CRLF.

@@ -179,7 +179,7 @@ def _read_echo(self, fp):
         return line, headers, echo.parse_response(body)

     def test_date_and_server(self):
-        to_send = "GET / HTTP/1.0\n" "Content-Length: 0\n\n"
+        to_send = "GET / HTTP/1.0\r\nContent-Length: 0\r\n\r\n"
         to_send = tobytes(to_send)
         self.connect()
         self.sock.send(to_send)
@@ -191,7 +191,7 @@ def test_date_and_server(self):

     def test_bad_host_header(self):
         # https://corte.si/posts/code/pathod/pythonservers/index.html
-        to_send = "GET / HTTP/1.0\n" " Host: 0\n\n"
+        to_send = "GET / HTTP/1.0\r\n Host: 0\r\n\r\n"
         to_send = tobytes(to_send)
         self.connect()
         self.sock.send(to_send)
@@ -202,7 +202,7 @@ def test_bad_host_header(self):
         self.assertTrue(headers.get("date"))

     def test_send_with_body(self):
-        to_send = "GET / HTTP/1.0\n" "Content-Length: 5\n\n"
+        to_send = "GET / HTTP/1.0\r\nContent-Length: 5\r\n\r\n"
         to_send += "hello"
         to_send = tobytes(to_send)
         self.connect()
@@ -214,7 +214,7 @@ def test_send_with_body(self):
         self.assertEqual(echo.body, b"hello")

     def test_send_empty_body(self):
-        to_send = "GET / HTTP/1.0\n" "Content-Length: 0\n\n"
+        to_send = "GET / HTTP/1.0\r\nContent-Length: 0\r\n\r\n"
         to_send = tobytes(to_send)
         self.connect()
         self.sock.send(to_send)
@@ -241,12 +241,12 @@ def test_multiple_requests_without_body(self):
         self.sock = orig_sock

     def test_without_crlf(self):
-        data = "Echo\nthis\r\nplease"
+        data = "Echo\r\nthis\r\nplease"
         s = tobytes(
-            "GET / HTTP/1.0\n"
-            "Connection: close\n"
-            "Content-Length: %d\n"
-            "\n"
+            "GET / HTTP/1.0\r\n"
+            "Connection: close\r\n"
+            "Content-Length: %d\r\n"
+            "\r\n"
             "%s" % (len(data), data)
         )
         self.connect()
@@ -262,7 +262,7 @@ def test_large_body(self):
         # 1024 characters.
         body = "This string has 32 characters.\r\n" * 32
         s = tobytes(
-            "GET / HTTP/1.0\n" "Content-Length: %d\n" "\n" "%s" % (len(body), body)
+            "GET / HTTP/1.0\r\nContent-Length: %d\r\n\r\n%s" % (len(body), body)
         )
         self.connect()
         self.sock.send(s)
@@ -289,7 +289,7 @@ def test_many_clients(self):
             h.close()

     def test_chunking_request_without_content(self):
-        header = tobytes("GET / HTTP/1.1\n" "Transfer-Encoding: chunked\n\n")
+        header = tobytes("GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n")
         self.connect()
         self.sock.send(header)
         self.sock.send(b"0\r\n\r\n")
@@ -304,13 +304,14 @@ def test_chunking_request_with_content(self):
         control_line = b"20;\r\n"  # 20 hex = 32 dec
         s = b"This string has 32 characters.\r\n"
         expected = s * 12
-        header = tobytes("GET / HTTP/1.1\n" "Transfer-Encoding: chunked\n\n")
+        header = tobytes("GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n")
         self.connect()
         self.sock.send(header)
         fp = self.sock.makefile("rb", 0)
         for n in range(12):
             self.sock.send(control_line)
             self.sock.send(s)
+            self.sock.send(b"\r\n")  # End the chunk
         self.sock.send(b"0\r\n\r\n")
         line, headers, echo = self._read_echo(fp)
         self.assertline(line, "200", "OK", "HTTP/1.1")
@@ -321,11 +322,34 @@ def test_chunking_request_with_content(self):
     def test_broken_chunked_encoding(self):
         control_line = "20;\r\n"  # 20 hex = 32 dec
         s = "This string has 32 characters.\r\n"
-        to_send = "GET / HTTP/1.1\nTransfer-Encoding: chunked\n\n"
-        to_send += control_line + s
+        to_send = "GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
+        to_send += control_line + s + "\r\n"
         # garbage in input
-        to_send += "GET / HTTP/1.1\nTransfer-Encoding: chunked\n\n"
+        to_send += "garbage\r\n"
+        to_send = tobytes(to_send)
+        self.connect()
+        self.sock.send(to_send)
+        fp = self.sock.makefile("rb", 0)
+        line, headers, response_body = read_http(fp)
+        # receiver caught garbage and turned it into a 400
+        self.assertline(line, "400", "Bad Request", "HTTP/1.1")
+        cl = int(headers["content-length"])
+        self.assertEqual(cl, len(response_body))
+        self.assertEqual(
+            sorted(headers.keys()), ["connection", "content-length", "content-type", "date", "server"]
+        )
+        self.assertEqual(headers["content-type"], "text/plain")
+        # connection has been closed
+        self.send_check_error(to_send)
+        self.assertRaises(ConnectionClosed, read_http, fp)
+
+    def test_broken_chunked_encoding_missing_chunk_end(self):
+        control_line = "20;\r\n"  # 20 hex = 32 dec
+        s = "This string has 32 characters.\r\n"
+        to_send = "GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
         to_send += control_line + s
+        # garbage in input
+        to_send += "garbage"
         to_send = tobytes(to_send)
         self.connect()
         self.sock.send(to_send)
@@ -335,6 +359,7 @@ def test_broken_chunked_encoding(self):
         self.assertline(line, "400", "Bad Request", "HTTP/1.1")
         cl = int(headers["content-length"])
         self.assertEqual(cl, len(response_body))
+        self.assertTrue(b"Chunk not properly terminated" in response_body)
         self.assertEqual(
             sorted(headers.keys()), ["connection", "content-length", "content-type", "date", "server"]
         )
@@ -347,7 +372,7 @@ def test_keepalive_http_10(self):
         # Handling of Keep-Alive within HTTP 1.0
         data = "Default: Don't keep me alive"
         s = tobytes(
-            "GET / HTTP/1.0\n" "Content-Length: %d\n" "\n" "%s" % (len(data), data)
+            "GET / HTTP/1.0\r\nContent-Length: %d\r\n\r\n%s" % (len(data), data)
         )
         self.connect()
         self.sock.send(s)
@@ -365,10 +390,10 @@ def test_keepalive_http10_explicit(self):
         # the corresponding header
         data = "Keep me alive"
         s = tobytes(
-            "GET / HTTP/1.0\n"
-            "Connection: Keep-Alive\n"
-            "Content-Length: %d\n"
-            "\n"
+            "GET / HTTP/1.0\r\n"
+            "Connection: Keep-Alive\r\n"
+            "Content-Length: %d\r\n"
+            "\r\n"
             "%s" % (len(data), data)
         )
         self.connect()
@@ -385,7 +410,7 @@ def test_keepalive_http_11(self):
         # All connections are kept alive, unless stated otherwise
         data = "Default: Keep me alive"
         s = tobytes(
-            "GET / HTTP/1.1\n" "Content-Length: %d\n" "\n" "%s" % (len(data), data)
+            "GET / HTTP/1.1\r\nContent-Length: %d\r\n\r\n%s" % (len(data), data)
         )
         self.connect()
         self.sock.send(s)
@@ -398,10 +423,10 @@ def test_keepalive_http11_explicit(self):
         # Explicitly set keep-alive
         data = "Default: Keep me alive"
         s = tobytes(
-            "GET / HTTP/1.1\n"
-            "Connection: keep-alive\n"
-            "Content-Length: %d\n"
-            "\n"
+            "GET / HTTP/1.1\r\n"
+            "Connection: keep-alive\r\n"
+            "Content-Length: %d\r\n"
+            "\r\n"
             "%s" % (len(data), data)
         )
         self.connect()
@@ -415,10 +440,10 @@ def test_keepalive_http11_connclose(self):
         # specifying Connection: close explicitly
         data = "Don't keep me alive"
         s = tobytes(
-            "GET / HTTP/1.1\n"
-            "Connection: close\n"
-            "Content-Length: %d\n"
-            "\n"
+            "GET / HTTP/1.1\r\n"
+            "Connection: close\r\n"
+            "Content-Length: %d\r\n"
+            "\r\n"
             "%s" % (len(data), data)
         )
         self.connect()
@@ -430,12 +455,12 @@ def test_keepalive_http11_connclose(self):

     def test_proxy_headers(self):
         to_send = (
-            "GET / HTTP/1.0\n"
-            "Content-Length: 0\n"
-            "Host: www.google.com:8080\n"
-            "X-Forwarded-For: 192.168.1.1\n"
-            "X-Forwarded-Proto: https\n"
-            "X-Forwarded-Port: 5000\n\n"
+            "GET / HTTP/1.0\r\n"
+            "Content-Length: 0\r\n"
+            "Host: www.google.com:8080\r\n"
+            "X-Forwarded-For: 192.168.1.1\r\n"
+            "X-Forwarded-Proto: https\r\n"
+            "X-Forwarded-Port: 5000\r\n\r\n"
         )
         to_send = tobytes(to_send)
         self.connect()
@@ -507,11 +532,11 @@ def test_expect_continue(self):
         # specifying Connection: close explicitly
         data = "I have expectations"
         to_send = tobytes(
-            "GET / HTTP/1.1\n"
-            "Connection: close\n"
-            "Content-Length: %d\n"
-            "Expect: 100-continue\n"
-            "\n"
+            "GET / HTTP/1.1\r\n"
+            "Connection: close\r\n"
+            "Content-Length: %d\r\n"
+            "Expect: 100-continue\r\n"
+            "\r\n"
             "%s" % (len(data), data)
         )
         self.connect()
@@ -546,10 +571,10 @@ def test_short_body(self):
         # check to see if server closes connection when body is too short
         # for cl header
         to_send = tobytes(
-            "GET /short_body HTTP/1.0\n"
-            "Connection: Keep-Alive\n"
-            "Content-Length: 0\n"
-            "\n"
+            "GET /short_body HTTP/1.0\r\n"
+            "Connection: Keep-Alive\r\n"
+            "Content-Length: 0\r\n"
+            "\r\n"
         )
         self.connect()
         self.sock.send(to_send)
@@ -572,10 +597,10 @@ def test_long_body(self):
         # check server doesnt close connection when body is too short
         # for cl header
         to_send = tobytes(
-            "GET /long_body HTTP/1.0\n"
-            "Connection: Keep-Alive\n"
-            "Content-Length: 0\n"
-            "\n"
+            "GET /long_body HTTP/1.0\r\n"
+            "Connection: Keep-Alive\r\n"
+            "Content-Length: 0\r\n"
+            "\r\n"
         )
         self.connect()
         self.sock.send(to_send)
@@ -611,9 +636,9 @@ def tearDown(self):
     def test_http10_generator(self):
         body = string.ascii_letters
         to_send = (
-            "GET / HTTP/1.0\n"
-            "Connection: Keep-Alive\n"
-            "Content-Length: %d\n\n" % len(body)
+            "GET / HTTP/1.0\r\n"
+            "Connection: Keep-Alive\r\n"
+            "Content-Length: %d\r\n\r\n" % len(body)
         )
         to_send += body
         to_send = tobytes(to_send)
@@ -633,9 +658,9 @@ def test_http10_generator(self):
     def test_http10_list(self):
         body = string.ascii_letters
         to_send = (
-            "GET /list HTTP/1.0\n"
-            "Connection: Keep-Alive\n"
-            "Content-Length: %d\n\n" % len(body)
+            "GET /list HTTP/1.0\r\n"
+            "Connection: Keep-Alive\r\n"
+            "Content-Length: %d\r\n\r\n" % len(body)
         )
         to_send += body
         to_send = tobytes(to_send)
@@ -656,9 +681,9 @@ def test_http10_list(self):
     def test_http10_listlentwo(self):
         body = string.ascii_letters
         to_send = (
-            "GET /list_lentwo HTTP/1.0\n"
-            "Connection: Keep-Alive\n"
-            "Content-Length: %d\n\n" % len(body)
+            "GET /list_lentwo HTTP/1.0\r\n"
+            "Connection: Keep-Alive\r\n"
+            "Content-Length: %d\r\n\r\n" % len(body)
         )
         to_send += body
         to_send = tobytes(to_send)
@@ -677,7 +702,7 @@ def test_http10_listlentwo(self):

     def test_http11_generator(self):
         body = string.ascii_letters
-        to_send = "GET / HTTP/1.1\n" "Content-Length: %s\n\n" % len(body)
+        to_send = "GET / HTTP/1.1\r\nContent-Length: %s\r\n\r\n" % len(body)
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -698,7 +723,7 @@ def test_http11_generator(self):

     def test_http11_list(self):
         body = string.ascii_letters
-        to_send = "GET /list HTTP/1.1\n" "Content-Length: %d\n\n" % len(body)
+        to_send = "GET /list HTTP/1.1\r\nContent-Length: %d\r\n\r\n" % len(body)
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -716,7 +741,7 @@ def test_http11_list(self):

     def test_http11_listlentwo(self):
         body = string.ascii_letters
-        to_send = "GET /list_lentwo HTTP/1.1\n" "Content-Length: %s\n\n" % len(body)
+        to_send = "GET /list_lentwo HTTP/1.1\r\nContent-Length: %s\r\n\r\n" % len(body)
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -749,10 +774,10 @@ def test_short_body(self):
         # check to see if server closes connection when body is too short
         # for cl header
         to_send = tobytes(
-            "GET /short_body HTTP/1.0\n"
-            "Connection: Keep-Alive\n"
-            "Content-Length: 0\n"
-            "\n"
+            "GET /short_body HTTP/1.0\r\n"
+            "Connection: Keep-Alive\r\n"
+            "Content-Length: 0\r\n"
+            "\r\n"
         )
         self.connect()
         self.sock.send(to_send)
@@ -773,10 +798,10 @@ def test_long_body(self):
         # check server doesnt close connection when body is too long
         # for cl header
         to_send = tobytes(
-            "GET /long_body HTTP/1.0\n"
-            "Connection: Keep-Alive\n"
-            "Content-Length: 0\n"
-            "\n"
+            "GET /long_body HTTP/1.0\r\n"
+            "Connection: Keep-Alive\r\n"
+            "Content-Length: 0\r\n"
+            "\r\n"
         )
         self.connect()
         self.sock.send(to_send)
@@ -796,10 +821,10 @@ def test_equal_body(self):
         # check server doesnt close connection when body is equal to
         # cl header
         to_send = tobytes(
-            "GET /equal_body HTTP/1.0\n"
-            "Connection: Keep-Alive\n"
-            "Content-Length: 0\n"
-            "\n"
+            "GET /equal_body HTTP/1.0\r\n"
+            "Connection: Keep-Alive\r\n"
+            "Content-Length: 0\r\n"
+            "\r\n"
         )
         self.connect()
         self.sock.send(to_send)
@@ -819,10 +844,10 @@ def test_equal_body(self):
     def test_no_content_length(self):
         # wtf happens when there's no content-length
         to_send = tobytes(
-            "GET /no_content_length HTTP/1.0\n"
-            "Connection: Keep-Alive\n"
-            "Content-Length: 0\n"
-            "\n"
+            "GET /no_content_length HTTP/1.0\r\n"
+            "Connection: Keep-Alive\r\n"
+            "Content-Length: 0\r\n"
+            "\r\n"
         )
         self.connect()
         self.sock.send(to_send)
@@ -853,7 +878,7 @@ def tearDown(self):

     def test_request_body_too_large_with_wrong_cl_http10(self):
         body = "a" * self.toobig
-        to_send = "GET / HTTP/1.0\n" "Content-Length: 5\n\n"
+        to_send = "GET / HTTP/1.0\r\nContent-Length: 5\r\n\r\n"
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -872,7 +897,7 @@ def test_request_body_too_large_with_wrong_cl_http10(self):

     def test_request_body_too_large_with_wrong_cl_http10_keepalive(self):
         body = "a" * self.toobig
-        to_send = "GET / HTTP/1.0\n" "Content-Length: 5\n" "Connection: Keep-Alive\n\n"
+        to_send = "GET / HTTP/1.0\r\nContent-Length: 5\r\nConnection: Keep-Alive\r\n\r\n"
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -893,7 +918,7 @@ def test_request_body_too_large_with_wrong_cl_http10_keepalive(self):

     def test_request_body_too_large_with_no_cl_http10(self):
         body = "a" * self.toobig
-        to_send = "GET / HTTP/1.0\n\n"
+        to_send = "GET / HTTP/1.0\r\n\r\n"
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -909,7 +934,7 @@ def test_request_body_too_large_with_no_cl_http10(self):

     def test_request_body_too_large_with_no_cl_http10_keepalive(self):
         body = "a" * self.toobig
-        to_send = "GET / HTTP/1.0\nConnection: Keep-Alive\n\n"
+        to_send = "GET / HTTP/1.0\r\nConnection: Keep-Alive\r\n\r\n"
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -932,7 +957,7 @@ def test_request_body_too_large_with_no_cl_http10_keepalive(self):

     def test_request_body_too_large_with_wrong_cl_http11(self):
         body = "a" * self.toobig
-        to_send = "GET / HTTP/1.1\n" "Content-Length: 5\n\n"
+        to_send = "GET / HTTP/1.1\r\nContent-Length: 5\r\n\r\n"
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -954,7 +979,7 @@ def test_request_body_too_large_with_wrong_cl_http11(self):

     def test_request_body_too_large_with_wrong_cl_http11_connclose(self):
         body = "a" * self.toobig
-        to_send = "GET / HTTP/1.1\nContent-Length: 5\nConnection: close\n\n"
+        to_send = "GET / HTTP/1.1\r\nContent-Length: 5\r\nConnection: close\r\n\r\n"
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -971,7 +996,7 @@ def test_request_body_too_large_with_wrong_cl_http11_connclose(self):

     def test_request_body_too_large_with_no_cl_http11(self):
         body = "a" * self.toobig
-        to_send = "GET / HTTP/1.1\n\n"
+        to_send = "GET / HTTP/1.1\r\n\r\n"
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -996,7 +1021,7 @@ def test_request_body_too_large_with_no_cl_http11(self):

     def test_request_body_too_large_with_no_cl_http11_connclose(self):
         body = "a" * self.toobig
-        to_send = "GET / HTTP/1.1\nConnection: close\n\n"
+        to_send = "GET / HTTP/1.1\r\nConnection: close\r\n\r\n"
         to_send += body
         to_send = tobytes(to_send)
         self.connect()
@@ -1014,7 +1039,7 @@ def test_request_body_too_large_with_no_cl_http11_connclose(self):
     def test_request_body_too_large_chunked_encoding(self):
         control_line = "20;\r\n"  # 20 hex = 32 dec
         s = "This string has 32 characters.\r\n"
-        to_send = "GET / HTTP/1.1\nTransfer-Encoding: chunked\n\n"
+        to_send = "GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
         repeat = control_line + s
         to_send += repeat * ((self.toobig // len(repeat)) + 1)
         to_send = tobytes(to_send)
@@ -1042,7 +1067,7 @@ def tearDown(self):
         self.stop_subprocess()

     def test_before_start_response_http_10(self):
-        to_send = "GET /before_start_response HTTP/1.0\n\n"
+        to_send = "GET /before_start_response HTTP/1.0\r\n\r\n"
         to_send = tobytes(to_send)
         self.connect()
         self.sock.send(to_send)
@@ -1058,7 +1083,7 @@ def test_before_start_response_http_10(self):
         self.assertRaises(ConnectionClosed, read_http, fp)

     def test_before_start_response_http_11(self):
-        to_send = "GET /before_start_response HTTP/1.1\n\n"
+        to_send = "GET /before_start_response HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)
         self.connect()
         self.sock.send(to_send)
@@ -1077,7 +1102,7 @@ def test_before_start_response_http_11(self):

     def test_before_start_response_http_11_close(self):
         to_send = tobytes(
-            "GET /before_start_response HTTP/1.1\n" "Connection: close\n\n"
+            "GET /before_start_response HTTP/1.1\r\nConnection: close\r\n\r\n"
         )
         self.connect()
         self.sock.send(to_send)
@@ -1097,7 +1122,7 @@ def test_before_start_response_http_11_close(self):
         self.assertRaises(ConnectionClosed, read_http, fp)

     def test_after_start_response_http10(self):
-        to_send = "GET /after_start_response HTTP/1.0\n\n"
+        to_send = "GET /after_start_response HTTP/1.0\r\n\r\n"
         to_send = tobytes(to_send)
         self.connect()
         self.sock.send(to_send)
@@ -1117,7 +1142,7 @@ def test_after_start_response_http10(self):
         self.assertRaises(ConnectionClosed, read_http, fp)

     def test_after_start_response_http11(self):
-        to_send = "GET /after_start_response HTTP/1.1\n\n"
+        to_send = "GET /after_start_response HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)
         self.connect()
         self.sock.send(to_send)
@@ -1136,7 +1161,7 @@ def test_after_start_response_http11(self):

     def test_after_start_response_http11_close(self):
         to_send = tobytes(
-            "GET /after_start_response HTTP/1.1\n" "Connection: close\n\n"
+            "GET /after_start_response HTTP/1.1\r\nConnection: close\r\n\r\n"
         )
         self.connect()
         self.sock.send(to_send)
@@ -1156,7 +1181,7 @@ def test_after_start_response_http11_close(self):
         self.assertRaises(ConnectionClosed, read_http, fp)

     def test_after_write_cb(self):
-        to_send = "GET /after_write_cb HTTP/1.1\n\n"
+        to_send = "GET /after_write_cb HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)
         self.connect()
         self.sock.send(to_send)
@@ -1169,7 +1194,7 @@ def test_after_write_cb(self):
         self.assertRaises(ConnectionClosed, read_http, fp)

     def test_in_generator(self):
-        to_send = "GET /in_generator HTTP/1.1\n\n"
+        to_send = "GET /in_generator HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)
         self.connect()
         self.sock.send(to_send)
@@ -1192,7 +1217,7 @@ def tearDown(self):
         self.stop_subprocess()

     def test_filelike_http11(self):
-        to_send = "GET /filelike HTTP/1.1\n\n"
+        to_send = "GET /filelike HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1210,7 +1235,7 @@ def test_filelike_http11(self):
             # connection has not been closed

     def test_filelike_nocl_http11(self):
-        to_send = "GET /filelike_nocl HTTP/1.1\n\n"
+        to_send = "GET /filelike_nocl HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1228,7 +1253,7 @@ def test_filelike_nocl_http11(self):
             # connection has not been closed

     def test_filelike_shortcl_http11(self):
-        to_send = "GET /filelike_shortcl HTTP/1.1\n\n"
+        to_send = "GET /filelike_shortcl HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1247,7 +1272,7 @@ def test_filelike_shortcl_http11(self):
             # connection has not been closed

     def test_filelike_longcl_http11(self):
-        to_send = "GET /filelike_longcl HTTP/1.1\n\n"
+        to_send = "GET /filelike_longcl HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1265,7 +1290,7 @@ def test_filelike_longcl_http11(self):
             # connection has not been closed

     def test_notfilelike_http11(self):
-        to_send = "GET /notfilelike HTTP/1.1\n\n"
+        to_send = "GET /notfilelike HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1283,7 +1308,7 @@ def test_notfilelike_http11(self):
             # connection has not been closed

     def test_notfilelike_iobase_http11(self):
-        to_send = "GET /notfilelike_iobase HTTP/1.1\n\n"
+        to_send = "GET /notfilelike_iobase HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1301,7 +1326,7 @@ def test_notfilelike_iobase_http11(self):
             # connection has not been closed

     def test_notfilelike_nocl_http11(self):
-        to_send = "GET /notfilelike_nocl HTTP/1.1\n\n"
+        to_send = "GET /notfilelike_nocl HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1318,7 +1343,7 @@ def test_notfilelike_nocl_http11(self):
         self.assertRaises(ConnectionClosed, read_http, fp)

     def test_notfilelike_shortcl_http11(self):
-        to_send = "GET /notfilelike_shortcl HTTP/1.1\n\n"
+        to_send = "GET /notfilelike_shortcl HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1337,7 +1362,7 @@ def test_notfilelike_shortcl_http11(self):
             # connection has not been closed

     def test_notfilelike_longcl_http11(self):
-        to_send = "GET /notfilelike_longcl HTTP/1.1\n\n"
+        to_send = "GET /notfilelike_longcl HTTP/1.1\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1356,7 +1381,7 @@ def test_notfilelike_longcl_http11(self):
         self.assertRaises(ConnectionClosed, read_http, fp)

     def test_filelike_http10(self):
-        to_send = "GET /filelike HTTP/1.0\n\n"
+        to_send = "GET /filelike HTTP/1.0\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1375,7 +1400,7 @@ def test_filelike_http10(self):
         self.assertRaises(ConnectionClosed, read_http, fp)

     def test_filelike_nocl_http10(self):
-        to_send = "GET /filelike_nocl HTTP/1.0\n\n"
+        to_send = "GET /filelike_nocl HTTP/1.0\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1394,7 +1419,7 @@ def test_filelike_nocl_http10(self):
         self.assertRaises(ConnectionClosed, read_http, fp)

     def test_notfilelike_http10(self):
-        to_send = "GET /notfilelike HTTP/1.0\n\n"
+        to_send = "GET /notfilelike HTTP/1.0\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1413,7 +1438,7 @@ def test_notfilelike_http10(self):
         self.assertRaises(ConnectionClosed, read_http, fp)

     def test_notfilelike_nocl_http10(self):
-        to_send = "GET /notfilelike_nocl HTTP/1.0\n\n"
+        to_send = "GET /notfilelike_nocl HTTP/1.0\r\n\r\n"
         to_send = tobytes(to_send)

         self.connect()
@@ -1582,7 +1607,7 @@ def read_http(fp):  # pragma: no cover
     header_lines = []
     while True:
         line = fp.readline()
-        if line in (b"\r\n", b"\n", b""):
+        if line in (b"\r\n", b"\r\n", b""):
             break
         else:
             header_lines.append(line)