[3.2.x] Fixed CVE-2022-23833 -- Fixed DoS possiblity in file uploads.

Thanks Alan Ryan for the report and initial patch.

Backport of fc18f36c4ab94399366ca2f2007b3692559a6f23 from main.

@@ -143,6 +143,26 @@ def test_big_base64_upload(self):
     def test_big_base64_newlines_upload(self):
         self._test_base64_upload("Big data" * 68000, encode=base64.encodebytes)

+    def test_base64_invalid_upload(self):
+        payload = client.FakePayload('\r\n'.join([
+            '--' + client.BOUNDARY,
+            'Content-Disposition: form-data; name="file"; filename="test.txt"',
+            'Content-Type: application/octet-stream',
+            'Content-Transfer-Encoding: base64',
+            ''
+        ]))
+        payload.write(b'\r\n!\r\n')
+        payload.write('--' + client.BOUNDARY + '--\r\n')
+        r = {
+            'CONTENT_LENGTH': len(payload),
+            'CONTENT_TYPE': client.MULTIPART_CONTENT,
+            'PATH_INFO': '/echo_content/',
+            'REQUEST_METHOD': 'POST',
+            'wsgi.input': payload,
+        }
+        response = self.client.request(**r)
+        self.assertEqual(response.json()['file'], '')
+
     def test_unicode_file_name(self):
         with sys_tempfile.TemporaryDirectory() as temp_dir:
             # This file contains Chinese symbols and an accented char in the name.


Replaced use of tempfile.mktemp() with tempfile.mkstemp() (#3358)

Signed-off-by: Abin Shahab <ashahab@linkedin.com>

@@ -640,7 +640,7 @@ def test(output, expected, exit_code=0):
         test(("HYDRA build details:\n"
               "    Version:           3.3a2\n"
               "    Configure options: 'MPICHLIB_CFLAGS=-g -O2'\n"), _MPICH_IMPL)
-
+
         test("Intel(R) MPI", _IMPI_IMPL)

         test("Unknown MPI v1.00", _UNKNOWN_IMPL)


security: reject whitespace in HTTP/1 header names

This commit fixes GHSA-gcx2-gvj7-pxv3 by making mitmproxy
reject header names that contain whitespace characters by default.
A new `validate_inbound_headers` option is provided to turn this behavior
off at the expense of allowing HTTP smuggling vulnerabilities.

@@ -1261,6 +1261,37 @@ def test_request_smuggling(tctx):
     assert b"Received both a Transfer-Encoding and a Content-Length header" in err()

+def test_request_smuggling_whitespace(tctx):
+    """Test that we reject header names with whitespace"""
+    err = Placeholder(bytes)
+    assert (
+        Playbook(http.HttpLayer(tctx, HTTPMode.regular), hooks=False)
+        >> DataReceived(tctx.client, b"GET http://example.com/ HTTP/1.1\r\n"
+                                     b"Host: example.com\r\n"
+                                     b"Content-Length : 42\r\n\r\n")
+        << SendData(tctx.client, err)
+        << CloseConnection(tctx.client)
+    )
+    assert b"Received an invalid header name" in err()
+
+
+def test_request_smuggling_validation_disabled(tctx):
+    """Test that we don't reject request smuggling when validation is disabled."""
+    tctx.options.validate_inbound_headers = False
+    assert (
+        Playbook(http.HttpLayer(tctx, HTTPMode.regular), hooks=False)
+        >> DataReceived(tctx.client, b"GET http://example.com/ HTTP/1.1\r\n"
+                                     b"Host: example.com\r\n"
+                                     b"Content-Length: 4\r\n"
+                                     b"Transfer-Encoding: chunked\r\n\r\n"
+                                     b"4\r\n"
+                                     b"abcd\r\n"
+                                     b"0\r\n"
+                                     b"\r\n")
+        << OpenConnection(Placeholder(Server))
+    )
+
+
 def test_request_smuggling_te_te(tctx):
     """Test that we reject transfer-encoding headers that are weird in some way"""
     err = Placeholder(bytes)


Merge pull request from GHSA-4f7p-27jc-3c36

Fix for HTTP request smuggling due to incorrect validation

@@ -155,7 +155,7 @@ def test_received_chunked_completed_sets_content_length(self):
             b"Transfer-Encoding: chunked\r\n"
             b"X-Foo: 1\r\n"
             b"\r\n"
-            b"1d;\r\n"
+            b"1d\r\n"
             b"This string has 29 characters\r\n"
             b"0\r\n\r\n"
         )
@@ -193,6 +193,26 @@ def test_parse_header_bad_content_length(self):
         else:  # pragma: nocover
             self.assertTrue(False)

+    def test_parse_header_bad_content_length_plus(self):
+        data = b"GET /foobar HTTP/8.4\r\ncontent-length: +10\r\n"
+
+        try:
+            self.parser.parse_header(data)
+        except ParsingError as e:
+            self.assertIn("Content-Length is invalid", e.args[0])
+        else:  # pragma: nocover
+            self.assertTrue(False)
+
+    def test_parse_header_bad_content_length_minus(self):
+        data = b"GET /foobar HTTP/8.4\r\ncontent-length: -10\r\n"
+
+        try:
+            self.parser.parse_header(data)
+        except ParsingError as e:
+            self.assertIn("Content-Length is invalid", e.args[0])
+        else:  # pragma: nocover
+            self.assertTrue(False)
+
     def test_parse_header_multiple_content_length(self):
         data = b"GET /foobar HTTP/8.4\r\ncontent-length: 10\r\ncontent-length: 20\r\n"


Require manage orders for fetching `user.orders` (#9128)

* Require manage orders for fetching customer.orders

* Update changelog

Co-authored-by: Marcin Gębala <5421321+maarcingebala@users.noreply.github.com>

@@ -364,9 +364,13 @@ def resolve_orders(root: models.User, info, **kwargs):
         def _resolve_orders(orders):
             requester = get_user_or_app_from_context(info.context)
             if not requester.has_perm(OrderPermissions.MANAGE_ORDERS):
-                orders = list(
-                    filter(lambda order: order.status != OrderStatus.DRAFT, orders)
-                )
+                # allow fetch requestor orders (except drafts)
+                if root == info.context.user:
+                    orders = list(
+                        filter(lambda order: order.status != OrderStatus.DRAFT, orders)
+                    )
+                else:
+                    raise PermissionDenied()

             return create_connection_slice(
                 orders, info, kwargs, OrderCountableConnection


@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-

 from django.contrib.auth.decorators import login_required
-from django.shortcuts import render, redirect, get_object_or_404
+from django.shortcuts import render, get_object_or_404

-from ...core.utils.views import is_post, post_data
+from spirit.core.utils.http import safe_redirect
+from spirit.core.utils.views import is_post, post_data
 from ..models import Comment
 from .forms import FlagForm

@@ -18,7 +19,7 @@ def create(request, comment_id):

     if is_post(request) and form.is_valid():
         form.save()
-        return redirect(request.POST.get('next', comment.get_absolute_url()))
+        return safe_redirect(request, 'next', comment.get_absolute_url(), method='POST')

     return render(
         request=request,


@@ -6,6 +6,7 @@

 from djconfig import config

+from spirit.core.utils.http import safe_redirect
 from spirit.core.utils.views import is_post, post_data
 from spirit.core.utils.paginator import paginate, yt_paginate
 from spirit.core.utils.ratelimit.decorators import ratelimit
@@ -38,9 +39,9 @@ def publish(request, category_id=None):
             all([form.is_valid(), cform.is_valid()]) and
             not request.is_limited()):
         if not user.st.update_post_hash(form.get_topic_hash()):
-            return redirect(
-                request.POST.get('next', None) or
-                form.get_category().get_absolute_url())
+            default_url = lambda: form.get_category().get_absolute_url()
+            return safe_redirect(
+                request, 'next', default_url, method='POST')
         # wrap in transaction.atomic?
         topic = form.save()
         cform.topic = topic
@@ -66,7 +67,7 @@ def update(request, pk):
         if topic.category_id != category_id:
             Comment.create_moderation_action(
                 user=request.user, topic=topic, action=Comment.MOVED)
-        return redirect(request.POST.get('next', topic.get_absolute_url()))
+        return safe_redirect(request,'next', topic.get_absolute_url(), method='POST')
     return render(
         request=request,
         template_name='spirit/topic/update.html',


Fixed exposed credentials in exception

Under certain conditions, sensitive connection parameters are shown in tracebacks when Ansible receives an unexpected response (in my case `None`) from `set_options` in ansible-connection. This patch creates a list of sensitive connection parameter names from `ansible.constants`, pulls a list of those values in the `params` that are about to be dumped to the screen and log file, and sends the `params` and list of `sensitive_values` to the `module_utils.common.parameters.remove_values` function for processing to hide them before being exposed.

@@ -38,10 +38,13 @@
 import uuid

 from functools import partial
+from ansible import constants as C
 from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.common.json import AnsibleJSONEncoder
+from ansible.module_utils.common.parameters import remove_values
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.six.moves import cPickle
+from ansible.utils.helpers import deduplicate_list

 def write_to_file_descriptor(fd, obj):
@@ -163,12 +166,26 @@ def _exec_jsonrpc(self, name, *args, **kwargs):
         try:
             response = json.loads(out)
         except ValueError:
-            params = [repr(arg) for arg in args] + ['{0}={1!r}'.format(k, v) for k, v in iteritems(kwargs)]
+            sensitive_keys = list(
+                    C.MAGIC_VARIABLE_MAPPING["password"]
+                    + C.MAGIC_VARIABLE_MAPPING["private_key_file"]
+                    + C.MAGIC_VARIABLE_MAPPING["become_pass"]
+            )
+            sensitive_values = [
+                v2
+                for k, v in iteritems(kwargs)
+                for k2, v2 in iteritems(v)
+                if k2 in sensitive_keys
+            ]
+            params = [repr(remove_values(arg, sensitive_values)) for arg in args] + [
+                "{0}={1!r}".format(k, remove_values(v, sensitive_values))
+                for k, v in iteritems(kwargs)]
             params = ', '.join(params)
             raise ConnectionError(
                 "Unable to decode JSON from response to {0}({1}). Received '{2}'.".format(name, params, out)
             )

+
         if response['id'] != reqid:
             raise ConnectionError('invalid json-rpc id received')
         if "result_type" in response:


fix(toolbar): only redirect to hosts in app_urls (#9268)

* fix(toolbar): only redirect to hosts in app_urls

* fix mypy and regex

@@ -56,6 +56,15 @@ def test_unauthenticated_routes_get_loaded_on_the_frontend(self):
         self.assertEqual(response.status_code, status.HTTP_200_OK)

     def test_authorize_and_redirect_domain(self):
+        self.team.app_urls = ["https://domain.com", "https://not.com"]
+        self.team.save()
+
+        response = self.client.get(
+            "/authorize_and_redirect/?redirect=https://not-permitted.com", HTTP_REFERER="https://not-permitted.com"
+        )
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertTrue("Can only redirect to a permitted domain." in str(response.content))
+
         response = self.client.get(
             "/authorize_and_redirect/?redirect=https://domain.com", HTTP_REFERER="https://not.com"
         )


Merge pull request from GHSA-j2x6-9323-fp7h

This commit addresses two issues in validating returndata, both related
to the inferred type of the external call return.

First, it addresses an issue with interfaces imported from JSON. The
JSON_ABI encoding type was added in 0.3.0 as part of the calling
convention refactor to mimic the old code's behavior when the signature
of a function had `is_from_json` toggled to True. However, both
implementations were a workaround for the fact that in
FunctionSignatures from JSON with Bytes return types, length is set to 1
as a hack to ensure they always typecheck - almost always resulting in a
runtime revert.

This commit removes the JSON_ABI encoding type, so that dynamic
returndata from an interface defined with .json ABI file cannot result
in a buffer overrun(!). To avoid the issue with always runtime
reverting, codegen uses the uses the inferred ContractFunction type of
the Call.func member (which is both more accurate than the inferred type
of the Call expression, and the return type on the FunctionSignature!)
to calculate the length of the external Bytes array.

Second, this commit addresses an issue with validating call returns in
complex expressions. In the following examples, the type of the call
return is either inferred incorrectly or it takes a path through codegen
which avoids generating runtime clamps:

```
interface Foo:
    def returns_int128() -> int128: view
    def returns_Bytes3() -> Bytes[3]: view

foo: Foo
...
x: uint256 = convert(self.foo.returns_int128(), uint256)
y: Bytes[32] = concat(self.foo.returns_Bytes3(), b"")
```

To address this issue, if the type of returndata needs validation, this
commit decodes the returndata "strictly" into a newly allocated buffer
at the time of the call, to avoid unvalidated data accidentally getting
into the runtime. This does result in extra memory traffic which is a
performance hit, but the performance issue can be addressed at a later
date with a zero-copy buffering scheme (parent Expr allocates the
buffer).

Additional minor fixes and cleanup:
- fix compiler panic in new_type_to_old_type for Tuples
- remove `_should_decode` helper function as it duplicates `needs_clamp`
- minor optimization in returndatasize check - assert ge uses one fewer
  instruction than assert gt.

@@ -47,8 +47,6 @@ class Encoding(Enum):
     VYPER = auto()
     # abi encoded, default for args/return values from external funcs
     ABI = auto()
-    # abi encoded, same as ABI but no clamps for bytestrings
-    JSON_ABI = auto()
     # future: packed


Merge pull request from GHSA-7vrm-3jc8-5wwm

* add more tests for string comparison

explicitly test the codepath with <= 32 bytes

* refactor keccak256 helper a bit

* fix bytestring equality

existing bytestring equality checks do not check length equality or for
dirty bytes.

@@ -210,6 +210,36 @@ def ret10_slice() -> Bytes[10]:
     assert c.ret10_slice() == b"A"

+def test_slice_equality(get_contract):
+    # test for equality with dirty bytes
+    code = """
+@external
+def assert_eq() -> bool:
+    dirty_bytes: String[4] = "abcd"
+    dirty_bytes = slice(dirty_bytes, 0, 3)
+    clean_bytes: String[4] = "abc"
+    return dirty_bytes == clean_bytes
+    """
+
+    c = get_contract(code)
+    assert c.assert_eq()
+
+
+def test_slice_inequality(get_contract):
+    # test for equality with dirty bytes
+    code = """
+@external
+def assert_ne() -> bool:
+    dirty_bytes: String[4] = "abcd"
+    dirty_bytes = slice(dirty_bytes, 0, 3)
+    clean_bytes: String[4] = "abcd"
+    return dirty_bytes != clean_bytes
+    """
+
+    c = get_contract(code)
+    assert c.assert_ne()
+
+
 def test_slice_convert(get_contract):
     # test slice of converting between bytes32 and Bytes
     code = """


Better epub cover parsing with multiple cover-image items
Code cosmetics
renamed variables
refactored xml page generation
refactored prepare author

@@ -53,11 +53,11 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):

     txt = epub_zip.read('META-INF/container.xml')
     tree = etree.fromstring(txt)
-    cfname = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
-    cf = epub_zip.read(cfname)
+    cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
+    cf = epub_zip.read(cf_name)
     tree = etree.fromstring(cf)

-    coverpath = os.path.dirname(cfname)
+    cover_path = os.path.dirname(cf_name)

     p = tree.xpath('/pkg:package/pkg:metadata', namespaces=ns)[0]

@@ -90,7 +90,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):

     epub_metadata = parse_epub_series(ns, tree, epub_metadata)

-    cover_file = parse_epub_cover(ns, tree, epub_zip, coverpath, tmp_file_path)
+    cover_file = parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path)

     if not epub_metadata['title']:
         title = original_file_name
@@ -114,9 +114,12 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
 def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path):
     cover_section = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns)
     cover_file = None
-    if len(cover_section) > 0:
-        cover_file = _extract_cover(epub_zip, cover_section[0], cover_path, tmp_file_path)
-    else:
+    # if len(cover_section) > 0:
+    for cs in cover_section:
+        cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path)
+        if cover_file:
+            break
+    if not cover_file:
         meta_cover = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='cover']/@content", namespaces=ns)
         if len(meta_cover) > 0:
             cover_section = tree.xpath(
@@ -143,8 +146,7 @@ def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path):
                     cover_file = _extract_cover(epub_zip, filename, "", tmp_file_path)
             else:
                 cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path)
-            if cover_file:
-                break
+            if cover_file: break
     return cover_file


@@ -17,13 +17,13 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.

-import copy
 import os
 import re
 import ast
 import json
 from datetime import datetime
 from urllib.parse import quote
+import unidecode

 from sqlalchemy import create_engine
 from sqlalchemy import Table, Column, ForeignKey, CheckConstraint
@@ -49,11 +49,6 @@

 from weakref import WeakSet

-try:
-    import unidecode
-    use_unidecode = True
-except ImportError:
-    use_unidecode = False

 log = logger.create()

@@ -93,7 +88,7 @@
                               )

-class Library_Id(Base):
+class LibraryId(Base):
     __tablename__ = 'library_id'
     id = Column(Integer, primary_key=True)
     uuid = Column(String, nullable=False)
@@ -112,7 +107,7 @@ def __init__(self, val, id_type, book):
         self.type = id_type
         self.book = book

-    def formatType(self):
+    def format_type(self):
         format_type = self.type.lower()
         if format_type == 'amazon':
             return u"Amazon"
@@ -184,8 +179,8 @@ class Comments(Base):
     book = Column(Integer, ForeignKey('books.id'), nullable=False, unique=True)
     text = Column(String(collation='NOCASE'), nullable=False)

-    def __init__(self, text, book):
-        self.text = text
+    def __init__(self, comment, book):
+        self.text = comment
         self.book = book

     def get(self):
@@ -367,18 +362,17 @@ def __init__(self, title, sort, author_sort, timestamp, pubdate, series_index, l
         self.path = path
         self.has_cover = (has_cover != None)

-
     def __repr__(self):
         return u"<Books('{0},{1}{2}{3}{4}{5}{6}{7}{8}')>".format(self.title, self.sort, self.author_sort,
                                                                  self.timestamp, self.pubdate, self.series_index,
                                                                  self.last_modified, self.path, self.has_cover)

     @property
     def atom_timestamp(self):
-        return (self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or '')
+        return self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or ''

-class Custom_Columns(Base):
+class CustomColumns(Base):
     __tablename__ = 'custom_columns'

     id = Column(Integer, primary_key=True)
@@ -436,7 +430,7 @@ def default(self, o):
         return json.JSONEncoder.default(self, o)

-class CalibreDB():
+class CalibreDB:
     _init = False
     engine = None
     config = None
@@ -450,17 +444,17 @@ def __init__(self, expire_on_commit=True):
         """
         self.session = None
         if self._init:
-            self.initSession(expire_on_commit)
+            self.init_session(expire_on_commit)

         self.instances.add(self)

-    def initSession(self, expire_on_commit=True):
+    def init_session(self, expire_on_commit=True):
         self.session = self.session_factory()
         self.session.expire_on_commit = expire_on_commit
         self.update_title_sort(self.config)

     @classmethod
-    def setup_db_cc_classes(self, cc):
+    def setup_db_cc_classes(cls, cc):
         cc_ids = []
         books_custom_column_links = {}
         for row in cc:
@@ -539,16 +533,16 @@ def check_valid_db(cls, config_calibre_dir, app_db_path, config_calibre_uuid):
             return False, False
         try:
             check_engine = create_engine('sqlite://',
-                          echo=False,
-                          isolation_level="SERIALIZABLE",
-                          connect_args={'check_same_thread': False},
-                          poolclass=StaticPool)
+                                         echo=False,
+                                         isolation_level="SERIALIZABLE",
+                                         connect_args={'check_same_thread': False},
+                                         poolclass=StaticPool)
             with check_engine.begin() as connection:
                 connection.execute(text("attach database '{}' as calibre;".format(dbpath)))
                 connection.execute(text("attach database '{}' as app_settings;".format(app_db_path)))
                 local_session = scoped_session(sessionmaker())
                 local_session.configure(bind=connection)
-                database_uuid = local_session().query(Library_Id).one_or_none()
+                database_uuid = local_session().query(LibraryId).one_or_none()
                 # local_session.dispose()

             check_engine.connect()
@@ -603,7 +597,7 @@ def setup_db(cls, config_calibre_dir, app_db_path):
                                                           autoflush=True,
                                                           bind=cls.engine))
         for inst in cls.instances:
-            inst.initSession()
+            inst.init_session()

         cls._init = True
         return True
@@ -644,12 +638,10 @@ def get_book_format(self, book_id, file_format):
     # Language and content filters for displaying in the UI
     def common_filters(self, allow_show_archived=False, return_all_languages=False):
         if not allow_show_archived:
-            archived_books = (
-                ub.session.query(ub.ArchivedBook)
-                    .filter(ub.ArchivedBook.user_id == int(current_user.id))
-                    .filter(ub.ArchivedBook.is_archived == True)
-                    .all()
-            )
+            archived_books = (ub.session.query(ub.ArchivedBook)
+                              .filter(ub.ArchivedBook.user_id == int(current_user.id))
+                              .filter(ub.ArchivedBook.is_archived == True)
+                              .all())
             archived_book_ids = [archived_book.book_id for archived_book in archived_books]
             archived_filter = Books.id.notin_(archived_book_ids)
         else:
@@ -668,11 +660,11 @@ def common_filters(self, allow_show_archived=False, return_all_languages=False):
                 pos_cc_list = current_user.allowed_column_value.split(',')
                 pos_content_cc_filter = true() if pos_cc_list == [''] else \
                     getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
-                        any(cc_classes[self.config.config_restricted_column].value.in_(pos_cc_list))
+                    any(cc_classes[self.config.config_restricted_column].value.in_(pos_cc_list))
                 neg_cc_list = current_user.denied_column_value.split(',')
                 neg_content_cc_filter = false() if neg_cc_list == [''] else \
                     getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
-                        any(cc_classes[self.config.config_restricted_column].value.in_(neg_cc_list))
+                    any(cc_classes[self.config.config_restricted_column].value.in_(neg_cc_list))
             except (KeyError, AttributeError):
                 pos_content_cc_filter = false()
                 neg_content_cc_filter = true()
@@ -728,7 +720,7 @@ def fill_indexpage_with_archived_books(self, page, database, pagesize, db_filter
                 query = (self.session.query(database, ub.ReadBook.read_status, ub.ArchivedBook.is_archived)
                          .select_from(Books)
                          .outerjoin(ub.ReadBook,
-                               and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == Books.id)))
+                                    and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == Books.id)))
             else:
                 try:
                     read_column = cc_classes[config_read_column]
@@ -738,7 +730,7 @@ def fill_indexpage_with_archived_books(self, page, database, pagesize, db_filter
                 except (KeyError, AttributeError):
                     log.error("Custom Column No.%d is not existing in calibre database", read_column)
                     # Skip linking read column and return None instead of read status
-                    query =self.session.query(database, None, ub.ArchivedBook.is_archived)
+                    query = self.session.query(database, None, ub.ArchivedBook.is_archived)
             query = query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id,
                                                           int(current_user.id) == ub.ArchivedBook.user_id))
         else:
@@ -812,7 +804,6 @@ def order_authors(self, entries, list_return=False, combined=False):
                 return authors_ordered
         return entries

-
     def get_typeahead(self, database, query, replace=('', ''), tag_filter=true()):
         query = query or ''
         self.session.connection().connection.connection.create_function("lower", 1, lcase)
@@ -872,7 +863,7 @@ def search_query(self, term, config_read_column, *join):
                 ))

     # read search results from calibre-database and return it (function is used for feed and simple search
-    def get_search_results(self, term, offset=None, order=None, limit=None, allow_show_archived=False,
+    def get_search_results(self, term, offset=None, order=None, limit=None,
                            config_read_column=False, *join):
         order = order[0] if order else [Books.sort]
         pagination = None
@@ -915,7 +906,6 @@ def speaking_language(self, languages=None, return_all_languages=False, with_cou
                 lang.name = isoLanguages.get_language_name(get_locale(), lang.lang_code)
             return sorted(languages, key=lambda x: x.name, reverse=reverse_order)

-
     def update_title_sort(self, config, conn=None):
         # user defined sort function for calibre databases (Series, etc.)
         def _title_sort(title):
@@ -973,6 +963,6 @@ def lcase(s):
     try:
         return unidecode.unidecode(s.lower())
     except Exception as ex:
-        log = logger.create()
-        log.error_or_exception(ex)
+        _log = logger.create()
+        _log.error_or_exception(ex)
         return s.lower()


Merge pull request #377 from Pylons/bugfix/select-closed-socket-race

Bugfix: Retry if a thread closes a socket before we select() on it

@@ -426,15 +426,16 @@ def accept(self):
         else:
             return conn, addr

-    def send(self, data):
+    def send(self, data, do_close=True):
         try:
             result = self.socket.send(data)
             return result
         except OSError as why:
             if why.args[0] == EWOULDBLOCK:
                 return 0
             elif why.args[0] in _DISCONNECTED:
-                self.handle_close()
+                if do_close:
+                    self.handle_close()
                 return 0
             else:
                 raise


Remove use of `eval` when evaluating the input example.

Use `ast.eval_literal` instead which safely evaluates the expression.

PiperOrigin-RevId: 400012249
Change-Id: I5ff98608ea2d736d093aa488af723ff4f6707e02

@@ -20,6 +20,7 @@
 """

 import argparse
+import ast
 import os
 import re
 import sys
@@ -521,16 +522,18 @@ def preprocess_inputs_arg_string(inputs_str):
   return input_dict

-def preprocess_input_exprs_arg_string(input_exprs_str):
+def preprocess_input_exprs_arg_string(input_exprs_str, safe=True):
   """Parses input arg into dictionary that maps input key to python expression.

   Parses input string in the format of 'input_key=<python expression>' into a
   dictionary that maps each input_key to its python expression.

   Args:
     input_exprs_str: A string that specifies python expression for input keys.
-    Each input is separated by semicolon. For each input key:
+      Each input is separated by semicolon. For each input key:
         'input_key=<python expression>'
+    safe: Whether to evaluate the python expression as literals or allow
+      arbitrary calls (e.g. numpy usage).

   Returns:
     A dictionary that maps input keys to their values.
@@ -545,8 +548,15 @@ def preprocess_input_exprs_arg_string(input_exprs_str):
       raise RuntimeError('--input_exprs "%s" format is incorrect. Please follow'
                          '"<input_key>=<python expression>"' % input_exprs_str)
     input_key, expr = input_raw.split('=', 1)
-    # ast.literal_eval does not work with numpy expressions
-    input_dict[input_key] = eval(expr)  # pylint: disable=eval-used
+    if safe:
+      try:
+        input_dict[input_key] = ast.literal_eval(expr)
+      except:
+        raise RuntimeError(
+            f'Expression "{expr}" is not a valid python literal.')
+    else:
+      # ast.literal_eval does not work with numpy expressions
+      input_dict[input_key] = eval(expr)  # pylint: disable=eval-used
   return input_dict

@@ -659,7 +669,7 @@ def load_inputs_from_input_arg_string(inputs_str, input_exprs_str,
   tensor_key_feed_dict = {}

   inputs = preprocess_inputs_arg_string(inputs_str)
-  input_exprs = preprocess_input_exprs_arg_string(input_exprs_str)
+  input_exprs = preprocess_input_exprs_arg_string(input_exprs_str, safe=False)
   input_examples = preprocess_input_examples_arg_string(input_examples_str)

   for input_tensor_key, (filename, variable_name) in inputs.items():
@@ -923,8 +933,10 @@ def add_run_subparser(subparsers):
   parser_run.add_argument('--inputs', type=str, default='', help=msg)
   msg = ('Specifying inputs by python expressions, in the format of'
          ' "<input_key>=\'<python expression>\'", separated by \';\'. '
-         'numpy module is available as \'np\'. '
-         'Will override duplicate input keys from --inputs option.')
+         'numpy module is available as \'np\'. Please note that the expression '
+         'will be evaluated as-is, and is susceptible to code injection. '
+         'When this is set, the value will override duplicate input keys from '
+         '--inputs option.')
   parser_run.add_argument('--input_exprs', type=str, default='', help=msg)
   msg = (
       'Specifying tf.Example inputs as list of dictionaries. For example: '


Fix tf.raw_ops.EditDistance vulnerability with negative indices.

Check that indices are non-negative. Fix several identical code sites.
Clean up grammar in error message.

PiperOrigin-RevId: 445442017

@@ -207,6 +207,24 @@ def testEditDistanceZeroLengthHypothesisAndTruth(self):
         normalize=True,
         expected_output=expected_output)

+  def testEditDistanceBadIndices(self):
+    hypothesis_indices = np.full((3, 3), -1250999896764, dtype=np.int64)
+    hypothesis_values = np.empty(3, dtype=np.int64)
+    hypothesis_shape = np.empty(3, dtype=np.int64)
+    truth_indices = np.full((3, 3), -1250999896764, dtype=np.int64)
+    truth_values = np.full([3], 2, dtype=np.int64)
+    truth_shape = np.full([3], 2, dtype=np.int64)
+    expected_output = []  # dummy; ignored
+
+    self._testEditDistance(
+        hypothesis=(hypothesis_indices, hypothesis_values, hypothesis_shape),
+        truth=(truth_indices, truth_values, truth_shape),
+        normalize=False,
+        expected_output=expected_output,
+        expected_err_re=(r"inner product -\d+ which would require writing "
+                         "to outside of the buffer for the output tensor")
+    )
+

 if __name__ == "__main__":
   test.main()


Fix security vulnerability with SpaceToBatchNDOp.

PiperOrigin-RevId: 445527615

@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/core/util/overflow.h"

 namespace tensorflow {
 namespace {
@@ -60,10 +61,14 @@ void SpaceToBatch(XlaOpKernelContext* ctx, const xla::XlaOp& input,
     int64_t pad_end = paddings.Get<int64_t>({i, 1});
     OP_REQUIRES(ctx, pad_start >= 0 && pad_end >= 0,
                 errors::InvalidArgument("Paddings must be non-negative"));
+    OP_REQUIRES(ctx, block_shape[i] >= 1,
+                errors::InvalidArgument(
+                    "All values in block_shape must be positive, got value, ",
+                    block_shape[i], " at index ", i, "."));
     dim->set_edge_padding_low(pad_start);
     dim->set_edge_padding_high(pad_end);
     padded_shape[1 + i] += pad_start + pad_end;
-    block_num_elems *= block_shape[i];
+    block_num_elems = MultiplyWithoutOverflow(block_num_elems, block_shape[i]);
   }
   // Don't pad the remainder dimensions.
   for (int i = 0; i < remainder_shape.size(); ++i) {
@@ -72,6 +77,16 @@ void SpaceToBatch(XlaOpKernelContext* ctx, const xla::XlaOp& input,
   OP_REQUIRES(ctx, block_num_elems > 0,
               errors::InvalidArgument(
                   "The product of the block dimensions must be positive"));
+  const int64_t batch_size = input_shape[0];
+  const int64_t output_dim =
+      MultiplyWithoutOverflow(batch_size, block_num_elems);
+  if (output_dim < 0) {
+    OP_REQUIRES(
+        ctx, output_dim >= 0,
+        errors::InvalidArgument("Negative output dimension size caused by "
+                                "overflow when multiplying ",
+                                batch_size, " and ", block_num_elems));
+  }

   xla::XlaOp padded =
       xla::Pad(input, XlaHelpers::Zero(b, input_dtype), padding_config);
@@ -85,7 +100,6 @@ void SpaceToBatch(XlaOpKernelContext* ctx, const xla::XlaOp& input,
   //       padded_shape[M] / block_shape[M-1],
   //       block_shape[M-1]] +
   //      remaining_shape
-  const int64_t batch_size = input_shape[0];
   std::vector<int64_t> reshaped_padded_shape(input_rank + block_rank);
   reshaped_padded_shape[0] = batch_size;
   for (int i = 0; i < block_rank; ++i) {
@@ -134,7 +148,7 @@ void SpaceToBatch(XlaOpKernelContext* ctx, const xla::XlaOp& input,
   // Determine the length of the prefix of block dims that can be combined
   // into the batch dimension due to having no padding and block_shape=1.
   std::vector<int64_t> output_shape(input_rank);
-  output_shape[0] = batch_size * block_num_elems;
+  output_shape[0] = output_dim;
   for (int i = 0; i < block_rank; ++i) {
     output_shape[1 + i] = padded_shape[1 + i] / block_shape[i];
   }


Fix empty resource handle vulnerability.

Some ops that attempt to extract a resource handle from user input
can lead to nullptr dereferences.  This returns an error in such
a case.

PiperOrigin-RevId: 445571938

@@ -304,6 +304,9 @@ Status GetDeviceForInput(const EagerOperation& op, const EagerContext& ctx,
     const Tensor* tensor;
     // TODO(fishx): Avoid blocking here.
     TF_RETURN_IF_ERROR(tensor_handle->Tensor(&tensor));
+    if (tensor->NumElements() == 0) {
+      return errors::InvalidArgument("Empty resource handle");
+    }
     const ResourceHandle& handle = tensor->flat<ResourceHandle>()(0);
     device_name = handle.device();


Fix tf.raw_ops.StagePeek vulnerability with invalid `index`.

Check that input is actually a scalar before treating it as such.

PiperOrigin-RevId: 445524908

@@ -258,6 +258,8 @@ class StagePeekOp : public OpKernel {
     core::ScopedUnref scope(buf);
     Buffer::Tuple tuple;

+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(ctx->input(0).shape()),
+                errors::InvalidArgument("index must be scalar"));
     std::size_t index = ctx->input(0).scalar<int>()();

     OP_REQUIRES_OK(ctx, buf->Peek(index, &tuple));


tenant, verifier: let the tenant provide the AK and mTLS certificate

This eliminates the need for the verifier to connect to the registrar.

Signed-off-by: Thore Sommer <mail@thson.de>

@@ -631,6 +631,8 @@ def do_cv(self):
             'accept_tpm_hash_algs': self.accept_tpm_hash_algs,
             'accept_tpm_encryption_algs': self.accept_tpm_encryption_algs,
             'accept_tpm_signing_algs': self.accept_tpm_signing_algs,
+            'ak_tpm': self.registrar_data['aik_tpm'],
+            'mtls_cert': self.registrar_data.get('mtls_cert', None),
             'supported_version': self.supported_version,
         }
         json_message = json.dumps(data)


Remove reliance on django-markdownx (#3231)

* Remove reliance on django-markdownx

- We are now rendering notes on the client side using easymde
- No longer any need to utilize the markdownx integration
- Adds character limit for notes fields`

* Adjust legacy migrations - remove references to markdownx

* Fix bug for company notes field

@@ -126,9 +126,6 @@

     re_path(r'^api/', include(apipatterns)),
     re_path(r'^api-doc/', include_docs_urls(title='InvenTree API')),
-
-    # 3rd party endpoints
-    re_path(r'^markdownx/', include('markdownx.urls')),
 ]

 frontendpatterns = [


@@ -6,7 +6,6 @@
 import django.core.validators
 from django.db import migrations, models
 import django.db.models.deletion
-import markdownx.models

 class Migration(migrations.Migration):
@@ -29,7 +28,7 @@ class Migration(migrations.Migration):
                 ('status', models.PositiveIntegerField(choices=[(10, 'Pending'), (20, 'Placed'), (30, 'Complete'), (40, 'Cancelled'), (50, 'Lost'), (60, 'Returned')], default=10, help_text='Order status')),
                 ('issue_date', models.DateField(blank=True, null=True)),
                 ('complete_date', models.DateField(blank=True, null=True)),
-                ('notes', markdownx.models.MarkdownxField(blank=True, help_text='Order notes')),
+                ('notes', models.TextField(blank=True, help_text='Order notes')),
                 ('customer_reference', models.CharField(blank=True, help_text='Customer order reference code', max_length=64)),
                 ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
                 ('customer', models.ForeignKey(help_text='Customer', limit_choices_to={True, 'is_supplier'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='sales_orders', to='company.Company')),


@@ -25,7 +25,6 @@
 from djmoney.contrib.exchange.exceptions import MissingRate
 from djmoney.contrib.exchange.models import convert_money
 from jinja2 import Template
-from markdownx.models import MarkdownxField
 from mptt.exceptions import InvalidMove
 from mptt.managers import TreeManager
 from mptt.models import MPTTModel, TreeForeignKey
@@ -41,7 +40,7 @@
 from common.settings import currency_code_default
 from company.models import SupplierPart
 from InvenTree import helpers, validators
-from InvenTree.fields import InvenTreeURLField
+from InvenTree.fields import InvenTreeNotesField, InvenTreeURLField
 from InvenTree.helpers import decimal2money, decimal2string, normalize
 from InvenTree.models import (DataImportMixin, InvenTreeAttachment,
                               InvenTreeTree)
@@ -920,11 +919,7 @@ def get_default_supplier(self):
         verbose_name=_('Virtual'),
         help_text=_('Is this a virtual part, such as a software product or license?'))

-    notes = MarkdownxField(
-        blank=True, null=True,
-        verbose_name=_('Notes'),
-        help_text=_('Part notes - supports Markdown formatting')
-    )
+    notes = InvenTreeNotesField(help_text=_('Part notes'))

     bom_checksum = models.CharField(max_length=128, blank=True, verbose_name=_('BOM checksum'), help_text=_('Stored BOM checksum'))


Back porting of security patches (#3197)

* Merge pull request from GHSA-fr2w-mp56-g4xp

* Enforce file download for attachments table(s)

* Enforce file download for attachment in 'StockItemTestResult' table

(cherry picked from commit 76aa3a75f2e5b93877a229e29326b8b4ea815aea)

* Merge pull request from GHSA-7rq4-qcpw-74gq

* Merge pull request from GHSA-rm89-9g65-4ffr

* Enable HTML escaping for all tables by default

* Enable HTML escaping for all tables by default

* Adds automatic escaping for bootstrap tables where custom formatter function is specified

- Intercept the row data *before* it is provided to the renderer function
- Adds a function for sanitizing nested data structure

* Sanitize form data before processing

(cherry picked from commit cd418d6948e6bf5f428cec5b4a7a1f0618a482a3)

* Increment version number for release

* Fix sanitization for array case - was missing a return value

@@ -3,17 +3,17 @@
 import import_export.widgets as widgets
 from import_export.admin import ImportExportModelAdmin
 from import_export.fields import Field
-from import_export.resources import ModelResource

+from InvenTree.admin import InvenTreeResource
 from part.models import Part

 from .models import (Company, ManufacturerPart, ManufacturerPartAttachment,
                      ManufacturerPartParameter, SupplierPart,
                      SupplierPriceBreak)

-class CompanyResource(ModelResource):
-    """ Class for managing Company data import/export """
+class CompanyResource(InvenTreeResource):
+    """Class for managing Company data import/export."""

     class Meta:
         model = Company
@@ -34,10 +34,8 @@ class CompanyAdmin(ImportExportModelAdmin):
     ]

-class SupplierPartResource(ModelResource):
-    """
-    Class for managing SupplierPart data import/export
-    """
+class SupplierPartResource(InvenTreeResource):
+    """Class for managing SupplierPart data import/export."""

     part = Field(attribute='part', widget=widgets.ForeignKeyWidget(Part))

@@ -70,10 +68,8 @@ class SupplierPartAdmin(ImportExportModelAdmin):
     autocomplete_fields = ('part', 'supplier', 'manufacturer_part',)

-class ManufacturerPartResource(ModelResource):
-    """
-    Class for managing ManufacturerPart data import/export
-    """
+class ManufacturerPartResource(InvenTreeResource):
+    """Class for managing ManufacturerPart data import/export."""

     part = Field(attribute='part', widget=widgets.ForeignKeyWidget(Part))

@@ -118,10 +114,8 @@ class ManufacturerPartAttachmentAdmin(ImportExportModelAdmin):
     autocomplete_fields = ('manufacturer_part',)

-class ManufacturerPartParameterResource(ModelResource):
-    """
-    Class for managing ManufacturerPartParameter data import/export
-    """
+class ManufacturerPartParameterResource(InvenTreeResource):
+    """Class for managing ManufacturerPartParameter data import/export."""

     class Meta:
         model = ManufacturerPartParameter
@@ -148,8 +142,8 @@ class ManufacturerPartParameterAdmin(ImportExportModelAdmin):
     autocomplete_fields = ('manufacturer_part',)

-class SupplierPriceBreakResource(ModelResource):
-    """ Class for managing SupplierPriceBreak data import/export """
+class SupplierPriceBreakResource(InvenTreeResource):
+    """Class for managing SupplierPriceBreak data import/export."""

     part = Field(attribute='part', widget=widgets.ForeignKeyWidget(SupplierPart))


@@ -3,19 +3,19 @@
 import import_export.widgets as widgets
 from import_export.admin import ImportExportModelAdmin
 from import_export.fields import Field
-from import_export.resources import ModelResource

 from build.models import Build
 from company.models import Company, SupplierPart
+from InvenTree.admin import InvenTreeResource
 from order.models import PurchaseOrder, SalesOrder
 from part.models import Part

 from .models import (StockItem, StockItemAttachment, StockItemTestResult,
                      StockItemTracking, StockLocation)

-class LocationResource(ModelResource):
-    """ Class for managing StockLocation data import/export """
+class LocationResource(InvenTreeResource):
+    """Class for managing StockLocation data import/export."""

     parent = Field(attribute='parent', widget=widgets.ForeignKeyWidget(StockLocation))

@@ -65,8 +65,8 @@ class LocationAdmin(ImportExportModelAdmin):
     ]

-class StockItemResource(ModelResource):
-    """ Class for managing StockItem data import/export """
+class StockItemResource(InvenTreeResource):
+    """Class for managing StockItem data import/export."""

     # Custom managers for ForeignKey fields
     part = Field(attribute='part', widget=widgets.ForeignKeyWidget(Part))


Merge pull request from GHSA-h85r-xv4w-cg8g

Fix GHSA-h85r-xv4w-cg8g - perform backend file type validation

@@ -95,19 +95,18 @@ def create_from_upload(
                 False,
             )
         except self.model.DoesNotExist:
-            return (
-                self.create(
+            with upload.content.open("rb") as content:
+                instance = self.create(
                     submission_step=submission_step,
                     temporary_file=upload,
                     form_key=form_key,
                     # wrap in File() so it will be physically copied
-                    content=File(upload.content, name=upload.file_name),
+                    content=File(content, name=upload.file_name),
                     content_type=upload.content_type,
                     original_name=upload.file_name,
                     file_name=file_name,
-                ),
-                True,
-            )
+                )
+            return (instance, True)

 class SubmissionFileAttachment(DeleteFileFieldFilesMixin, models.Model):


fixed shell injection in os auth provider

@@ -5,6 +5,7 @@
 import subprocess
 import syslog
 from jadi import component, service, interface
+from six import PY3

 import aj
 from aj.api.http import BaseHttpHandler
@@ -99,8 +100,14 @@ class OSAuthenticationProvider(AuthenticationProvider):

     def authenticate(self, username, password):
         child = None
+
+        if PY3:
+            from shlex import quote
+        else:
+            from pipes import quote
+
         try:
-            child = pexpect.spawn('/bin/sh', ['-c', '/bin/su -c "/bin/echo SUCCESS" - %s' % username], timeout=5)
+            child = pexpect.spawn('/bin/sh', ['-c', '/bin/su -c "/bin/echo SUCCESS" - %s' % quote(username)], timeout=5)
             child.expect('.*:')
             child.sendline(password)
             result = child.expect(['su: .*', 'SUCCESS'])


Fix CVE-XXXX-XXXX -- Fix Path Traversal security vulnerability

@@ -1,12 +1,14 @@
-import os
 import tempfile
+from pathlib import Path

 import pytest
 from django.core.files.base import ContentFile
 from django.utils.encoding import force_str
 from selenium import webdriver
 from selenium.common.exceptions import WebDriverException

+from s3file.storages import storage
+

 @pytest.fixture(scope="session")
 def driver():
@@ -22,30 +24,49 @@ def driver():

 @pytest.fixture
-def upload_file(request):
-    path = tempfile.mkdtemp()
-    file_name = os.path.join(path, "%s.txt" % request.node.name)
-    with open(file_name, "w") as f:
+def freeze_upload_folder(monkeypatch):
+    """Freeze datetime and UUID."""
+    upload_folder = Path(storage.aws_location) / "tmp" / "s3file"
+    monkeypatch.setattr(
+        "s3file.forms.S3FileInputMixin.upload_folder",
+        str(upload_folder),
+    )
+    return upload_folder
+
+
+@pytest.fixture
+def upload_file(request, freeze_upload_folder):
+    path = Path(tempfile.mkdtemp()) / freeze_upload_folder / f"{request.node.name}.txt"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w") as f:
         f.write(request.node.name)
-    return file_name
+    return str(path.absolute())

 @pytest.fixture
-def another_upload_file(request):
-    path = tempfile.mkdtemp()
-    file_name = os.path.join(path, "another_%s.txt" % request.node.name)
-    with open(file_name, "w") as f:
+def another_upload_file(request, freeze_upload_folder):
+    path = (
+        Path(tempfile.mkdtemp())
+        / freeze_upload_folder
+        / f"another_{request.node.name}.txt"
+    )
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w") as f:
         f.write(request.node.name)
-    return file_name
+    return str(path.absolute())

 @pytest.fixture
-def yet_another_upload_file(request):
-    path = tempfile.mkdtemp()
-    file_name = os.path.join(path, "yet_another_%s.txt" % request.node.name)
-    with open(file_name, "w") as f:
+def yet_another_upload_file(request, freeze_upload_folder):
+    path = (
+        Path(tempfile.mkdtemp())
+        / freeze_upload_folder
+        / f"yet_another_{request.node.name}.txt"
+    )
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w") as f:
         f.write(request.node.name)
-    return file_name
+    return str(path.absolute())

 @pytest.fixture


avoid use of |safe filter in templates

Explicitly mark those few places where we need to pass in data that
should not be escaped on the Python side.

@@ -12,14 +12,15 @@
 from typing import MutableMapping
 from typing import TypeVar

-import flask
 from beancount.core import compare
 from beancount.core import realization
 from beancount.core.account import ACCOUNT_RE
 from beancount.core.data import Directive
 from beancount.core.inventory import Inventory
 from beancount.core.number import Decimal
 from beancount.core.number import ZERO
+from flask import url_for
+from markupsafe import Markup

 from fava.context import g
 from fava.core.conversion import cost
@@ -145,14 +146,14 @@ def basename(file_path: str) -> str:
     return unicodedata.normalize("NFC", os.path.basename(file_path))

-def format_errormsg(message: str) -> str:
+def format_errormsg(message: str) -> Markup:
     """Match account names in error messages and insert HTML links for them."""
     match = re.search(ACCOUNT_RE, message)
     if not match:
-        return message
+        return Markup(message)
     account = match.group()
-    url = flask.url_for("account", name=account)
-    return (
+    url = url_for("account", name=account)
+    return Markup(
         message.replace(account, f'{account}')
         .replace("for '", "for ")
         .replace("': ", ": ")


Eth account support (#361)

* Create separate execute function

* Add is_valid_eth_signature to account library

* Add eth_execute to account library

* Create eth account mock and test

* Add missing dependencies

* Create TestEthSigner

* Update used private key

* Update implicit parameters

* Update execute parameters

* Update all implicit arguments

* Update signature values and hash

* Update variable name

* Update documentation

* Fix merge error

* Improve format

* Update tests/utils.py

Co-authored-by: Andrew Fleming <fleming.andrew@protonmail.com>

* Update docs/Account.md

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update docs/Account.md

Co-authored-by: Martín Triay <martriay@gmail.com>

* Rename test and fix documentation

* Add documentation

* Update src/openzeppelin/account/library.cairo

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update src/openzeppelin/account/library.cairo

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update src/openzeppelin/account/library.cairo

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update src/openzeppelin/account/library.cairo

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update tests/utils.py

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update tests/utils.py

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update tests/mocks/eth_account.cairo

Co-authored-by: Martín Triay <martriay@gmail.com>

* Create eth account preset

* Create signers module

* use assert_revert to test nonce

* Add test for valid signature

* use internal hash

* Update validity test

* Update docs/Account.md

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update docs/Account.md

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update tests/signers.py

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update src/openzeppelin/account/library.cairo

Co-authored-by: Martín Triay <martriay@gmail.com>

* Fix after merge

* Improve tests

* Update account library

* Update Account.md

* update format

* Update src/openzeppelin/account/library.cairo

Co-authored-by: Andrew Fleming <fleming.andrew@protonmail.com>

* Update tests/access/test_Ownable.py

Co-authored-by: Andrew Fleming <fleming.andrew@protonmail.com>

* Update src/openzeppelin/account/library.cairo

Co-authored-by: Andrew Fleming <fleming.andrew@protonmail.com>

* Update eth test

* Update Account.md

* Update test

* Update tests/signers.py

Co-authored-by: Andrew Fleming <fleming.andrew@protonmail.com>

* Fix typo

* Update signers

* Update test

* Update docs/Account.md

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update docs/Account.md

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update tests/signers.py

Co-authored-by: Martín Triay <martriay@gmail.com>

* Update docs/Account.md

Co-authored-by: Martín Triay <martriay@gmail.com>

* update test

* Update documenation for Account

* Update docs/Account.md

* Update docs/Account.md

Co-authored-by: Martín Triay <martriay@gmail.com>

Co-authored-by: Andrew Fleming <fleming.andrew@protonmail.com>
Co-authored-by: Martín Triay <martriay@gmail.com>

@@ -0,0 +1,95 @@
+from nile.signer import Signer, from_call_to_call_array, get_transaction_hash
+from utils import to_uint
+import eth_keys
+
+class MockSigner():
+    """
+    Utility for sending signed transactions to an Account on Starknet.
+
+    Parameters
+    ----------
+
+    private_key : int
+
+    Examples
+    ---------
+    Constructing a MockSigner object
+
+    >>> signer = MockSigner(1234)
+
+    Sending a transaction
+
+    >>> await signer.send_transaction(
+            account, contract_address, 'contract_method', [arg_1]
+        )
+
+    Sending multiple transactions
+
+    >>> await signer.send_transaction(
+            account, [
+                (contract_address, 'contract_method', [arg_1]),
+                (contract_address, 'another_method', [arg_1, arg_2])
+            ]
+        )
+
+    """
+    def __init__(self, private_key):
+        self.signer = Signer(private_key)
+        self.public_key = self.signer.public_key
+
+    async def send_transaction(self, account, to, selector_name, calldata, nonce=None, max_fee=0):
+        return await self.send_transactions(account, [(to, selector_name, calldata)], nonce, max_fee)
+
+    async def send_transactions(self, account, calls, nonce=None, max_fee=0):
+        if nonce is None:
+            execution_info = await account.get_nonce().call()
+            nonce, = execution_info.result
+
+        build_calls = []
+        for call in calls:
+            build_call = list(call)
+            build_call[0] = hex(build_call[0])
+            build_calls.append(build_call)
+
+        (call_array, calldata, sig_r, sig_s) = self.signer.sign_transaction(hex(account.contract_address), build_calls, nonce, max_fee)
+        return await account.__execute__(call_array, calldata, nonce).invoke(signature=[sig_r, sig_s])
+
+class MockEthSigner():
+    """
+    Utility for sending signed transactions to an Account on Starknet, like MockSigner, but using a secp256k1 signature.
+    Parameters
+    ----------
+    private_key : int
+
+    """
+    def __init__(self, private_key):
+        self.signer = eth_keys.keys.PrivateKey(private_key)
+        self.eth_address = int(self.signer.public_key.to_checksum_address(),0)
+
+    async def send_transaction(self, account, to, selector_name, calldata, nonce=None, max_fee=0):
+        return await self.send_transactions(account, [(to, selector_name, calldata)], nonce, max_fee)
+
+    async def send_transactions(self, account, calls, nonce=None, max_fee=0):
+        if nonce is None:
+            execution_info = await account.get_nonce().call()
+            nonce, = execution_info.result
+
+        build_calls = []
+        for call in calls:
+            build_call = list(call)
+            build_call[0] = hex(build_call[0])
+            build_calls.append(build_call)
+
+        (call_array, calldata) = from_call_to_call_array(build_calls)
+        message_hash = get_transaction_hash(
+            account.contract_address, call_array, calldata, nonce, max_fee
+        )
+
+        signature = self.signer.sign_msg_hash((message_hash).to_bytes(32, byteorder="big"))
+        sig_r = to_uint(signature.r)
+        sig_s = to_uint(signature.s)
+
+        # the hash and signature are returned for other tests to use
+        return await account.__execute__(call_array, calldata, nonce).invoke(
+            signature=[signature.v, *sig_r, *sig_s]
+        ), message_hash, [signature.v, *sig_r, *sig_s]


@@ -1,7 +1,7 @@
 import pytest
 from starkware.starknet.testing.starknet import Starknet
+from signers import MockSigner
 from utils import (
-    MockSigner,
     assert_revert,
     get_contract_class,
     cached_contract,


Fix path traversal issues in all dataretriever backends

@@ -24,6 +24,7 @@
 from flask import Blueprint, url_for, Response, stream_with_context, send_file, \
     jsonify
 from werkzeug.datastructures import Headers
+from werkzeug.security import safe_join

 BASEURL = 'cocktail'
 STYLE = False
@@ -249,11 +250,11 @@ def _get_obj_path(obj_path):
     return obj_path.replace(DATAROOT+'/', '')

 def _get_obj_absolute_path(obj_path):
-    return os.path.join(DATAROOT, obj_path)
+    return safe_join(DATAROOT, obj_path)

 def _get_index_absolute_path(index):
-    return os.path.join(INDEXDIR, index)
+    return safe_join(INDEXDIR, index)

 @scope_blueprint.route('/obj/<path:obj_path>')


Merge pull request #2173 from bookwyrm-social/html-sanitizer

Html sanitizer

@@ -16,7 +16,7 @@

 from bookwyrm import activitypub
 from bookwyrm.connectors import get_image
-from bookwyrm.sanitize_html import InputHtmlParser
+from bookwyrm.utils.sanitizer import clean
 from bookwyrm.settings import MEDIA_FULL_URL

@@ -497,9 +497,7 @@ class HtmlField(ActivitypubFieldMixin, models.TextField):
     def field_from_activity(self, value):
         if not value or value == MISSING:
             return None
-        sanitizer = InputHtmlParser()
-        sanitizer.feed(value)
-        return sanitizer.get_output()
+        return clean(value)

 class ArrayField(ActivitypubFieldMixin, DjangoArrayField):


Merge pull request #5 from mike-arnica/security-review

Fix insecure/undeclared raised errors; initial commit of scripts for atheris fuzzer

@@ -221,118 +221,127 @@ def from_bytes(

         Raises:
             ValueError: The provided byte string is not an ``openssh-key-v1``
-                key list or the declared key count is negative.
+                key list, when the declared key count is negative, or when an
+                EOF is found while parsing the key.
+
             UserWarning: The check numbers in the decrypted private byte string
                 do not match (likely due to an incorrect passphrase), the key
                 type or parameter values of a private key do not match that of
                 the corresponding public key in the list, or the padding bytes
                 at the end of the decrypted private byte string are not as
                 expected.
         """
-        byte_stream = PascalStyleByteStream(byte_string)
+        try:
+            byte_stream = PascalStyleByteStream(byte_string)

-        header = byte_stream.read_from_format_instructions_dict(
-            cls.HEADER_FORMAT_INSTRUCTIONS_DICT
-        )
+            header = byte_stream.read_from_format_instructions_dict(
+                cls.HEADER_FORMAT_INSTRUCTIONS_DICT
+            )
+
+            if header['auth_magic'] != b'openssh-key-v1\x00':
+                raise ValueError('Not an openssh-key-v1 key')

-        if header['auth_magic'] != b'openssh-key-v1\x00':
-            raise ValueError('Not an openssh-key-v1 key')
+            num_keys = header['num_keys']

-        num_keys = header['num_keys']
+            if num_keys < 0:
+                raise ValueError('Cannot parse negative number of keys')

-        if num_keys < 0:
-            raise ValueError('Cannot parse negative number of keys')
+            public_key_list = []
+            for i in range(num_keys):
+                public_key_bytes = byte_stream.read_from_format_instruction(
+                    PascalStyleFormatInstruction.BYTES
+                )
+                public_key_list.append(
+                    PublicKey.from_bytes(public_key_bytes)
+                )

-        public_key_list = []
-        for i in range(num_keys):
-            public_key_bytes = byte_stream.read_from_format_instruction(
+            cipher_bytes = byte_stream.read_from_format_instruction(
                 PascalStyleFormatInstruction.BYTES
             )
-            public_key_list.append(
-                PublicKey.from_bytes(public_key_bytes)
-            )

-        cipher_bytes = byte_stream.read_from_format_instruction(
-            PascalStyleFormatInstruction.BYTES
-        )
-
-        kdf_class = get_kdf_options_class(header['kdf'])
-        kdf_options = kdf_class(
-            PascalStyleByteStream(
-                header['kdf_options']
-            ).read_from_format_instructions_dict(
-                kdf_class.FORMAT_INSTRUCTIONS_DICT
+            kdf_class = get_kdf_options_class(header['kdf'])
+            kdf_options = kdf_class(
+                PascalStyleByteStream(
+                    header['kdf_options']
+                ).read_from_format_instructions_dict(
+                    kdf_class.FORMAT_INSTRUCTIONS_DICT
+                )
             )
-        )

-        cipher_class = get_cipher_class(header['cipher'])
+            cipher_class = get_cipher_class(header['cipher'])

-        if kdf_class == NoneKDFOptions:
-            passphrase = ''
-        elif passphrase is None:
-            passphrase = getpass.getpass('Key passphrase: ')
+            if kdf_class == NoneKDFOptions:
+                passphrase = ''
+            elif passphrase is None:
+                passphrase = getpass.getpass('Key passphrase: ')

-        if issubclass(cipher_class, ConfidentialityIntegrityCipher):
-            cipher_bytes += byte_stream.read_fixed_bytes(
-                cipher_class.TAG_LENGTH
-            )
-
-        decipher_bytes = cipher_class.decrypt(
-            kdf_class(kdf_options),
-            passphrase,
-            cipher_bytes
-        )
-
-        decipher_byte_stream = PascalStyleByteStream(decipher_bytes)
+            if issubclass(cipher_class, ConfidentialityIntegrityCipher):
+                cipher_bytes += byte_stream.read_fixed_bytes(
+                    cipher_class.TAG_LENGTH
+                )

-        decipher_bytes_header = \
-            decipher_byte_stream.read_from_format_instructions_dict(
-                cls.DECIPHER_BYTES_HEADER_FORMAT_INSTRUCTIONS_DICT
+            decipher_bytes = cipher_class.decrypt(
+                kdf_class(kdf_options),
+                passphrase,
+                cipher_bytes
             )

-        if decipher_bytes_header['check_int_1'] \
-                != decipher_bytes_header['check_int_2']:
-            warnings.warn('Cipher header check numbers do not match')
+            decipher_byte_stream = PascalStyleByteStream(decipher_bytes)

-        initlist = []
-        for i in range(num_keys):
-            initlist.append(
-                PublicPrivateKeyPair(
-                    public_key_list[i],
-                    PrivateKey.from_byte_stream(decipher_byte_stream)
-                )
-            )
-            if initlist[i].public.header['key_type'] \
-                    != initlist[i].private.header['key_type']:
-                warnings.warn(
-                    f'Inconsistency between private and public '
-                    f'key types for key {i}'
+            decipher_bytes_header = \
+                decipher_byte_stream.read_from_format_instructions_dict(
+                    cls.DECIPHER_BYTES_HEADER_FORMAT_INSTRUCTIONS_DICT
                 )
-            if not all(
-                (
-                    initlist[i].public.params[k] ==
-                    initlist[i].private.params[k]
-                ) for k in (
-                    initlist[i].public.params.keys() &
-                    initlist[i].private.params.keys()
+
+            if decipher_bytes_header['check_int_1'] \
+                    != decipher_bytes_header['check_int_2']:
+                warnings.warn('Cipher header check numbers do not match')
+
+            initlist = []
+            for i in range(num_keys):
+                initlist.append(
+                    PublicPrivateKeyPair(
+                        public_key_list[i],
+                        PrivateKey.from_byte_stream(decipher_byte_stream)
+                    )
                 )
+                if initlist[i].public.header['key_type'] \
+                        != initlist[i].private.header['key_type']:
+                    warnings.warn(
+                        f'Inconsistency between private and public '
+                        f'key types for key {i}'
+                    )
+                if not all(
+                    (
+                        initlist[i].public.params[k] ==
+                        initlist[i].private.params[k]
+                    ) for k in (
+                        initlist[i].public.params.keys() &
+                        initlist[i].private.params.keys()
+                    )
+                ):
+                    warnings.warn(
+                        f'Inconsistency between private and public '
+                        f'values for key {i}'
+                    )
+
+            decipher_padding = decipher_byte_stream.read()
+
+            if (
+                len(decipher_byte_stream.getvalue()) %
+                    cipher_class.BLOCK_SIZE != 0
+            ) or not (
+                bytes(
+                    range(1, 1 + cipher_class.BLOCK_SIZE)
+                ).startswith(decipher_padding)
             ):
-                warnings.warn(
-                    f'Inconsistency between private and public '
-                    f'values for key {i}'
-                )
-
-        decipher_padding = decipher_byte_stream.read()
-
-        if (
-            len(decipher_byte_stream.getvalue()) %
-                cipher_class.BLOCK_SIZE != 0
-        ) or not (
-            bytes(
-                range(1, 1 + cipher_class.BLOCK_SIZE)
-            ).startswith(decipher_padding)
-        ):
-            warnings.warn('Incorrect padding at end of ciphertext')
+                warnings.warn('Incorrect padding at end of ciphertext')
+        except ValueError as e:
+            raise e
+        except EOFError as e:
+            raise ValueError('Premature EOF detected while parsing key.')
+        except e:
+            raise ValueError('Unexpected error condition reached.')

         return cls(
             initlist,


[4.0.x] Fixed CVE-2022-34265 -- Protected Trunc(kind)/Extract(lookup_name) against SQL injection.

Thanks Takuto Yoshikai (Aeye Security Lab) for the report.

@@ -51,6 +51,8 @@ def __init__(self, expression, lookup_name=None, tzinfo=None, **extra):
         super().__init__(expression, **extra)

     def as_sql(self, compiler, connection):
+        if not connection.ops.extract_trunc_lookup_pattern.fullmatch(self.lookup_name):
+            raise ValueError("Invalid lookup_name: %s" % self.lookup_name)
         sql, params = compiler.compile(self.lhs)
         lhs_output_field = self.lhs.output_field
         if isinstance(lhs_output_field, DateTimeField):
@@ -235,6 +237,8 @@ def __init__(
         super().__init__(expression, output_field=output_field, **extra)

     def as_sql(self, compiler, connection):
+        if not connection.ops.extract_trunc_lookup_pattern.fullmatch(self.kind):
+            raise ValueError("Invalid kind: %s" % self.kind)
         inner_sql, inner_params = compiler.compile(self.lhs)
         tzname = None
         if isinstance(self.lhs.output_field, DateTimeField):


Merge forward 3004.2 (#62200)

* Check only ssh-rsa encyption for set_known_host

* Windows test fix

* Fix pre-commit

* add CentOS Stream to _OS_FAMILY_MAP, fix #59161

* added changelog and test

* fix syntax

* Use centosstream 8 for testing

* Use ? for matching spaces

Technically this isn't *quite* right as 'CentOSyStream' would also
match, but it's pretty reasonable:

- OS grains shouldn't ever be that kind of close
- This test is only swapping out spaces, and only for the os grain. That
  would mean there would have to be two OSes with grains that only
  differ by one having a space where another one has any other
  character.
- This test really isn't even about matching grains, we're just using
  compound matching and that's a reasonable one to use.

* Add centos stream when detecting package manager name

* Fix pre-commit

* Remove tests for fedora 32/33 EOL

* Remove tests for fedora 32/33 EOL

* Remove tests for fedora 33 EOL

* Use centosstream 8 for testing

* Use ? for matching spaces

Technically this isn't *quite* right as 'CentOSyStream' would also
match, but it's pretty reasonable:

- OS grains shouldn't ever be that kind of close
- This test is only swapping out spaces, and only for the os grain. That
  would mean there would have to be two OSes with grains that only
  differ by one having a space where another one has any other
  character.
- This test really isn't even about matching grains, we're just using
  compound matching and that's a reasonable one to use.

* 3002.9: Fix pre-commit

* 3003.5 Fix pre-commit

* [3002.9] Replace use of 'sl' with 'paper' for Arch tests, due to 'sl' having key issues

* Remove mojave testing

* Remove mojave and high sierra testing

* Remove mojave testing

* [3002.9] Fix cloud vultr size issue

* Update package name to aspnetcore-runtime-6.0 for redhat 8 pkg tests

* Update package name to aspnetcore-runtime-6.0 for redhat 8 pkg tests

* change amazon linux AMI

* Migrate `unit.modules.test_gpg` to PyTest

* Don't leave any `gpg-agent`'s running behind

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* Start a background process to generate entropy.

Some tests have failed because of not enough entropy which then makes
the test timeout.

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* A different approach at generating entropy

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* Turn entropy generation into a helper

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* change amazon linux AMI

* change amazon linux AMI

* [3004.2] Fix cloud vultr size issue

* Fix cloud requirements

* Skip pam tests on windows

* Update ami to try to get the tests running

* Update amis to try to get the tests running

* Fixing test_publish_to_pubserv_ipc_tcp, moving the call to socket.socket into the while loop.

* Add static requirements for 3.8 and 3.9 on Windows

* Fix requirements

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* The whole CI process is already slower than GH Actions, no caches.

* Pre-commit must not run with ``PIP_EXTRA_INDEX_URL`` set.

* Lint fixes

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* Compile cloud requirements

* Run add requirements files for 3.8 and 3.9

* Fix docs and cloud requirements

* [3003.5] Fix cloud vultr size issue

* Windows test fix

* Skip test if docker not running

* [3003.5] Fix pre-commit

* Update Markup and contextfunction imports for jinja versions >=3.1.

* update bootstrap to 2022.03.15

* update bootstrap to 2022.03.15

* skipping tests/pytests/integration/modules/test_virt.py on 3002.x and 3003.x branches.

* Windows test fix

* Skip PAM tests on Windows

Windows has no ctypes with the PAM bits, so we should go ahead and skip
on Windows.

* Skip PAM auth tests on Windows

Windows lacks the correct bits, so...

* Fix pre-commit

* Skipping tests since they're also skipped on the master branch

Fixes #403

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* Skip test that only runs because the patch binary is now available.

The feature though, was only added in 3004.

Fixes #404

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* Skip test which is only supposed to run in Linux

Fixes https://github.com/saltstack/salt-priv/issues/405

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* GPG tests do not work on windows yet

* Fix tests

* Fix pre-commit

* skip tests.integration.modules.test_mac_brew_pkg.BrewModuleTest.test_list_upgrades and tests.integration.modules.test_state.StateModuleTest.test_get_file_from_env_in_top_match on Mac OS.

* skip tests.integration.modules.test_mac_brew_pkg.BrewModuleTest.test_list_upgrades and tests.integration.modules.test_state.StateModuleTest.test_get_file_from_env_in_top_match on Mac OS.

* Removing skip, moving it to different PR.

* Skipping tests on 3002.9.

* test fix

* Do not run patch tests on 3003.5. Feature not added till 3004

* skipping tests/pytests/integration/modules/test_virt.py on 3002.x and 3003.x branches.

* Fix pre-commit

* [3004.2] Update freebsd ami

* Bump the git version for freebsd CI tests

* removing versions that are no longer available from the tests.pytests.scenarios.compat.test_with_versions tests.

* Skip tests on windows when NOT using static requirements

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* removing versions that are no longer available from the tests.pytests.scenarios.compat.test_with_versions tests.

* test_issue_36469_tcp causes a fatal python error when run on Mac OS, so skipping.

* Fix tests

* Fix pre-commit

* Do not run patch tests on 3003.5. Feature not added till 3004

* Skip archive tar tests on windows

* [3002.9] Skip archive tar tests on windows

* GPG tests do not work on windows yet

* Skip test which is only supposed to run in Linux

Fixes https://github.com/saltstack/salt-priv/issues/405

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* Skip test that only runs because the patch binary is now available.

The feature though, was only added in 3004.

Fixes #404

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* Skipping tests since they're also skipped on the master branch

Fixes #403

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* Fix pre-commit

* Fix pre-commit

* Fix pre-commit

* Fix pre-commit

* retry sdb.get if it returns None

None is an entirely valid return - see EtcdClient.get in
salt/utils/etcd_util.py

* drop py2/six

* fix etcd sdb.set as well

* Fix etdcd-sdb test failure

If docker container is up and running, but etcd isn't responding yet
it's possible that we get some failing tests. This should wait a
reasonable amount of time for things to come up. Or just skip the test.

* Fix etdcd-sdb test failure

If docker container is up and running, but etcd isn't responding yet
it's possible that we get some failing tests. This should wait a
reasonable amount of time for things to come up. Or just skip the test.

* Skip the tests from unit/transport/test_zero.py that are hanging on Mac.

* skip tests in tests/pytests/unit/states/test_archive.py for 3002.9

* 3002.9 Skipping  CA permissions tests on Windows, similar to 3003.5 and 3004.2

* change skipif to skip

* Rollback Windows AMIs to use Python 3.7

* Rollback AMI's to Python 3.7... fix tests

* Fix failing test_archive tests

* Build using pyenv

* Add symlinks to openssl and rpath

* Add shasum for zeromq 4.3.4

* Fix docs on scripts

* Build zeromq earlier, fix symlinks

* Bring 61446 to 3004.1 branch

* Add changelog and tests

* Fix schedule test flakiness

* Retry with new port if in use

* fixing failing tests, ensuring that the correct path is used.

* fixing failing tests, ensuring that the correct path is used.

* fixing failing tests, ensuring that the correct path is used.

* Re-enable tiamat-pip on windows

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

* Bump duration time for windwos for test_retry_option_success

* Skip test cauing hangs

* go go pylint disable

* more pre-commit

* oh lint

* so many weird hook failures

* Add unit tests for PAM auth CVE

We could add functional tests if it's important enough, but this is the
narrowest place to test.

* Fix PAM auth CVE

Credit to @ysf

Previously we weren't checking the result of PAM_ACCT_MGMT.

* pylint disable

* rewrite hook changes

* Skip PAM auth tests on Windows

Since Windows ends out lacking the correct bits, no need to run tests
there.

* pre-commit fixes

* docs 3004.2 release

* Fix bug in tcp transport

* Fix the test_zeromq_filtering test

* skip test_npm_install_url_referenced_package on centos 7 and 8.

* Swapping CentOS Linux-8 for CentOS Stream-8

* Update build scripts to use pyenv

* Fix tests on MacOS

* Fix bug in tcp transport

* Fix test failures

* Update release notes and man pages for 3003.5

* Add 3002.9 changelog, release notes, man pages

* Update doc/topics/releases/3002.9.rst

Co-authored-by: Megan Wilhite <mwilhite@vmware.com>

* Fix requirements

* Fix imports

* Test fixup

* Fix merge warts

* fix merge wart in changelog

* Fix merge warts in tests

Co-authored-by: krionbsd <krion@FreeBSD.org>
Co-authored-by: Megan Wilhite <megan.wilhite@gmail.com>
Co-authored-by: Alexander Kriventsov <akriventsov@nic.ru>
Co-authored-by: Megan Wilhite <mwilhite@vmware.com>
Co-authored-by: Wayne Werner <wwerner@vmware.com>
Co-authored-by: Gareth J. Greenaway <gareth@saltstack.com>
Co-authored-by: David Murphy < dmurphy@saltstack.com>
Co-authored-by: Twangboy <leesh@vmware.com>
Co-authored-by: MKLeb <calebb@vmware.com>
Co-authored-by: Pedro Algarvio <pedro@algarvio.me>
Co-authored-by: Pedro Algarvio <palgarvio@vmware.com>
Co-authored-by: Thomas Phipps <tphipps@vmware.com>
Co-authored-by: Frode Gundersen <frogunder@gmail.com>
Co-authored-by: Alyssa Rock <alyssa.rock@gmail.com>
Co-authored-by: Alyssa Rock <43180546+barbaricyawps@users.noreply.github.com>

@@ -737,7 +737,10 @@ def start(self):
             # must be after channels
             log.info("Creating master maintenance process")
             self.process_manager.add_process(
-                Maintenance, args=(self.opts,), name="Maintenance"
+                Maintenance,
+                args=(self.opts,),
+                kwargs={"master_secrets": SMaster.secrets},
+                name="Maintenance",
             )

             if self.opts.get("event_return"):


@@ -488,6 +488,11 @@ def _realpath_windows(path):
                 base = os.path.abspath(os.path.sep.join([base, part]))
         else:
             base = part
+    # Python 3.8 added support for directory junctions which prefixes the
+    # return with `\\?\`. We need to strip that off.
+    # https://docs.python.org/3/library/os.html#os.readlink
+    if base.startswith("\\\\?\\"):
+        base = base[4:]
     return base


Fix regex used to mask password

Some use cases are poorly handled by the regex used
to mask password. Indeed when the password contains
quotes or double quotes in the middle such as `pass"word`,
the mask_password method will return `***"word`.

For more details please see
https://bugs.launchpad.net/oslo.utils/+bug/1949623

Closes-Bug: #1949623
Change-Id: I941750b4d49d2d75f0831b24d6dd17f4040f70a2

@@ -73,6 +73,7 @@
 # for XML and JSON automatically.
 _SANITIZE_PATTERNS_2 = {}
 _SANITIZE_PATTERNS_1 = {}
+_SANITIZE_PATTERNS_WILDCARD = {}

 # NOTE(amrith): Some regular expressions have only one parameter, some
 # have two parameters. Use different lists of patterns here.
@@ -88,13 +89,15 @@
                       r'([\'"][^\'"]*%(key)s[0-9]*[\'"]\s*,\s*\'--?[A-z]+'
                       r'\'\s*,\s*u?[\'"])[^\"\']*([\'"])',
                       r'(%(key)s[0-9]*\s*--?[A-z]+\s*)\S+(\s*)']
+_FORMAT_PATTERNS_WILDCARD = [r'([\'\"][^\"\']*%(key)s[0-9]*[\'\"]\s*:\s*u?[\'\"].*[\'\"])[^\"\']*([\'\"])']  # noqa: E501

 # NOTE(dhellmann): Keep a separate list of patterns by key so we only
 # need to apply the substitutions for keys we find using a quick "in"
 # test.
 for key in _SANITIZE_KEYS:
     _SANITIZE_PATTERNS_1[key] = []
     _SANITIZE_PATTERNS_2[key] = []
+    _SANITIZE_PATTERNS_WILDCARD[key] = []

     for pattern in _FORMAT_PATTERNS_2:
         reg_ex = re.compile(pattern % {'key': key}, re.DOTALL | re.IGNORECASE)
@@ -104,6 +107,10 @@
         reg_ex = re.compile(pattern % {'key': key}, re.DOTALL | re.IGNORECASE)
         _SANITIZE_PATTERNS_1[key].append(reg_ex)

+    for pattern in _FORMAT_PATTERNS_WILDCARD:
+        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL | re.IGNORECASE)
+        _SANITIZE_PATTERNS_WILDCARD[key].append(reg_ex)
+

 def int_from_bool_as_string(subject):
     """Interpret a string as a boolean and return either 1 or 0.
@@ -331,6 +338,7 @@ def mask_password(message, secret="***"):  # nosec

     substitute1 = r'\g<1>' + secret
     substitute2 = r'\g<1>' + secret + r'\g<2>'
+    substitute_wildcard = r'\g<1>'

     # NOTE(ldbragst): Check to see if anything in message contains any key
     # specified in _SANITIZE_KEYS, if not then just return the message since
@@ -341,7 +349,12 @@ def mask_password(message, secret="***"):  # nosec
                 message = re.sub(pattern, substitute2, message)
             for pattern in _SANITIZE_PATTERNS_1[key]:
                 message = re.sub(pattern, substitute1, message)
-
+            # NOTE(hberaud): Those case are poorly handled by previous
+            # patterns. They are passwords with quotes or double quotes.
+            # They also needs a different way to substitute group this is why
+            # they aren't fix in the pattern 1 or 2.
+            for pattern in _SANITIZE_PATTERNS_WILDCARD[key]:
+                message = re.sub(pattern, substitute_wildcard, message)
     return message


Merge pull request #742 from abadger/secure-tempfiles

Secure tempfiles

Looking through the code, I found two separate race conditions with tempfile handling which can be exploited.  One of those is mentioned in #738. The other one happens because we create a tempdir and then delete it, then later reuse its name. An attacker can pre-create the tempdir in between the deletion and the second creation with the reused name.  This PR attempts to fix both.
Fixes #738

Reviewed-by: Shane McDonald <me@shanemcd.com>
Reviewed-by: Matthew Jones <bsdmatburt@gmail.com>

@@ -3,13 +3,13 @@
 from functools import partial
 import os
 import re
+from tempfile import gettempdir

 import six
 from pexpect import TIMEOUT, EOF

 import pytest

-from tempfile import gettempdir
 from unittest.mock import (Mock, patch, PropertyMock)

 from ansible_runner.config._base import BaseConfig, BaseExecutionMode
@@ -47,7 +47,12 @@ def test_base_config_init_defaults():
 def test_base_config_with_artifact_dir():
     rc = BaseConfig(artifact_dir='/tmp/this-is-some-dir')
     assert rc.artifact_dir == os.path.join('/tmp/this-is-some-dir', rc.ident)
-    assert rc.private_data_dir == os.path.join(gettempdir(), ".ansible-runner")
+
+    # Check that the private data dir is placed in our default location with our default prefix
+    # and has some extra uniqueness on the end.
+    base_private_data_dir = os.path.join(gettempdir(), '.ansible-runner-')
+    assert rc.private_data_dir.startswith(base_private_data_dir)
+    assert len(rc.private_data_dir) > len(base_private_data_dir)

 def test_base_config_init_with_ident():


🔒 Rate limit failed login attempts

Should reduce likelihood of brute force attacks
on unsecure networks if users decide to deploy/make
OctoPrint accessible there against all advice to
the contrary.

@@ -281,6 +281,11 @@ def serverStatus():

 @api.route("/login", methods=["POST"])
+@octoprint.server.limiter.limit(
+    "3/minute;5/10 minutes;10/hour",
+    deduct_when=lambda response: response.status_code == 403,
+    error_message="You have made too many failed login attempts. Please try again later.",
+)
 def login():
     data = request.get_json()
     if not data:


Limit user's fullname and Token name field length

@@ -76,7 +76,13 @@ class UserForm(CherryForm):
             validators.length(max=256, message=_('Username too long.')),
         ],
     )
-    fullname = StringField(_('Fullname'), validators=[validators.optional()])
+    fullname = StringField(
+        _('Fullname'),
+        validators=[
+            validators.optional(),
+            validators.length(max=256, message=_('Fullname too long.')),
+        ],
+    )
     email = EmailField(
         _('Email'),
         validators=[


@@ -209,18 +209,6 @@ class UserForm(CherryForm):
         _('Quota Used'), validators=[validators.optional()], description=_("Disk spaces (in bytes) used by this user.")
     )

-    def validate_password(self, field):
-        validator = validators.length(
-            min=self.app.cfg.password_min_length,
-            max=self.app.cfg.password_max_length,
-            message=_('Password must have between %(min)d and %(max)d characters.'),
-        )
-        validator(self, field)
-
-    @property
-    def app(self):
-        return cherrypy.request.app
-
     def validate_role(self, field):
         # Don't allow the user to changes it's "role" state.
         currentuser = cherrypy.request.currentuser


@@ -111,6 +111,23 @@ def test_add_get_method(self):
         # Then ssh key is not added
         self.assertEqual(0, len(list(user.authorizedkeys)))

+    def test_add_with_title_too_long(self):
+        # Given an authenticated user without any ssh keys
+        user = self.app.store.get_user('admin')
+        for key in user.authorizedkeys:
+            user.delete_authorizedkey(key.fingerprint)
+        self.assertEqual(0, len(list(user.authorizedkeys)))
+        # When adding a key with title too long.
+        self._add_ssh_key(
+            "title" * 52,
+            "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSEN5VTn9MLituZvdYTZMbZEaMxe0UuU7BelxHkvxzSpVWtazrIBEc3KZjtVoK9F3+0kd26P4DzSQuPUl3yZDgyZZeXrF6p2GlEA7A3tPuOEsAQ9c0oTiDYktq5/Go8vD+XAZKLd//qmCWW1Jg4datkWchMKJzbHUgBrBH015FDbGvGDWYTfVyb8I9H+LQ0GmbTHsuTu63DhPODncMtWPuS9be/flb4EEojMIx5Vce0SNO9Eih38W7jTvNWxZb75k5yfPJxBULRnS5v/fPnDVVtD3JSGybSwKoMdsMX5iImAeNhqnvd8gBu1f0IycUQexTbJXk1rPiRcF13SjKrfXz ikus060@ikus060-t530",
+        )
+        # Then page return with error
+        self.assertStatus('200 OK')
+        self.assertInBody('Title too long.')
+        # Then key is not added
+        self.assertEqual(0, len(list(user.authorizedkeys)))
+
     def test_delete(self):
         # Delete existing keys
         user = self.app.store.get_user('admin')


Define field limit for username, email and root directory #223

@@ -239,6 +239,33 @@ def test_add_without_user_root(self):
         user = self.app.store.get_user('test6')
         self.assertEqual('', user.user_root)

+    def test_add_with_username_too_long(self):
+        # Given a too long username
+        username = "test2" * 52
+        # When trying to create the user
+        self._add_user(username, None, "password", "/tmp/", USER_ROLE)
+        # Then an error is raised
+        self.assertStatus(200)
+        self.assertInBody("Username too long.")
+
+    def test_add_with_email_too_long(self):
+        # Given a too long username
+        email = ("test2" * 50) + "@test.com"
+        # When trying to create the user
+        self._add_user("test2", email, "password", "/tmp/", USER_ROLE)
+        # Then an error is raised
+        self.assertStatus(200)
+        self.assertInBody("Email too long.")
+
+    def test_add_with_user_root_too_long(self):
+        # Given a too long user root
+        user_root = "/temp/" * 50
+        # When trying to create the user
+        self._add_user("test2", "test@test,com", "password", user_root, USER_ROLE)
+        # Then an error is raised
+        self.assertStatus(200)
+        self.assertInBody("Root directory too long.")
+
     def test_delete_user_with_not_existing_username(self):
         """
         Verify failure to delete invalid username.


FIX VULNERABILITY

jwcrypto accepts both compact and JSON formats.
It was possible to use this to present a token with arbitrary
claims with a signature from another valid token.

See test/vulnerability_vows.py for an example.

@@ -0,0 +1,38 @@
+""" Test claim forgery vulnerability fix """
+from datetime import timedelta
+from json import loads, dumps
+from test.common import generated_keys
+from test import python_jwt as jwt
+from pyvows import Vows, expect
+from jwcrypto.common import base64url_decode, base64url_encode
+
+@Vows.batch
+class ForgedClaims(Vows.Context):
+    """ Check we get an error when payload is forged using mix of compact and JSON formats """
+    def topic(self):
+        """ Generate token """
+        payload = {'sub': 'alice'}
+        return jwt.generate_jwt(payload, generated_keys['PS256'], 'PS256', timedelta(minutes=60))
+
+    class PolyglotToken(Vows.Context):
+        """ Make a forged token """
+        def topic(self, topic):
+            """ Use mix of JSON and compact format to insert forged claims including long expiration """
+            [header, payload, signature] = topic.split('.')
+            parsed_payload = loads(base64url_decode(payload))
+            parsed_payload['sub'] = 'bob'
+            parsed_payload['exp'] = 2000000000
+            fake_payload = base64url_encode((dumps(parsed_payload, separators=(',', ':'))))
+            return '{"  ' + header + '.' + fake_payload + '.":"","protected":"' + header + '", "payload":"' + payload + '","signature":"' + signature + '"}'
+
+        class Verify(Vows.Context):
+            """ Check the forged token fails to verify """
+            @Vows.capture_error
+            def topic(self, topic):
+                """ Verify the forged token """
+                return jwt.verify_jwt(topic, generated_keys['PS256'], ['PS256'])
+
+            def token_should_not_verify(self, r):
+                """ Check the token doesn't verify due to mixed format being detected """
+                expect(r).to_be_an_error()
+                expect(str(r)).to_equal('invalid JWT format')


@@ -139,6 +139,18 @@ def test_update_notification(self):
         repo_obj = self.app.store.get_user(self.USERNAME).get_repo(self.REPO)
         self.assertEqual(7, repo_obj.maxage)

+    def test_update_notification_method_get(self):
+        # Given a user with repositories
+        # When trying to update notification with GET method
+        self.getPage("/prefs/notification?action=set_notification_info&testcases=7")
+        # Then page return with success
+        self.assertStatus(200)
+        # Then page doesn't update values
+        self.assertNotInBody('Notification settings updated successfully.')
+        # Then database is not updated
+        repo_obj = self.app.store.get_user(self.USERNAME).get_repo(self.REPO)
+        self.assertEqual(0, repo_obj.maxage)
+
     def test_get_page(self):
         self.getPage("/prefs/", method='GET')
         self.assertInBody("SSH")


Validate user ID in all public interfaces

The user ID is read from the config file, or from some public REST API.
We should validate that is composed with valid set of chars.

Signed-off-by: Alberto Planas <aplanas@suse.com>

@@ -676,6 +676,11 @@ def main():
         agent_uuid = os.getenv("KEYLIME_AGENT_UUID", None)
         if agent_uuid is None:
             raise RuntimeError("Env variable KEYLIME_AGENT_UUID is empty, but agent_uuid is set to 'environment'")
+    elif not validators.valid_uuid(agent_uuid):
+        raise RuntimeError("The UUID is not valid")
+
+    if not validators.valid_agent_id(agent_uuid):
+        raise RuntimeError("The agent ID set via agent uuid parameter use invalid characters")

     if config.STUB_VTPM and config.TPM_CANNED_VALUES is not None:
         # Use canned values for stubbing


secure_mount: use /proc/self/mountinfo

The function `check_mounted()` is parsing the output of the `mount`
command.  This command can have an unknown number of spaces, that can
break the parsing logic (and produces security issues)

This commit take the mount information data from `/proc/self/mountinfo`,
that is more friendly parsed as it will escape the spaces.

Signed-off-by: Alberto Planas <aplanas@suse.com>

@@ -13,29 +13,45 @@

 def check_mounted(secdir):
-    whatsmounted = cmd_exec.run("mount")['retout']
-    whatsmounted_converted = config.convert(whatsmounted)
-    for line in whatsmounted_converted:
-        tokens = line.split()
-        tmpfs = False
-        if len(tokens) < 3:
-            continue
-        if tokens[0] == 'tmpfs':
-            tmpfs = True
-        if tokens[2] == secdir:
-            if not tmpfs:
-                logger.error("secure storage location %s already mounted "
-                             "on wrong file system type: %s.  Unmount to"
-                             "continue.", secdir, tokens[0])
-                raise Exception(
-                    f"secure storage location {secdir} already mounted on "
-                    f"wrong file system type: {tokens[0]}.  Unmount to "
-                    f"continue.")
+    """Inspect mountinfo to detect if a directory is mounted."""
+    secdir_escaped = secdir.replace(" ", r"\040")
+    for line in open("/proc/self/mountinfo", "r"):
+        # /proc/[pid]/mountinfo have 10+ elements separated with
+        # spaces (check proc (5) for a complete description)
+        #
+        # At position 7 there are some optional fields, so we need
+        # first to determine the separator mark, and validate the
+        # final total number of fields.
+        elements = line.split()
+        try:
+            separator = elements.index("-")
+        except ValueError:
+            msg = "Separator filed not found. " \
+                "Information line cannot be parsed"
+            logger.error(msg)
+            raise Exception(msg)
+
+        if len(elements) < 10 or len(elements) - separator < 4:
+            msg = "Mount information line cannot be parsed"
+            logger.error(msg)
+            raise Exception(msg)
+
+        mount_point = elements[4]
+        filesystem_type = elements[separator + 1]
+        if mount_point == secdir_escaped:
+            if filesystem_type != "tmpfs":
+                msg = f"Secure storage location {secdir} already mounted " \
+                    f"on wrong file system type: {filesystem_type}. " \
+                    "Unmount to continue."
+                logger.error(msg)
+                raise Exception(msg)

             logger.debug(
-                "secure storage location %s already mounted on tmpfs" % secdir)
+                "Secure storage location %s already mounted on tmpfs", secdir
+            )
             return True
-    logger.debug("secure storage location %s not mounted " % secdir)
+
+    logger.debug("Secure storage location %s not mounted", secdir)
     return False


🔒 Make session handling more secure

* 🔒 Tie remember_me cookie to password hash

That way when the password gets changed,
existing cookies automatically get
invalidated.

* 🔒 Invalidate user sessions on password change

This will require a user to log in again after
their password is changed, either by themselves
or through an admin.

This is a slight change in behaviour compared
to before, but makes sure password changes
are enforced across all existing sessions.

* ♻ remember_key_for_user => signature_key_for_user

* 🔒 Invalidate sessions after 15min of inactivity

The server has been keeping track of login
session for a while, but now it also only
considers such (active login) sessions fresh
that have been active within the past 15min.

That way, a session cookie's lifetime is now
also restricted server side. An active
socket connection suffices to keep the
session alive, as do any other requests
using the cookie.

As the session tracking happens in-memory,
all sessions will become invalid on server
restart. However, if the remember me cookie
is set, after server restart a passive
login will succeed.

* 🔒 Limit remember_me validity to 90 days

So far that was only done through the cookie
duration, and with a year to boot. Now we
encode the creation date into the cookie
and check against that, if it exceeds the
cookie duration the cookie is considered
unset.

Will prevent reuse of ancient remember me
cookies (and force the user to login at
least once every 90d).

* 💚 Fix unit tests

New cookie handling requires a faked flask
config object.

Related: https://huntr.dev/bounties/d27d232b-2578-4b32-b3b4-74aabdadf629/

@@ -38,6 +38,7 @@
     limit,
     no_firstrun_access,
     passive_login,
+    session_signature,
 )
 from octoprint.settings import settings as s
 from octoprint.settings import valid_boolean_trues
@@ -312,6 +313,9 @@ def login():

                 user = octoprint.server.userManager.login_user(user)
                 session["usersession.id"] = user.session
+                session["usersession.signature"] = session_signature(
+                    username, user.session
+                )
                 g.user = user

                 login_user(user, remember=remember)


Mitigate CSRF on repository deletion and user deletion #214 #215

@@ -54,15 +54,13 @@ class UserPasswordForm(CherryForm):
         _('Confirm new password'), validators=[InputRequired(_("Confirmation password is missing."))]
     )

-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.new.validators += [
-            Length(
-                min=self.app.cfg.password_min_length,
-                max=self.app.cfg.password_max_length,
-                message=_('Password must have between %(min)d and %(max)d characters.'),
-            )
-        ]
+    def validate_new(self, field):
+        validator = Length(
+            min=self.app.cfg.password_min_length,
+            max=self.app.cfg.password_max_length,
+            message=_('Password must have between %(min)d and %(max)d characters.'),
+        )
+        validator(self, field)

     @property
     def app(self):