Merge pull request from GHSA-p6p2-qq95-vq5h

* Removing custom connector template functions

* Updating connector template modal description

* Adding function upload tests

@@ -1,22 +1,17 @@
 # pylint: disable=protected-access
 import os
 from abc import ABC, abstractmethod
-from ast import AST, AnnAssign
-from operator import getitem
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
+from typing import Dict, Iterable, List, Optional, Type
 from zipfile import ZipFile

-from AccessControl.ZopeGuards import safe_builtins
 from fideslang.models import Dataset
 from loguru import logger
 from packaging.version import Version
 from packaging.version import parse as parse_version
-from RestrictedPython import compile_restricted
-from RestrictedPython.transformer import RestrictingNodeTransformer
 from sqlalchemy.orm import Session

 from fides.api.api.deps import get_api_session
-from fides.api.common_exceptions import FidesopsException, ValidationError
+from fides.api.common_exceptions import ValidationError
 from fides.api.cryptography.cryptographic_util import str_to_b64_str
 from fides.api.models.connectionconfig import (
     AccessLevel,
@@ -41,7 +36,6 @@
     replace_version,
 )
 from fides.api.util.unsafe_file_util import verify_svg, verify_zip
-from fides.config import CONFIG

 class ConnectorTemplateLoader(ABC):
@@ -96,7 +90,6 @@ def _load_connector_templates(self) -> None:
                             f"data/saas/dataset/{connector_type}_dataset.yml"
                         ),
                         icon=icon,
-                        functions=None,
                         human_readable=human_readable,
                     )
                 except Exception:
@@ -151,24 +144,16 @@ def _register_template(
         template: CustomConnectorTemplate,
     ) -> None:
         """
-        Registers a custom connector template by converting it to a ConnectorTemplate,
-        registering any custom functions, and adding it to the loader's template dictionary.
+        Registers a custom connector template by converting it to a ConnectorTemplate
+        and adding it to the loader's template dictionary.
         """
         connector_template = ConnectorTemplate(
             config=template.config,
             dataset=template.dataset,
             icon=template.icon,
-            functions=template.functions,
             human_readable=template.name,
         )

-        # register custom functions if available
-        if template.functions:
-            register_custom_functions(template.functions)
-            logger.info(
-                f"Loaded functions from the custom connector template '{template.key}'"
-            )
-
         # register the template in the loader's template dictionary
         CustomConnectorTemplateLoader.get_connector_templates()[
             template.key
@@ -220,13 +205,6 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> None:
                     raise ValidationError(
                         "Multiple svg files found, only one is allowed."
                     )
-            elif info.filename.endswith(".py"):
-                if not function_contents:
-                    function_contents = file_contents
-                else:
-                    raise ValidationError(
-                        "Multiple Python (.py) files found, only one is allowed."
-                    )

         if not config_contents:
             raise ValidationError("Zip file does not contain a config.yml file.")
@@ -266,7 +244,6 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> None:
             config=config_contents,
             dataset=dataset_contents,
             icon=icon_contents,
-            functions=function_contents,
             replaceable=replaceable,
         )

@@ -447,80 +424,3 @@ def update_saas_instance(
     connection_config.update_saas_config(db, SaaSConfig(**config_from_template))

     upsert_dataset_config_from_template(db, connection_config, template, template_vals)
-
-
-def register_custom_functions(script: str) -> None:
-    """
-    Registers custom functions by executing the given script in a restricted environment.
-
-    The script is compiled and executed with RestrictedPython, which is designed to reduce
-    the risk of executing untrusted code. It provides a set of safe builtins to prevent
-    malicious or unintended behavior.
-
-    Args:
-        script (str): The Python script containing the custom functions to be registered.
-
-    Raises:
-        FidesopsException: If allow_custom_connector_functions is disabled.
-        SyntaxError: If the script contains a syntax error or uses restricted language features.
-        Exception: If an exception occurs during the execution of the script.
-    """
-
-    if CONFIG.security.allow_custom_connector_functions:
-        restricted_code = compile_restricted(
-            script, "<string>", "exec", policy=CustomRestrictingNodeTransformer
-        )
-        safe_builtins["__import__"] = custom_guarded_import
-        safe_builtins["_getitem_"] = getitem
-        safe_builtins["staticmethod"] = staticmethod
-
-        # pylint: disable=exec-used
-        exec(
-            restricted_code,
-            {
-                "__metaclass__": type,
-                "__name__": "restricted_module",
-                "__builtins__": safe_builtins,
-            },
-        )
-    else:
-        raise FidesopsException(
-            message="The import of connector templates with custom functions is disabled by the 'security.allow_custom_connector_functions' setting."
-        )
-
-
-class CustomRestrictingNodeTransformer(RestrictingNodeTransformer):
-    """
-    Custom node transformer class that extends RestrictedPython's RestrictingNodeTransformer
-    to allow the use of type annotations (AnnAssign) in restricted code.
-    """
-
-    def visit_AnnAssign(self, node: AnnAssign) -> AST:
-        return self.node_contents_visit(node)
-
-
-def custom_guarded_import(
-    name: str,
-    _globals: Optional[dict] = None,
-    _locals: Optional[dict] = None,
-    fromlist: Optional[Tuple[str, ...]] = None,
-    level: int = 0,
-) -> Any:
-    """
-    A custom import function that prevents the import of certain potentially unsafe modules.
-    """
-    if name in [
-        "os",
-        "sys",
-        "subprocess",
-        "shutil",
-        "socket",
-        "importlib",
-        "tempfile",
-        "glob",
-    ]:
-        # raising SyntaxError to be consistent with exceptions thrown from other guarded functions
-        raise SyntaxError(f"Import of '{name}' module is not allowed.")
-    if fromlist is None:
-        fromlist = ()
-    return __import__(name, _globals, _locals, fromlist, level)


perf: 修改默认的邮箱地址

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.10 on 2023-09-27 06:56
+
+from django.db import migrations
+
+
+def migrate_user_default_email(apps, *args):
+    user_cls = apps.get_model('users', 'User')
+    user_cls.objects\
+        .filter(email='admin@mycomany.com')\
+        .update(email='admin@example.com')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0045_delete_usersession'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_user_default_email),
+    ]


Fix `Meta` inheritance in nested serializer classes (#4692)

* Add test coverage

* Fix issue

* Change fragment

* Add more change fragments

* Refine test implementation

* Add reference to GHSA

* Add admonition about use of fields = __all__ on serializers

* Add PR number

@@ -314,14 +314,9 @@ def nested_serializer_factory(relation_info, nested_depth):
         base_serializer_class = get_serializer_for_model(relation_info.related_model)

         class NautobotNestedSerializer(base_serializer_class):
-            class Meta:
-                model = relation_info.related_model
+            class Meta(base_serializer_class.Meta):
                 is_nested = True
                 depth = nested_depth - 1
-                if hasattr(base_serializer_class.Meta, "fields"):
-                    fields = base_serializer_class.Meta.fields
-                if hasattr(base_serializer_class.Meta, "exclude"):
-                    exclude = base_serializer_class.Meta.exclude

         NautobotNestedSerializer.__name__ = nested_serializer_name
         NESTED_SERIALIZER_CACHE[nested_serializer_name] = NautobotNestedSerializer


Merge pull request from GHSA-jq3w-9mgf-43m4

* init

* feat: add non-dev-mode verification for IP addresses

* fix: static_checks

* feat: add tests

* cleanup

* feat: add better error messaging if the hostname can't resolve

* fix: docstring

* fix: handle special cases for templated values

* feat: extract host checking and apply it to the authenticated client class

* Fixing the values being passed into deny_unsafe_hosts and moving the check to before the send is executed

* Removing unused import

* Fixing typo and updating change log

---------

Co-authored-by: Adrian Galvan <adrian@ethyca.com>

@@ -6,6 +6,7 @@
 from functools import wraps
 from time import sleep
 from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union
+from urllib.parse import urlparse

 from loguru import logger
 from requests import PreparedRequest, Request, Response, Session
@@ -20,6 +21,7 @@
     RateLimiterPeriod,
     RateLimiterRequest,
 )
+from fides.api.util.saas_util import deny_unsafe_hosts
 from fides.config import CONFIG

 if TYPE_CHECKING:
@@ -195,7 +197,7 @@ def send(
         Builds and executes an authenticated request.
         Optionally ignores:
           - all non-2xx/3xx responses if ignore_errors is set to True
-          - no non-2xx/3xx repsones if ignore_errors is set to False
+          - no non-2xx/3xx responses if ignore_errors is set to False
           - specific non-2xx/3xx responses if ignore_errors is set to a list of status codes
         """
         rate_limit_requests = self.build_rate_limit_requests()
@@ -204,6 +206,12 @@ def send(
         prepared_request: PreparedRequest = self.get_authenticated_request(
             request_params
         )
+        if not prepared_request.url:
+            raise ValueError("The URL for the prepared request is missing.")
+
+        # extract the hostname from the complete URL and verify its safety
+        deny_unsafe_hosts(urlparse(prepared_request.url).netloc)
+
         response = self.session.send(prepared_request)

         log_request_and_response_for_debugging(


Redirect away from user bulk actions when user has no permissions on users

@@ -1,11 +1,17 @@
 from django.contrib.auth import get_user_model

 from wagtail.admin.views.bulk_action import BulkAction
+from wagtail.admin.views.generic.permissions import PermissionCheckedMixin
+from wagtail.permission_policies import ModelPermissionPolicy
 from wagtail.users.views.users import get_users_filter_query

+User = get_user_model()

-class UserBulkAction(BulkAction):
-    models = [get_user_model()]
+
+class UserBulkAction(PermissionCheckedMixin, BulkAction):
+    models = [User]
+    permission_policy = ModelPermissionPolicy(User)
+    any_permission_required = ["add", "change", "delete"]

     def get_all_objects_in_listing_query(self, parent_id):
         listing_objects = self.model.objects.all().values_list("pk", flat=True)


@@ -1,6 +1,6 @@
 import logging
-import os
 import sys
+import subprocess

 try:
     if sys.version_info[0] == 2:
@@ -36,4 +36,4 @@ def __init__(self):
     def emit(self, record):
         if tkinter:
             msg = self.format(record)
-            os.system("notify-send '%s'" % msg)
+            subprocess.run(["notify-send", msg], check=False, timeout=2)


Merge pull request from GHSA-5m22-cfq9-86x6

Removing pickle serialization/deserialization

@@ -1,115 +0,0 @@
-"""
-Module for deserialization of algorithm results.
-
-TODO: Merge with `vantage6.tools.deserialization` in `vantage6-toolkit` and move to `vantage6-common`
-"""
-
-import json
-import logging
-import pickle
-from .exceptions import DeserializationException
-
-_DATA_FORMAT_SEPARATOR = '.'
-_MAX_FORMAT_STRING_LENGTH = 10
-
-logger = logging.getLogger(__name__)
-
-_deserializers = {}
-
-
-def deserialize(file, data_format):
-    """
-    Lookup data_format in deserializer mapping and return the associated
-    :param file:
-    :param data_format:
-    :return:
-    """
-    try:
-        return _deserializers[data_format.lower()](file)
-    except KeyError:
-        raise Exception(f'Deserialization of {data_format} has not been implemented.')
-
-
-def deserializer(data_format):
-    """
-    Register function as deserializer by adding it to the `_deserializers` map with key `data_format`.
-
-    :param data_format:
-    :return:
-    """
-
-    def decorator_deserializer(func):
-        # Register deserialization function
-        _deserializers[data_format] = func
-
-        # Return function without modifications so it can also be run without retrieving it from `_deserializers`.
-        return func
-
-    return decorator_deserializer
-
-
-@deserializer('json')
-def deserialize_json(file):
-    return json.loads(file)
-
-
-@deserializer('pickle')
-def deserialize_pickle(file):
-    return pickle.loads(file)
-
-
-def unpack_legacy_results(result):
-    return pickle.loads(result.get("result"))
-
-
-def load_data(input_bytes: bytes):
-    """
-    Try to read the specified data format and deserialize the rest of the stream accordingly. If this fails, assume
-    the data format is pickle.
-
-    :param input_bytes:
-    :return:
-    """
-    try:
-        input_data = _read_formatted(input_bytes)
-    except DeserializationException:
-        logger.info('No data format specified. Assuming input data is pickle format')
-        try:
-            input_data = pickle.loads(input_bytes)
-        except pickle.UnpicklingError:
-            raise DeserializationException('Could not deserialize input')
-    return input_data
-
-
-def _read_formatted(input_bytes):
-    data_format = str.join('', list(_read_data_format(input_bytes)))
-    return deserialize(input_bytes[len(data_format) + 1:], data_format)
-
-
-def _read_data_format(input_bytes):
-    """
-    Try to read the prescribed data format. The data format should be specified as follows: DATA_FORMAT.ACTUAL_BYTES.
-    This function will attempt to read the string before the period. It will fail if the file is not in the right
-    format.
-
-    :param input_bytes: Input file received from vantage infrastructure.
-    :return:
-    """
-    success = False
-
-    for i in range(_MAX_FORMAT_STRING_LENGTH):
-        try:
-            char = input_bytes[i:i+1].decode()
-        except UnicodeDecodeError:
-            # We aren't reading a unicode string
-            raise DeserializationException('No data format specified')
-
-        if char == _DATA_FORMAT_SEPARATOR:
-            success = True
-            break
-        else:
-            yield char
-
-    if not success:
-        # The file didn't have a format prepended
-        raise DeserializationException('No data format specified')


fix code injection vuln (#11233)

- **Description:** Fix a code injection vuln by adding one more keyword
into the filtering list
  - **Issue:** N/A
  - **Dependencies:** N/A
  - **Tag maintainer:**
  - **Twitter handle:**

Co-authored-by: Eugene Yurtsev <eyurtsev@gmail.com>

@@ -20,7 +20,7 @@
 from langchain_experimental.pal_chain.math_prompt import MATH_PROMPT
 from langchain_experimental.pydantic_v1 import Extra, Field

-COMMAND_EXECUTION_FUNCTIONS = ["system", "exec", "execfile", "eval"]
+COMMAND_EXECUTION_FUNCTIONS = ["system", "exec", "execfile", "eval", "__import__"]

 class PALValidation:


Backport GHSA-v845-jxx5-vc9f (#3139)

Co-authored-by: Quentin Pradet <quentin.pradet@gmail.com>
Co-authored-by: Illia Volochii <illia.volochii@gmail.com>

@@ -235,7 +235,7 @@ class Retry(object):
     RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])

     #: Default headers to be used for ``remove_headers_on_redirect``
-    DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"])
+    DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"])

     #: Maximum backoff time.
     DEFAULT_BACKOFF_MAX = 120


41.0.6 release (#9927)

* Fixed crash when loading a PKCS#7 bundle with no certificates (#9926)

* Version bump for 41.0.6

* Temporarily allow a new clippy warning (#9835)

* Temporarily allow a new clippy warning

* Update lib.rs

* Update lib.rs

* Need to accept this to skip test

* It's a word

@@ -10,7 +10,7 @@
     "__copyright__",
 ]

-__version__ = "41.0.5"
+__version__ = "41.0.6"

 __author__ = "The Python Cryptographic Authority and individual contributors"


[2.0] Fix unsafe `mark_safe` use in various features (#4832)

* Audit all uses of mark_safe() and change to format_html() where appropriate

* Test updates and change fragment

* Add ruff for linting, add S308 disables to audited code

* Test fixes

* Update change fragments

* One more change fragment

---------

@@ -20,9 +20,9 @@
 from nautobot.core.utils import color, config, data, lookup
 from nautobot.core.utils.navigation import is_route_new_ui_ready

-HTML_TRUE = ''
-HTML_FALSE = ''
-HTML_NONE = '—'
+HTML_TRUE = mark_safe('')  # noqa: S308
+HTML_FALSE = mark_safe('')  # noqa: S308
+HTML_NONE = mark_safe('—')  # noqa: S308

 DEFAULT_SUPPORT_MESSAGE = (
     "If further assistance is required, please join the `#nautobot` channel "
@@ -99,7 +99,7 @@ def placeholder(value):
     """
     if value:
         return value
-    return mark_safe(HTML_NONE)
+    return HTML_NONE

 @library.filter()
@@ -123,7 +123,7 @@ def add_html_id(element_str, id_str):
     match = re.match(r"^(.*?<\w+) ?(.*)$", element_str, flags=re.DOTALL)
     if not match:
         return element_str
-    return mark_safe(match.group(1) + format_html(' id="{}" ', id_str) + match.group(2))
+    return mark_safe(match.group(1) + format_html(' id="{}" ', id_str) + match.group(2))  # noqa: S308

 @library.filter()
@@ -154,10 +154,10 @@ def render_boolean(value):
         ''
     """
     if value is None:
-        return mark_safe(HTML_NONE)
+        return HTML_NONE
     if bool(value):
-        return mark_safe(HTML_TRUE)
-    return mark_safe(HTML_FALSE)
+        return HTML_TRUE
+    return HTML_FALSE

 @library.filter()
@@ -180,7 +180,7 @@ def render_markdown(value):
     # Render Markdown
     html = markdown(value, extensions=["fenced_code", "tables"])

-    return mark_safe(html)
+    return mark_safe(html)  # noqa: S308

 @library.filter()


@@ -1,7 +1,6 @@
 import django_tables2 as tables
 from django.conf import settings
 from django.utils.html import format_html
-from django.utils.safestring import mark_safe
 from django_tables2.utils import Accessor
 from jsonschema.exceptions import ValidationError as JSONSchemaValidationError

@@ -236,7 +235,7 @@ class Meta(BaseTable.Meta):

     def render_description(self, record):
         if record.description:
-            return mark_safe(render_markdown(record.description))
+            return render_markdown(record.description)
         return self.default


@@ -10,9 +10,8 @@
 from django.templatetags.static import static
 from django.shortcuts import get_object_or_404, redirect, render
 from django.urls import reverse
-from django.utils.html import escape
+from django.utils.html import format_html
 from django.utils.http import urlencode
-from django.utils.safestring import mark_safe
 from django.views.generic import View
 from django_tables2 import RequestConfig

@@ -575,20 +574,27 @@ def successful_post(self, request, obj, created, _logger):
         """Check for data that will be invalid in a future Nautobot release and warn the user if found."""
         # 3.0 TODO: remove these checks after enabling strict enforcement of the equivalent logic in Prefix.save()
         edit_url = reverse("ipam:prefix_edit", kwargs={"pk": obj.pk})
-        warning_msg = (
-            '
This will be considered invalid data in a future release.
'
+        warning_msg = format_html(
+            '
This will be considered invalid data in a future release.
',
+            static("docs/models/ipam/prefix.html"),
         )
         if obj.parent and obj.parent.type != constants.PREFIX_ALLOWED_PARENT_TYPES[obj.type]:
             parent_edit_url = reverse("ipam:prefix_edit", kwargs={"pk": obj.parent.pk})
             messages.warning(
                 request,
-                mark_safe(
-                    f'{obj} is a {obj.type.title()} prefix but its parent '
-                    f"{obj.parent} is a {obj.parent.type.title()}. {warning_msg} "
-                    f'Consider changing the type of {obj} and/or '
-                    f'{obj.parent} to resolve this issue.'
+                format_html(
+                    '{} is a {} prefix but its parent {} is a {}. {} Consider '
+                    'changing the type of {} and/or {} to resolve this issue.',
+                    obj,
+                    obj.type.title(),
+                    obj.parent.get_absolute_url(),
+                    obj.parent,
+                    obj.parent.type.title(),
+                    warning_msg,
+                    edit_url,
+                    obj,
+                    parent_edit_url,
+                    obj.parent,
                 ),
             )

@@ -597,45 +603,59 @@ def successful_post(self, request, obj, created, _logger):
         )

         if invalid_children.exists():
-            children_link = 'its children'
+            children_link = format_html('its children', reverse("ipam:prefix_list"), obj.pk)
             if obj.type == choices.PrefixTypeChoices.TYPE_CONTAINER:
                 messages.warning(
                     request,
-                    mark_safe(
-                        f"{obj} is a Container prefix and should not contain child prefixes of type Pool. "
-                        f"{warning_msg} Consider creating an intermediary Network prefix, or changing "
-                        f"the type of {children_link} to Network, to resolve this issue."
+                    format_html(
+                        "{} is a Container prefix and should not contain child prefixes of type Pool. {} "
+                        "Consider creating an intermediary Network prefix, or changing the type of {} to Network, "
+                        "to resolve this issue.",
+                        obj,
+                        warning_msg,
+                        children_link,
                     ),
                 )
             elif obj.type == choices.PrefixTypeChoices.TYPE_NETWORK:
                 messages.warning(
                     request,
-                    mark_safe(
-                        f"{obj} is a Network prefix and should not contain child prefixes of types Container or "
-                        f'Network. {warning_msg} Consider changing the type of {obj} '
-                        f"to Container, or changing the type of {children_link} to Pool, to resolve this issue."
+                    format_html(
+                        "{} is a Network prefix and should not contain child prefixes of types Container or Network. "
+                        '{} Consider changing the type of {} to Container, '
+                        "or changing the type of {} to Pool, to resolve this issue.",
+                        obj,
+                        warning_msg,
+                        edit_url,
+                        obj,
+                        children_link,
                     ),
                 )
             else:  # TYPE_POOL
                 messages.warning(
                     request,
-                    mark_safe(
-                        f"{obj} is a Pool prefix and should not contain other prefixes. {warning_msg} "
-                        f'Consider either changing the type of {obj} '
-                        f"to Container or Network, or deleting {children_link}, to resolve this issue."
+                    format_html(
+                        "{} is a Pool prefix and should not contain other prefixes. {} "
+                        'Consider either changing the type of {} to Container or Network, '
+                        "or deleting {}, to resolve this issue.",
+                        obj,
+                        warning_msg,
+                        edit_url,
+                        obj,
+                        children_link,
                     ),
                 )

         if obj.ip_addresses.exists() and obj.type == choices.PrefixTypeChoices.TYPE_CONTAINER:
-            ip_warning_msg = (
-                '
This will be considered invalid data '
-                "in a future release.
"
+            ip_warning_msg = format_html(
+                '
This will be considered invalid data '
+                "in a future release.
",
+                static("docs/models/ipam/ipaddress.html"),
             )
             shortest_child_mask_length = min([ip.mask_length for ip in obj.ip_addresses.all()])
             if shortest_child_mask_length > obj.prefix_length:
-                ip_link = 'these IP addresses'
+                ip_link = format_html(
+                    'these IP addresses', reverse("ipam:ipaddress_list"), obj.pk
+                )
                 create_url = reverse("ipam:prefix_add") + urlencode(
                     {
                         "namespace": obj.namespace.pk,
@@ -645,20 +665,29 @@ def successful_post(self, request, obj, created, _logger):
                 )
                 messages.warning(
                     request,
-                    mark_safe(
-                        f"{obj} is a Container prefix and should not directly contain IP addresses. {ip_warning_msg} "
-                        f'Consider either changing the type of {obj} to Network, or '
-                        f'creating one or more child prefix(es) of type Network to contain '
-                        f"{ip_link}, to resolve this issue."
+                    format_html(
+                        "{} is a Container prefix and should not directly contain IP addresses. {} "
+                        'Consider either changing the type of {} to Network, '
+                        'or creating one or more child prefix(es) of type Network to contain {}, '
+                        "to resolve this issue.",
+                        obj,
+                        ip_warning_msg,
+                        edit_url,
+                        obj,
+                        create_url,
+                        ip_link,
                     ),
                 )
             else:
                 messages.warning(
                     request,
-                    mark_safe(
-                        f"{obj} is a Container prefix and should not directly contain IP addresses. {ip_warning_msg} "
-                        f'Consider changing the type of {obj} to Network '
-                        "to resolve this issue."
+                    format_html(
+                        "{} is a Container prefix and should not directly contain IP addresses. {} "
+                        'Consider changing the type of {} to Network to resolve this issue.',
+                        obj,
+                        ip_warning_msg,
+                        edit_url,
+                        obj,
                     ),
                 )

@@ -763,13 +792,12 @@ def successful_post(self, request, obj, created, _logger):
         """Check for data that will be invalid in a future Nautobot release and warn the user if found."""
         # 3.0 TODO: remove this check after enabling strict enforcement of the equivalent logic in IPAddress.save()
         if obj.parent.type == choices.PrefixTypeChoices.TYPE_CONTAINER:
-            warning_msg = (
-                '
This will be considered invalid data '
-                "in a future release.
"
+            warning_msg = format_html(
+                '
This will be considered invalid data '
+                "in a future release.
",
+                static("docs/models/ipam/ipaddress.html"),
             )
-            parent_link = f'{obj.parent}'
+            parent_link = format_html('{}', obj.parent.get_absolute_url(), obj.parent)
             if obj.parent.prefix_length < obj.mask_length:
                 create_url = (
                     reverse("ipam:prefix_add")
@@ -784,20 +812,27 @@ def successful_post(self, request, obj, created, _logger):
                 )
                 messages.warning(
                     request,
-                    mark_safe(
-                        f"IP address {obj} currently has prefix {parent_link} as its parent, which is a Container. "
-                        f'{warning_msg} Consider creating an intermediate /{obj.mask_length} '
-                        "prefix of type Network to resolve this issue."
+                    format_html(
+                        "IP address {} currently has prefix {} as its parent, which is a Container. {} "
+                        'Consider creating an intermediate /{} prefix of type Network '
+                        "to resolve this issue.",
+                        obj,
+                        parent_link,
+                        warning_msg,
+                        create_url,
+                        obj.mask_length,
                     ),
                 )
             else:
                 messages.warning(
                     request,
-                    mark_safe(
-                        f"IP address {obj} currently has prefix {parent_link} as its parent, which is a Container. "
-                        f'{warning_msg} Consider changing the prefix to type Network or Pool to resolve this issue.'
+                    format_html(
+                        "IP address {} currently has prefix {} as its parent, which is a Container. {} "
+                        'Consider changing the prefix to type Network or Pool to resolve this issue.',
+                        obj,
+                        parent_link,
+                        warning_msg,
+                        reverse("ipam:prefix_edit", kwargs={"pk": obj.parent.pk}),
                     ),
                 )

@@ -1023,9 +1058,12 @@ def post(self, request):
                         logger.info("Caught ProtectedError while attempting to delete objects")
                         handle_protectederror(collapsed_ips, request, e)
                         return redirect(self.get_return_url(request))
-                    msg = (
-                        f"Merged {deleted_count} {self.queryset.model._meta.verbose_name} "
-                        f'into {escape(merged_ip)}'
+                    msg = format_html(
+                        'Merged {} {} into {}',
+                        deleted_count,
+                        self.queryset.model._meta.verbose_name,
+                        merged_ip.get_absolute_url(),
+                        merged_ip,
                     )
                     logger_msg = f"Merged {deleted_count} {self.queryset.model._meta.verbose_name} into {merged_ip}"
                     merged_ip.validated_save()
@@ -1041,7 +1079,7 @@ def post(self, request):
                     for service in services:
                         Service.objects.get(pk=service).ip_addresses.add(merged_ip)
                     logger.info(logger_msg)
-                    messages.success(request, mark_safe(msg))
+                    messages.success(request, msg)
         return self.find_duplicate_ips(request, merged_attributes)


[1.6] Fix unsafe `mark_safe` use in various features (#4833)

* Audit all uses of mark_safe() and change to format_html() where appropriate

* Test updates and change fragment

* Add ruff for linting, add S308 disables to audited code

* Test fixes

* Update change fragments

* One more change fragment

* add get_absolute_url check back to handle_protectederror

* fix ruff

---------

@@ -326,12 +326,15 @@ class CustomLink(BaseModel, ChangeLoggedModel, NotesMixin):
     name = models.CharField(max_length=100, unique=True)
     text = models.CharField(
         max_length=500,
-        help_text="Jinja2 template code for link text. Reference the object as {{ obj }} such as {{ obj.platform.slug }}. Links which render as empty text will not be displayed.",
+        help_text="Jinja2 template code for link text. "
+        "Reference the object as {{ obj }} such as {{ obj.platform.slug }}. "
+        "Links which render as empty text will not be displayed.",
     )
     target_url = models.CharField(
         max_length=500,
         verbose_name="URL",
-        help_text="Jinja2 template code for link URL. Reference the object as {{ obj }} such as {{ obj.platform.slug }}.",
+        help_text="Jinja2 template code for link URL. "
+        "Reference the object as {{ obj }} such as {{ obj.platform.slug }}.",
     )
     weight = models.PositiveSmallIntegerField(default=100)
     group_name = models.CharField(


@@ -13,9 +13,8 @@
 from django.shortcuts import get_object_or_404, redirect, render
 from django.urls import reverse
 from django.utils import timezone
-from django.utils.html import escape
+from django.utils.html import format_html
 from django.utils.http import is_safe_url
-from django.utils.safestring import mark_safe
 from django.views.generic import View
 from django.template.loader import get_template, TemplateDoesNotExist
 from django_tables2 import RequestConfig
@@ -407,10 +406,10 @@ def post(self, request, *args, **kwargs):
                 msg = f"{verb} {self.queryset.model._meta.verbose_name}"
                 logger.info(f"{msg} {obj} (PK: {obj.pk})")
                 if hasattr(obj, "get_absolute_url"):
-                    msg = f'{msg} {escape(obj)}'
+                    msg = format_html('{} {}', msg, obj.get_absolute_url(), obj)
                 else:
-                    msg = f"{msg} {escape(obj)}"
-                messages.success(request, mark_safe(msg))
+                    msg = format_html("{} {}", msg, obj)
+                messages.success(request, msg)

                 if "_addanother" in request.POST:
                     # If the object has clone_fields, pre-populate a new instance of the form
@@ -645,10 +644,10 @@ def post(self, request, *args, **kwargs):
                 msg = f"{verb} {self.queryset.model._meta.verbose_name}"
                 logger.info(f"{msg} {obj} (PK: {obj.pk})")
                 if hasattr(obj, "get_absolute_url"):
-                    msg = f'{msg} {escape(obj)}'
+                    msg = format_html('{} {}', msg, obj.get_absolute_url(), obj)
                 else:
-                    msg = f"{msg} {escape(obj)}"
-                messages.success(request, mark_safe(msg))
+                    msg = format_html("{} {}", msg, obj)
+                messages.success(request, msg)

                 if "_addanother" in request.POST:
                     # If the object has clone_fields, pre-populate a new instance of the form
@@ -1616,8 +1615,8 @@ def post(self, request, pk):
             request=copy_safe_request(request),
             commit=True,
         )
-        msg = f'Job enqueued. Click here for the results.'
-        messages.info(request=request, message=mark_safe(msg))
+        msg = format_html('Job enqueued. Click here for the results.', result.get_absolute_url())
+        messages.info(request=request, message=msg)
         return redirect(post_data["redirect_path"])

@@ -1973,10 +1972,10 @@ def post(self, request, *args, **kwargs):
                 msg = f"{verb} {self.queryset.model._meta.verbose_name}"
                 logger.info(f"{msg} {obj} (PK: {obj.pk})")
                 if hasattr(obj, "get_absolute_url"):
-                    msg = f'{msg} {escape(obj)}'
+                    msg = format_html('{} {}', msg, obj.get_absolute_url(), obj)
                 else:
-                    msg = f"{msg} {escape(obj)}"
-                messages.success(request, mark_safe(msg))
+                    msg = format_html("{} {}", msg, obj)
+                messages.success(request, msg)

                 if "_addanother" in request.POST:
                     # If the object has clone_fields, pre-populate a new instance of the form


Merge pull request #3 from ubertidavide/develop

Develop

@@ -1,6 +1,6 @@
 import logging
 from abc import ABC, abstractmethod
-from typing import Type, Union
+from typing import Type, Union, Dict

 from selenium.webdriver.common.by import By
 from selenium.webdriver.remote.webelement import WebElement
@@ -58,7 +58,29 @@ def __locator__(self, locator_name: str) -> tuple:

         """
         # load the locators from file and interprete that as code
-        return eval(self._bot.locator(self._page_name, locator_name))
+        full_locator: str = self._bot.locator(self._page_name, locator_name)
+
+        if not full_locator.startswith('(') or not full_locator.endswith(')'):
+            raise ValueError('The locator must be enclosed in round brackets.')
+
+        # declared locators
+        locator_list: Dict[str, By] = ['By.ID', 'By.XPATH', 'By.NAME', 'By.CLASS_NAME', 'By.CSS_SELECTOR',
+                                       'By.LINK_TEXT', 'By.PARTIAL_LINK_TEXT', 'By.TAG_NAME']
+
+        # check the used locator
+        parsed_locator: tuple = None
+        for locator in locator_list:
+            # check that the first characters are them of the locators and the next one of the comma
+            if full_locator[1:-1].strip().startswith(locator) and full_locator[1:-1].strip()[len(locator):].strip().startswith(','):
+                # extract the tuple required as locator
+                parsed_locator = (eval(locator), full_locator[1:-1].strip()[len(locator):].strip()[1:].strip()[1:-1])
+
+                logging.info(f'{parsed_locator}')
+
+                return parsed_locator
+
+        else:
+            raise ValueError('The specified locator is unknown or worng, check by, brackets and commas.')

     @abstractmethod
     def forward(self) -> Union[Type['Page'], None]:


CVE-2023-47642: Invalid metadata access for formerly subscribed streams.

It was discovered by the Zulip development team that active users who
had previously been subscribed to a stream incorrectly continued being
able to use the Zulip API to access metadata for that stream. As a
result, users who had been removed from a stream, but still had an
account in the organization, could still view metadata for that
stream (including the stream name, description, settings, and an email
address used to send emails into the stream via the incoming email
integration). This potentially allowed users to see changes to a
stream’s metadata after they had lost access to the stream.

This bug was present in all Zulip releases prior to today's Zulip
Server 7.5.

@@ -6135,14 +6135,47 @@ def test_previously_subscribed_private_streams(self) -> None:
         sub_data = gather_subscriptions_helper(non_admin_user)
         self.verify_sub_fields(sub_data)
         unsubscribed_streams = sub_data.unsubscribed
-        self.assert_length(unsubscribed_streams, 1)
-        self.assertEqual(unsubscribed_streams[0]["subscribers"], [])
+        self.assert_length(unsubscribed_streams, 0)

+        sub_data = gather_subscriptions_helper(guest_user)
+        self.verify_sub_fields(sub_data)
+        unsubscribed_streams = sub_data.unsubscribed
+        self.assert_length(unsubscribed_streams, 0)
+
+    def test_previously_subscribed_public_streams(self) -> None:
+        public_stream_name = "public_stream"
+        web_public_stream_name = "web_public_stream"
+        guest_user = self.example_user("polonius")
+        member_user = self.example_user("hamlet")
+
+        self.make_stream(public_stream_name, realm=get_realm("zulip"))
+        self.make_stream(web_public_stream_name, realm=get_realm("zulip"), is_web_public=True)
+
+        for stream_name in [public_stream_name, web_public_stream_name]:
+            self.subscribe(guest_user, stream_name)
+            self.subscribe(member_user, stream_name)
+            self.subscribe(self.example_user("othello"), stream_name)
+
+        for stream_name in [public_stream_name, web_public_stream_name]:
+            self.unsubscribe(guest_user, stream_name)
+            self.unsubscribe(member_user, stream_name)
+
+        # Test member user gets previously subscribed public stream and its subscribers.
+        sub_data = gather_subscriptions_helper(member_user)
+        self.verify_sub_fields(sub_data)
+        unsubscribed_streams = sub_data.unsubscribed
+        self.assert_length(unsubscribed_streams, 2)
+        self.assert_length(unsubscribed_streams[0]["subscribers"], 1)
+        self.assert_length(unsubscribed_streams[1]["subscribers"], 1)
+
+        # Test guest users cannot get previously subscribed public stream but can get
+        # web-public stream and its subscribers.
         sub_data = gather_subscriptions_helper(guest_user)
         self.verify_sub_fields(sub_data)
         unsubscribed_streams = sub_data.unsubscribed
         self.assert_length(unsubscribed_streams, 1)
-        self.assertEqual(unsubscribed_streams[0]["subscribers"], [])
+        self.assertEqual(unsubscribed_streams[0]["is_web_public"], True)
+        self.assert_length(unsubscribed_streams[0]["subscribers"], 1)

     def test_gather_subscriptions_mit(self) -> None:
         """


[ie] Do not smuggle `http_headers`

See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x

Authored by: coletdjnz

@@ -123,6 +123,7 @@ def clean_headers(headers: HTTPHeaderDict):
     if 'Youtubedl-No-Compression' in headers:  # compat
         del headers['Youtubedl-No-Compression']
         headers['Accept-Encoding'] = 'identity'
+    headers.pop('Ytdl-socks-proxy', None)

 def remove_dot_segments(path):


fix: LEAP-237: Patch ORM Leak vulnerability in open source (#5012)

* fix: LEAP-237: Patch ORM Leak vulnerability in open source

* add further detail to docstring about security concern

* fix bug where string starts with desc marker

@@ -15,7 +15,7 @@
 import re
 from datetime import timedelta

-from label_studio.core.utils.params import get_bool_env
+from label_studio.core.utils.params import get_bool_env, get_env_list

 formatter = 'standard'
 JSON_LOG = get_bool_env('JSON_LOG', False)
@@ -667,3 +667,9 @@ def collect_versions_dummy(**kwargs):
 REAL_HOSTNAME = os.getenv('HOSTNAME')  # we have to use getenv, because we don't use LABEL_STUDIO_ prefix
 GCS_CLOUD_STORAGE_FORCE_DEFAULT_CREDENTIALS = get_bool_env('GCS_CLOUD_STORAGE_FORCE_DEFAULT_CREDENTIALS', False)
 PUBLIC_API_DOCS = get_bool_env('PUBLIC_API_DOCS', False)
+
+# By default, we disallow filters with foreign keys in data manager for security reasons.
+# Add to this list (either here in code, or via the env) to allow specific filters that rely on foreign keys.
+DATA_MANAGER_FILTER_ALLOWLIST = list(
+    set(get_env_list('DATA_MANAGER_FILTER_ALLOWLIST') + ['updated_by__active_organization'])
+)


Merge pull request from GHSA-xq59-7jf3-rjc6

Co-authored-by: skelmis <ethan.mckee-harris@zxsecurity.co.nz>

@@ -4,7 +4,7 @@
 import typing as t
 from dataclasses import dataclass

-from piccolo.engine.base import Batch, Engine
+from piccolo.engine.base import Batch, Engine, validate_savepoint_name
 from piccolo.engine.exceptions import TransactionError
 from piccolo.query.base import DDL, Query
 from piccolo.querystring import QueryString
@@ -129,11 +129,13 @@ def __init__(self, name: str, transaction: PostgresTransaction):
         self.transaction = transaction

     async def rollback_to(self):
+        validate_savepoint_name(self.name)
         await self.transaction.connection.execute(
             f"ROLLBACK TO SAVEPOINT {self.name}"
         )

     async def release(self):
+        validate_savepoint_name(self.name)
         await self.transaction.connection.execute(
             f"RELEASE SAVEPOINT {self.name}"
         )
@@ -236,6 +238,7 @@ def get_savepoint_id(self) -> int:

     async def savepoint(self, name: t.Optional[str] = None) -> Savepoint:
         name = name or f"savepoint_{self.get_savepoint_id()}"
+        validate_savepoint_name(name)
         await self.connection.execute(f"SAVEPOINT {name}")
         return Savepoint(name=name, transaction=self)


Fixed CVE-2023-43665 -- Mitigated potential DoS in django.utils.text.Truncator when truncating HTML text.

Thanks Wenchao Li of Alibaba Group for the report.

@@ -83,8 +83,14 @@ def add_truncation_text(text, truncate=None):
 class Truncator(SimpleLazyObject):
     """
     An object used to truncate text, either by characters or words.
+
+    When truncating HTML text (either chars or words), input will be limited to
+    at most `MAX_LENGTH_HTML` characters.
     """

+    # 5 million characters are approximately 4000 text pages or 3 web pages.
+    MAX_LENGTH_HTML = 5_000_000
+
     def __init__(self, text):
         super().__init__(lambda: str(text))

@@ -165,6 +171,11 @@ def _truncate_html(self, length, truncate, text, truncate_len, words):
         if words and length <= 0:
             return ""

+        size_limited = False
+        if len(text) > self.MAX_LENGTH_HTML:
+            text = text[: self.MAX_LENGTH_HTML]
+            size_limited = True
+
         html4_singlets = (
             "br",
             "col",
@@ -221,10 +232,14 @@ def _truncate_html(self, length, truncate, text, truncate_len, words):
                 # Add it to the start of the open tags list
                 open_tags.insert(0, tagname)

+        truncate_text = add_truncation_text("", truncate)
+
         if current_len <= length:
+            if size_limited and truncate_text:
+                text += truncate_text
             return text
+
         out = text[:end_text_pos]
-        truncate_text = add_truncation_text("", truncate)
         if truncate_text:
             out += truncate_text
         # Close any tags still open