@@ -35,9 +35,9 @@ LOGGER = log_util.get_logger(__name__)
 class ViolationDao(dao.Dao):
     """Data access object (DAO) for rule violations."""

-    violation_attribute_list = ['resource_type', 'resource_id', 'rule_name',
-                                'rule_index', 'violation_type',
-                                'violation_data']
+    violation_attribute_list = [
+        'violation_hash', 'resource_type', 'resource_id', 'rule_name',
+        'rule_index', 'violation_type', 'violation_data', 'created_at_datetime']
     frozen_violation_attribute_list = frozenset(violation_attribute_list)
     Violation = namedtuple('Violation', frozen_violation_attribute_list)

# Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Provides the data access object (DAO) for Organizations."""

import json

from collections import defaultdict
from collections import namedtuple

import MySQLdb

from google.cloud.security.common.data_access import dao
from google.cloud.security.common.data_access import errors as db_errors
from google.cloud.security.common.data_access import violation_map as vm
from google.cloud.security.common.data_access.sql_queries import load_data
from google.cloud.security.common.data_access.sql_queries import select_data
from google.cloud.security.common.util import log_util

LOGGER = log_util.get_logger(__name__)

class ViolationDao(dao.Dao):
    """Data access object (DAO) for rule violations."""

    violation_attribute_list = ['resource_type', 'resource_id', 'rule_name',
                                'rule_index', 'violation_type',
                                'violation_data']
    frozen_violation_attribute_list = frozenset(violation_attribute_list)
    Violation = namedtuple('Violation', frozen_violation_attribute_list)

    def insert_violations(self, violations,
                          snapshot_timestamp=None):
        """Import violations into database.

        Args:
            violations (iterator): An iterator of RuleViolations.
            snapshot_timestamp (str): The snapshot timestamp to associate
                these violations with.

        Returns:
            tuple: A tuple of (int, list) containing the count of inserted
                rows and a list of violations that encountered an error during
                insert.

        Raise:
            MySQLError: is raised when the snapshot table can not be created.
        """

        resource_name = 'violations'

        try:
            # Make sure to have a reasonable timestamp to use.
            if not snapshot_timestamp:
                snapshot_timestamp = self.get_latest_snapshot_timestamp(
                    ('PARTIAL_SUCCESS', 'SUCCESS'))

            # Create the violations snapshot table.
            snapshot_table = self.create_snapshot_table(
                resource_name, snapshot_timestamp)
        # TODO: Remove this exception handling by moving the check for
        # violations table outside of the scanners.
        except MySQLdb.OperationalError, e:
            if e[0] == 1050:
                LOGGER.debug('Violations table already exists: %s', e)
                snapshot_table = self._create_snapshot_table_name(
                    resource_name, snapshot_timestamp)
            else:
                raise db_errors.MySQLError(resource_name, e)
        except MySQLdb.Error, e:
            raise db_errors.MySQLError(resource_name, e)

        inserted_rows = 0
        violation_errors = []
        for violation in violations:
            violation = self.Violation(
                resource_type=violation['resource_type'],
                resource_id=violation['resource_id'],
                rule_name=violation['rule_name'],
                rule_index=violation['rule_index'],
                violation_type=violation['violation_type'],
                violation_data=violation['violation_data'])
            for formatted_violation in _format_violation(violation,
                                                         resource_name):
                try:
                    self.execute_sql_with_commit(
                        resource_name,
                        load_data.INSERT_VIOLATION.format(snapshot_table),
                        formatted_violation)
                    inserted_rows += 1
                except MySQLdb.Error, e:
                    LOGGER.error('Unable to insert violation %s due to %s',
                                 formatted_violation, e)
                    violation_errors.append(formatted_violation)

        return (inserted_rows, violation_errors)

    def get_all_violations(self, timestamp, violation_type=None):
        """Get all the violations.

        Args:
            timestamp (str): The timestamp of the snapshot.
            violation_type (str): The violation type.

        Returns:
            list: A list of dict of the violations data.
        """
        if not violation_type:
            resource_name = 'all_violations'
            query = select_data.SELECT_ALL_VIOLATIONS
            params = ()
        else:
            resource_name = violation_type
            query = select_data.SELECT_VIOLATIONS_BY_TYPE
            params = (violation_type,)

        violations_sql = query.format(timestamp)
        rows = self.execute_sql_with_fetch(
            resource_name, violations_sql, params)
        return rows

def _format_violation(violation, resource_name):
    """Violation formating stub that uses a map to call the formating
    function for the resource.

    Args:
        violation (iterator): An iterator of RuleViolations.
        resource_name (str): String that defines a resource.

    Returns:
        tuple: A tuple of formatted violation.
    """
    formatted_output = vm.VIOLATION_MAP[resource_name](violation)
    return formatted_output

def map_by_resource(violation_rows):
    """Create a map of violation types to violations of that resource.

    Args:
        violation_rows (list): A list of dict of violation data.

    Returns:
        dict: A dict of violation types mapped to the list of corresponding
            violation types, i.e. { resource => [violation_data...] }.
    """
    v_by_type = defaultdict(list)

    for v_data in violation_rows:
        try:
            v_data['violation_data'] = json.loads(v_data['violation_data'])
        except ValueError:
            LOGGER.warn('Invalid violation data, unable to parse json for %s',
                        v_data['violation_data'])

        v_resource = vm.VIOLATION_RESOURCES.get(v_data['violation_type'])
        if v_resource:
            v_by_type[v_resource].append(v_data)

    return dict(v_by_type)


@@ -101,6 +101,7 @@ def venv_python(*args, output=False):
 def test_toolchain():
     """Test if imports work properly."""
     utils.print_title("Checking toolchain")
+
     packages = ['sip', 'PyQt5.QtCore', 'PyQt5.QtWebKit', 'qutebrowser.app']
     if g_args.dev:
         packages += get_dev_packages(short=True)

#!/usr/bin/env python3
# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:

# Copyright 2014-2015 Florian Bruhin (The Compiler) <mail@qutebrowser.org>

# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.

"""Initialize a virtualenv suitable to be used for qutebrowser."""

import os
import re
import sys
import glob
import os.path
import shutil
import argparse
import subprocess
import distutils.sysconfig  # pylint: disable=import-error
# see https://bitbucket.org/logilab/pylint/issue/73/

sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir))
from scripts import utils

g_path = None
g_args = None

def parse_args():
    """Parse the commandline arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument('--force', help="Force creating a new virtualenv.",
                        action='store_true')
    parser.add_argument('--dev', help="Set up an environment suitable for "
                                      "developing qutebrowser.",
                        action='store_true')
    parser.add_argument('path', help="Path to the virtualenv folder",
                        default='.venv', nargs='?')
    return parser.parse_args()

def check_exists():
    """Check if the virtualenv already exists."""
    if os.path.exists(g_path):
        if g_args.force:
            print("Deleting old virtualenv at {}".format(g_path))
            shutil.rmtree(g_path)
        else:
            print("virtualenv at {} does already exist!".format(g_path),
                  file=sys.stderr)
            sys.exit(1)

def get_dev_packages(short=False):
    """Get a list of packages to install.

    Args:
        short: Remove the version specification.
    """
    packages = ['colorlog', 'flake8', 'astroid==1.2.1', 'pylint==1.3.1',
                'pep257', 'colorama', 'beautifulsoup4']
    if short:
        packages = [re.split(r'[<>=]', p)[0] for p in packages]
    return packages

def install_dev_packages():
    """Install the packages needed for development."""
    for pkg in get_dev_packages():
        utils.print_subtitle("Installing {}".format(pkg))
        if os.name == 'nt':
            venv_python('-m', 'pip', 'install', '--no-clean', pkg)
        else:
            venv_python('-m', 'pip', 'install', pkg)

def venv_python(*args, output=False):
    """Call the virtualenv's python with the given arguments."""
    subdir = 'Scripts' if os.name == 'nt' else 'bin'
    executable = os.path.join(g_path, subdir, os.path.basename(sys.executable))
    if output:
        return subprocess.check_output([executable] + list(args),
                                       universal_newlines=True)
    else:
        subprocess.check_call([executable] + list(args))

def test_toolchain():
    """Test if imports work properly."""
    utils.print_title("Checking toolchain")
    packages = ['sip', 'PyQt5.QtCore', 'PyQt5.QtWebKit', 'qutebrowser.app']
    if g_args.dev:
        packages += get_dev_packages(short=True)
    for pkg in packages:
        if pkg == 'beautifulsoup4':
            pkg = 'bs4'
        print("Importing {}".format(pkg))
        venv_python('-c', 'import {}'.format(pkg))

def link_pyqt():
    """Symlink the systemwide PyQt/sip into the virtualenv."""
    if os.name == 'nt':
        return
    utils.print_title("Softlinking PyQt5")
    sys_path = distutils.sysconfig.get_python_lib()
    venv_path = venv_python(
        '-c', 'from distutils.sysconfig import get_python_lib\n'
              'print(get_python_lib())', output=True).rstrip()
    globbed_sip = (glob.glob(os.path.join(sys_path, 'sip*.so')) +
                   glob.glob(os.path.join(sys_path, 'sip*.pyd')))
    if not globbed_sip:
        print("Did not find sip in {}!".format(sys_path), file=sys.stderr)
        sys.exit(1)
    files = [
        'PyQt5',
    ]
    files += [os.path.basename(e) for e in globbed_sip]
    for fn in files:
        source = os.path.join(sys_path, fn)
        link_name = os.path.join(venv_path, fn)
        if not os.path.exists(source):
            raise FileNotFoundError(source)
        print('{} -> {}'.format(source, link_name))
        os.symlink(source, link_name)

def create_venv():
    """Create a new virtualenv."""
    utils.print_title("Creating virtualenv")
    if os.name == 'nt':
        sys_site = ['--system-site-packages']
    else:
        sys_site = []
    subprocess.check_call(['virtualenv'] + sys_site +
                          ['-p', sys.executable, g_path])

def main():
    """Main entry point."""
    global g_path, g_args
    g_args = parse_args()
    if not g_args.path:
        print("Refusing to run with empty path!", file=sys.stderr)
        sys.exit(1)
    g_path = os.path.abspath(g_args.path)
    check_exists()
    create_venv()
    utils.print_title("Calling setup.py")
    venv_python('setup.py', 'develop')
    if g_args.dev:
        utils.print_title("Installing developer packages")
        install_dev_packages()
    link_pyqt()
    test_toolchain()

if __name__ == '__main__':
    main()


@@ -63,12 +63,12 @@ def build_extensions_nupic():
   os.chdir(buildScriptsDir)

   # Generate build files with CMake
-  return_code = subprocess.call("cmake " + sourceDir + ' ' + cmakeOptions, shell=True)
+  return_code = subprocess.call('cmake ' + sourceDir + ' ' + cmakeOptions, shell=True)
   if (return_code != 0):
     sys.exit("Unable to generate build scripts!")

   # Build library with Make
-  return_code = subprocess.call("make " + makeOptions, shell=True)
+  return_code = subprocess.call('make ' + makeOptions, shell=True)
   if (return_code != 0):
     sys.exit("Unable to build the library!")

import sys
import os
import subprocess
from setuptools import setup

"""
This file only will call CMake process to generate scripts, build, and then install the NuPIC binaries.
ANY EXTRA code related to build process MUST be put into CMake file.
"""

repositoryDir = os.getcwd()

# Read command line options looking for extra options for CMake and Make
# For example, an user could type:
#   python setup.py install make_options='-j3'
# which will add '-j3' option to Make commandline
cmakeOptions = ""
makeOptions = ""
setupOptions = ""
mustBuildExtensions = False
for arg in sys.argv:
  if ("cmake_options" in arg) or ("make_options" in arg):
    (option, _, rhs) = arg.partition("=")
    if option[0] == "--cmake_options":
      cmakeOptions = rhs
    if option[0] == "--make_options":
      makeOptions = rhs
  elif (not "setup.py" in arg):
    if ("build" in arg) or ("install" in arg):
      mustBuildExtensions = True
    setupOptions += arg + " "

# Get properties of the project like version, notes, etc
properties = {}
execfile(os.path.join(repositoryDir, "nupic", "__init__.py"), {}, properties)

def findPackages(repositoryDir):
  """
  Traverse nupic directory and create packages for each subdir containing a
  __init__.py file
  """
  packages = []
  for root, dirs, files in os.walk(repositoryDir + '/nupic'):
    if '__init__.py' in files:
      subdir = root.replace(repositoryDir + '/', '')
      packages.append(subdir.replace('/', '.'))
  return packages

def build_extensions_nupic():
  """
  CMake-specific build operations
  """

  # Prepare directories to the CMake process
  sourceDir = repositoryDir
  buildScriptsDir = repositoryDir + '/build/scripts'
  if not os.path.exists(buildScriptsDir):
    os.makedirs(buildScriptsDir)
  os.chdir(buildScriptsDir)

  # Generate build files with CMake
  return_code = subprocess.call("cmake " + sourceDir + ' ' + cmakeOptions, shell=True)
  if (return_code != 0):
    sys.exit("Unable to generate build scripts!")

  # Build library with Make
  return_code = subprocess.call("make " + makeOptions, shell=True)
  if (return_code != 0):
    sys.exit("Unable to build the library!")

def setup_nupic():
  """
  Package setup operations
  """

  # Setup library
  os.chdir(repositoryDir)
  setup(
    name = 'nupic',
    version = properties["__version__"],
    packages = findPackages(repositoryDir),
    package_data = {
      'nupic': ['README.md', 'LICENSE.txt', '*.so', '*.dll', '*.dylib'],
      'nupic.bindings': ['_*.so', '_*.dll'],
      'nupic.data': ['*.json'],
      'nupic.frameworks.opf.exp_generator': ['*.json', '*.tpl'],
      'nupic.frameworks.opf.jsonschema': ['*.json'],
      'nupic.support.resources.images': ['*.png', '*.gif', '*.ico', '*.graffle'],
      'nupic.swarming.jsonschema': ['*.json']},
    description = 'Numenta Platform for Intelligent Computing',
    author='Numenta',
    author_email='help@numenta.org',
    url='https://github.com/numenta/nupic',
    classifiers=[
      'Programming Language :: Python',
      'Programming Language :: Python :: 2',
      'License :: OSI Approved :: GNU General Public License (GPL)',
      'Operating System :: OS Independent',
      'Development Status :: 5 - Production/Stable',
      'Environment :: Console',
      'Intended Audience :: Science/Research',
      'Topic :: Scientific/Engineering :: Artificial Intelligence'
    ],
    long_description = """\
NuPIC is a library that provides the building blocks for online prediction systems. The library contains the Cortical Learning Algorithm (CLA), but also the Online Prediction Framework (OPF) that allows clients to build prediction systems out of encoders, models, and metrics.

For more information, see numenta.org or the NuPIC wiki (https://github.com/numenta/nupic/wiki).
"""
  )

# Build and setup NuPIC
if mustBuildExtensions:
  build_extensions_nupic()
setup_nupic()


@@ -50,10 +50,10 @@ def scope_logged_job2():

     # end_get_logger
     @graph
-    def thing():
+    def thing_two():
         ambitious_op()

-    return thing
+    return thing_two

 @repository

from dagster import repository

def scope_logged_job():

    from dagster import graph

    # start_python_logger

    import logging
    from dagster import op

    @op
    def ambitious_op():
        my_logger = logging.getLogger("my_logger")
        try:
            x = 1 / 0
            return x
        except ZeroDivisionError:
            my_logger.error("Couldn't divide by zero!")

        return None

    # end_python_logger
    @graph
    def thing():
        ambitious_op()

    return thing

def scope_logged_job2():

    from dagster import graph

    # start_get_logger

    from dagster import get_dagster_logger, op

    @op
    def ambitious_op():
        my_logger = get_dagster_logger()
        try:
            x = 1 / 0
            return x
        except ZeroDivisionError:
            my_logger.error("Couldn't divide by zero!")

        return None

    # end_get_logger
    @graph
    def thing():
        ambitious_op()

    return thing

@repository
def python_logging_repo():
    return [scope_logged_job(), scope_logged_job2()]


@@ -15,7 +15,7 @@ from astroid import nodes

 from pylint.checkers import BaseTokenChecker
 from pylint.checkers.utils import check_messages
-from pylint.interfaces import IAstroidChecker, ITokenChecker
+from pylint.interfaces import HIGH, IAstroidChecker, ITokenChecker

 class ElseifUsedChecker(BaseTokenChecker):

# Copyright (c) 2015 LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr>
# Copyright (c) 2016-2020 Claudiu Popa <pcmanticore@gmail.com>
# Copyright (c) 2016 Glenn Matthews <glmatthe@cisco.com>
# Copyright (c) 2018 Ville Skyttä <ville.skytta@iki.fi>
# Copyright (c) 2019-2021 Pierre Sassoulas <pierre.sassoulas@gmail.com>
# Copyright (c) 2020 hippo91 <guillaume.peillex@gmail.com>
# Copyright (c) 2020 Anthony Sottile <asottile@umich.edu>
# Copyright (c) 2021 Daniël van Noord <13665637+DanielNoord@users.noreply.github.com>
# Copyright (c) 2021 Marc Mueller <30130371+cdce8p@users.noreply.github.com>

# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE

from astroid import nodes

from pylint.checkers import BaseTokenChecker
from pylint.checkers.utils import check_messages
from pylint.interfaces import IAstroidChecker, ITokenChecker

class ElseifUsedChecker(BaseTokenChecker):
    """Checks for use of "else if" when an "elif" could be used"""

    __implements__ = (ITokenChecker, IAstroidChecker)
    name = "else_if_used"
    msgs = {
        "R5501": (
            'Consider using "elif" instead of "else if"',
            "else-if-used",
            "Used when an else statement is immediately followed by "
            "an if statement and does not contain statements that "
            "would be unrelated to it.",
        )
    }

    def __init__(self, linter=None):
        super().__init__(linter)
        self._init()

    def _init(self):
        self._elifs = []
        self._if_counter = 0

    def process_tokens(self, tokens):
        # Process tokens and look for 'if' or 'elif'
        for _, token, _, _, _ in tokens:
            if token == "elif":
                self._elifs.append(True)
            elif token == "if":
                self._elifs.append(False)

    def leave_module(self, _: nodes.Module) -> None:
        self._init()

    def visit_ifexp(self, node: nodes.IfExp) -> None:
        if isinstance(node.parent, nodes.FormattedValue):
            return
        self._if_counter += 1

    def visit_comprehension(self, node: nodes.Comprehension) -> None:
        self._if_counter += len(node.ifs)

    @check_messages("else-if-used")
    def visit_if(self, node: nodes.If) -> None:
        if isinstance(node.parent, nodes.If):
            orelse = node.parent.orelse
            # current if node must directly follow an "else"
            if orelse and orelse == [node]:
                if not self._elifs[self._if_counter]:
                    self.add_message("else-if-used", node=node)
        self._if_counter += 1

def register(linter):
    """Required method to auto register this checker.

    :param linter: Main interface object for Pylint plugins
    :type linter: Pylint object
    """
    linter.register_checker(ElseifUsedChecker(linter))


@@ -106,7 +106,7 @@ bind_layers(SNAP, DTP, code=0x2004, OUI=0xc)

 def negotiate_trunk(iface=conf.iface, mymac=str(RandMAC())):
-    print "Trying to negotiate a trunk on interface %s" % iface
+    print("Trying to negotiate a trunk on interface %s" % iface)
     p = Dot3(src=mymac, dst="01:00:0c:cc:cc:cc")/LLC()/SNAP()/DTP(tlvlist=[DTPDomain(),DTPStatus(),DTPType(),DTPNeighbor(neighbor=mymac)])
     sendp(p)

#!/usr/bin/env python

# scapy.contrib.description = DTP
# scapy.contrib.status = loads

"""
    DTP Scapy Extension
    ~~~~~~~~~~~~~~~~~~~

    :version: 2008-12-22
    :author: Jochen Bartl <lobo@c3a.de>

    :Thanks:

    - TLV code derived from the CDP implementation of scapy. (Thanks to Nicolas Bareil and Arnaud Ebalard)
        http://trac.secdev.org/scapy/ticket/18
"""

from scapy.packet import *
from scapy.fields import *
from scapy.layers.l2 import SNAP,Dot3,LLC
from scapy.sendrecv import sendp

class DtpGenericTlv(Packet):
    name = "DTP Generic TLV"
    fields_desc = [ XShortField("type", 0x0001),
            FieldLenField("length", None, length_of=lambda pkt:pkt.value + 4),
            StrLenField("value", "", length_from=lambda pkt:pkt.length - 4)
            ]

    def guess_payload_class(self, p):
        return conf.padding_layer

class RepeatedTlvListField(PacketListField):
    def __init__(self, name, default, cls):
        PacketField.__init__(self, name, default, cls)

    def getfield(self, pkt, s):
        lst = []
        remain = s
        while len(remain) > 0:
            p = self.m2i(pkt,remain)
            if conf.padding_layer in p:
                pad = p[conf.padding_layer]
                remain = pad.load
                del(pad.underlayer.payload)
            else:
                remain = ""
            lst.append(p)
        return remain,lst

    def addfield(self, pkt, s, val):
        return s+reduce(str.__add__, map(str, val),"")

_DTP_TLV_CLS = {
                    0x0001 : "DTPDomain",
                    0x0002 : "DTPStatus",
                    0x0003 : "DTPType",
                    0x0004 : "DTPNeighbor"
                   }

class DTPDomain(DtpGenericTlv):
    name = "DTP Domain"
    fields_desc = [ ShortField("type", 1),
            FieldLenField("length", None, "domain", adjust=lambda pkt,x:x + 4),
            StrLenField("domain", b"\x00", length_from=lambda pkt:pkt.length - 4)
            ]

class DTPStatus(DtpGenericTlv):
    name = "DTP Status"
    fields_desc = [ ShortField("type", 2),
            FieldLenField("length", None, "status", adjust=lambda pkt,x:x + 4),
            StrLenField("status", b"\x03", length_from=lambda pkt:pkt.length - 4)
            ]

class DTPType(DtpGenericTlv):
    name = "DTP Type"
    fields_desc = [ ShortField("type", 3),
            FieldLenField("length", None, "dtptype", adjust=lambda pkt,x:x + 4),
            StrLenField("dtptype", b"\xa5", length_from=lambda pkt:pkt.length - 4)
            ]

class DTPNeighbor(DtpGenericTlv):
    name = "DTP Neighbor"
    fields_desc = [ ShortField("type", 4),
            #FieldLenField("length", None, "neighbor", adjust=lambda pkt,x:x + 4),
            ShortField("len", 10),
            MACField("neighbor", None)
            ]

def _DTPGuessPayloadClass(p, **kargs):
    cls = conf.raw_layer
    if len(p) >= 2:
        t = struct.unpack("!H", p[:2])[0]
        clsname = _DTP_TLV_CLS.get(t, "DtpGenericTlv")
        cls = globals()[clsname]
    return cls(p, **kargs)

class DTP(Packet):
    name = "DTP"
    fields_desc = [ ByteField("ver", 1),
                    RepeatedTlvListField("tlvlist", [], _DTPGuessPayloadClass)
                ]

bind_layers(SNAP, DTP, code=0x2004, OUI=0xc)

def negotiate_trunk(iface=conf.iface, mymac=str(RandMAC())):
    print "Trying to negotiate a trunk on interface %s" % iface
    p = Dot3(src=mymac, dst="01:00:0c:cc:cc:cc")/LLC()/SNAP()/DTP(tlvlist=[DTPDomain(),DTPStatus(),DTPType(),DTPNeighbor(neighbor=mymac)])
    sendp(p)

if __name__ == "__main__":
    from scapy.main import interact
    interact(mydict=globals(), mybanner="DTP")


@@ -114,27 +114,43 @@ def det2json(dataset, results):
                 data['score'] = float(bboxes[i][4])
                 data['category_id'] = dataset.cat_ids[label]
                 json_results.append(data)
-    return json_results
+    return dict(bbox=json_results)

 def segm2json(dataset, results):
-    json_results = []
+    bbox_json_results = []
+    segm_json_results = []
     for idx in range(len(dataset)):
         img_id = dataset.img_ids[idx]
-        det, seg = results[idx]
+        det, (seg, mask_scores) = results[idx]
         for label in range(len(det)):
+            # bbox results
             bboxes = det[label]
-            segms = seg[label]
             for i in range(bboxes.shape[0]):
                 data = dict()
                 data['image_id'] = img_id
                 data['bbox'] = xyxy2xywh(bboxes[i])
                 data['score'] = float(bboxes[i][4])
                 data['category_id'] = dataset.cat_ids[label]
+                bbox_json_results.append(data)
+
+            # segm results
+            segms = seg[label]
+            if mask_scores is not None:
+                mask_score = mask_scores[label]
+            for i in range(bboxes.shape[0]):
+                data = dict()
+                data['image_id'] = img_id
+                data['bbox'] = xyxy2xywh(bboxes[i])
+                if mask_scores is not None:
+                    data['score'] = float(mask_score[i])
+                else:
+                    data['score'] = float(bboxes[i][4])
+                data['category_id'] = dataset.cat_ids[label]
                 segms[i]['counts'] = segms[i]['counts'].decode()
                 data['segmentation'] = segms[i]
-                json_results.append(data)
-    return json_results
+                segm_json_results.append(data)
+    return dict(bbox=bbox_json_results, segm=segm_json_results)

 def results2json(dataset, results, out_file):

import mmcv
import numpy as np
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval

from .recall import eval_recalls

def coco_eval(result_file, result_types, coco, max_dets=(100, 300, 1000)):
    for res_type in result_types:
        assert res_type in [
            'proposal', 'proposal_fast', 'bbox', 'segm', 'keypoints'
        ]

    if mmcv.is_str(coco):
        coco = COCO(coco)
    assert isinstance(coco, COCO)

    if result_types == ['proposal_fast']:
        ar = fast_eval_recall(result_file, coco, np.array(max_dets))
        for i, num in enumerate(max_dets):
            print('AR@{}\t= {:.4f}'.format(num, ar[i]))
        return

    assert result_file.endswith('.json')
    coco_dets = coco.loadRes(result_file)

    img_ids = coco.getImgIds()
    for res_type in result_types:
        iou_type = 'bbox' if res_type == 'proposal' else res_type
        cocoEval = COCOeval(coco, coco_dets, iou_type)
        cocoEval.params.imgIds = img_ids
        if res_type == 'proposal':
            cocoEval.params.useCats = 0
            cocoEval.params.maxDets = list(max_dets)
        cocoEval.evaluate()
        cocoEval.accumulate()
        cocoEval.summarize()

def fast_eval_recall(results,
                     coco,
                     max_dets,
                     iou_thrs=np.arange(0.5, 0.96, 0.05)):
    if mmcv.is_str(results):
        assert results.endswith('.pkl')
        results = mmcv.load(results)
    elif not isinstance(results, list):
        raise TypeError(
            'results must be a list of numpy arrays or a filename, not {}'.
            format(type(results)))

    gt_bboxes = []
    img_ids = coco.getImgIds()
    for i in range(len(img_ids)):
        ann_ids = coco.getAnnIds(imgIds=img_ids[i])
        ann_info = coco.loadAnns(ann_ids)
        if len(ann_info) == 0:
            gt_bboxes.append(np.zeros((0, 4)))
            continue
        bboxes = []
        for ann in ann_info:
            if ann.get('ignore', False) or ann['iscrowd']:
                continue
            x1, y1, w, h = ann['bbox']
            bboxes.append([x1, y1, x1 + w - 1, y1 + h - 1])
        bboxes = np.array(bboxes, dtype=np.float32)
        if bboxes.shape[0] == 0:
            bboxes = np.zeros((0, 4))
        gt_bboxes.append(bboxes)

    recalls = eval_recalls(
        gt_bboxes, results, max_dets, iou_thrs, print_summary=False)
    ar = recalls.mean(axis=1)
    return ar

def xyxy2xywh(bbox):
    _bbox = bbox.tolist()
    return [
        _bbox[0],
        _bbox[1],
        _bbox[2] - _bbox[0] + 1,
        _bbox[3] - _bbox[1] + 1,
    ]

def proposal2json(dataset, results):
    json_results = []
    for idx in range(len(dataset)):
        img_id = dataset.img_ids[idx]
        bboxes = results[idx]
        for i in range(bboxes.shape[0]):
            data = dict()
            data['image_id'] = img_id
            data['bbox'] = xyxy2xywh(bboxes[i])
            data['score'] = float(bboxes[i][4])
            data['category_id'] = 1
            json_results.append(data)
    return json_results

def det2json(dataset, results):
    json_results = []
    for idx in range(len(dataset)):
        img_id = dataset.img_ids[idx]
        result = results[idx]
        for label in range(len(result)):
            bboxes = result[label]
            for i in range(bboxes.shape[0]):
                data = dict()
                data['image_id'] = img_id
                data['bbox'] = xyxy2xywh(bboxes[i])
                data['score'] = float(bboxes[i][4])
                data['category_id'] = dataset.cat_ids[label]
                json_results.append(data)
    return json_results

def segm2json(dataset, results):
    json_results = []
    for idx in range(len(dataset)):
        img_id = dataset.img_ids[idx]
        det, seg = results[idx]
        for label in range(len(det)):
            bboxes = det[label]
            segms = seg[label]
            for i in range(bboxes.shape[0]):
                data = dict()
                data['image_id'] = img_id
                data['bbox'] = xyxy2xywh(bboxes[i])
                data['score'] = float(bboxes[i][4])
                data['category_id'] = dataset.cat_ids[label]
                segms[i]['counts'] = segms[i]['counts'].decode()
                data['segmentation'] = segms[i]
                json_results.append(data)
    return json_results

def results2json(dataset, results, out_file):
    if isinstance(results[0], list):
        json_results = det2json(dataset, results)
    elif isinstance(results[0], tuple):
        json_results = segm2json(dataset, results)
    elif isinstance(results[0], np.ndarray):
        json_results = proposal2json(dataset, results)
    else:
        raise TypeError('invalid type of results')
    mmcv.dump(json_results, out_file)


@@ -6,6 +6,7 @@ from kaitaistruct import KaitaiStream
 from mitmproxy.contrib.kaitaistruct import png
 from mitmproxy.contrib.kaitaistruct import gif
 from mitmproxy.contrib.kaitaistruct import jpeg
+from mitmproxy.contrib.kaitaistruct import ico

 Metadata = typing.List[typing.Tuple[str, str]]

import io
import typing

from kaitaistruct import KaitaiStream

from mitmproxy.contrib.kaitaistruct import png
from mitmproxy.contrib.kaitaistruct import gif
from mitmproxy.contrib.kaitaistruct import jpeg

Metadata = typing.List[typing.Tuple[str, str]]

def parse_png(data: bytes) -> Metadata:
    img = png.Png(KaitaiStream(io.BytesIO(data)))
    parts = [
        ('Format', 'Portable network graphics'),
        ('Size', "{0} x {1} px".format(img.ihdr.width, img.ihdr.height))
    ]
    for chunk in img.chunks:
        if chunk.type == 'gAMA':
            parts.append(('gamma', str(chunk.body.gamma_int / 100000)))
        elif chunk.type == 'pHYs':
            aspectx = chunk.body.pixels_per_unit_x
            aspecty = chunk.body.pixels_per_unit_y
            parts.append(('aspect', "{0} x {1}".format(aspectx, aspecty)))
        elif chunk.type == 'tEXt':
            parts.append((chunk.body.keyword, chunk.body.text))
        elif chunk.type == 'iTXt':
            parts.append((chunk.body.keyword, chunk.body.text))
        elif chunk.type == 'zTXt':
            parts.append((chunk.body.keyword, chunk.body.text_datastream.decode('iso8859-1')))
    return parts

def parse_gif(data: bytes) -> Metadata:
    img = gif.Gif(KaitaiStream(io.BytesIO(data)))
    descriptor = img.logical_screen_descriptor
    parts = [
        ('Format', 'Compuserve GIF'),
        ('Version', "GIF{}".format(img.hdr.version)),
        ('Size', "{} x {} px".format(descriptor.screen_width, descriptor.screen_height)),
        ('background', str(descriptor.bg_color_index))
    ]
    ext_blocks = []
    for block in img.blocks:
        if block.block_type.name == 'extension':
            ext_blocks.append(block)
    comment_blocks = []
    for block in ext_blocks:
        if block.body.label._name_ == 'comment':
            comment_blocks.append(block)
    for block in comment_blocks:
        entries = block.body.body.entries
        for entry in entries:
            comment = entry.bytes
            if comment is not b'':
                parts.append(('comment', str(comment)))
    return parts

def parse_jpeg(data: bytes) -> Metadata:
    img = jpeg.Jpeg(KaitaiStream(io.BytesIO(data)))
    parts = [
        ('Format', 'JPEG (ISO 10918)')
    ]
    for segment in img.segments:
        if segment.marker._name_ == 'sof0':
            parts.append(('Size', "{0} x {1} px".format(segment.data.image_width, segment.data.image_height)))
        if segment.marker._name_ == 'app0':
            parts.append(('jfif_version', "({0}, {1})".format(segment.data.version_major, segment.data.version_minor)))
            parts.append(('jfif_density', "({0}, {1})".format(segment.data.density_x, segment.data.density_y)))
            parts.append(('jfif_unit', str(segment.data.density_units._value_)))
        if segment.marker._name_ == 'com':
            parts.append(('comment', str(segment.data)))
        if segment.marker._name_ == 'app1':
            if hasattr(segment.data, 'body'):
                for field in segment.data.body.data.body.ifd0.fields:
                    if field.data is not None:
                        parts.append((field.tag._name_, field.data.decode('UTF-8').strip('\x00')))
    return parts


@@ -104,6 +104,22 @@ class TemporalMemoryPerformanceTest(unittest.TestCase):
   # Helper functions
   # ==============================

+  def _generateSequence(self):
+    sequence = []
+    with open (_INPUT_FILE_PATH) as fin:
+      reader = csv.reader(fin)
+      headers = reader.next()
+      types = reader.next()
+      reader.next()
+      for _ in xrange(NUM_PATTERNS):
+        record = reader.next()
+        value = float(record[1])
+        encodedValue = self.scalarEncoder.encode(value)
+        activeBits = set(encodedValue.nonzero()[0])
+        sequence.append(activeBits)
+    return sequence
+
+
   def _feedAll(self, sequence, learn=True, num=1):
     repeatedSequence = sequence * num
     times = []

#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import time
import unittest

import numpy

from nupic.data.generators.pattern_machine import PatternMachine
from nupic.data.generators.sequence_machine import SequenceMachine
from nupic.research.temporal_memory import TemporalMemory as TemporalMemoryPy
from nupic.bindings.algorithms import TemporalMemory as TemporalMemoryCPP
from nupic.research.TP import TP
from nupic.research.TP10X2 import TP10X2

# ==============================
# Tests
# ==============================

class TemporalMemoryPerformanceTest(unittest.TestCase):

  def setUp(self):
    self.tmPy = TemporalMemoryPy(columnDimensions=[2048],
                                 cellsPerColumn=32,
                                 initialPermanence=0.5,
                                 connectedPermanence=0.8,
                                 minThreshold=10,
                                 maxNewSynapseCount=12,
                                 permanenceIncrement=0.1,
                                 permanenceDecrement=0.05,
                                 activationThreshold=15)

    self.tmCPP = TemporalMemoryCPP(columnDimensions=[2048],
                                   cellsPerColumn=32,
                                   initialPermanence=0.5,
                                   connectedPermanence=0.8,
                                   minThreshold=10,
                                   maxNewSynapseCount=12,
                                   permanenceIncrement=0.1,
                                   permanenceDecrement=0.05,
                                   activationThreshold=15)

    self.tp = TP(numberOfCols=2048,
                 cellsPerColumn=32,
                 initialPerm=0.5,
                 connectedPerm=0.8,
                 minThreshold=10,
                 newSynapseCount=12,
                 permanenceInc=0.1,
                 permanenceDec=0.05,
                 activationThreshold=15,
                 globalDecay=0, burnIn=1,
                 checkSynapseConsistency=False,
                 pamLength=1)

    self.tp10x2 = TP10X2(numberOfCols=2048,
                         cellsPerColumn=32,
                         initialPerm=0.5,
                         connectedPerm=0.8,
                         minThreshold=10,
                         newSynapseCount=12,
                         permanenceInc=0.1,
                         permanenceDec=0.05,
                         activationThreshold=15,
                         globalDecay=0, burnIn=1,
                         checkSynapseConsistency=False,
                         pamLength=1)

    self.patternMachine = PatternMachine(2048, 40, num=100)
    self.sequenceMachine = SequenceMachine(self.patternMachine)

  def testSingleSequence(self):
    print "Test: Single sequence"
    sequence = self.sequenceMachine.generateFromNumbers(range(50))
    times = self._feedAll(sequence)

    self.assertTrue(times[1] < times[0])
    self.assertTrue(times[3] < times[2])

  # ==============================
  # Helper functions
  # ==============================

  def _feedAll(self, sequence, learn=True, num=1):
    repeatedSequence = sequence * num
    times = []

    def tmComputeFn(pattern, instance):
      instance.compute(pattern, learn)

    def tpComputeFn(pattern, instance):
      array = self._patternToNumpyArray(pattern)
      instance.compute(array, enableLearn=learn, computeInfOutput=True)

    elapsed = self._feedOne(repeatedSequence, self.tmPy, tmComputeFn)
    times.append(elapsed)
    print "TM (py):\t{0}s".format(elapsed)

    elapsed = self._feedOne(repeatedSequence, self.tmCPP, tmComputeFn)
    times.append(elapsed)
    print "TM (C++):\t{0}s".format(elapsed)

    elapsed = self._feedOne(repeatedSequence, self.tp, tpComputeFn)
    times.append(elapsed)
    print "TP:\t\t{0}s".format(elapsed)

    elapsed = self._feedOne(repeatedSequence, self.tp10x2, tpComputeFn)
    times.append(elapsed)
    print "TP10X2:\t\t{0}s".format(elapsed)

    return times

  @staticmethod
  def _feedOne(sequence, instance, computeFn):
    start = time.clock()

    for pattern in sequence:
      if pattern == None:
        instance.reset()
      else:
        computeFn(pattern, instance)

    elapsed = time.clock() - start

    return elapsed

  @staticmethod
  def _patternToNumpyArray(pattern):
    array = numpy.zeros(2048, dtype='int32')
    array[list(pattern)] = 1

    return array

# ==============================
# Main
# ==============================

if __name__ == "__main__":
  unittest.main()


@@ -1,9 +1,13 @@
 import colander

-from kinto.core import resource
+from pyramid import httpexceptions
+from pyramid.security import NO_PERMISSION_REQUIRED
+
+from kinto.core import resource, Service
 from kinto.core.utils import instance_uri
-from kinto.core.storage import Filter
+from kinto.core.storage import Filter, Sort
 from kinto.core.resource.viewset import ViewSet
+from kinto.core.utils import COMPARISON

 class HistorySchema(resource.ResourceSchema):

import colander

from kinto.core import resource
from kinto.core.utils import instance_uri
from kinto.core.storage import Filter
from kinto.core.resource.viewset import ViewSet

class HistorySchema(resource.ResourceSchema):
    user_id = colander.SchemaNode(colander.String())
    uri = colander.SchemaNode(colander.String())
    action = colander.SchemaNode(colander.String())
    date = colander.SchemaNode(colander.String())
    resource_name = colander.SchemaNode(colander.String())
    bucket_id = colander.SchemaNode(colander.String(), missing=colander.drop)
    collection_id = colander.SchemaNode(colander.String(), missing=colander.drop)
    group_id = colander.SchemaNode(colander.String(), missing=colander.drop)
    record_id = colander.SchemaNode(colander.String(), missing=colander.drop)
    target = colander.SchemaNode(colander.Mapping())

    class Options:
        preserve_unknown = False

# Add custom OpenAPI tags/operation ids
collection_get_arguments = getattr(ViewSet, "collection_get_arguments", {})
collection_delete_arguments = getattr(ViewSet, "collection_delete_arguments", {})

get_history_arguments = {'tags': ['History'], 'operation_id': 'get_history',
                         **collection_get_arguments}
delete_history_arguments = {'tags': ['History'], 'operation_id': 'delete_history',
                            **collection_delete_arguments}

@resource.register(name='history',
                   collection_path='/buckets/{{bucket_id}}/history',
                   record_path=None,
                   collection_methods=('GET', 'DELETE'),
                   default_arguments={'tags': ['History']},
                   collection_get_arguments=get_history_arguments,
                   collection_delete_arguments=delete_history_arguments)
class History(resource.ShareableResource):

    schema = HistorySchema

    def get_parent_id(self, request):
        self.bucket_id = request.matchdict['bucket_id']
        return instance_uri(request, 'bucket', id=self.bucket_id)

    def _extract_filters(self):
        filters = super()._extract_filters()
        filters_str_id = []
        for filt in filters:
            if filt.field in ('record_id', 'collection_id', 'bucket_id'):
                if isinstance(filt.value, int):
                    filt = Filter(filt.field, str(filt.value), filt.operator)
            filters_str_id.append(filt)

        return filters_str_id


@@ -125,5 +125,21 @@ class ServerTestRun(unittest.TestCase):
         self.assertEqual(work['task_id'], 'A')

+class URLLibServerTestRun(ServerTestRun):
+
+    @mock.patch.object(luigi.rpc, 'HAS_REQUESTS', False)
+    def start_server(self, *args, **kwargs):
+        super(URLLibServerTestRun, self).start_server(*args, **kwargs)
+
+
+@attr('unix')
+class UNIXServerTestRun(unittest.TestCase):
+    server_client_class = UNIXServerClient
+
+    def tearDown(self):
+        super(self, ServerTestRun).tearDown()
+        shutil.rmtree(self.server_client.tempdir)
+
+
 if __name__ == '__main__':
     unittest.main()

# -*- coding: utf-8 -*-
#
# Copyright 2012-2015 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import multiprocessing
import random
import signal
import time
import tempfile

from helpers import unittest, with_config, skipOnTravis
import luigi.rpc
import luigi.server
from luigi.scheduler import CentralPlannerScheduler

from tornado.testing import AsyncHTTPTestCase

class ServerTestBase(AsyncHTTPTestCase):

    def get_app(self):
        return luigi.server.app(CentralPlannerScheduler())

    def setUp(self):
        super(ServerTestBase, self).setUp()

        self._old_fetch = luigi.rpc.RemoteScheduler._fetch

        def _fetch(obj, url, body, *args, **kwargs):
            response = self.fetch(url, body=body, method='POST')
            if response.code >= 400:
                raise luigi.rpc.RPCError(
                    'Errror when connecting to remote scheduler'
                )
            return response.body.decode('utf-8')

        luigi.rpc.RemoteScheduler._fetch = _fetch

    def tearDown(self):
        super(ServerTestBase, self).tearDown()
        luigi.rpc.RemoteScheduler._fetch = self._old_fetch

class ServerTest(ServerTestBase):

    def test_visualizer(self):
        page = self.fetch('/').body
        self.assertTrue(page.find(b'<title>') != -1)

    def _test_404(self, path):
        response = self.fetch(path)

        self.assertEqual(response.code, 404)

    def test_404(self):
        self._test_404('/foo')

    def test_api_404(self):
        self._test_404('/api/foo')

class ServerTestRun(unittest.TestCase):
    """Test to start and stop the server in a more "standard" way
    """

    def run_server(self):
        luigi.server.run(api_port=self._api_port, address='127.0.0.1')

    def start_server(self):
        self._api_port = random.randint(1024, 9999)
        self._process = multiprocessing.Process(target=self.run_server)
        self._process.start()
        time.sleep(0.1)  # wait for server to start
        self.sch = luigi.rpc.RemoteScheduler(host='localhost', port=self._api_port)
        self.sch._wait = lambda: None

    def stop_server(self):
        self._process.terminate()
        self._process.join(1)
        if self._process.is_alive():
            os.kill(self._process.pid, signal.SIGKILL)

    def setUp(self):
        state_path = tempfile.mktemp(suffix=self.id())
        luigi.configuration.get_config().set('scheduler', 'state_path', state_path)
        self.start_server()

    def tearDown(self):
        self.stop_server()

    def test_ping(self):
        self.sch.ping(worker='xyz')

    def test_raw_ping(self):
        self.sch._request('/api/ping', {'worker': 'xyz'})

    def test_raw_ping_extended(self):
        self.sch._request('/api/ping', {'worker': 'xyz', 'foo': 'bar'})

    def test_404(self):
        with self.assertRaises(luigi.rpc.RPCError):
            self.sch._request('/api/fdsfds', {'dummy': 1})

    @skipOnTravis('https://travis-ci.org/spotify/luigi/jobs/72953884')
    def test_save_state(self):
        self.sch.add_task('X', 'B', deps=('A',))
        self.sch.add_task('X', 'A')
        self.assertEqual(self.sch.get_work('X')['task_id'], 'A')
        self.stop_server()
        self.start_server()
        work = self.sch.get_work('X')['running_tasks'][0]
        self.assertEqual(work['task_id'], 'A')

if __name__ == '__main__':
    unittest.main()


@@ -74,22 +74,29 @@ class BibSortWasher(object):
         """
         if not val:
             return ''
-        val_tokens = str(val).split(" ", 1) #split in leading_word, phrase_without_leading_word
-        if len(val_tokens) == 2 and val_tokens[0].lower() in LEADING_ARTICLES:
-            return val_tokens[1].strip().lower()
-        return val.lower()
+        val = decode_to_unicode(val).lower().encode('UTF-8')
+        val_tokens = val.split(" ", 1) #split in leading_word, phrase_without_leading_word
+        if len(val_tokens) == 2 and val_tokens[0].strip() in LEADING_ARTICLES:
+            return val_tokens[1].strip()
+        return val.strip()

     def _sort_case_insensitive_strip_accents(self, val):
         """Remove accents and convert to lower case"""
         if not val:
             return ''
-        return strip_accents(str(val).lower())
+        return translate_to_ascii(val).pop().lower()
+
+    def _sort_nosymbols_case_insensitive_strip_accents(self, val):
+        """Remove accents, remove symbols, and convert to lower case"""
+        if not val:
+            return ''
+        return ''.join(_RE_NOSYMBOLS.findall(translate_to_ascii(val).pop().lower()))

     def _sort_case_insensitive(self, val):
         """Conversion to lower case"""
         if not val:
             return ''
-        return str(val).lower()
+        return decode_to_unicode(val).lower().encode('UTF-8')

     def _sort_dates(self, val):
         """

## -*- mode: python; coding: utf-8; -*-
##
## This file is part of Invenio.
## Copyright (C) 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""Applies a transformation function to a value"""

from time import strptime
from invenio.dateutils import strftime
from invenio.textutils import strip_accents

LEADING_ARTICLES = ['the', 'a', 'an', 'at', 'on', 'of']

class InvenioBibSortWasherNotImplementedError(Exception):
    """Exception raised when a washer method
    defined in the bibsort config file is not implemented"""
    pass

class BibSortWasher(object):
    """Implements all the washer methods"""

    def __init__(self, washer):
        self.washer = washer
        fnc_name = '_' + washer
        try:
            self.washer_fnc = self.__getattribute__(fnc_name)
        except AttributeError, err:
            raise InvenioBibSortWasherNotImplementedError(err)

    def get_washer(self):
        """Returns the washer name"""
        return self.washer

    def get_transformed_value(self, val):
        """Returns the value"""
        return self.washer_fnc(val)

    def _sort_alphanumerically_remove_leading_articles_strip_accents(self, val):
        """
        Convert:
        'The title' => 'title'
        'A title' => 'title'
        'Title' => 'title'
        """
        if not val:
            return ''
        val_tokens = str(val).split(" ", 1) #split in leading_word, phrase_without_leading_word
        if len(val_tokens) == 2 and val_tokens[0].lower() in LEADING_ARTICLES:
            return strip_accents(val_tokens[1].strip().lower())
        return strip_accents(val.lower())

    def _sort_alphanumerically_remove_leading_articles(self, val):
        """
        Convert:
        'The title' => 'title'
        'A title' => 'title'
        'Title' => 'title'
        """
        if not val:
            return ''
        val_tokens = str(val).split(" ", 1) #split in leading_word, phrase_without_leading_word
        if len(val_tokens) == 2 and val_tokens[0].lower() in LEADING_ARTICLES:
            return val_tokens[1].strip().lower()
        return val.lower()

    def _sort_case_insensitive_strip_accents(self, val):
        """Remove accents and convert to lower case"""
        if not val:
            return ''
        return strip_accents(str(val).lower())

    def _sort_case_insensitive(self, val):
        """Conversion to lower case"""
        if not val:
            return ''
        return str(val).lower()

    def _sort_dates(self, val):
        """
        Convert:
        '8 nov 2010' => '2010-11-08'
        'nov 2010' => '2010-11-01'
        '2010' => '2010-01-01'
        """
        datetext_format = "%Y-%m-%d"
        try:
            datestruct = strptime(val, datetext_format)
        except ValueError:
            try:
                datestruct = strptime(val, "%d %b %Y")
            except ValueError:
                try:
                    datestruct = strptime(val, "%b %Y")
                except ValueError:
                    try:
                        datestruct = strptime(val, "%Y")
                    except ValueError:
                        return val
        return strftime(datetext_format, datestruct)

    def _sort_numerically(self, val):
        """
        Convert:
        1245 => float(1245)
        """
        try:
            return float(val)
        except ValueError:
            return 0

def get_all_available_washers():
    """
    Returns all the available washer functions without the leading '_'
    """
    method_list = dir(BibSortWasher)
    return [method[1:] for method in method_list if method.startswith('_') and method.find('__') < 0]


@@ -11,6 +11,7 @@ from localstack.services.cloudformation.service_models import (
 )
 from localstack.utils import common
 from localstack.utils.aws import aws_stack
+from localstack.utils.common import short_uid

 class EventConnection(GenericBaseModel):

import json

from localstack.services.cloudformation.deployment_utils import (
    PLACEHOLDER_RESOURCE_NAME,
    select_parameters,
)
from localstack.services.cloudformation.service_models import (
    REF_ATTRS,
    REF_ID_ATTRS,
    GenericBaseModel,
)
from localstack.utils import common
from localstack.utils.aws import aws_stack

class EventConnection(GenericBaseModel):
    @staticmethod
    def cloudformation_type():
        return "AWS::Events::Connection"

    def fetch_state(self, stack_name, resources):
        client = aws_stack.connect_to_service("events")
        conn_name = self.resolve_refs_recursively(stack_name, self.props.get("Name"), resources)
        return client.describe_connection(Name=conn_name)

    def get_cfn_attribute(self, attribute_name):
        props = self.props
        if attribute_name in REF_ID_ATTRS:
            return props.get("Name")
        if attribute_name == "Arn":
            return props.get("ConnectionArn")
        # TODO: handle "SecretArn" attribute
        return super(EventConnection, self).get_cfn_attribute(attribute_name)

    @classmethod
    def get_deploy_templates(cls):
        return {
            "create": {"function": "create_connection"},
            "delete": {"function": "delete_connection", "parameters": {"Name": "Name"}},
        }

class EventBus(GenericBaseModel):
    @staticmethod
    def cloudformation_type():
        return "AWS::Events::EventBus"

    def fetch_state(self, stack_name, resources):
        event_bus_name = self.resolve_refs_recursively(
            stack_name, self.props.get("Name"), resources
        )
        client = aws_stack.connect_to_service("events")
        return client.describe_event_bus(Name=event_bus_name)

    def get_cfn_attribute(self, attribute_name):
        props = self.props
        if attribute_name in REF_ATTRS + ["Name"]:
            return props.get("Name")
        if attribute_name == "Arn":
            return props.get("Arn")
        return super(EventBus, self).get_cfn_attribute(attribute_name)

    @classmethod
    def get_deploy_templates(cls):
        return {
            "create": {"function": "create_event_bus", "parameters": {"Name": "Name"}},
            "delete": {"function": "delete_event_bus", "parameters": {"Name": "Name"}},
        }

class EventsRule(GenericBaseModel):
    @staticmethod
    def cloudformation_type():
        return "AWS::Events::Rule"

    def get_cfn_attribute(self, attribute_name):
        if attribute_name == "Arn":
            return self.params.get("Arn") or aws_stack.events_rule_arn(self.params.get("Name"))
        return super(EventsRule, self).get_cfn_attribute(attribute_name)

    def get_physical_resource_id(self, attribute=None, **kwargs):
        return self.props.get("Name")

    def fetch_state(self, stack_name, resources):
        rule_name = self.resolve_refs_recursively(stack_name, self.props.get("Name"), resources)
        result = aws_stack.connect_to_service("events").describe_rule(Name=rule_name) or {}
        return result if result.get("Name") else None

    @classmethod
    def get_deploy_templates(cls):
        def events_put_rule_params(params, **kwargs):
            attrs = [
                "ScheduleExpression",
                "EventPattern",
                "State",
                "Description",
                "Name",
                "EventBusName",
            ]
            result = select_parameters(*attrs)(params, **kwargs)
            result["Name"] = result.get("Name") or PLACEHOLDER_RESOURCE_NAME

            def wrap_in_lists(o, **kwargs):
                if isinstance(o, dict):
                    for k, v in o.items():
                        if not isinstance(v, (dict, list)):
                            o[k] = [v]
                return o

            pattern = result.get("EventPattern")
            if isinstance(pattern, dict):
                wrapped = common.recurse_object(pattern, wrap_in_lists)
                result["EventPattern"] = json.dumps(wrapped)
            return result

        return {
            "create": [
                {"function": "put_rule", "parameters": events_put_rule_params},
                {
                    "function": "put_targets",
                    "parameters": {
                        "Rule": PLACEHOLDER_RESOURCE_NAME,
                        "EventBusName": "EventBusName",
                        "Targets": "Targets",
                    },
                },
            ],
            "delete": {
                "function": "delete_rule",
                "parameters": {"Name": "PhysicalResourceId"},
            },
        }


@@ -44,6 +44,15 @@ class DetectionBlock(nn.Module):
         self.conv2 = ConvModule(
             out_channels, double_out_channels, 3, padding=1, **cfg)
         self.conv3 = ConvModule(double_out_channels, out_channels, 1, **cfg)
+
+        if self.spp_on:
+            self.poolers = [
+                nn.MaxPool2d(size, 1, padding=(size - 1) // 2)
+                for size in spp_scales
+            ]
+            self.conv_spp = ConvModule(out_channels * (len(spp_scales) + 1),
+                                       out_channels, 1, **cfg)
+
         self.conv4 = ConvModule(
             out_channels, double_out_channels, 3, padding=1, **cfg)
         self.conv5 = ConvModule(double_out_channels, out_channels, 1, **cfg)

# Copyright (c) 2019 Western Digital Corporation or its affiliates.

import torch
import torch.nn as nn
import torch.nn.functional as F
from mmcv.cnn import ConvModule

from ..builder import NECKS

class DetectionBlock(nn.Module):
    """Detection block in YOLO neck.

    Let out_channels = n, the DetectionBlock contains:
    Six ConvLayers, 1 Conv2D Layer and 1 YoloLayer.
    The first 6 ConvLayers are formed the following way:
        1x1xn, 3x3x2n, 1x1xn, 3x3x2n, 1x1xn, 3x3x2n.
    The Conv2D layer is 1x1x255.
    Some block will have branch after the fifth ConvLayer.
    The input channel is arbitrary (in_channels)

    Args:
        in_channels (int): The number of input channels.
        out_channels (int): The number of output channels.
        conv_cfg (dict): Config dict for convolution layer. Default: None.
        norm_cfg (dict): Dictionary to construct and config norm layer.
            Default: dict(type='BN', requires_grad=True)
        act_cfg (dict): Config dict for activation layer.
            Default: dict(type='LeakyReLU', negative_slope=0.1).
    """

    def __init__(self,
                 in_channels,
                 out_channels,
                 conv_cfg=None,
                 norm_cfg=dict(type='BN', requires_grad=True),
                 act_cfg=dict(type='LeakyReLU', negative_slope=0.1)):
        super(DetectionBlock, self).__init__()
        double_out_channels = out_channels * 2

        # shortcut
        cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)
        self.conv1 = ConvModule(in_channels, out_channels, 1, **cfg)
        self.conv2 = ConvModule(
            out_channels, double_out_channels, 3, padding=1, **cfg)
        self.conv3 = ConvModule(double_out_channels, out_channels, 1, **cfg)
        self.conv4 = ConvModule(
            out_channels, double_out_channels, 3, padding=1, **cfg)
        self.conv5 = ConvModule(double_out_channels, out_channels, 1, **cfg)

    def forward(self, x):
        tmp = self.conv1(x)
        tmp = self.conv2(tmp)
        tmp = self.conv3(tmp)
        tmp = self.conv4(tmp)
        out = self.conv5(tmp)
        return out

@NECKS.register_module()
class YOLOV3Neck(nn.Module):
    """The neck of YOLOV3.

    It can be treated as a simplified version of FPN. It
    will take the result from Darknet backbone and do some upsampling and
    concatenation. It will finally output the detection result.

    Note:
        The input feats should be from top to bottom.
            i.e., from high-lvl to low-lvl
        But YOLOV3Neck will process them in reversed order.
            i.e., from bottom (high-lvl) to top (low-lvl)

    Args:
        num_scales (int): The number of scales / stages.
        in_channels (int): The number of input channels.
        out_channels (int): The number of output channels.
        conv_cfg (dict): Config dict for convolution layer. Default: None.
        norm_cfg (dict): Dictionary to construct and config norm layer.
            Default: dict(type='BN', requires_grad=True)
        act_cfg (dict): Config dict for activation layer.
            Default: dict(type='LeakyReLU', negative_slope=0.1).
    """

    def __init__(self,
                 num_scales,
                 in_channels,
                 out_channels,
                 conv_cfg=None,
                 norm_cfg=dict(type='BN', requires_grad=True),
                 act_cfg=dict(type='LeakyReLU', negative_slope=0.1)):
        super(YOLOV3Neck, self).__init__()
        assert (num_scales == len(in_channels) == len(out_channels))
        self.num_scales = num_scales
        self.in_channels = in_channels
        self.out_channels = out_channels

        # shortcut
        cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)

        # To support arbitrary scales, the code looks awful, but it works.
        # Better solution is welcomed.
        self.detect1 = DetectionBlock(in_channels[0], out_channels[0], **cfg)
        for i in range(1, self.num_scales):
            in_c, out_c = self.in_channels[i], self.out_channels[i]
            self.add_module(f'conv{i}', ConvModule(in_c, out_c, 1, **cfg))
            # in_c + out_c : High-lvl feats will be cat with low-lvl feats
            self.add_module(f'detect{i+1}',
                            DetectionBlock(in_c + out_c, out_c, **cfg))

    def forward(self, feats):
        assert len(feats) == self.num_scales

        # processed from bottom (high-lvl) to top (low-lvl)
        outs = []
        out = self.detect1(feats[-1])
        outs.append(out)

        for i, x in enumerate(reversed(feats[:-1])):
            conv = getattr(self, f'conv{i+1}')
            tmp = conv(out)

            # Cat with low-lvl feats
            tmp = F.interpolate(tmp, scale_factor=2)
            tmp = torch.cat((tmp, x), 1)

            detect = getattr(self, f'detect{i+2}')
            out = detect(tmp)
            outs.append(out)

        return tuple(outs)

    def init_weights(self):
        """Initialize the weights of module."""
        # init is done in ConvModule
        pass


@@ -71,8 +71,10 @@ def _file_lines(path):

 DOT_FILES = ["packages_No_Name.dot", "classes_No_Name.dot"]
+COLORIZED_DOT_FILES = ["packages_colorized.dot", "classes_colorized.dot"]
 VCG_FILES = ["packages_No_Name.vcg", "classes_No_Name.vcg"]
 PUML_FILES = ["packages_No_Name.puml", "classes_No_Name.puml"]
+COLORIZED_PUML_FILES = ["packages_colorized.puml", "classes_colorized.puml"]

 @pytest.fixture()

# Copyright (c) 2008, 2010, 2013 LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr>
# Copyright (c) 2014-2018, 2020 Claudiu Popa <pcmanticore@gmail.com>
# Copyright (c) 2014 Google, Inc.
# Copyright (c) 2014 Arun Persaud <arun@nubati.net>
# Copyright (c) 2015 Ionel Cristian Maries <contact@ionelmc.ro>
# Copyright (c) 2016 Derek Gustafson <degustaf@gmail.com>
# Copyright (c) 2018 Ville Skyttä <ville.skytta@iki.fi>
# Copyright (c) 2019-2021 Pierre Sassoulas <pierre.sassoulas@gmail.com>
# Copyright (c) 2019 Ashley Whetter <ashley@awhetter.co.uk>
# Copyright (c) 2020 hippo91 <guillaume.peillex@gmail.com>
# Copyright (c) 2020 Anthony Sottile <asottile@umich.edu>
# Copyright (c) 2021 Marc Mueller <30130371+cdce8p@users.noreply.github.com>
# Copyright (c) 2021 Mark Byrne <31762852+mbyrnepr2@users.noreply.github.com>
# Copyright (c) 2021 Andreas Finkler <andi.finkler@gmail.com>

# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE

"""
Unit test for ``DiagramWriter``
"""

import codecs
import os
from difflib import unified_diff

import pytest

from pylint.pyreverse.diadefslib import DefaultDiadefGenerator, DiadefsHandler
from pylint.pyreverse.inspector import Linker
from pylint.pyreverse.writer import DiagramWriter

_DEFAULTS = {
    "all_ancestors": None,
    "show_associated": None,
    "module_names": None,
    "output_format": "dot",
    "diadefs_file": None,
    "quiet": 0,
    "show_ancestors": None,
    "classes": (),
    "all_associated": None,
    "mode": "PUB_ONLY",
    "show_builtin": False,
    "only_classnames": False,
    "output_directory": "",
}

class Config:
    """config object for tests"""

    def __init__(self):
        for attr, value in _DEFAULTS.items():
            setattr(self, attr, value)

def _file_lines(path):
    # we don't care about the actual encoding, but python3 forces us to pick one
    with codecs.open(path, encoding="latin1") as stream:
        lines = [
            line.strip()
            for line in stream.readlines()
            if (
                line.find("squeleton generated by ") == -1
                and not line.startswith('__revision__ = "$Id:')
            )
        ]
    return [line for line in lines if line]

DOT_FILES = ["packages_No_Name.dot", "classes_No_Name.dot"]
VCG_FILES = ["packages_No_Name.vcg", "classes_No_Name.vcg"]
PUML_FILES = ["packages_No_Name.puml", "classes_No_Name.puml"]

@pytest.fixture()
def setup_dot(default_config, get_project):
    writer = DiagramWriter(default_config)
    project = get_project(os.path.join(os.path.dirname(__file__), "..", "data"))
    yield from _setup(project, default_config, writer)

@pytest.fixture()
def setup_vcg(vcg_config, get_project):
    writer = DiagramWriter(vcg_config)
    project = get_project(os.path.join(os.path.dirname(__file__), "..", "data"))
    yield from _setup(project, vcg_config, writer)

@pytest.fixture()
def setup_puml(puml_config, get_project):
    writer = DiagramWriter(puml_config)
    project = get_project(os.path.join(os.path.dirname(__file__), "..", "data"))
    yield from _setup(project, puml_config, writer)

def _setup(project, config, writer):
    linker = Linker(project)
    handler = DiadefsHandler(config)
    dd = DefaultDiadefGenerator(linker, handler).visit(project)
    for diagram in dd:
        diagram.extract_relationships()
    writer.write(dd)
    yield
    for fname in DOT_FILES + VCG_FILES + PUML_FILES:
        try:
            os.remove(fname)
        except FileNotFoundError:
            continue

@pytest.mark.usefixtures("setup_dot")
@pytest.mark.parametrize("generated_file", DOT_FILES)
def test_dot_files(generated_file):
    _assert_files_are_equal(generated_file)

@pytest.mark.usefixtures("setup_vcg")
@pytest.mark.parametrize("generated_file", VCG_FILES)
def test_vcg_files(generated_file):
    _assert_files_are_equal(generated_file)

@pytest.mark.usefixtures("setup_puml")
@pytest.mark.parametrize("generated_file", PUML_FILES)
def test_puml_files(generated_file):
    _assert_files_are_equal(generated_file)

def _assert_files_are_equal(generated_file):
    expected_file = os.path.join(os.path.dirname(__file__), "data", generated_file)
    generated = _file_lines(generated_file)
    expected = _file_lines(expected_file)
    generated = "\n".join(generated)
    expected = "\n".join(expected)
    files = f"\n *** expected : {expected_file}, generated : {generated_file} \n"
    diff = "\n".join(
        line for line in unified_diff(expected.splitlines(), generated.splitlines())
    )
    assert expected == generated, f"{files}{diff}"


@@ -29,4 +29,45 @@ def pre_process_resource_data(resource_data_iterator,
         DataFrame: DataFrame table with all the resource_data.
     """

-    return None
+    df = pd.DataFrame(resource_data_json)
+
+    existing_columns = df.columns
+
+    new_columns = list(set(existing_columns).intersection(selected_features))
+
+    df = df[new_columns]
+
+    return df
+
+
+if __name__ == '__main__':
+    import json
+    from os import sys, path
+    import sys
+
+
+    sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
+    print sys.path
+
+    from resources.firewall_rule import FirewallRule
+
+
+    with open('../sample_datasets/dataset_firewall.json') as firewall_dataset:
+        firewall_rules = json.load(firewall_dataset)
+        firewall_rules_data = [i.get('data') for i in firewall_rules]
+
+        flattened_firewall_rules = FirewallRule.flatten_firewall_rules(firewall_rules_data)
+        flattened_firewall_rules_dict = [i.to_dict() for i in flattened_firewall_rules]
+
+        df_filtered = pre_process_firewall_data(
+            flattened_firewall_rules_dict,
+            ['priority',
+             'ip_addr',
+             'ip_bits',
+             'identifier',
+             'action',
+             'ip_protocol',
+             'ports',
+             'direction',
+             'disabled'])
+        print df_filtered

# Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

def pre_process_resource_data(resource_data_iterator,
                              resource_type,
                              selected_features):
    """Pre process resource data.

    Args:
        resource_data_iterator (list): An iterator that will iterate through
            all the resource data. Resource data is in JSON string format.
        resource_type (ResourceType): The type of the resource data.
        selected_features (list): A list of selected features, if the
            list is empty, we will include all the features.

    Returns:
        DataFrame: DataFrame table with all the resource_data.
    """

    return None


@@ -80,6 +80,13 @@ class EvalHook(Hook):
         for name, val in eval_res.items():
             runner.log_buffer.output[name] = val
         runner.log_buffer.ready = True
+        if self.save_best is not None:
+            if self.key_indicator == 'auto':
+                # infer from eval_results
+                self._init_rule(self.rule, list(eval_res.keys())[0])
+            return eval_res[self.key_indicator]
+        else:
+            return None

 class DistEvalHook(EvalHook):

import os.path as osp
import warnings

from mmcv.runner import Hook
from torch.utils.data import DataLoader

class EvalHook(Hook):
    """Evaluation hook.

    Notes:
        If new arguments are added for EvalHook, tools/test.py may be
    effected.

    Attributes:
        dataloader (DataLoader): A PyTorch dataloader.
        start (int, optional): Evaluation starting epoch. It enables evaluation
            before the training starts if ``start`` <= the resuming epoch.
            If None, whether to evaluate is merely decided by ``interval``.
            Default: None.
        interval (int): Evaluation interval (by epochs). Default: 1.
        **eval_kwargs: Evaluation arguments fed into the evaluate function of
            the dataset.
    """

    def __init__(self, dataloader, start=None, interval=1, **eval_kwargs):
        if not isinstance(dataloader, DataLoader):
            raise TypeError('dataloader must be a pytorch DataLoader, but got'
                            f' {type(dataloader)}')
        if not interval > 0:
            raise ValueError(f'interval must be positive, but got {interval}')
        if start is not None and start < 0:
            warnings.warn(
                f'The evaluation start epoch {start} is smaller than 0, '
                f'use 0 instead', UserWarning)
            start = 0
        self.dataloader = dataloader
        self.interval = interval
        self.start = start
        self.eval_kwargs = eval_kwargs
        self.initial_epoch_flag = True

    def before_train_epoch(self, runner):
        """Evaluate the model only at the start of training."""
        if not self.initial_epoch_flag:
            return
        if self.start is not None and runner.epoch >= self.start:
            self.after_train_epoch(runner)
        self.initial_epoch_flag = False

    def evaluation_flag(self, runner):
        """Judge whether to perform_evaluation after this epoch.

        Returns:
            bool: The flag indicating whether to perform evaluation.
        """
        if self.start is None:
            if not self.every_n_epochs(runner, self.interval):
                # No evaluation during the interval epochs.
                return False
        elif (runner.epoch + 1) < self.start:
            # No evaluation if start is larger than the current epoch.
            return False
        else:
            # Evaluation only at epochs 3, 5, 7... if start==3 and interval==2
            if (runner.epoch + 1 - self.start) % self.interval:
                return False
        return True

    def after_train_epoch(self, runner):
        if not self.evaluation_flag(runner):
            return
        from mmdet.apis import single_gpu_test
        results = single_gpu_test(runner.model, self.dataloader, show=False)
        self.evaluate(runner, results)

    def evaluate(self, runner, results):
        eval_res = self.dataloader.dataset.evaluate(
            results, logger=runner.logger, **self.eval_kwargs)
        for name, val in eval_res.items():
            runner.log_buffer.output[name] = val
        runner.log_buffer.ready = True

class DistEvalHook(EvalHook):
    """Distributed evaluation hook.

    Notes:
        If new arguments are added, tools/test.py may be effected.

    Attributes:
        dataloader (DataLoader): A PyTorch dataloader.
        start (int, optional): Evaluation starting epoch. It enables evaluation
            before the training starts if ``start`` <= the resuming epoch.
            If None, whether to evaluate is merely decided by ``interval``.
            Default: None.
        interval (int): Evaluation interval (by epochs). Default: 1.
        tmpdir (str | None): Temporary directory to save the results of all
            processes. Default: None.
        gpu_collect (bool): Whether to use gpu or cpu to collect results.
            Default: False.
        **eval_kwargs: Evaluation arguments fed into the evaluate function of
            the dataset.
    """

    def __init__(self,
                 dataloader,
                 start=None,
                 interval=1,
                 tmpdir=None,
                 gpu_collect=False,
                 **eval_kwargs):
        super().__init__(
            dataloader, start=start, interval=interval, **eval_kwargs)
        self.tmpdir = tmpdir
        self.gpu_collect = gpu_collect

    def after_train_epoch(self, runner):
        if not self.evaluation_flag(runner):
            return
        from mmdet.apis import multi_gpu_test
        tmpdir = self.tmpdir
        if tmpdir is None:
            tmpdir = osp.join(runner.work_dir, '.eval_hook')
        results = multi_gpu_test(
            runner.model,
            self.dataloader,
            tmpdir=tmpdir,
            gpu_collect=self.gpu_collect)
        if runner.rank == 0:
            print('\n')
            self.evaluate(runner, results)


@@ -17,6 +17,7 @@

 from selenium.webdriver.chromium.options import ChromiumOptions
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from typing import Optional, NoReturn

 class Options(ChromiumOptions):

# Licensed to the Software Freedom Conservancy (SFC) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The SFC licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from selenium.webdriver.chromium.options import ChromiumOptions
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

class Options(ChromiumOptions):

    @property
    def default_capabilities(self) -> dict:
        return DesiredCapabilities.CHROME.copy()

    def enable_mobile(self, android_package="com.android.chrome", android_activity=None, device_serial=None):
        super().enable_mobile(android_package, android_activity, device_serial)


@@ -71,6 +71,9 @@ public class GraphqlHandlerTest {

     SessionMap sessions = new LocalSessionMap(tracer, events);
     distributor = new LocalDistributor(tracer, events, clientFactory, sessions, registrationSecret);
+    stereotype = new ImmutableCapabilities("browserName", "cheese");
+    caps = new ImmutableCapabilities("browserName", "cheese");
+    payload = NewSessionPayload.create(caps);
   }

   @Test

// Licensed to the Software Freedom Conservancy (SFC) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The SFC licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package org.openqa.selenium.grid.graphql;

import org.junit.Before;
import org.junit.Test;
import org.openqa.selenium.Capabilities;
import org.openqa.selenium.ImmutableCapabilities;
import org.openqa.selenium.events.EventBus;
import org.openqa.selenium.events.local.GuavaEventBus;
import org.openqa.selenium.grid.data.CreateSessionRequest;
import org.openqa.selenium.grid.distributor.Distributor;
import org.openqa.selenium.grid.distributor.local.LocalDistributor;
import org.openqa.selenium.grid.node.ActiveSession;
import org.openqa.selenium.grid.node.Node;
import org.openqa.selenium.grid.node.SessionFactory;
import org.openqa.selenium.grid.node.local.LocalNode;
import org.openqa.selenium.grid.security.Secret;
import org.openqa.selenium.grid.sessionmap.SessionMap;
import org.openqa.selenium.grid.sessionmap.local.LocalSessionMap;
import org.openqa.selenium.json.Json;
import org.openqa.selenium.remote.http.Contents;
import org.openqa.selenium.remote.http.HttpClient;
import org.openqa.selenium.remote.http.HttpHandler;
import org.openqa.selenium.remote.http.HttpRequest;
import org.openqa.selenium.remote.http.HttpResponse;
import org.openqa.selenium.remote.tracing.DefaultTestTracer;
import org.openqa.selenium.remote.tracing.Tracer;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import java.util.Optional;

import static org.assertj.core.api.Assertions.assertThat;
import static org.openqa.selenium.json.Json.MAP_TYPE;
import static org.openqa.selenium.remote.http.HttpMethod.GET;

public class GraphqlHandlerTest {

  private final Secret registrationSecret = new Secret("stilton");
  private final URI publicUri = new URI("http://example.com/grid-o-matic");
  private Distributor distributor;
  private Tracer tracer;
  private EventBus events;

  public GraphqlHandlerTest() throws URISyntaxException {
  }

  @Before
  public void setupGrid() {
    tracer = DefaultTestTracer.createTracer();
    events = new GuavaEventBus();
    HttpClient.Factory clientFactory = HttpClient.Factory.createDefault();

    SessionMap sessions = new LocalSessionMap(tracer, events);
    distributor = new LocalDistributor(tracer, events, clientFactory, sessions, registrationSecret);
  }

  @Test
  public void shouldBeAbleToGetGridUri() {
    GraphqlHandler handler = new GraphqlHandler(distributor, publicUri);

    Map<String, Object> topLevel = executeQuery(handler, "{ grid { uri } }");

    assertThat(topLevel).isEqualTo(Map.of("data", Map.of("grid", Map.of("uri", publicUri.toString()))));
  }

  @Test
  public void shouldReturnAnEmptyListForNodesIfNoneAreRegistered() {
    GraphqlHandler handler = new GraphqlHandler(distributor, publicUri);

    Map<String, Object> topLevel = executeQuery(handler, "{ grid { nodes { uri } } }");

    assertThat(topLevel)
      .describedAs(topLevel.toString())
      .isEqualTo(Map.of("data", Map.of("grid", Map.of("nodes", List.of()))));
  }

  @Test
  public void shouldBeAbleToGetUrlsOfAllNodes() throws URISyntaxException {
    Capabilities stereotype = new ImmutableCapabilities("cheese", "stilton");
    String nodeUri = "http://localhost:5556";
    Node node = LocalNode.builder(tracer, events, new URI(nodeUri), publicUri, registrationSecret)
      .add(stereotype, new SessionFactory() {
        @Override
        public Optional<ActiveSession> apply(CreateSessionRequest createSessionRequest) {
          return Optional.empty();
        }

        @Override
        public boolean test(Capabilities capabilities) {
          return false;
        }
      })
      .build();
    distributor.add(node);

    GraphqlHandler handler = new GraphqlHandler(distributor, publicUri);
    Map<String, Object> topLevel = executeQuery(handler, "{ grid { nodes { uri } } }");

    assertThat(topLevel)
      .describedAs(topLevel.toString())
      .isEqualTo(Map.of("data", Map.of("grid", Map.of("nodes", List.of(Map.of("uri", nodeUri))))));
  }

  private Map<String, Object> executeQuery(HttpHandler handler, String query) {
    HttpResponse res = handler.execute(
      new HttpRequest(GET, "/graphql")
        .setContent(Contents.asJson(Map.of("query", query))));

    return new Json().toType(Contents.string(res), MAP_TYPE);
  }
}


@@ -24,13 +24,15 @@ Unit test for ``DiagramWriter``
 import codecs
 import os
 from difflib import unified_diff
+from typing import Callable, Iterator, List
 from unittest.mock import Mock

 import pytest

 from pylint.pyreverse.diadefslib import DefaultDiadefGenerator, DiadefsHandler
-from pylint.pyreverse.inspector import Linker
+from pylint.pyreverse.inspector import Linker, Project
 from pylint.pyreverse.writer import DiagramWriter
+from pylint.testutils.pyreverse import PyreverseConfig

 _DEFAULTS = {
     "all_ancestors": None,

# Copyright (c) 2008, 2010, 2013 LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr>
# Copyright (c) 2014-2018, 2020 Claudiu Popa <pcmanticore@gmail.com>
# Copyright (c) 2014 Google, Inc.
# Copyright (c) 2014 Arun Persaud <arun@nubati.net>
# Copyright (c) 2015 Ionel Cristian Maries <contact@ionelmc.ro>
# Copyright (c) 2016 Derek Gustafson <degustaf@gmail.com>
# Copyright (c) 2018 Ville Skyttä <ville.skytta@iki.fi>
# Copyright (c) 2019-2021 Pierre Sassoulas <pierre.sassoulas@gmail.com>
# Copyright (c) 2019 Ashley Whetter <ashley@awhetter.co.uk>
# Copyright (c) 2020 hippo91 <guillaume.peillex@gmail.com>
# Copyright (c) 2020 Anthony Sottile <asottile@umich.edu>
# Copyright (c) 2021 Marc Mueller <30130371+cdce8p@users.noreply.github.com>
# Copyright (c) 2021 Mark Byrne <31762852+mbyrnepr2@users.noreply.github.com>
# Copyright (c) 2021 Andreas Finkler <andi.finkler@gmail.com>

# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE

"""
Unit test for ``DiagramWriter``
"""

import codecs
import os
from difflib import unified_diff
from unittest.mock import Mock

import pytest

from pylint.pyreverse.diadefslib import DefaultDiadefGenerator, DiadefsHandler
from pylint.pyreverse.inspector import Linker
from pylint.pyreverse.writer import DiagramWriter

_DEFAULTS = {
    "all_ancestors": None,
    "show_associated": None,
    "module_names": None,
    "output_format": "dot",
    "diadefs_file": None,
    "quiet": 0,
    "show_ancestors": None,
    "classes": (),
    "all_associated": None,
    "mode": "PUB_ONLY",
    "show_builtin": False,
    "only_classnames": False,
    "output_directory": "",
}

class Config:
    """config object for tests"""

    def __init__(self):
        for attr, value in _DEFAULTS.items():
            setattr(self, attr, value)

def _file_lines(path):
    # we don't care about the actual encoding, but python3 forces us to pick one
    with codecs.open(path, encoding="latin1") as stream:
        lines = [
            line.strip()
            for line in stream.readlines()
            if (
                line.find("squeleton generated by ") == -1
                and not line.startswith('__revision__ = "$Id:')
            )
        ]
    return [line for line in lines if line]

DOT_FILES = ["packages_No_Name.dot", "classes_No_Name.dot"]
COLORIZED_DOT_FILES = ["packages_colorized.dot", "classes_colorized.dot"]
VCG_FILES = ["packages_No_Name.vcg", "classes_No_Name.vcg"]
PUML_FILES = ["packages_No_Name.puml", "classes_No_Name.puml"]
COLORIZED_PUML_FILES = ["packages_colorized.puml", "classes_colorized.puml"]

@pytest.fixture()
def setup_dot(default_config, get_project):
    writer = DiagramWriter(default_config)
    project = get_project(os.path.join(os.path.dirname(__file__), "..", "data"))
    yield from _setup(project, default_config, writer)

@pytest.fixture()
def setup_colorized_dot(colorized_dot_config, get_project):
    writer = DiagramWriter(colorized_dot_config)
    project = get_project(
        os.path.join(os.path.dirname(__file__), "..", "data"), name="colorized"
    )
    yield from _setup(project, colorized_dot_config, writer)

@pytest.fixture()
def setup_vcg(vcg_config, get_project):
    writer = DiagramWriter(vcg_config)
    project = get_project(os.path.join(os.path.dirname(__file__), "..", "data"))
    yield from _setup(project, vcg_config, writer)

@pytest.fixture()
def setup_puml(puml_config, get_project):
    writer = DiagramWriter(puml_config)
    project = get_project(os.path.join(os.path.dirname(__file__), "..", "data"))
    yield from _setup(project, puml_config, writer)

@pytest.fixture()
def setup_colorized_puml(colorized_puml_config, get_project):
    writer = DiagramWriter(colorized_puml_config)
    project = get_project(
        os.path.join(os.path.dirname(__file__), "..", "data"), name="colorized"
    )
    yield from _setup(project, colorized_puml_config, writer)

def _setup(project, config, writer):
    linker = Linker(project)
    handler = DiadefsHandler(config)
    dd = DefaultDiadefGenerator(linker, handler).visit(project)
    for diagram in dd:
        diagram.extract_relationships()
    writer.write(dd)
    yield
    for fname in (
        DOT_FILES + COLORIZED_DOT_FILES + VCG_FILES + PUML_FILES + COLORIZED_PUML_FILES
    ):
        try:
            os.remove(fname)
        except FileNotFoundError:
            continue

@pytest.mark.usefixtures("setup_dot")
@pytest.mark.parametrize("generated_file", DOT_FILES)
def test_dot_files(generated_file):
    _assert_files_are_equal(generated_file)

@pytest.mark.usefixtures("setup_colorized_dot")
@pytest.mark.parametrize("generated_file", COLORIZED_DOT_FILES)
def test_colorized_dot_files(generated_file):
    _assert_files_are_equal(generated_file)

@pytest.mark.usefixtures("setup_vcg")
@pytest.mark.parametrize("generated_file", VCG_FILES)
def test_vcg_files(generated_file):
    _assert_files_are_equal(generated_file)

@pytest.mark.usefixtures("setup_puml")
@pytest.mark.parametrize("generated_file", PUML_FILES)
def test_puml_files(generated_file):
    _assert_files_are_equal(generated_file)

@pytest.mark.usefixtures("setup_colorized_puml")
@pytest.mark.parametrize("generated_file", COLORIZED_PUML_FILES)
def test_colorized_puml_files(generated_file):
    _assert_files_are_equal(generated_file)

def _assert_files_are_equal(generated_file):
    expected_file = os.path.join(os.path.dirname(__file__), "data", generated_file)
    generated = _file_lines(generated_file)
    expected = _file_lines(expected_file)
    generated = "\n".join(generated)
    expected = "\n".join(expected)
    files = f"\n *** expected : {expected_file}, generated : {generated_file} \n"
    diff = "\n".join(
        line for line in unified_diff(expected.splitlines(), generated.splitlines())
    )
    assert expected == generated, f"{files}{diff}"

def test_color_for_stdlib_module(default_config):
    writer = DiagramWriter(default_config)
    obj = Mock()
    obj.node = Mock()
    obj.node.qname.return_value = "collections"
    assert writer.get_shape_color(obj) == "grey"


@@ -11,9 +11,9 @@ def get_listening_challenge(source, course):
     return [
         {
             "type": "listeningExercise",
-            "answer": source.in_target_language[0],
-            "meaning": source.in_source_language[0],
-            "audio": audio_id(course.target_language, source.in_target_language[0]),
+            "answer": _remove_special_characters_for_display(source.in_target_language[0]),
+            "meaning": _remove_special_characters_for_display(source.in_source_language[0]),
+            "audio": audio_id(course.target_language, _remove_special_characters_for_display(source.in_target_language[0])),
             "id": get_dumb_opaque_id("Word", source, "listeningExercise"),
             "priority": 1,
             "group": get_dumb_opaque_id("Group", source),

import editdistance  # type: ignore

from librelingo_utils import get_dumb_opaque_id, audio_id, clean_word, iterate_phrases
from .dictionary import _define_words_in_sentence

def get_listening_challenge(source, course):
    if not course.settings.audio_files_enabled:
        return []

    return [
        {
            "type": "listeningExercise",
            "answer": source.in_target_language[0],
            "meaning": source.in_source_language[0],
            "audio": audio_id(course.target_language, source.in_target_language[0]),
            "id": get_dumb_opaque_id("Word", source, "listeningExercise"),
            "priority": 1,
            "group": get_dumb_opaque_id("Group", source),
        }
    ]

def get_short_input_challenge(source, course):
    return [
        {
            "type": "shortInput",
            "pictures": [pic + ".jpg" for pic in source.pictures]
            if source.pictures
            else None,
            "formInTargetLanguage": source.in_target_language,
            "phrase": _define_words_in_sentence(
                course, source.in_source_language[0], reverse=False
            ),
            "id": get_dumb_opaque_id("Word", source, "shortInput"),
            "priority": 1,
            "group": get_dumb_opaque_id("Group", source),
        }
    ]

def get_cards_challenge(word, _):
    return [
        {
            "type": "cards",
            "pictures": [pic + ".jpg" for pic in word.pictures]
            if word.pictures
            else None,
            "formInTargetLanguage": word.in_target_language[0],
            "meaningInSourceLanguage": word.in_source_language[0],
            "id": get_dumb_opaque_id("Word", word, "cards"),
            "priority": 0,
            "group": get_dumb_opaque_id("Group", word),
        }
    ]

def get_options_challenge(phrase, _):
    return [
        {
            "type": "options",
            "formInTargetLanguage": phrase.in_target_language[0],
            "meaningInSourceLanguage": phrase.in_source_language[0],
            "id": get_dumb_opaque_id("Options", phrase, "options"),
            "priority": 0,
            "group": get_dumb_opaque_id("Group", phrase),
        }
    ]

def get_chips_from_string(phrase):
    return list(map(clean_word, phrase.split()))

def get_chips_from_phrase(get_input_texts, phrase, course):
    extra_chips = []
    solution_chips = get_chips_from_string(get_input_texts(phrase)[0])

    for phrase in iterate_phrases(course):
        for variant in get_input_texts(phrase):
            for chip in get_chips_from_string(variant):
                if chip not in solution_chips:
                    extra_chips.append(chip)

    chips_already_added = set()
    deduplicated_chips = []

    for chip in extra_chips:
        if chip.lower() not in chips_already_added:
            deduplicated_chips.append(chip)
            chips_already_added.add(chip.lower())

    extra_chips = sorted(
        deduplicated_chips,
        key=lambda chip: sum(
            editdistance.eval(other_chip, chip) for other_chip in solution_chips
        ),
    )

    return solution_chips + extra_chips[0 : max(len(solution_chips) - 1, 2)]

def get_solutions_from_phrase(get_input_texts, phrase):
    return [get_chips_from_string(x) for x in get_input_texts(phrase)]

def create_chips_challenge_generator(reverse):
    def get_input_texts(phrase):
        return phrase.in_source_language if reverse else phrase.in_target_language

    def get_phrase_texts(phrase):
        return phrase.in_target_language if reverse else phrase.in_source_language

    def get_input_text(phrase):
        return get_input_texts(phrase)[0]

    def get_phrase_text(phrase):
        return get_phrase_texts(phrase)[0]

    def is_long_enough_to_have_chips(phrase):
        if len(phrase.in_source_language[0].split()) < 2:
            return False
        if len(phrase.in_target_language[0].split()) < 2:
            return False
        return True

    def get_chips_challenge(phrase, course):
        if not is_long_enough_to_have_chips(phrase):
            return []

        return [
            {
                "type": "chips",
                "translatesToSourceLanguage": reverse,
                "phrase": _define_words_in_sentence(
                    course, get_phrase_text(phrase), reverse
                ),
                "chips": get_chips_from_phrase(get_input_texts, phrase, course),
                "solutions": get_solutions_from_phrase(get_input_texts, phrase),
                "formattedSolution": get_input_text(phrase),
                "id": get_dumb_opaque_id(
                    "Chips", phrase, "reverse chips" if reverse else "chips"
                ),
                "priority": 2,
                "group": get_dumb_opaque_id("Group", phrase),
            }
        ]

    return get_chips_challenge

get_chips_challenge = create_chips_challenge_generator(False)
get_reverse_chips_challenge = create_chips_challenge_generator(True)