@@ -55,7 +55,7 @@ def translate(tableList, inbuf, typeform=None, cursorPos=None, mode=0):
 	* returns a list of integers instead of an string with cells, and
 	* distinguishes between cursor position 0 (cursor at first character) and None (no cursor at all)
 	"""
-	text = unicode(inbuf).replace('\0','')
+	text = inbuf.replace('\0','')
 	braille, brailleToRawPos, rawToBraillePos, brailleCursorPos = louis.translate(
 		tableList,
 		text,

#louisHelper.py
#A part of NonVisual Desktop Access (NVDA)
#This file is covered by the GNU General Public License.
#See the file COPYING for more details.
#Copyright (C) 2018 NV Access Limited, Babbage B.V.

"""Helper module to ease communication to and from liblouis."""

import louis
from logHandler import log
import config

LOUIS_TO_NVDA_LOG_LEVELS = {
	louis.LOG_ALL: log.DEBUG,
	louis.LOG_DEBUG: log.DEBUG,
	louis.LOG_INFO: log.INFO,
	louis.LOG_WARN: log.WARNING,
	louis.LOG_ERROR: log.ERROR,
	louis.LOG_FATAL: log.ERROR,
}

@louis.LogCallback
def louis_log(level, message):
	if not _isDebug():
		return
	NVDALevel = LOUIS_TO_NVDA_LOG_LEVELS.get(level, log.DEBUG)
	if not log.isEnabledFor(NVDALevel):
		return
	message = message.decode("ASCII")
	codepath = "liblouis at internal log level %d" % level
	log._log(NVDALevel, message, [], codepath=codepath)

def _isDebug():
	return config.conf["debugLog"]["louis"]

def initialize():
	# Register the liblouis logging callback.
	louis.registerLogCallback(louis_log)
	# Set the log level to debug.
	# The NVDA logging callback will filter messages appropriately,
	# i.e. error messages will be logged at the error level.
	louis.setLogLevel(louis.LOG_DEBUG)

def terminate():
	# Set the log level to off.
	louis.setLogLevel(louis.LOG_OFF)
	# Unregister the liblouis logging callback.
	louis.registerLogCallback(None)
	# Free liblouis resources
	louis.liblouis.lou_free()

def translate(tableList, inbuf, typeform=None, cursorPos=None, mode=0):
	"""
	Convenience wrapper for louis.translate that:
	* returns a list of integers instead of an string with cells, and
	* distinguishes between cursor position 0 (cursor at first character) and None (no cursor at all)
	"""
	text = unicode(inbuf).replace('\0','')
	braille, brailleToRawPos, rawToBraillePos, brailleCursorPos = louis.translate(
		tableList,
		text,
		# liblouis mutates typeform if it is a list.
		typeform=tuple(typeform) if isinstance(typeform, list) else typeform,
		cursorPos=cursorPos or 0,
		mode=mode
	)
	# liblouis gives us back a character string of cells, so convert it to a list of ints.
	# For some reason, the highest bit is set, so only grab the lower 8 bits.
	braille = [ord(cell) & 255 for cell in braille]
	if cursorPos is None:
		brailleCursorPos = None
	return braille, brailleToRawPos, rawToBraillePos, brailleCursorPos

Ugh, this should be a string instead of an string.


@@ -19,6 +19,14 @@ Exceptions/Errors used in Koalas.
 """

+class GroupByError(Exception):
+    pass
+
+
+class DataError(GroupByError):
+    pass
+
+
 class SparkPandasIndexingError(Exception):
     pass

#
# Copyright (C) 2019 Databricks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Exceptions/Errors used in Koalas.
"""

class SparkPandasIndexingError(Exception):
    pass

def code_change_hint(pandas_function, spark_target_function):
    if pandas_function is not None and spark_target_function is not None:
        return "You are trying to use pandas function {}, use spark function {}".format(
            pandas_function, spark_target_function
        )
    elif pandas_function is not None and spark_target_function is None:
        return (
            "You are trying to use pandas function {}, checkout the spark "
            "user guide to find a relevant function"
        ).format(pandas_function)
    elif pandas_function is None and spark_target_function is not None:
        return "Use spark function {}".format(spark_target_function)
    else:  # both none
        return "Checkout the spark user guide to find a relevant function"

class SparkPandasNotImplementedError(NotImplementedError):
    def __init__(self, pandas_function=None, spark_target_function=None, description=""):
        self.pandas_source = pandas_function
        self.spark_target = spark_target_function
        hint = code_change_hint(pandas_function, spark_target_function)
        if len(description) > 0:
            description += " " + hint
        else:
            description = hint
        super(SparkPandasNotImplementedError, self).__init__(description)

class PandasNotImplementedError(NotImplementedError):
    def __init__(
        self,
        class_name,
        method_name=None,
        arg_name=None,
        property_name=None,
        deprecated=False,
        reason="",
    ):
        assert (method_name is None) != (property_name is None)
        self.class_name = class_name
        self.method_name = method_name
        self.arg_name = arg_name
        if method_name is not None:
            if arg_name is not None:
                msg = "The method `{0}.{1}()` does not support `{2}` parameter. {3}".format(
                    class_name, method_name, arg_name, reason
                )
            else:
                if deprecated:
                    msg = (
                        "The method `{0}.{1}()` is deprecated in pandas and will therefore "
                        + "not be supported in Koalas. {2}"
                    ).format(class_name, method_name, reason)
                else:
                    if reason == "":
                        reason = " yet."
                    else:
                        reason = ". " + reason
                    msg = "The method `{0}.{1}()` is not implemented{2}".format(
                        class_name, method_name, reason
                    )
        else:
            if deprecated:
                msg = (
                    "The property `{0}.{1}()` is deprecated in pandas and will therefore "
                    + "not be supported in Koalas. {2}"
                ).format(class_name, property_name, reason)
            else:
                if reason == "":
                    reason = " yet."
                else:
                    reason = ". " + reason
                msg = "The property `{0}.{1}()` is not implemented{2}".format(
                    class_name, property_name, reason
                )
        super(NotImplementedError, self).__init__(msg)

Does pandas throw an exception like this?


@@ -14,10 +14,43 @@

 """ Forseti Installer.

-A stub to call gcp/install_setup.py which installs into GCP.
+A stub to call gcp/run_forseti_installer.py which installs into GCP.
 """

-from gcp import install_setup
+import site
+
+import pip
+
+
+INSTALLER_REQUIRED_PACKAGES = [
+    'ruamel.yaml'
+]
+
+def install(package_name):
+    """Install package.
+
+    Args:
+        package_name (str): Name of the package to install.
+    """
+    pip.main(['install', package_name, '--user'])
+
+
+def install_required_packages():
+    """Install required packages."""
+    installed_pkgs = [pkg.key for pkg in pip.get_installed_distributions()]
+    for package in INSTALLER_REQUIRED_PACKAGES:
+        if package not in installed_pkgs:
+            install(package)
+
+

 if __name__ == '__main__':
-    install_setup.run()
+    # We need to install all the required packages before importing our modules
+
+    # Installing required packages
+    install_required_packages()
+    site.main() # Load up the package
+
+    # Importing our own modules
+    from gcp import run_forseti_installer
+    run_forseti_installer.run()

# Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Forseti Installer.

A stub to call gcp/install_setup.py which installs into GCP.
"""

from gcp import install_setup

if __name__ == '__main__':
    install_setup.run()

If this is a stub, why should these be here? The only job of this stub is to alias off `run_forseti_installer.run()`


@@ -111,6 +111,27 @@ class QuickmarkCompletionModel(base.BaseCompletionModel):
                 match_field))

+class BookmarkCompletionModel(base.BaseCompletionModel):
+
+    """A CompletionModel filled with all bookmarks."""
+
+    # pylint: disable=abstract-method
+
+    def __init__(self, match_field='url', parent=None):
+        super().__init__(parent)
+        cat = self.new_category("Bookmarks")
+        bookmarks = objreg.get('bookmark-manager').bookmarks.items()
+        if match_field == 'url':
+            for bm_url, bm_title in bookmarks:
+                self.new_item(cat, bm_url, bm_title)
+        elif match_field == 'title':
+            for bm_url, bm_title in bookmarks:
+                self.new_item(cat, bm_title, bm_url)
+        else:
+            raise ValueError("Invalid value '{}' for match_field!".format(
+                match_field))
+
+
 class SessionCompletionModel(base.BaseCompletionModel):

     """A CompletionModel filled with session names."""

# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:

# Copyright 2014-2015 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
#
# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.

"""Misc. CompletionModels."""

from qutebrowser.config import config, configdata
from qutebrowser.utils import objreg, log
from qutebrowser.commands import cmdutils
from qutebrowser.completion.models import base

class CommandCompletionModel(base.BaseCompletionModel):

    """A CompletionModel filled with all commands and descriptions."""

    # pylint: disable=abstract-method

    def __init__(self, parent=None):
        super().__init__(parent)
        assert cmdutils.cmd_dict
        cmdlist = []
        for obj in set(cmdutils.cmd_dict.values()):
            if (obj.hide or (obj.debug and not objreg.get('args').debug) or
                    obj.deprecated):
                pass
            else:
                cmdlist.append((obj.name, obj.desc))
        for name, cmd in config.section('aliases').items():
            cmdlist.append((name, "Alias for '{}'".format(cmd)))
        cat = self.new_category("Commands")
        for (name, desc) in sorted(cmdlist):
            self.new_item(cat, name, desc)

class HelpCompletionModel(base.BaseCompletionModel):

    """A CompletionModel filled with help topics."""

    # pylint: disable=abstract-method

    def __init__(self, parent=None):
        super().__init__(parent)
        self._init_commands()
        self._init_settings()

    def _init_commands(self):
        """Fill completion with :command entries."""
        assert cmdutils.cmd_dict
        cmdlist = []
        for obj in set(cmdutils.cmd_dict.values()):
            if (obj.hide or (obj.debug and not objreg.get('args').debug) or
                    obj.deprecated):
                pass
            else:
                cmdlist.append((':' + obj.name, obj.desc))
        cat = self.new_category("Commands")
        for (name, desc) in sorted(cmdlist):
            self.new_item(cat, name, desc)

    def _init_settings(self):
        """Fill completion with section->option entries."""
        cat = self.new_category("Settings")
        for sectname, sectdata in configdata.DATA.items():
            for optname in sectdata.keys():
                try:
                    desc = sectdata.descriptions[optname]
                except (KeyError, AttributeError):
                    # Some stuff (especially ValueList items) don't have a
                    # description.
                    desc = ""
                else:
                    desc = desc.splitlines()[0]
                name = '{}->{}'.format(sectname, optname)
                self.new_item(cat, name, desc)

class QuickmarkCompletionModel(base.BaseCompletionModel):

    """A CompletionModel filled with all quickmarks."""

    # pylint: disable=abstract-method

    def __init__(self, match_field='url', parent=None):
        super().__init__(parent)
        cat = self.new_category("Quickmarks")
        quickmarks = objreg.get('quickmark-manager').marks.items()
        if match_field == 'url':
            for qm_name, qm_url in quickmarks:
                self.new_item(cat, qm_url, qm_name)
        elif match_field == 'name':
            for qm_name, qm_url in quickmarks:
                self.new_item(cat, qm_name, qm_url)
        else:
            raise ValueError("Invalid value '{}' for match_field!".format(
                match_field))

class SessionCompletionModel(base.BaseCompletionModel):

    """A CompletionModel filled with session names."""

    # pylint: disable=abstract-method

    def __init__(self, parent=None):
        super().__init__(parent)
        cat = self.new_category("Sessions")
        try:
            for name in objreg.get('session-manager').list_sessions():
                if not name.startswith('_'):
                    self.new_item(cat, name)
        except OSError:
            log.completion.exception("Failed to list sessions!")

Maybe this could be merged into `QuickmarkCompletionModel` as well, with a parameter (similar to `match_field` which says which object to use to get the marks (i.e. `bookmark-manager` or `quickmark-manager`)?


@@ -54,6 +54,7 @@ DEFAULT_SETTINGS = {
         'kinto.core.events.setup_transaction_hook',
     ),
     'event_listeners': '',
+    'heartbeat_timeout_seconds': 5,
     'logging_renderer': 'kinto.core.logs.ClassicLogRenderer',
     'newrelic_config': None,
     'newrelic_env': 'dev',

"""Main entry point
"""
import pkg_resources

from cornice import Service as CorniceService
from pyramid.settings import aslist

from kinto.core import authentication
from kinto.core import errors
from kinto.core import events
from kinto.core.initialization import (  # NOQA
    initialize, install_middlewares,
    load_default_settings)
from kinto.core.utils import (
    follow_subrequest, current_service, current_resource_name)
from kinto.core.logs import logger

# Module version, as defined in PEP-0396.
__version__ = pkg_resources.get_distribution('kinto').version  # FIXME?

DEFAULT_SETTINGS = {
    'backoff': None,
    'batch_max_requests': 25,
    'cache_backend': '',
    'cache_url': '',
    'cache_pool_size': 25,
    'cache_prefix': '',
    'cors_origins': '*',
    'cors_max_age_seconds': 3600,
    'eos': None,
    'eos_message': None,
    'eos_url': None,
    'error_info_link': 'https://github.com/Kinto/kinto/issues/',
    'http_host': None,
    'http_scheme': None,
    'id_generator': 'kinto.core.storage.generators.UUID4',
    'includes': '',
    'initialization_sequence': (
        'kinto.core.initialization.setup_request_bound_data',
        'kinto.core.initialization.setup_json_serializer',
        'kinto.core.initialization.setup_logging',
        'kinto.core.initialization.setup_storage',
        'kinto.core.initialization.setup_permission',
        'kinto.core.initialization.setup_cache',
        'kinto.core.initialization.setup_requests_scheme',
        'kinto.core.initialization.setup_version_redirection',
        'kinto.core.initialization.setup_deprecation',
        'kinto.core.initialization.setup_authentication',
        'kinto.core.initialization.setup_backoff',
        'kinto.core.initialization.setup_statsd',
        'kinto.core.initialization.setup_listeners',
        'kinto.core.events.setup_transaction_hook',
    ),
    'event_listeners': '',
    'logging_renderer': 'kinto.core.logs.ClassicLogRenderer',
    'newrelic_config': None,
    'newrelic_env': 'dev',
    'paginate_by': None,
    'permission_backend': '',
    'permission_url': '',
    'permission_pool_size': 25,
    'profiler_dir': '/tmp',
    'profiler_enabled': False,
    'project_docs': '',
    'project_name': '',
    'project_version': '',
    'readonly': False,
    'retry_after_seconds': 30,
    'statsd_prefix': 'kinto.core',
    'statsd_url': None,
    'storage_backend': '',
    'storage_url': '',
    'storage_max_fetch_size': 10000,
    'storage_pool_size': 25,
    'tm.annotate_user': False,  # Do annotate transactions with the user-id.
    'transaction_per_request': True,
    'userid_hmac_secret': '',
    'version_prefix_redirect_enabled': True,
    'trailing_slash_redirect_enabled': True,
    'multiauth.groupfinder': 'kinto.core.authorization.groupfinder',
    'multiauth.policies': 'basicauth',
    'multiauth.policy.basicauth.use': ('kinto.core.authentication.'
                                       'BasicAuthAuthenticationPolicy'),
    'multiauth.authorization_policy': ('kinto.core.authorization.'
                                       'AuthorizationPolicy')
}

class Service(CorniceService):
    """Subclass of the default cornice service.

    This is useful in order to attach specific behaviours without monkey
    patching the default cornice service (which would impact other uses of it)
    """
    default_cors_headers = ('Backoff', 'Retry-After', 'Alert',
                            'Content-Length')

    def error_handler(self, error):
        return errors.json_error_handler(error)

    @classmethod
    def init_from_settings(cls, settings):
        cls.cors_origins = tuple(aslist(settings['cors_origins']))
        cors_max_age = settings['cors_max_age_seconds']
        cls.cors_max_age = int(cors_max_age) if cors_max_age else None

def includeme(config):
    settings = config.get_settings()

    # Heartbeat registry.
    config.registry.heartbeats = {}

    # Public settings registry.
    config.registry.public_settings = {'batch_max_requests', 'readonly'}

    # Directive to declare arbitrary API capabilities.
    def add_api_capability(config, identifier, description="", url="", **kw):
        existing = config.registry.api_capabilities.get(identifier)
        if existing:
            error_msg = "The '%s' API capability was already registered (%s)."
            raise ValueError(error_msg % (identifier, existing))

        capability = dict(description=description, url=url, **kw)
        config.registry.api_capabilities[identifier] = capability

    config.add_directive('add_api_capability', add_api_capability)
    config.registry.api_capabilities = {}

    # Resource events helpers.
    config.add_request_method(events.get_resource_events,
                              name='get_resource_events')
    config.add_request_method(events.notify_resource_event,
                              name='notify_resource_event')

    # Setup cornice.
    config.include("cornice")

    # Per-request transaction.
    config.include("pyramid_tm")

    # Add CORS settings to the base kinto.core Service class.
    Service.init_from_settings(settings)

    # Setup components.
    for step in aslist(settings['initialization_sequence']):
        step_func = config.maybe_dotted(step)
        step_func(config)

    # Custom helpers.
    config.add_request_method(follow_subrequest)
    config.add_request_method(authentication.prefixed_userid, property=True)
    config.add_request_method(lambda r: {'id': r.prefixed_userid},
                              name='get_user_info')
    config.add_request_method(current_resource_name, reify=True)
    config.add_request_method(current_service, reify=True)
    config.commit()

    # Include plugins after init, unlike pyramid includes.
    includes = aslist(settings['includes'])
    for app in includes:
        config.include(app)

    # # Show settings to output.
    # for key, value in settings.items():
    #     logger.info('Using %s = %s' % (key, value))

    # Scan views.
    config.scan("kinto.core.views")

    # Give sign of life.
    msg = "%(project_name)s %(project_version)s starting."
    logger.info(msg % settings)

I am confused, I've read 2, 3 and 5 seconds in various places


@@ -158,6 +158,10 @@ class CollectionDeletionTest(BaseWebTest, unittest.TestCase):
         self.app.put_json(self.collection_url, MINIMALIST_COLLECTION,
                           headers=headers, status=201)

+    def test_records_permissions_are_removed_after_collection_deleted(self):
+        self.assertDictEqual(self.permission.get_object_permissions(
+                             self.record_url), {})
+

 class CollectionCreationTest(BaseWebTest, unittest.TestCase):

import unittest

from kinto.core.testing import get_user_headers

from .support import (BaseWebTest, MINIMALIST_BUCKET,
                      MINIMALIST_COLLECTION, MINIMALIST_RECORD)

class CollectionViewTest(BaseWebTest, unittest.TestCase):

    collections_url = '/buckets/beers/collections'
    collection_url = '/buckets/beers/collections/barley'

    def setUp(self):
        super().setUp()
        self.app.put_json('/buckets/beers', MINIMALIST_BUCKET,
                          headers=self.headers)
        resp = self.app.put_json(self.collection_url,
                                 MINIMALIST_COLLECTION,
                                 headers=self.headers)
        self.record = resp.json['data']

    def test_collection_endpoint_lists_them_all(self):
        resp = self.app.get(self.collections_url, headers=self.headers)
        records = resp.json['data']
        self.assertEqual(len(records), 1)
        self.assertEqual(records[0]['id'], 'barley')

    def test_collections_can_be_put_with_simple_name(self):
        self.assertEqual(self.record['id'], 'barley')

    def test_collections_name_should_be_simple(self):
        self.app.put_json('/buckets/beers/collections/__barley__',
                          MINIMALIST_COLLECTION,
                          headers=self.headers,
                          status=400)

    def test_collections_should_reject_unaccepted_request_content_type(self):
        headers = {**self.headers, 'Content-Type': 'text/plain'}
        self.app.put('/buckets/beers/collections/barley',
                     MINIMALIST_COLLECTION,
                     headers=headers,
                     status=415)

    def test_unknown_bucket_raises_403(self):
        other_bucket = self.collections_url.replace('beers', 'sodas')
        self.app.get(other_bucket, headers=self.headers, status=403)

    def test_unknown_collection_raises_404(self):
        other_collection = self.collection_url.replace('barley', 'pills')
        resp = self.app.get(other_collection, headers=self.headers, status=404)
        self.assertEqual(resp.json['details']['id'], 'pills')
        self.assertEqual(resp.json['details']['resource_name'], 'collection')

    def test_collections_are_isolated_by_bucket(self):
        other_bucket = self.collection_url.replace('beers', 'sodas')
        self.app.put_json('/buckets/sodas',
                          MINIMALIST_BUCKET,
                          headers=self.headers)
        self.app.get(other_bucket, headers=self.headers, status=404)

    def test_create_permissions_can_be_added_on_collections(self):
        collection = {**MINIMALIST_COLLECTION, 'permissions': {'record:create': ['fxa:user']}}
        resp = self.app.put_json('/buckets/beers/collections/barley',
                                 collection,
                                 headers=self.headers,
                                 status=200)
        permissions = resp.json['permissions']
        self.assertIn('fxa:user', permissions['record:create'])

    def test_wrong_create_permissions_cannot_be_added_on_collections(self):
        collection = {**MINIMALIST_COLLECTION, 'permissions': {'collection:create': ['fxa:user']}}
        self.app.put_json('/buckets/beers/collections/barley',
                          collection,
                          headers=self.headers,
                          status=400)

    def test_collections_can_handle_arbitrary_attributes(self):
        fingerprint = "5866f245a00bb3a39100d31b2f14d453"
        collection = {**MINIMALIST_COLLECTION, 'data': {'fingerprint': fingerprint}}
        resp = self.app.put_json('/buckets/beers/collections/barley',
                                 collection,
                                 headers=self.headers,
                                 status=200)
        data = resp.json['data']
        self.assertIn('fingerprint', data)
        self.assertEqual(data['fingerprint'], fingerprint)

    def test_collections_can_be_filtered_by_arbitrary_attribute(self):
        collection = {**MINIMALIST_COLLECTION, 'data': {'size': 3}}
        self.app.put_json('/buckets/beers/collections/moderator',
                          collection,
                          headers=self.headers)
        resp = self.app.get('/buckets/beers/collections?has_size=true&min_size=2',
                            headers=self.headers)
        data = resp.json['data']
        self.assertEqual(len(data), 1)

class CollectionDeletionTest(BaseWebTest, unittest.TestCase):

    collection_url = '/buckets/beers/collections/barley'

    def setUp(self):
        super().setUp()
        bucket = {**MINIMALIST_BUCKET,
                  'permissions': {'collection:create': ['system.Everyone'],
                                  'read': ['system.Everyone']}}
        self.app.put_json('/buckets/beers', bucket,
                          headers=self.headers)
        self.app.put_json(self.collection_url, MINIMALIST_COLLECTION,
                          headers=self.headers)
        resp = self.app.post_json(self.collection_url + '/records',
                                  MINIMALIST_RECORD,
                                  headers=self.headers)
        self.previous_ts = resp.headers['ETag']
        record_id = resp.json['data']['id']
        self.record_url = self.collection_url + '/records/{}'.format(record_id)
        self.app.delete(self.collection_url, headers=self.headers)

    def test_collections_can_be_deleted(self):
        self.app.get(self.collection_url, headers=self.headers, status=404)

    def test_collections_can_be_deleted_in_bulk(self):
        alice_headers = get_user_headers('alice')
        self.app.put_json('/buckets/beers/collections/1',
                          MINIMALIST_COLLECTION, headers=self.headers)
        self.app.put_json('/buckets/beers/collections/2',
                          MINIMALIST_COLLECTION, headers=alice_headers)
        self.app.put_json('/buckets/beers/collections/3',
                          MINIMALIST_COLLECTION, headers=alice_headers)
        self.app.delete('/buckets/beers/collections',
                        headers=alice_headers)
        resp = self.app.get('/buckets/beers/collections', headers=self.headers)
        self.assertEqual(len(resp.json['data']), 1)

    def test_records_of_collection_are_deleted_too(self):
        self.app.put_json(self.collection_url, MINIMALIST_COLLECTION,
                          headers=self.headers)
        self.app.get(self.record_url, headers=self.headers, status=404)

        # Verify tombstones
        resp = self.app.get('{}/records?_since=0'.format(self.collection_url),
                            headers=self.headers)
        self.assertEqual(len(resp.json['data']), 0)

    def test_timestamps_are_refreshed_when_collection_is_recreated(self):
        # Kinto/kinto#1223
        # Recreate with same name.
        self.app.put_json(self.collection_url, MINIMALIST_COLLECTION,
                          headers=self.headers)
        resp = self.app.get(self.collection_url + '/records', headers=self.headers)
        records_ts = resp.headers['ETag']
        self.assertNotEqual(self.previous_ts, records_ts)

    def test_can_be_created_after_deletion_with_if_none_match_star(self):
        headers = {**self.headers, 'If-None-Match': '*'}
        self.app.put_json(self.collection_url, MINIMALIST_COLLECTION,
                          headers=headers, status=201)

class CollectionCreationTest(BaseWebTest, unittest.TestCase):

    collections_url = '/buckets/beers/collections'

    def setUp(self):
        super().setUp()
        self.app.put_json('/buckets/beers', MINIMALIST_BUCKET,
                          headers=self.headers)

    def test_collection_can_be_created_with_post(self):
        r = self.app.post_json(self.collections_url,
                               MINIMALIST_COLLECTION,
                               headers=self.headers)
        self.assertEqual(r.status_code, 201)
        self.assertTrue(len(r.json['data']['id']) == 8)

    def test_collection_can_be_specified_in_post(self):
        collection = 'barley'
        r = self.app.post_json(self.collections_url,
                               {'data': {'id': collection}},
                               headers=self.headers)
        self.assertEqual(r.status_code, 201)
        self.assertEqual(r.json['data']['id'], collection)

    def test_collection_already_exists_post(self):
        collection = "barley"
        self.app.post_json(self.collections_url,
                           {'data': {'id': collection}},
                           headers=self.headers)
        r = self.app.post_json(self.collections_url,
                               {'data': {'id': collection}},
                               headers=self.headers)
        self.assertEqual(r.json['data']['id'], collection)
        self.assertEqual(r.status_code, 200)

To test this, you would need to actually create someting and delete it from the test function itself.


@@ -71,6 +71,7 @@ setup(
         'tqdm>=4.26.0',
         'urllib3<1.25,>=1.21.1',            # required by requests
         'requests_futures==1.0.0',
+        'git-pylint-commit-hook',
     ],
     extras_require={
         'pyarrow': [

import os
import sys

from pathlib import Path

from setuptools import setup, find_packages
from setuptools.command.install import install

VERSION = Path(Path(__file__).parent, "quilt3", "VERSION").read_text().strip()

def readme():
    readme_short = """
    Quilt manages data like code (with packages, repositories, browsing and
    revision history) so that teams can experiment faster in machine learning,
    biotech, and other data-driven domains.

    The `quilt3` PyPi package allows you to build, push, and install data packages.
    Visit the `documentation quickstart <https://docs.quiltdata.com/quickstart>`_
    to learn more.
    """
    return readme_short

class VerifyVersionCommand(install):
    """Custom command to verify that the git tag matches our version"""
    description = 'verify that the git tag matches our version'

    def run(self):
        tag = os.getenv('CIRCLE_TAG')

        if tag != VERSION:
            info = "Git tag: {0} does not match the version of this app: {1}".format(
                tag, VERSION
            )
            sys.exit(info)

setup(
    name="quilt3",
    version=VERSION,
    packages=find_packages(),
    description='Quilt: where data comes together',
    long_description=readme(),
    python_requires='>=3.6',
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
    ],
    author='quiltdata',
    author_email='contact@quiltdata.io',
    license='LICENSE',
    url='https://github.com/quiltdata/quilt',
    keywords='',
    install_requires=[
        'appdirs>=1.4.0',
        'aws-requests-auth>=0.4.2',
        'boto3>=1.10.0',
        'dnspython>=1.16.0',
        'flask',
        'flask_cors',
        'flask_json',
        'jsonlines==1.2.0',
        'packaging>=16.8',
        'python-dateutil<=2.8.0',           # 2.8.1 conflicts with botocore
        'PyYAML>=5.3',
        'requests>=2.12.4',
        'tenacity>=5.1.1',
        'tqdm>=4.26.0',
        'urllib3<1.25,>=1.21.1',            # required by requests
        'requests_futures==1.0.0',
    ],
    extras_require={
        'pyarrow': [
            'numpy>=1.14.0',                # required by pandas, but missing from its dependencies.
            'pandas>=0.19.2',
            'pyarrow>=0.14.1',              # as of 7/5/19: linux/circleci bugs on 0.14.0
        ],
        'tests': [
            'codecov',
            'numpy>=1.14.0',                # required by pandas, but missing from its dependencies.
            'pandas>=0.19.2',
            'pyarrow>=0.14.1',              # as of 7/5/19: linux/circleci bugs on 0.14.0
            'pytest<5.1.0',                 # TODO: Fix pytest.ensuretemp in conftest.py
            'pytest-cov',
            'pytest-env',
            'responses',
            'tox',
            'detox',
            'tox-pytest-summary',
        ],
    },
    include_package_data=True,
    entry_points={
        'console_scripts': ['quilt3=quilt3.main:main'],
    },
    cmdclass={
        'verify': VerifyVersionCommand,
    }
)


@@ -18,13 +18,47 @@ def sum(iterable, start=0):
     else:
         return _builtin_sum(iterable, start)

-def count(iterable, start=0):
+
+def count(iterable=None):
     if isinstance(iterable, BaseExpr):
         return ReduceExpr("count", iterable)
+    elif iterable is None:
+        return CountExpr()
+    else:
+        return _builtin_sum(1 for x in iterable)

-def first(iterable, start=0):
+
+def first(iterable):
     if isinstance(iterable, BaseExpr):
         return ReduceExpr("first", iterable)
+    else:
+        for x in iterable:
+            if x:
+                return x
+
+
+class CountExpr(BaseExpr):
+    def __init__(self):
+        super().__init__()
+
+    def is_reduce_expr(self, ee):
+        return True
+
+    def resolve(self):
+       self._stype = int
+#        self._expr.resolve()
+#         if self._stype is None:
+#             raise ValueError(
+#                 "Cannot compute %s of a variable of type %s"
+#                 % (self._op, expr_stype))
+
+    def evaluate_eager(self, ee):
+        return core.expr_count(ee.groupby)
+
+    def __str__(self):
+        return ""
+
+

 class ReduceExpr(BaseExpr):
     __slots__ = ["_op", "_expr"]

#!/usr/bin/env python3
# © H2O.ai 2018; -*- encoding: utf-8 -*-
#   This Source Code Form is subject to the terms of the Mozilla Public
#   License, v. 2.0. If a copy of the MPL was not distributed with this
#   file, You can obtain one at http://mozilla.org/MPL/2.0/.
#-------------------------------------------------------------------------------
# mean_expr, sd_expr, minmax_expr can eventually be merged into here too

from .base_expr import BaseExpr
from .consts import reduce_opcodes, ops_rules
from datatable.lib import core

_builtin_sum = sum

def sum(iterable, start=0):
    if isinstance(iterable, BaseExpr):
        return ReduceExpr("sum", iterable)
    else:
        return _builtin_sum(iterable, start)

def count(iterable, start=0):
    if isinstance(iterable, BaseExpr):
        return ReduceExpr("count", iterable)

def first(iterable, start=0):
    if isinstance(iterable, BaseExpr):
        return ReduceExpr("first", iterable)

class ReduceExpr(BaseExpr):
    __slots__ = ["_op", "_expr"]

    def __init__(self, op, expr):
        super().__init__()
        self._op = op
        self._expr = expr

    def is_reduce_expr(self, ee):
        return True

    def resolve(self):
        self._expr.resolve()
        expr_stype = self._expr.stype
        self._stype = ops_rules.get((self._op, expr_stype))
        if self._stype is None:
            raise ValueError(
                "Cannot compute %s of a variable of type %s"
                % (self._op, expr_stype))

    def evaluate_eager(self, ee):
        col = self._expr.evaluate_eager(ee)
        opcode = reduce_opcodes[self._op]
        return core.expr_reduceop(opcode, col, ee.groupby)

    def __str__(self):
        return "%s(%s)" % (self._op, self._expr)

Remove this `if`, so that we're consistent with functionality for the `Frame`


@@ -66,6 +66,10 @@ def get_argparser():
                         "session even if one would be restored.",
                         action='store_true')
     parser.add_argument('--json-args', help=argparse.SUPPRESS)
+    parser.add_argument('--target', choices=['auto', 'tab', 'tab-bg',
+                        'tab-silent', 'tab-bg-silent', 'window'],
+                        help="How the urls should be opened if there is "
+                        "already a qutebrowser instance running.")

     debug = parser.add_argument_group('debug arguments')
     debug.add_argument('-l', '--loglevel', dest='loglevel',

# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:

# Copyright 2014-2015 Florian Bruhin (The Compiler) <mail@qutebrowser.org>

# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.

"""Early initialization and main entry point."""

import sys
import json

import qutebrowser
try:
    from qutebrowser.misc.checkpyver import check_python_version
except ImportError:
    try:
        # python2
        from .misc.checkpyver import check_python_version
    except (SystemError, ValueError):
        # Import without module - SystemError on Python3, ValueError (?!?) on
        # Python2
        sys.stderr.write("Please don't run this script directly, do something "
                         "like   python3 -m qutebrowser   instead.\n")
        sys.stderr.flush()
        sys.exit(100)
check_python_version()

import argparse
from qutebrowser.misc import earlyinit

def get_argparser():
    """Get the argparse parser."""
    parser = argparse.ArgumentParser("usage: qutebrowser",
                                     description=qutebrowser.__description__)
    parser.add_argument('-c', '--confdir', help="Set config directory (empty "
                        "for no config storage).")
    parser.add_argument('--datadir', help="Set data directory (empty for "
                        "no data storage).")
    parser.add_argument('--cachedir', help="Set cache directory (empty for "
                        "no cache storage).")
    parser.add_argument('--basedir', help="Base directory for all storage. "
                        "Other --*dir arguments are ignored if this is given.")
    parser.add_argument('-V', '--version', help="Show version and quit.",
                        action='store_true')
    parser.add_argument('-s', '--set', help="Set a temporary setting for "
                        "this session.", nargs=3, action='append',
                        dest='temp_settings', default=[],
                        metavar=('SECTION', 'OPTION', 'VALUE'))
    parser.add_argument('-r', '--restore', help="Restore a named session.",
                        dest='session')
    parser.add_argument('-R', '--override-restore', help="Don't restore a "
                        "session even if one would be restored.",
                        action='store_true')
    parser.add_argument('--json-args', help=argparse.SUPPRESS)

    debug = parser.add_argument_group('debug arguments')
    debug.add_argument('-l', '--loglevel', dest='loglevel',
                       help="Set loglevel", default='info')
    debug.add_argument('--logfilter',
                       help="Comma-separated list of things to be logged "
                       "to the debug log on stdout.")
    debug.add_argument('--loglines',
                       help="How many lines of the debug log to keep in RAM "
                       "(-1: unlimited).",
                       default=2000, type=int)
    debug.add_argument('--debug', help="Turn on debugging options.",
                       action='store_true')
    debug.add_argument('--nocolor', help="Turn off colored logging.",
                       action='store_false', dest='color')
    debug.add_argument('--harfbuzz', choices=['old', 'new', 'system', 'auto'],
                       default='auto', help="HarfBuzz engine version to use. "
                       "Default: auto.")
    debug.add_argument('--relaxed-config', action='store_true',
                       help="Silently remove unknown config options.")
    debug.add_argument('--nowindow', action='store_true', help="Don't show "
                       "the main window.")
    debug.add_argument('--debug-exit', help="Turn on debugging of late exit.",
                       action='store_true')
    debug.add_argument('--pdb-postmortem', action='store_true',
                       help="Drop into pdb on exceptions.")
    debug.add_argument('--temp-basedir', action='store_true', help="Use a "
                       "temporary basedir.")
    debug.add_argument('--no-err-windows', action='store_true', help="Don't "
                       "show any error windows (used for tests/smoke.py).")
    # For the Qt args, we use store_const with const=True rather than
    # store_true because we want the default to be None, to make
    # utils.qt:get_args easier.
    debug.add_argument('--qt-name', help="Set the window name.",
                       metavar='NAME')
    debug.add_argument('--qt-style', help="Set the Qt GUI style to use.",
                       metavar='STYLE')
    debug.add_argument('--qt-stylesheet', help="Override the Qt application "
                       "stylesheet.", metavar='STYLESHEET')
    debug.add_argument('--qt-widgetcount', help="Print debug message at the "
                       "end about number of widgets left undestroyed and "
                       "maximum number of widgets existed at the same time.",
                       action='store_const', const=True)
    debug.add_argument('--qt-reverse', help="Set the application's layout "
                       "direction to right-to-left.", action='store_const',
                       const=True)
    debug.add_argument('--qt-qmljsdebugger', help="Activate the QML/JS "
                       "debugger with a specified port. 'block' is optional "
                       "and will make the application wait until a debugger "
                       "connects to it.", metavar='port:PORT[,block]')
    parser.add_argument('command', nargs='*', help="Commands to execute on "
                        "startup.", metavar=':command')
    # URLs will actually be in command
    parser.add_argument('url', nargs='*', help="URLs to open on startup "
                        "(empty as a window separator).")
    return parser

def main():
    """Main entry point for qutebrowser."""
    parser = get_argparser()
    if sys.platform == 'darwin' and getattr(sys, 'frozen', False):
        # Ignore Mac OS X' idiotic -psn_* argument...
        # http://stackoverflow.com/questions/19661298/
        # http://sourceforge.net/p/cx-freeze/mailman/message/31041783/
        argv = [arg for arg in sys.argv[1:] if not arg.startswith('-psn_0_')]
    else:
        argv = sys.argv[1:]
    args = parser.parse_args(argv)
    if args.json_args is not None:
        # Restoring after a restart.
        # When restarting, we serialize the argparse namespace into json, and
        # construct a "fake" argparse.Namespace here based on the data loaded
        # from json.
        data = json.loads(args.json_args)
        args = argparse.Namespace(**data)
    earlyinit.earlyinit(args)
    # We do this imports late as earlyinit needs to be run first (because of
    # the harfbuzz fix and version checking).
    from qutebrowser import app
    return app.run(args)

I think "How URLs should be opened ..." would be better here.


@@ -1,8 +1,8 @@
-#vkCodes.py
-#A part of NonVisual Desktop Access (NVDA)
-#This file is covered by the GNU General Public License.
-#See the file COPYING for more details.
-#Copyright (C) 2007-2010 Michael Curran <mick@kulgan.net>, James Teh <jamie@jantrid.net>
+# vkCodes.py
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2007-2021 Michael Curran <mick@kulgan.net>, James Teh <jamie@jantrid.net>, Quin Marilyn.

 """Maps between Windows virtual key (vk) codes and NVDA key names.
 These names are used when binding keyboard gestures to scripts.

#vkCodes.py
#A part of NonVisual Desktop Access (NVDA)
#This file is covered by the GNU General Public License.
#See the file COPYING for more details.
#Copyright (C) 2007-2010 Michael Curran <mick@kulgan.net>, James Teh <jamie@jantrid.net>

"""Maps between Windows virtual key (vk) codes and NVDA key names.
These names are used when binding keyboard gestures to scripts.
"""

#: Maps vk codes to key names.
#: The dict key is a tuple of (vkCode, extended),
#: where vkCode is the vk code and extended is a bool specifying whether the key is an extended key.
#: If extended is C{None}, the extended state of the key is irrelevant to the mapping;
#: i.e. the name is the same in either case.
#: The dict value is the key name.
#: @type: dict with keys of tuple(int, bool) and values of str
byCode = {
	(0x01, None): "leftMouse",
	(0x02, None): "rightMouse",
	(0x03, None): "break",
	(0x04, None): "middleMouse",
	(0x08, None): "backspace",
	(0x09, None): "tab",
	(0x0C, None): "numpad5",
	(0x0D, False): "enter",
	(0x0D, True): "numpadEnter",
	(0x10, None): "shift",
	(0x11, None): "control",
	(0x12, None): "alt",
	(0x13, None): "pause",
	(0x14, None): "capsLock",
	(0x18, None): "IMEFinalMode",
	(0x1B, None): "escape",
	(0x1C, None): "IMEConvert",
	(0x1D, None): "IMENonconvert",
	(0x1E, None): "IMEAccept",
	(0x1F, None): "IMEModeChange",
	(0x20, None): "space",
	(0x21, True): "pageUp",
	(0x21, False): "numpad9",
	(0x22, True): "pageDown",
	(0x22, False): "numpad3",
	(0x23, True): "end",
	(0x23, False): "numpad1",
	(0x24, True): "home",
	(0x24, False): "numpad7",
	(0x25, True): "leftArrow",
	(0x25, False): "numpad4",
	(0x26, True): "upArrow",
	(0x26, False): "numpad8",
	(0x27, True): "rightArrow",
	(0x27, False): "numpad6",
	(0x28, True): "downArrow",
	(0x28, False): "numpad2",
	(0x29, None): "select",
	(0x2A, None): "print",
	(0x2B, None): "execute",
	(0x2C, None): "printScreen",
	(0x2D, True): "insert",
	(0x2D, False): "numpadInsert",
	(0x2E, True): "delete",
	(0x2E, False): "numpadDelete",
	(0x2F, None): "help",
	(0x5B, None): "leftWindows",
	(0x5C, None): "rightWindows",
	(0x5D, None): "applications",
	(0x5F, None): "sleep",
	(0x60, None): "numLockNumpad0",
	(0x61, None): "numLockNumpad1",
	(0x62, None): "numLockNumpad2",
	(0x63, None): "numLockNumpad3",
	(0x64, None): "numLockNumpad4",
	(0x65, None): "numLockNumpad5",
	(0x66, None): "numLockNumpad6",
	(0x67, None): "numLockNumpad7",
	(0x68, None): "numLockNumpad8",
	(0x69, None): "numLockNumpad9",
	(0x6A, None): "numpadMultiply",
	(0x6B, None): "numpadPlus",
	(0x6C, None): "numpadSeparator",
	(0x6D, None): "numpadMinus",
	(0x6E, None): "numpadDecimal",
	(0x6F, None): "numpadDivide",
	(0x70, None): "f1",
	(0x71, None): "f2",
	(0x72, None): "f3",
	(0x73, None): "f4",
	(0x74, None): "f5",
	(0x75, None): "f6",
	(0x76, None): "f7",
	(0x77, None): "f8",
	(0x78, None): "f9",
	(0x79, None): "f10",
	(0x7A, None): "f11",
	(0x7B, None): "f12",
	(0x7C, None): "f13",
	(0x7D, None): "f14",
	(0x7E, None): "f15",
	(0x7F, None): "f16",
	(0x80, None): "f17",
	(0x81, None): "f18",
	(0x82, None): "f19",
	(0x83, None): "f20",
	(0x84, None): "f21",
	(0x85, None): "f22",
	(0x86, None): "f23",
	(0x87, None): "f24",
	(0x90, None): "numLock",
	(0x91, None): "scrollLock",
	(0xA0, None): "leftShift",
	(0xA1, None): "rightShift",
	(0xA2, None): "leftControl",
	(0xA3, None): "rightControl",
	(0xA4, None): "leftAlt",
	(0xA5, None): "rightAlt",
	(0xA6, None): "browserBack",
	(0xA7, None): "browserForward",
	(0xA8, None): "browserRefresh",
	(0xA9, None): "browserStop",
	(0xAA, None): "browserSearch",
	(0xAB, None): "browserFavorites",
	(0xAC, None): "browserHome",
	(0xAD, None): "volumeMute",
	(0xAE, None): "volumeDown",
	(0xAF, None): "volumeUp",
	(0xB0, None): "mediaNextTrack",
	(0xB1, None): "mediaPrevTrack",
	(0xB2, None): "mediaStop",
	(0xB3, None): "mediaPlayPause",
	(0xB4, None): "launchMail",
	(0xB5, None): "launchMediaPlayer",
	(0xB6, None): "launchApp1",
	(0xB7, None): "launchApp2",
}

#: Maps key names to vk codes.
#: This is the inverse of the L{byCode} map
#: except that names are all lower case to make case insensitive lookup easier.
#: @type: dict with keys of str and values of tuple(int, bool)
byName = dict((name.lower(), code) for code, name in byCode.items())

# Used by SendInput for non-keyboard input to pass Unicode characters as if they were keystrokes.
# The scan code is the Unicode character.
# See https://msdn.microsoft.com/en-us/library/windows/desktop/ms646271(v=vs.85).aspx
VK_PACKET = 0xE7

This line, and all lines which just contains the file name should be removed according to the copyright header standards on the Wiki. Also it feels a bit weird to me to modify the file just to reformat its copyright header so this should probably be reverted.


@@ -127,3 +127,34 @@ def unescape(s):

   return s

+
+#############################################################################
+def parseSdr(s):
+  """Parses a string containing only 0's and 1's and return a Python list object.
+  """
+  assert isinstance(s, basestring)
+  try:
+    sdr = []
+    for c in s:
+      if c == "0":
+        sdr.append(0)
+      elif c == "1":
+        sdr.append(1)
+    return sdr
+  except ValueError:
+    pass
+  raise ValueError("The provided string %s is malformed. The string should "
+                   "have only 0's and 1's.")
+
+
+#############################################################################
+def serializeSdr(sdr):
+  """Serialize Python list object containing only 0's and 1's to string.
+  """
+  s = ""
+  for elem in sdr:
+    if elem == 0:
+      s += "0"
+    elif elem == 1:
+      s += "1"
+  return s

# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Collection of utilities to process input data
"""

import datetime
# Workaround for this error:
#  "ImportError: Failed to import _strptime because the import lockis held by
#     another thread"

# These are the supported timestamp formats to parse. The first is used for
# serializing datetimes. Functions in this file rely on specific formats from
# this tuple so be careful when changing the indices for existing formats.
DATETIME_FORMATS = ('%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d %H:%M:%S:%f',
                    '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M', '%Y-%m-%d',
                    '%m/%d/%Y %H:%M', '%m/%d/%y %H:%M',
                    '%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%SZ',
                    '%Y-%m-%dT%H:%M:%S')

#############################################################################
def parseTimestamp(s):
  """Parses a textual datetime format and return a Python datetime object.

  The supported format is: yyyy-mm-dd h:m:s.ms

  The time component is optional
  hours are 00..23 (no AM/PM)
  minutes are 00..59
  seconds are 00..59
  micro-seconds are 000000..999999
  """
  s = s.strip()
  for pattern in DATETIME_FORMATS:
    try:
      return datetime.datetime.strptime(s, pattern)
    except ValueError:
      pass
  raise ValueError('The provided timestamp %s is malformed. The supported '
                   'formats are: [%s]' % (s, ', '.join(DATETIME_FORMATS)))

#############################################################################
def serializeTimestamp(t):
  return t.strftime(DATETIME_FORMATS[0])

#############################################################################
def serializeTimestampNoMS(t):
  return t.strftime(DATETIME_FORMATS[2])

#############################################################################
def parseBool(s):
  l = s.lower()
  if l in ("true", "t", "1"):
    return True
  if l in ("false", "f", "0"):
    return False
  raise Exception("Unable to convert string '%s' to a boolean value" % s)

#############################################################################
def floatOrNone(f):
  if f == 'None':
    return None
  return float(f)

#############################################################################
def intOrNone(i):
  if i.strip() == 'None' or i.strip() == 'NULL':
    return None
  return int(i)

#############################################################################
def escape(s):
  """Escape commas, tabs, newlines and dashes in a string

  Commas are encoded as tabs
  """
  if s is None:
    return ''

  assert isinstance(s, basestring), \
        "expected %s but got %s; value=%s" % (basestring, type(s), s)
  s = s.replace('\\', '\\\\')
  s = s.replace('\n', '\\n')
  s = s.replace('\t', '\\t')
  s = s.replace(',', '\t')
  return s

#############################################################################
def unescape(s):
  """Unescapes a string that may contain commas, tabs, newlines and dashes

  Commas are decoded from tabs
  """
  #assert isinstance(s, str)
  assert isinstance(s, basestring)
  s = s.replace('\t', ',')
  s = s.replace('\\,', ',')
  s = s.replace('\\n', '\n')
  s = s.replace('\\\\', '\\')

  return s

I'd put this in an else block above and remove the try/except.


@@ -35,9 +35,11 @@ class ScalarSpaceEncoderTest(unittest.TestCase):
   def testScalarSpaceEncoder(self):
     """scalar space encoder"""
     # use of forced=True is not recommended, but used in the example for readibility, see scalar.py
-    sse = ScalarSpaceEncoder(1,1,2,False,2,1,1,None,0,False,"delta", forced=True)
+    sse = ScalarSpaceEncoder(w=21,minval=1,maxval=2,n=100,radius=1,
+            resolution=1,name="SP1",verbosity=0,clipInput=False,space="delta")
     self.assertTrue(sse.isDelta())
-    sse = ScalarSpaceEncoder(1,1,2,False,2,1,1,None,0,False,"absolute", forced=True)
+    sse = ScalarSpaceEncoder(w=21,minval=1,maxval=2,n=100,radius=1,
+            resolution=1,name="sp2",verbosity=0,clipInput=False,space="absolute")
     self.assertFalse(sse.isDelta())

#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Unit tests for scalar space encoder"""

import unittest2 as unittest

from nupic.encoders.scalarspace import ScalarSpaceEncoder

#########################################################################
class ScalarSpaceEncoderTest(unittest.TestCase):
  '''Unit tests for ScalarSpaceEncoder class'''

  def testScalarSpaceEncoder(self):
    """scalar space encoder"""
    # use of forced=True is not recommended, but used in the example for readibility, see scalar.py
    sse = ScalarSpaceEncoder(1,1,2,False,2,1,1,None,0,False,"delta", forced=True)
    self.assertTrue(sse.isDelta())
    sse = ScalarSpaceEncoder(1,1,2,False,2,1,1,None,0,False,"absolute", forced=True)
    self.assertFalse(sse.isDelta())

###########################################
if __name__ == '__main__':
  unittest.main()


@@ -13,7 +13,8 @@ def anchor_target(anchor_list,
                   cfg,
                   gt_labels_list=None,
                   cls_out_channels=1,
-                  sampling=True):
+                  sampling=True,
+                  need_unmap=True):
     """Compute regression and classification targets for anchors.

     Args:

import torch

from ..bbox import assign_and_sample, BBoxAssigner, SamplingResult, bbox2delta
from ..utils import multi_apply

def anchor_target(anchor_list,
                  valid_flag_list,
                  gt_bboxes_list,
                  img_metas,
                  target_means,
                  target_stds,
                  cfg,
                  gt_labels_list=None,
                  cls_out_channels=1,
                  sampling=True):
    """Compute regression and classification targets for anchors.

    Args:
        anchor_list (list[list]): Multi level anchors of each image.
        valid_flag_list (list[list]): Multi level valid flags of each image.
        gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image.
        img_metas (list[dict]): Meta info of each image.
        target_means (Iterable): Mean value of regression targets.
        target_stds (Iterable): Std value of regression targets.
        cfg (dict): RPN train configs.

    Returns:
        tuple
    """
    num_imgs = len(img_metas)
    assert len(anchor_list) == len(valid_flag_list) == num_imgs

    # anchor number of multi levels
    num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]]
    # concat all level anchors and flags to a single tensor
    for i in range(num_imgs):
        assert len(anchor_list[i]) == len(valid_flag_list[i])
        anchor_list[i] = torch.cat(anchor_list[i])
        valid_flag_list[i] = torch.cat(valid_flag_list[i])

    # compute targets for each image
    if gt_labels_list is None:
        gt_labels_list = [None for _ in range(num_imgs)]
    (all_labels, all_label_weights, all_bbox_targets, all_bbox_weights,
     pos_inds_list, neg_inds_list) = multi_apply(
         anchor_target_single,
         anchor_list,
         valid_flag_list,
         gt_bboxes_list,
         gt_labels_list,
         img_metas,
         target_means=target_means,
         target_stds=target_stds,
         cfg=cfg,
         cls_out_channels=cls_out_channels,
         sampling=sampling)
    # no valid anchors
    if any([labels is None for labels in all_labels]):
        return None
    # sampled anchors of all images
    num_total_pos = sum([max(inds.numel(), 1) for inds in pos_inds_list])
    num_total_neg = sum([max(inds.numel(), 1) for inds in neg_inds_list])
    # split targets to a list w.r.t. multiple levels
    labels_list = images_to_levels(all_labels, num_level_anchors)
    label_weights_list = images_to_levels(all_label_weights, num_level_anchors)
    bbox_targets_list = images_to_levels(all_bbox_targets, num_level_anchors)
    bbox_weights_list = images_to_levels(all_bbox_weights, num_level_anchors)
    return (labels_list, label_weights_list, bbox_targets_list,
            bbox_weights_list, num_total_pos, num_total_neg)

def images_to_levels(target, num_level_anchors):
    """Convert targets by image to targets by feature level.

    [target_img0, target_img1] -> [target_level0, target_level1, ...]
    """
    target = torch.stack(target, 0)
    level_targets = []
    start = 0
    for n in num_level_anchors:
        end = start + n
        level_targets.append(target[:, start:end].squeeze(0))
        start = end
    return level_targets

def anchor_target_single(flat_anchors,
                         valid_flags,
                         gt_bboxes,
                         gt_labels,
                         img_meta,
                         target_means,
                         target_stds,
                         cfg,
                         cls_out_channels=1,
                         sampling=True):
    inside_flags = anchor_inside_flags(flat_anchors, valid_flags,
                                       img_meta['img_shape'][:2],
                                       cfg.allowed_border)
    if not inside_flags.any():
        return (None, ) * 6
    # assign gt and sample anchors
    anchors = flat_anchors[inside_flags, :]

    if sampling:
        assign_result, sampling_result = assign_and_sample(
            anchors, gt_bboxes, None, None, cfg)
    else:
        bbox_assigner = BBoxAssigner(**cfg.assigner)
        assign_result = bbox_assigner.assign(anchors, gt_bboxes, None,
                                             gt_labels)
        pos_inds = torch.nonzero(
            assign_result.gt_inds > 0).squeeze(-1).unique()
        neg_inds = torch.nonzero(
            assign_result.gt_inds == 0).squeeze(-1).unique()
        gt_flags = anchors.new_zeros(anchors.shape[0], dtype=torch.uint8)
        sampling_result = SamplingResult(pos_inds, neg_inds, anchors,
                                         gt_bboxes, assign_result, gt_flags)

    num_valid_anchors = anchors.shape[0]
    bbox_targets = torch.zeros_like(anchors)
    bbox_weights = torch.zeros_like(anchors)
    labels = anchors.new_zeros(num_valid_anchors, dtype=torch.long)
    label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float)

    pos_inds = sampling_result.pos_inds
    neg_inds = sampling_result.neg_inds
    if len(pos_inds) > 0:
        pos_bbox_targets = bbox2delta(sampling_result.pos_bboxes,
                                      sampling_result.pos_gt_bboxes,
                                      target_means, target_stds)
        bbox_targets[pos_inds, :] = pos_bbox_targets
        bbox_weights[pos_inds, :] = 1.0
        if gt_labels is None:
            labels[pos_inds] = 1
        else:
            labels[pos_inds] = gt_labels[sampling_result.pos_assigned_gt_inds]
        if cfg.pos_weight <= 0:
            label_weights[pos_inds] = 1.0
        else:
            label_weights[pos_inds] = cfg.pos_weight
    if len(neg_inds) > 0:
        label_weights[neg_inds] = 1.0

    # map up to original set of anchors
    num_total_anchors = flat_anchors.size(0)
    labels = unmap(labels, num_total_anchors, inside_flags)
    label_weights = unmap(label_weights, num_total_anchors, inside_flags)
    if cls_out_channels > 1:
        labels, label_weights = expand_binary_labels(labels, label_weights,
                                                     cls_out_channels)
    bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags)
    bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags)

    return (labels, label_weights, bbox_targets, bbox_weights, pos_inds,
            neg_inds)

def expand_binary_labels(labels, label_weights, cls_out_channels):
    bin_labels = labels.new_full(
        (labels.size(0), cls_out_channels), 0, dtype=torch.float32)
    inds = torch.nonzero(labels >= 1).squeeze()
    if inds.numel() > 0:
        bin_labels[inds, labels[inds] - 1] = 1
    bin_label_weights = label_weights.view(-1, 1).expand(
        label_weights.size(0), cls_out_channels)
    return bin_labels, bin_label_weights

def anchor_inside_flags(flat_anchors, valid_flags, img_shape,
                        allowed_border=0):
    img_h, img_w = img_shape[:2]
    if allowed_border >= 0:
        inside_flags = valid_flags & \
            (flat_anchors[:, 0] >= -allowed_border) & \
            (flat_anchors[:, 1] >= -allowed_border) & \
            (flat_anchors[:, 2] < img_w + allowed_border) & \
            (flat_anchors[:, 3] < img_h + allowed_border)
    else:
        inside_flags = valid_flags
    return inside_flags

def unmap(data, count, inds, fill=0):
    """ Unmap a subset of item (data) back to the original set of items (of
    size count) """
    if data.dim() == 1:
        ret = data.new_full((count, ), fill)
        ret[inds] = data
    else:
        new_size = (count, ) + data.size()[1:]
        ret = data.new_full(new_size, fill)
        ret[inds, :] = data
    return ret


@@ -17,15 +17,10 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.

+# pylint: disable=unused-import
 import pytest
 import pytest_bdd as bdd

-# pylint: disable=unused-import
 from end2end.features.test_yankpaste_bdd import init_fake_clipboard

-
-pytestmark = pytest.mark.qtwebengine_todo("Caret mode is not implemented",
-                                          run=False)
-
-
 bdd.scenarios('caret.feature')

# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:

# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
#
# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.

import pytest
import pytest_bdd as bdd

# pylint: disable=unused-import
from end2end.features.test_yankpaste_bdd import init_fake_clipboard

pytestmark = pytest.mark.qtwebengine_todo("Caret mode is not implemented",
                                          run=False)

bdd.scenarios('caret.feature')

Why not simply remove the `pytest` import now that it's not needed anymore? :wink:


@@ -1,25 +1,35 @@
 import os
-from datetime import datetime
+from datetime import datetime, timedelta

 from dagster import Partition
 from dagster.core.definitions import PipelineDefinition
-from dagster.core.definitions.partition import PartitionScheduleDefinition
 from dagster.core.execution.api import create_execution_plan
-from hacker_news.pipelines.download_pipeline import download_pipeline
-from hacker_news.schedules.hourly_hn_download_schedule import hourly_hn_download_schedule
+from hacker_news.jobs.download_job import download_prod_job, download_staging_job

 def assert_partitioned_schedule_builds(
-    schedule_def: PartitionScheduleDefinition,
-    pipeline_def: PipelineDefinition,
-    partition: datetime,
+    job_def: PipelineDefinition,
+    start: datetime,
+    end: datetime,
 ):
-    run_config = schedule_def.get_partition_set().run_config_for_partition(Partition(partition))
-    create_execution_plan(pipeline_def, run_config=run_config, mode=schedule_def.mode)
+    mode = job_def.mode_definitions[0]
+    partition_set = mode.get_partition_set_def(job_def.name)
+    run_config = partition_set.run_config_for_partition(Partition((start, end)))
+    create_execution_plan(job_def, run_config=run_config)

 def test_daily_download_schedule():
     os.environ["SLACK_DAGSTER_ETL_BOT_TOKEN"] = "something"
+    start = datetime.strptime("2020-10-01", "%Y-%m-%d")
+    end = start + timedelta(hours=1)
+
+    assert_partitioned_schedule_builds(
+        download_prod_job,
+        start,
+        end,
+    )
     assert_partitioned_schedule_builds(
-        hourly_hn_download_schedule, download_pipeline, datetime.strptime("2020-10-01", "%Y-%m-%d")
+        download_staging_job,
+        start,
+        end,
     )

import os
from datetime import datetime

from dagster import Partition
from dagster.core.definitions import PipelineDefinition
from dagster.core.definitions.partition import PartitionScheduleDefinition
from dagster.core.execution.api import create_execution_plan
from hacker_news.pipelines.download_pipeline import download_pipeline
from hacker_news.schedules.hourly_hn_download_schedule import hourly_hn_download_schedule

def assert_partitioned_schedule_builds(
    schedule_def: PartitionScheduleDefinition,
    pipeline_def: PipelineDefinition,
    partition: datetime,
):
    run_config = schedule_def.get_partition_set().run_config_for_partition(Partition(partition))
    create_execution_plan(pipeline_def, run_config=run_config, mode=schedule_def.mode)

def test_daily_download_schedule():
    os.environ["SLACK_DAGSTER_ETL_BOT_TOKEN"] = "something"
    assert_partitioned_schedule_builds(
        hourly_hn_download_schedule, download_pipeline, datetime.strptime("2020-10-01", "%Y-%m-%d")
    )

Not in scope for this PR, but we need a Craggier set of APIs for doing this.


@@ -18,11 +18,13 @@ from .retina_head import RetinaHead
 from .retina_sepbn_head import RetinaSepBNHead
 from .rpn_head import RPNHead
 from .ssd_head import SSDHead
+from .yolact_head import YolactHead, YolactProtonet, YolactSegmHead

 __all__ = [
     'AnchorFreeHead', 'AnchorHead', 'GuidedAnchorHead', 'FeatureAdaption',
     'RPNHead', 'GARPNHead', 'RetinaHead', 'RetinaSepBNHead', 'GARetinaHead',
     'SSDHead', 'FCOSHead', 'RepPointsHead', 'FoveaHead',
     'FreeAnchorRetinaHead', 'ATSSHead', 'FSAFHead', 'NASFCOSHead',
-    'PISARetinaHead', 'PISASSDHead', 'GFLHead', 'CornerHead'
+    'PISARetinaHead', 'PISASSDHead', 'GFLHead', 'CornerHead', 'YolactHead',
+    'YolactSegmHead', 'YolactProtonet'
 ]

from .anchor_free_head import AnchorFreeHead
from .anchor_head import AnchorHead
from .atss_head import ATSSHead
from .corner_head import CornerHead
from .fcos_head import FCOSHead
from .fovea_head import FoveaHead
from .free_anchor_retina_head import FreeAnchorRetinaHead
from .fsaf_head import FSAFHead
from .ga_retina_head import GARetinaHead
from .ga_rpn_head import GARPNHead
from .gfl_head import GFLHead
from .guided_anchor_head import FeatureAdaption, GuidedAnchorHead
from .nasfcos_head import NASFCOSHead
from .pisa_retinanet_head import PISARetinaHead
from .pisa_ssd_head import PISASSDHead
from .reppoints_head import RepPointsHead
from .retina_head import RetinaHead
from .retina_sepbn_head import RetinaSepBNHead
from .rpn_head import RPNHead
from .ssd_head import SSDHead

__all__ = [
    'AnchorFreeHead', 'AnchorHead', 'GuidedAnchorHead', 'FeatureAdaption',
    'RPNHead', 'GARPNHead', 'RetinaHead', 'RetinaSepBNHead', 'GARetinaHead',
    'SSDHead', 'FCOSHead', 'RepPointsHead', 'FoveaHead',
    'FreeAnchorRetinaHead', 'ATSSHead', 'FSAFHead', 'NASFCOSHead',
    'PISARetinaHead', 'PISASSDHead', 'GFLHead', 'CornerHead'
]

Use upper case: YOLACTHead, YOLACTProtonet, YOLACTSegmHead


@@ -37,16 +37,18 @@ setup(
     keywords='quilt quiltdata shareable data dataframe package platform pandas',
     install_requires=[
         'appdirs>=1.4.0',
-        'future>=0.16.0',
+        'funcsigs; python_version<"3.4"',  # stdlib backport, test only
+        'future>=0.16.0',   # unused
         'packaging>=16.8',
         'pandas>=0.19.2',
+        'pathlib2; python_version<"3.6"',    # stdlib backport
         'pyarrow>=0.4.0,<0.8.0', # TODO(dima): Make unit tests work with 0.8.*.
         'pyOpenSSL>=16.2.0',  # Note: not actually used at the moment.
         'pyyaml>=3.12',
         'requests>=2.12.4',
-        'responses>=0.7.0',
+        'responses>=0.7.0',  # test only
         'six>=1.10.0',
-        'tables>=3.3.0',
+        'tables>=3.3.0',  # unused
         'tqdm>=4.11.2',
         'xlrd>=1.0.0',
     ],

from setuptools import setup, find_packages

def readme():
    readme_short = """
    ``quilt`` is a command-line utility that builds, pushes, and installs
    data packages. A `data package <https://blog.quiltdata.com/data-packages-for-fast-reproducible-python-analysis-c74b78015c7f>`_
    is a versioned bundle of serialized data wrapped in a Python module.

    ``quilt`` pushes to and pulls from the package registry at quiltdata.com.

    Visit `quiltdata.com <https://quiltdata.com>`_ for docs and more.
    """
    return readme_short

setup(
    name="quilt",
    version="2.8.4-dev",
    packages=find_packages(),
    description='Quilt is a data package manager',
    long_description=readme(),
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
    ],
    author='quiltdata',
    author_email='contact@quiltdata.io',
    license='LICENSE',
    url='https://github.com/quiltdata/quilt',
    download_url='https://github.com/quiltdata/quilt/releases/tag/2.8.0',
    keywords='quilt quiltdata shareable data dataframe package platform pandas',
    install_requires=[
        'appdirs>=1.4.0',
        'future>=0.16.0',
        'packaging>=16.8',
        'pandas>=0.19.2',
        'pyarrow>=0.4.0,<0.8.0', # TODO(dima): Make unit tests work with 0.8.*.
        'pyOpenSSL>=16.2.0',  # Note: not actually used at the moment.
        'pyyaml>=3.12',
        'requests>=2.12.4',
        'responses>=0.7.0',
        'six>=1.10.0',
        'tables>=3.3.0',
        'tqdm>=4.11.2',
        'xlrd>=1.0.0',
    ],
    include_package_data=True,
    entry_points={
        'console_scripts': ['quilt=quilt.tools.main:main'],
    }
)

Actually, it is used, but in subtle ways: `from builtins import input` relies on it. Might only be needed for Python 2, though.


@@ -46,7 +46,7 @@ def open_browser(url: str) -> bool:
     browsers = (
         "windows-default", "macosx",
         "wslview %s",
-        "x-www-browser %s", "gnome-open %s",
+        "x-www-browser %s", "gnome-open %s", "xdg-open",
         "google-chrome", "chrome", "chromium", "chromium-browser",
         "firefox", "opera", "safari",
     )

import webbrowser

from mitmproxy import ctx

class WebAddon:
    def load(self, loader):
        loader.add_option(
            "web_open_browser", bool, True,
            "Start a browser."
        )
        loader.add_option(
            "web_debug", bool, False,
            "Enable mitmweb debugging."
        )
        loader.add_option(
            "web_port", int, 8081,
            "Web UI port."
        )
        loader.add_option(
            "web_host", str, "127.0.0.1",
            "Web UI host."
        )

    def running(self):
        if hasattr(ctx.options, "web_open_browser") and ctx.options.web_open_browser:
            web_url = f"http://{ctx.options.web_host}:{ctx.options.web_port}/"
            success = open_browser(web_url)
            if not success:
                ctx.log.info(
                    f"No web browser found. Please open a browser and point it to {web_url}",
                )

def open_browser(url: str) -> bool:
    """
    Open a URL in a browser window.
    In contrast to webbrowser.open, we limit the list of suitable browsers.
    This gracefully degrades to a no-op on headless servers, where webbrowser.open
    would otherwise open lynx.

    Returns:
        True, if a browser has been opened
        False, if no suitable browser has been found.
    """
    browsers = (
        "windows-default", "macosx",
        "wslview %s",
        "x-www-browser %s", "gnome-open %s",
        "google-chrome", "chrome", "chromium", "chromium-browser",
        "firefox", "opera", "safari",
    )
    for browser in browsers:
        try:
            b = webbrowser.get(browser)
        except webbrowser.Error:
            pass
        else:
            if b.open(url):
                return True
    return False

Is this related to the Docker change? I don't think this even works when mitmproxy is running inside a container...


@@ -1,12 +1,40 @@
 # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
 # For details: https://github.com/PyCQA/pylint/blob/master/LICENSE

+from typing import Optional, Union, cast
+
 import astroid

 from pylint import checkers, interfaces
 from pylint.checkers import utils

+def _check_if_dict_keys_used(
+    node: Union[astroid.For, astroid.Comprehension]
+) -> Optional[str]:
+    if not isinstance(node.iter, astroid.Call):
+        # Is it a dictionary?
+        if not isinstance(node.iter, (astroid.Name, astroid.Attribute)):
+            return None
+        inferred = utils.safe_infer(node.iter)
+        if not isinstance(inferred, (astroid.Dict, astroid.DictComp)):
+            return None
+        iterating_object_name = node.iter.as_string()
+
+    else:
+        # Is it a proper keys call?
+        if (
+            isinstance(node.iter.func, astroid.Name)
+            or node.iter.func.attrname != "keys"
+        ):
+            return None
+        inferred = utils.safe_infer(node.iter.func)
+        if not isinstance(inferred, (astroid.BoundMethod, astroid.Dict)):
+            return None
+        iterating_object_name = node.iter.as_string().partition(".")[0]
+    return iterating_object_name
+
+
 class RecommendationChecker(checkers.BaseChecker):

     __implements__ = (interfaces.IAstroidChecker,)

# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
# For details: https://github.com/PyCQA/pylint/blob/master/LICENSE

import astroid

from pylint import checkers, interfaces
from pylint.checkers import utils

class RecommendationChecker(checkers.BaseChecker):

    __implements__ = (interfaces.IAstroidChecker,)
    name = "refactoring"
    msgs = {
        "C0200": (
            "Consider using enumerate instead of iterating with range and len",
            "consider-using-enumerate",
            "Emitted when code that iterates with range and len is "
            "encountered. Such code can be simplified by using the "
            "enumerate builtin.",
        ),
        "C0201": (
            "Consider iterating the dictionary directly instead of calling .keys()",
            "consider-iterating-dictionary",
            "Emitted when the keys of a dictionary are iterated through the .keys() "
            "method. It is enough to just iterate through the dictionary itself, as "
            'in "for key in dictionary".',
        ),
    }

    @staticmethod
    def _is_builtin(node, function):
        inferred = utils.safe_infer(node)
        if not inferred:
            return False
        return utils.is_builtin_object(inferred) and inferred.name == function

    @utils.check_messages("consider-iterating-dictionary")
    def visit_call(self, node):
        if not isinstance(node.func, astroid.Attribute):
            return
        if node.func.attrname != "keys":
            return
        if not isinstance(node.parent, (astroid.For, astroid.Comprehension)):
            return

        inferred = utils.safe_infer(node.func)
        if not isinstance(inferred, astroid.BoundMethod) or not isinstance(
            inferred.bound, astroid.Dict
        ):
            return

        if isinstance(node.parent, (astroid.For, astroid.Comprehension)):
            self.add_message("consider-iterating-dictionary", node=node)

    @utils.check_messages("consider-using-enumerate")
    def visit_for(self, node):
        """Emit a convention whenever range and len are used for indexing."""
        # Verify that we have a `range([start], len(...), [stop])` call and
        # that the object which is iterated is used as a subscript in the
        # body of the for.

        # Is it a proper range call?
        if not isinstance(node.iter, astroid.Call):
            return
        if not self._is_builtin(node.iter.func, "range"):
            return
        if not node.iter.args:
            return
        is_constant_zero = (
            isinstance(node.iter.args[0], astroid.Const)
            and node.iter.args[0].value == 0
        )
        if len(node.iter.args) == 2 and not is_constant_zero:
            return
        if len(node.iter.args) > 2:
            return

        # Is it a proper len call?
        if not isinstance(node.iter.args[-1], astroid.Call):
            return
        second_func = node.iter.args[-1].func
        if not self._is_builtin(second_func, "len"):
            return
        len_args = node.iter.args[-1].args
        if not len_args or len(len_args) != 1:
            return
        iterating_object = len_args[0]
        if not isinstance(iterating_object, astroid.Name):
            return
        # If we're defining __iter__ on self, enumerate won't work
        scope = node.scope()
        if iterating_object.name == "self" and scope.name == "__iter__":
            return

        # Verify that the body of the for loop uses a subscript
        # with the object that was iterated. This uses some heuristics
        # in order to make sure that the same object is used in the
        # for body.
        for child in node.body:
            for subscript in child.nodes_of_class(astroid.Subscript):
                if not isinstance(subscript.value, astroid.Name):
                    continue

                value = subscript.slice
                if isinstance(value, astroid.Index):
                    value = value.value
                if not isinstance(value, astroid.Name):
                    continue
                if value.name != node.target.name:
                    continue
                if iterating_object.name != subscript.value.name:
                    continue
                if subscript.value.scope() != node.scope():
                    # Ignore this subscript if it's not in the same
                    # scope. This means that in the body of the for
                    # loop, another scope was created, where the same
                    # name for the iterating object was used.
                    continue
                self.add_message("consider-using-enumerate", node=node)
                return

@cdce8p I placed this here as it's shared logic between visit_for as well as visit_comprehension so there's not too much repetition in code, however this feels a little out of place. Is this fine?


@@ -153,7 +153,7 @@ def qute_help(win_id, request):
             url=request.url().toDisplayString(),
             error="This most likely means the documentation was not generated "
                   "properly. If you are running qutebrowser from the git "
-                  "repository, please run scripts/asciidoc2html.py."
+                  "repository, please run scripts/asciidoc2html.py. "
                   "If you're running a released version this is a bug, please "
                   "use :report to report it.",
             icon='')

# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:

# Copyright 2014-2015 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
#
# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
#
# pylint complains when using .render() on jinja templates, so we make it shut
# up for this whole module.

# pylint: disable=no-member
# https://bitbucket.org/logilab/pylint/issue/490/

"""Handler functions for different qute:... pages.

Module attributes:
    pyeval_output: The output of the last :pyeval command.
"""

import functools
import configparser

from PyQt5.QtCore import pyqtSlot, QObject
from PyQt5.QtNetwork import QNetworkReply

import qutebrowser
from qutebrowser.browser.network import schemehandler, networkreply
from qutebrowser.utils import (version, utils, jinja, log, message, docutils,
                               objreg)
from qutebrowser.config import configexc, configdata

pyeval_output = ":pyeval was never called"

class QuteSchemeHandler(schemehandler.SchemeHandler):

    """Scheme handler for qute: URLs."""

    def createRequest(self, _op, request, _outgoing_data):
        """Create a new request.

        Args:
             request: const QNetworkRequest & req
             _op: Operation op
             _outgoing_data: QIODevice * outgoingData

        Return:
            A QNetworkReply.
        """
        path = request.url().path()
        host = request.url().host()
        # An url like "qute:foo" is split as "scheme:path", not "scheme:host".
        log.misc.debug("url: {}, path: {}, host {}".format(
            request.url().toDisplayString(), path, host))
        try:
            handler = HANDLERS[path]
        except KeyError:
            try:
                handler = HANDLERS[host]
            except KeyError:
                errorstr = "No handler found for {}!".format(
                    request.url().toDisplayString())
                return networkreply.ErrorNetworkReply(
                    request, errorstr, QNetworkReply.ContentNotFoundError,
                    self.parent())
        try:
            data = handler(self._win_id, request)
        except OSError as e:
            return networkreply.ErrorNetworkReply(
                request, str(e), QNetworkReply.ContentNotFoundError,
                self.parent())
        return networkreply.FixedDataNetworkReply(
            request, data, 'text/html', self.parent())

class JSBridge(QObject):

    """Javascript-bridge for special qute:... pages."""

    def __init__(self, parent=None):
        super().__init__(parent)

    @pyqtSlot(int, str, str, str)
    def set(self, win_id, sectname, optname, value):
        """Slot to set a setting from qute:settings."""
        try:
            objreg.get('config').set('conf', sectname, optname, value)
        except (configexc.Error, configparser.Error) as e:
            message.error(win_id, e)

def qute_pyeval(_win_id, _request):
    """Handler for qute:pyeval. Return HTML content as bytes."""
    html = jinja.env.get_template('pre.html').render(
        title='pyeval', content=pyeval_output)
    return html.encode('UTF-8', errors='xmlcharrefreplace')

def qute_version(_win_id, _request):
    """Handler for qute:version. Return HTML content as bytes."""
    html = jinja.env.get_template('version.html').render(
        title='Version info', version=version.version(),
        copyright=qutebrowser.__copyright__)
    return html.encode('UTF-8', errors='xmlcharrefreplace')

def qute_plainlog(_win_id, _request):
    """Handler for qute:plainlog. Return HTML content as bytes."""
    if log.ram_handler is None:
        text = "Log output was disabled."
    else:
        text = log.ram_handler.dump_log()
    html = jinja.env.get_template('pre.html').render(title='log', content=text)
    return html.encode('UTF-8', errors='xmlcharrefreplace')

def qute_log(_win_id, _request):
    """Handler for qute:log. Return HTML content as bytes."""
    if log.ram_handler is None:
        html_log = None
    else:
        html_log = log.ram_handler.dump_log(html=True)
    html = jinja.env.get_template('log.html').render(
        title='log', content=html_log)
    return html.encode('UTF-8', errors='xmlcharrefreplace')

def qute_gpl(_win_id, _request):
    """Handler for qute:gpl. Return HTML content as bytes."""
    return utils.read_file('html/COPYING.html').encode('ASCII')

def qute_help(win_id, request):
    """Handler for qute:help. Return HTML content as bytes."""
    try:
        utils.read_file('html/doc/index.html')
    except FileNotFoundError:
        html = jinja.env.get_template('error.html').render(
            title="Error while loading documentation",
            url=request.url().toDisplayString(),
            error="This most likely means the documentation was not generated "
                  "properly. If you are running qutebrowser from the git "
                  "repository, please run scripts/asciidoc2html.py."
                  "If you're running a released version this is a bug, please "
                  "use :report to report it.",
            icon='')
        return html.encode('UTF-8', errors='xmlcharrefreplace')
    urlpath = request.url().path()
    if not urlpath or urlpath == '/':
        urlpath = 'index.html'
    else:
        urlpath = urlpath.lstrip('/')
    if not docutils.docs_up_to_date(urlpath):
        message.error(win_id, "Your documentation is outdated! Please re-run "
                      "scripts/asciidoc2html.py.")
    path = 'html/doc/{}'.format(urlpath)
    return utils.read_file(path).encode('UTF-8', errors='xmlcharrefreplace')

def qute_settings(win_id, _request):
    """Handler for qute:settings. View/change qute configuration."""
    config_getter = functools.partial(objreg.get('config').get, raw=True)
    html = jinja.env.get_template('settings.html').render(
        win_id=win_id, title='settings', config=configdata,
        confget=config_getter)
    return html.encode('UTF-8', errors='xmlcharrefreplace')

HANDLERS = {
    'pyeval': qute_pyeval,
    'version': qute_version,
    'plainlog': qute_plainlog,
    'log': qute_log,
    'gpl': qute_gpl,
    'help': qute_help,
    'settings': qute_settings,
}

Seems like an unrelated change - is this actually fixing something or is it an accident?


@@ -55,6 +55,9 @@ def writeModelParamsToFile(modelParams, name):
   outDir = os.path.join(os.getcwd(), 'model_params')
   if not os.path.isdir(outDir):
     os.mkdir(outDir)
+    # Create an __init__.py so the params are recognized.
+    initPath = os.path.join(outDir, '__init__.py')
+    open(initPath, 'a').close()
   outPath = os.path.join(os.getcwd(), 'model_params', paramsName)
   with open(outPath, "wb") as outFile:
     modelParamsString = modelParamsToString(modelParams)

# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""
Groups together the code dealing with swarming.
(This is a component of the One Hot Gym Prediction Tutorial.)
"""
import os
import pprint

# add logging to output errors to stdout
import logging
logging.basicConfig()

from nupic.swarming import permutations_runner
from swarm_description import SWARM_DESCRIPTION

INPUT_FILE = "rec-center-hourly.csv"
DESCRIPTION = (
  "This script runs a swarm on the input data (rec-center-hourly.csv) and\n"
  "creates a model parameters file in the `model_params` directory containing\n"
  "the best model found by the swarm. Dumps a bunch of crud to stdout because\n"
  "that is just what swarming does at this point. You really don't need to\n"
  "pay any attention to it.\n"
  )

def modelParamsToString(modelParams):
  pp = pprint.PrettyPrinter(indent=2)
  return pp.pformat(modelParams)

def writeModelParamsToFile(modelParams, name):
  cleanName = name.replace(" ", "_").replace("-", "_")
  paramsName = "%s_model_params.py" % cleanName
  outDir = os.path.join(os.getcwd(), 'model_params')
  if not os.path.isdir(outDir):
    os.mkdir(outDir)
  outPath = os.path.join(os.getcwd(), 'model_params', paramsName)
  with open(outPath, "wb") as outFile:
    modelParamsString = modelParamsToString(modelParams)
    outFile.write("MODEL_PARAMS = \\\n%s" % modelParamsString)
  return outPath

def swarmForBestModelParams(swarmConfig, name, maxWorkers=4):
  outputLabel = name
  permWorkDir = os.path.abspath('swarm')
  if not os.path.exists(permWorkDir):
    os.mkdir(permWorkDir)
  modelParams = permutations_runner.runWithConfig(
    swarmConfig,
    {"maxWorkers": maxWorkers, "overwrite": True},
    outputLabel=outputLabel,
    outDir=permWorkDir,
    permWorkDir=permWorkDir,
    verbosity=0
  )
  modelParamsFile = writeModelParamsToFile(modelParams, name)
  return modelParamsFile

def printSwarmSizeWarning(size):
  if size is "small":
    print "= THIS IS A DEBUG SWARM. DON'T EXPECT YOUR MODEL RESULTS TO BE GOOD."
  elif size is "medium":
    print "= Medium swarm. Sit back and relax, this could take awhile."
  else:
    print "= LARGE SWARM! Might as well load up the Star Wars Trilogy."

def swarm(filePath):
  name = os.path.splitext(os.path.basename(filePath))[0]
  print "================================================="
  print "= Swarming on %s data..." % name
  printSwarmSizeWarning(SWARM_DESCRIPTION["swarmSize"])
  print "================================================="
  modelParams = swarmForBestModelParams(SWARM_DESCRIPTION, name)
  print "\nWrote the following model param files:"
  print "\t%s" % modelParams

if __name__ == "__main__":
  print DESCRIPTION
  swarm(INPUT_FILE)

Consider moving these lines outside of the `if not os.path.isdir(outDir):` block.


@@ -49,7 +49,7 @@ def stripControlChars(string):

 def compactHash(string):
   hash = md5()
-  hash.update(string)
+  hash.update(string.encode('unicode_escape'))
   return hash.hexdigest()

"""Copyright 2008 Orbitz WorldWide
   Copyright 2011 Chris Davis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License."""

from graphite.logger import log
import time
try:
  from hashlib import md5
except ImportError:
  from md5 import md5
import bisect

def hashRequest(request):
  # Normalize the request parameters so ensure we're deterministic
  queryParams = ["%s=%s" % (key, '&'.join(values))
                 for (key,values) in request.GET.lists()
                 if not key.startswith('_')]

  normalizedParams = ','.join( sorted(queryParams) ) or 'noParam'
  myHash = stripControlChars(normalizedParams) #memcached doesn't like unprintable characters in its keys

  return compactHash(myHash)

def hashData(targets, startTime, endTime):
  targetsString = ','.join(targets)
  startTimeString = startTime.strftime("%Y%m%d_%H%M")
  endTimeString = endTime.strftime("%Y%m%d_%H%M")
  myHash = targetsString + '@' + startTimeString + ':' + endTimeString
  myHash = stripControlChars(myHash)

  return compactHash(myHash)

def stripControlChars(string):
  return filter(lambda char: ord(char) >= 33, string)

def compactHash(string):
  hash = md5()
  hash.update(string)
  return hash.hexdigest()

class ConsistentHashRing:
  def __init__(self, nodes, replica_count=100):
    self.ring = []
    self.ring_len = len(self.ring)
    self.nodes = set()
    self.nodes_len = len(self.nodes)
    self.replica_count = replica_count
    for node in nodes:
      self.add_node(node)

  def compute_ring_position(self, key):
    big_hash = md5( str(key) ).hexdigest()
    small_hash = int(big_hash[:4], 16)
    return small_hash

  def add_node(self, key):
    self.nodes.add(key)
    self.nodes_len = len(self.nodes)
    for i in range(self.replica_count):
      replica_key = "%s:%d" % (key, i)
      position = self.compute_ring_position(replica_key)
      entry = (position, key)
      bisect.insort(self.ring, entry)
    self.ring_len = len(self.ring)

  def remove_node(self, key):
    self.nodes.discard(key)
    self.nodes_len = len(self.nodes)
    self.ring = [entry for entry in self.ring if entry[1] != key]
    self.ring_len = len(self.ring)

  def get_node(self, key):
    assert self.ring
    position = self.compute_ring_position(key)
    search_entry = (position, None)
    index = bisect.bisect_left(self.ring, search_entry) % self.ring_len
    entry = self.ring[index]
    return entry[1]

  def get_nodes(self, key):
    nodes = []
    position = self.compute_ring_position(key)
    search_entry = (position, None)
    index = bisect.bisect_left(self.ring, search_entry) % self.ring_len
    last_index = (index - 1) % self.ring_len
    nodes_len = len(nodes)
    while nodes_len < self.nodes_len and index != last_index:
      next_entry = self.ring[index]
      (position, next_node) = next_entry
      if next_node not in nodes:
        nodes.append(next_node)
        nodes_len += 1

      index = (index + 1) % self.ring_len

    return nodes

`string.encode('utf-8')` is more common but I guess this is mostly cosmetic :)