@@ -7,6 +7,7 @@ from mmdet.core.bbox import distance2bbox
 from mmdet.core.mask.structures import BitmapMasks, PolygonMasks
 from mmdet.core.utils import (center_of_mass, filter_scores_and_topk,
                               flip_tensor, mask2ndarray, select_single_mlvl)
+from mmdet.utils import find_latest_checkpoint

 def dummy_raw_polygon_masks(size):

# Copyright (c) OpenMMLab. All rights reserved.
import numpy as np
import pytest
import torch

from mmdet.core.bbox import distance2bbox
from mmdet.core.mask.structures import BitmapMasks, PolygonMasks
from mmdet.core.utils import (center_of_mass, filter_scores_and_topk,
                              flip_tensor, mask2ndarray, select_single_mlvl)

def dummy_raw_polygon_masks(size):
    """
    Args:
        size (tuple): expected shape of dummy masks, (N, H, W)

    Return:
        list[list[ndarray]]: dummy mask
    """
    num_obj, height, width = size
    polygons = []
    for _ in range(num_obj):
        num_points = np.random.randint(5) * 2 + 6
        polygons.append([np.random.uniform(0, min(height, width), num_points)])
    return polygons

def test_mask2ndarray():
    raw_masks = np.ones((3, 28, 28))
    bitmap_mask = BitmapMasks(raw_masks, 28, 28)
    output_mask = mask2ndarray(bitmap_mask)
    assert np.allclose(raw_masks, output_mask)

    raw_masks = dummy_raw_polygon_masks((3, 28, 28))
    polygon_masks = PolygonMasks(raw_masks, 28, 28)
    output_mask = mask2ndarray(polygon_masks)
    assert output_mask.shape == (3, 28, 28)

    raw_masks = np.ones((3, 28, 28))
    output_mask = mask2ndarray(raw_masks)
    assert np.allclose(raw_masks, output_mask)

    raw_masks = torch.ones((3, 28, 28))
    output_mask = mask2ndarray(raw_masks)
    assert np.allclose(raw_masks, output_mask)

    # test unsupported type
    raw_masks = []
    with pytest.raises(TypeError):
        output_mask = mask2ndarray(raw_masks)

def test_distance2bbox():
    point = torch.Tensor([[74., 61.], [-29., 106.], [138., 61.], [29., 170.]])

    distance = torch.Tensor([[0., 0, 1., 1.], [1., 2., 10., 6.],
                             [22., -29., 138., 61.], [54., -29., 170., 61.]])
    expected_decode_bboxes = torch.Tensor([[74., 61., 75., 62.],
                                           [0., 104., 0., 112.],
                                           [100., 90., 100., 120.],
                                           [0., 120., 100., 120.]])
    out_bbox = distance2bbox(point, distance, max_shape=(120, 100))
    assert expected_decode_bboxes.allclose(out_bbox)
    out = distance2bbox(point, distance, max_shape=torch.Tensor((120, 100)))
    assert expected_decode_bboxes.allclose(out)

    batch_point = point.unsqueeze(0).repeat(2, 1, 1)
    batch_distance = distance.unsqueeze(0).repeat(2, 1, 1)
    batch_out = distance2bbox(
        batch_point, batch_distance, max_shape=(120, 100))[0]
    assert out.allclose(batch_out)
    batch_out = distance2bbox(
        batch_point, batch_distance, max_shape=[(120, 100), (120, 100)])[0]
    assert out.allclose(batch_out)

    batch_out = distance2bbox(point, batch_distance, max_shape=(120, 100))[0]
    assert out.allclose(batch_out)

    # test max_shape is not equal to batch
    with pytest.raises(AssertionError):
        distance2bbox(
            batch_point,
            batch_distance,
            max_shape=[(120, 100), (120, 100), (32, 32)])

    rois = torch.zeros((0, 4))
    deltas = torch.zeros((0, 4))
    out = distance2bbox(rois, deltas, max_shape=(120, 100))
    assert rois.shape == out.shape

    rois = torch.zeros((2, 0, 4))
    deltas = torch.zeros((2, 0, 4))
    out = distance2bbox(rois, deltas, max_shape=(120, 100))
    assert rois.shape == out.shape

@pytest.mark.parametrize('mask', [
    torch.ones((28, 28)),
    torch.zeros((28, 28)),
    torch.rand(28, 28) > 0.5,
    torch.tensor([[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]])
])
def test_center_of_mass(mask):
    center_h, center_w = center_of_mass(mask)
    if mask.shape[0] == 4:
        assert center_h == 1.5
        assert center_w == 1.5
    assert isinstance(center_h, torch.Tensor) \
           and isinstance(center_w, torch.Tensor)
    assert 0 <= center_h <= 28 \
           and 0 <= center_w <= 28

def test_flip_tensor():
    img = np.random.random((1, 3, 10, 10))
    src_tensor = torch.from_numpy(img)

    # test flip_direction parameter error
    with pytest.raises(AssertionError):
        flip_tensor(src_tensor, 'flip')

    # test tensor dimension
    with pytest.raises(AssertionError):
        flip_tensor(src_tensor[0], 'vertical')

    hfilp_tensor = flip_tensor(src_tensor, 'horizontal')
    expected_hflip_tensor = torch.from_numpy(img[..., ::-1, :].copy())
    expected_hflip_tensor.allclose(hfilp_tensor)

    vfilp_tensor = flip_tensor(src_tensor, 'vertical')
    expected_vflip_tensor = torch.from_numpy(img[..., ::-1].copy())
    expected_vflip_tensor.allclose(vfilp_tensor)

    diag_filp_tensor = flip_tensor(src_tensor, 'diagonal')
    expected_diag_filp_tensor = torch.from_numpy(img[..., ::-1, ::-1].copy())
    expected_diag_filp_tensor.allclose(diag_filp_tensor)

def test_select_single_mlvl():
    mlvl_tensors = [torch.rand(2, 1, 10, 10)] * 5
    mlvl_tensor_list = select_single_mlvl(mlvl_tensors, 1)
    assert len(mlvl_tensor_list) == 5 and mlvl_tensor_list[0].ndim == 3

def test_filter_scores_and_topk():
    score = torch.tensor([[0.1, 0.3, 0.2], [0.12, 0.7, 0.9], [0.02, 0.8, 0.08],
                          [0.4, 0.1, 0.08]])
    bbox_pred = torch.tensor([[0.2, 0.3], [0.4, 0.7], [0.1, 0.1], [0.5, 0.1]])
    score_thr = 0.15
    nms_pre = 4
    # test results type error
    with pytest.raises(NotImplementedError):
        filter_scores_and_topk(score, score_thr, nms_pre, (score, ))

    filtered_results = filter_scores_and_topk(
        score, score_thr, nms_pre, results=dict(bbox_pred=bbox_pred))
    filtered_score, labels, keep_idxs, results = filtered_results
    assert filtered_score.allclose(torch.tensor([0.9, 0.8, 0.7, 0.4]))
    assert labels.allclose(torch.tensor([2, 1, 1, 0]))
    assert keep_idxs.allclose(torch.tensor([1, 2, 1, 3]))
    assert results['bbox_pred'].allclose(
        torch.tensor([[0.4, 0.7], [0.1, 0.1], [0.4, 0.7], [0.5, 0.1]]))


@@ -69,14 +69,13 @@ class UtilsTestCase(SparkTestCase):
                         'release_mbid': '1',
                         'listen_count': 1
                     }]
-                     }
-
-        self.assertDictEqual(user_utils.get_recordings('table'), dictionary)
-        self.assertEqual(user_utils.get_recordings('table')['user1'][0]['listen_count'], 2)
-
+                      }
+
+        self.assertDictEqual(user.utils.get_recordings(df), dictionary)
+        self.assertEqual(user.utils.get_recordings(df)['user1'][0]['listen_count'], 2)
+
     def test_get_releases(self):
         df = self.create_df()
-        df.createOrReplaceTempView('table')
         dictionary =  {
                     'user1' : [{
                         'artist_name': 'artist1',

import listenbrainz_spark.stats.user.utils as user_utils
from listenbrainz_spark import utils
from listenbrainz_spark.tests import SparkTestCase

from pyspark.sql import Row

class UtilsTestCase(SparkTestCase):

    def create_df(self):
        df = utils.create_dataframe(Row(user_name='user2', artist_name='artist1', artist_msid='1',artist_mbids='1',
            track_name='test', recording_msid='1', recording_mbid='1', release_name='test',release_msid='1',
            release_mbid='1'), schema=None)
        df1 = utils.create_dataframe(Row(user_name='user1',artist_name='artist1', artist_msid='1',artist_mbids='1',
            track_name='test', recording_msid='1', recording_mbid='1', release_name='test',release_msid='1',
             release_mbid='1'), schema=None)
        df = df.union(df1)
        df2 = utils.create_dataframe(Row(user_name='user1',artist_name='artist1', artist_msid='1',artist_mbids='1',
            track_name='test', recording_msid='1', recording_mbid='1', release_name='test',release_msid='1',
            release_mbid='1'), schema=None)
        df = df.union(df2)
        return df

    def test_get_artists(self):
        df = self.create_df()
        df.createOrReplaceTempView('table')
        dictionary = {
                    'user1': [{
                        'artist_name': 'artist1',
                        'artist_msid': '1',
                        'artist_mbids': '1',
                        'listen_count': 2
                    }],
                        'user2': [{
                        'artist_name': 'artist1',
                        'artist_msid': '1',
                        'artist_mbids': '1',
                        'listen_count': 1
                    }]
                     }

        self.assertDictEqual(user_utils.get_artists('table'), dictionary)
        self.assertEqual(user_utils.get_artists('table')['user1'][0]['listen_count'], 2)

    def test_get_recordings(self):
        df = self.create_df()
        df.createOrReplaceTempView('table')
        dictionary =  {
                    'user1' : [{
                        'track_name': 'test',
                        'recording_msid': '1',
                        'recording_mbid': '1',
                        'artist_name': 'artist1',
                        'artist_msid': '1',
                        'artist_mbids': '1',
                        'release_name': 'test',
                        'release_msid': '1',
                        'release_mbid': '1',
                        'listen_count': 2
                    }],
                    'user2' : [{
                        'track_name': 'test',
                        'recording_msid': '1',
                        'recording_mbid': '1',
                        'artist_name': 'artist1',
                        'artist_msid': '1',
                        'artist_mbids': '1',
                        'release_name': 'test',
                        'release_msid': '1',
                        'release_mbid': '1',
                        'listen_count': 1
                    }]
                     }

        self.assertDictEqual(user_utils.get_recordings('table'), dictionary)
        self.assertEqual(user_utils.get_recordings('table')['user1'][0]['listen_count'], 2)

    def test_get_releases(self):
        df = self.create_df()
        df.createOrReplaceTempView('table')
        dictionary =  {
                    'user1' : [{
                        'artist_name': 'artist1',
                        'artist_msid': '1',
                        'artist_mbids': '1',
                        'release_name': 'test',
                        'release_msid': '1',
                        'release_mbid': '1',
                        'listen_count': 2
                    }],
                    'user2' : [{
                        'artist_name': 'artist1',
                        'artist_msid': '1',
                        'artist_mbids': '1',
                        'release_name': 'test',
                        'release_msid': '1',
                        'release_mbid': '1',
                        'listen_count': 1
                    }]
                     }

        self.assertDictEqual(user_utils.get_releases('table'), dictionary)
        self.assertEqual(user_utils.get_releases('table')['user1'][0]['listen_count'], 2)


@@ -1,11 +1,37 @@
 import glob
-from mitmproxy import utils, script
+import json
+import os
+from contextlib import contextmanager
+
+from mitmproxy import utils, script, contentviews
 from mitmproxy.proxy import config
-from . import tservers
+from netlib import tutils as netutils
+from netlib.http import Headers
+from . import tservers, tutils
+
+example_dir = utils.Data(__name__).path("../../examples")
+
+
+class DummyContext(object):
+    """Emulate script.ScriptContext() functionality."""
+
+    def log(self, *args, **kwargs):
+        pass
+
+    def add_contentview(self, view_obj):
+        pass
+
+
+@contextmanager
+def example(command):
+    command = os.path.join(example_dir, command)
+    ctx = DummyContext()
+    s = script.Script(command, ctx)
+    yield s
+    s.unload()

 def test_load_scripts():
-    example_dir = utils.Data(__name__).path("../../examples")
     scripts = glob.glob("%s/*.py" % example_dir)

     tmaster = tservers.TestMaster(config.ProxyConfig())

import glob
from mitmproxy import utils, script
from mitmproxy.proxy import config
from . import tservers

def test_load_scripts():
    example_dir = utils.Data(__name__).path("../../examples")
    scripts = glob.glob("%s/*.py" % example_dir)

    tmaster = tservers.TestMaster(config.ProxyConfig())

    for f in scripts:
        if "har_extractor" in f:
            continue
        if "flowwriter" in f:
            f += " -"
        if "iframe_injector" in f:
            f += " foo"  # one argument required
        if "filt" in f:
            f += " ~a"
        if "modify_response_body" in f:
            f += " foo bar"  # two arguments required
        try:
            s = script.Script(f, script.ScriptContext(tmaster))  # Loads the script file.
        except Exception as v:
            if "ImportError" not in str(v):
                raise
        else:
            s.unload()


@@ -8,6 +8,7 @@ from bzt.modules.gatling import DataLogReader as GatlingLogReader
 from bzt.modules.grinder import DataLogReader as GrinderLogReader
 from bzt.modules.jmeter import JTLReader, XMLJTLReader
 from bzt.modules.pbench import PBenchKPIReader
+from bzt.utils import dehumanize_time

 class ExternalResultsLoader(ScenarioExecutor, AggregatorListener):

import re

from bzt import TaurusInternalException, TaurusConfigError
from bzt.engine import ScenarioExecutor
from bzt.modules.ab import TSVDataReader
from bzt.modules.aggregator import AggregatorListener, ConsolidatingAggregator, DataPoint
from bzt.modules.gatling import DataLogReader as GatlingLogReader
from bzt.modules.grinder import DataLogReader as GrinderLogReader
from bzt.modules.jmeter import JTLReader, XMLJTLReader
from bzt.modules.pbench import PBenchKPIReader

class ExternalResultsLoader(ScenarioExecutor, AggregatorListener):
    AB_HEADER = "starttime\tseconds\tctime\tdtime\tttime\twait"
    PBENCH_FORMAT = re.compile("^[0-9]+\.[0-9]{3}\t[^\t]*\t([0-9]+\t){9}[0-9]+$")

    def __init__(self):
        # TODO: document this executor
        super(ExternalResultsLoader, self).__init__()
        self.data_file = None
        self.errors_file = None
        self.reader = None
        self._last_ts = -1
        self._prev_ts = -1

    def prepare(self):
        exc = TaurusConfigError("Option is required for executor: data-file")
        self.data_file = self.execution.get("data-file", exc)
        self.data_file = self.engine.find_file(self.data_file)
        self.label = self.data_file
        self.errors_file = self.execution.get("errors-jtl", None)
        if self.errors_file:
            self.errors_file = self.engine.find_file(self.errors_file)

        self.reader = self._get_reader()
        if isinstance(self.engine.aggregator, ConsolidatingAggregator):
            self.engine.aggregator.add_underling(self.reader)

        if isinstance(self.engine.aggregator, ConsolidatingAggregator):
            self.engine.aggregator.add_listener(self)

    def _get_reader(self):
        with open(self.data_file) as fhd:
            header = fhd.readline(2048).strip()  # just header chunk of file

        # TODO: detect CSV dialect for JTLs

        if header.startswith(self.AB_HEADER):
            return TSVDataReader(self.data_file, self.log)
        elif header.startswith("<?xml"):
            return XMLJTLReader(self.data_file, self.log)
        elif self.PBENCH_FORMAT.match(header):
            return PBenchKPIReader(self.data_file, self.log, self.errors_file)
        elif header.startswith("RUN\t") or "\tRUN\t" in header:
            return GatlingLogReader(self.data_file, self.log, None)
        elif "timestamp" in header.lower() and "elapsed" in header.lower():
            return JTLReader(self.data_file, self.log, self.errors_file)
        elif "worker process" in header.lower() and header.startswith("worker."):
            return GrinderLogReader(self.data_file, self.log)
        else:
            self.log.info("Header line was: %s", header)
            raise TaurusInternalException("Unable to detect results format for: %s" % self.data_file)

    def aggregated_second(self, data):
        self._last_ts = data[DataPoint.TIMESTAMP]

    def check(self):
        if self._last_ts > 0 and self._last_ts == self._prev_ts:
            return True
        else:
            self._prev_ts = self._last_ts


@@ -192,3 +192,22 @@ def concurrent(fn):
         return _concurrent
     raise NotImplementedError(
         "Concurrent decorator not supported for '%s' method." % fn.func_name)
+
+
+class ScriptModified(PatternMatchingEventHandler):
+
+    def __init__(self, FlowMaster):
+        self.FlowMaster = FlowMaster
+        PatternMatchingEventHandler.__init__(self, ignore_directories=True, patterns=["*.py"])
+
+    def on_modified(self, event=FileModifiedEvent):
+        self.FlowMaster.reload_scripts()
+        signals.status_message.send(message="script: <{0}> reloaded.".format(event.src_path))
+
+
+def ObserveScripts(FlowMaster, path):
+    script_dir = os.path.dirname(path)
+    event_handler = ScriptModified(FlowMaster)
+    observer = Observer()
+    observer.schedule(event_handler, script_dir)
+    observer.start()

from __future__ import absolute_import
import os
import traceback
import threading
import shlex
import sys

class ScriptError(Exception):
    pass

class ScriptContext:
    """
    The script context should be used to interact with the global mitmproxy state from within a
    script.
    """
    def __init__(self, master):
        self._master = master

    def log(self, message, level="info"):
        """
            Logs an event.

            By default, only events with level "error" get displayed. This can be controlled with the "-v" switch.
            How log messages are handled depends on the front-end. mitmdump will print them to stdout,
            mitmproxy sends output to the eventlog for display ("e" keyboard shortcut).
        """
        self._master.add_event(message, level)

    def kill_flow(self, f):
        """
            Kills a flow immediately. No further data will be sent to the client or the server.
        """
        f.kill(self._master)

    def duplicate_flow(self, f):
        """
            Returns a duplicate of the specified flow. The flow is also
            injected into the current state, and is ready for editing, replay,
            etc.
        """
        self._master.pause_scripts = True
        f = self._master.duplicate_flow(f)
        self._master.pause_scripts = False
        return f

    def replay_request(self, f):
        """
            Replay the request on the current flow. The response will be added
            to the flow object.
        """
        return self._master.replay_request(f, block=True, run_scripthooks=False)

    @property
    def app_registry(self):
        return self._master.apps

class Script:
    """
        Script object representing an inline script.
    """

    def __init__(self, command, master):
        self.command = command
        self.args = self.parse_command(command)
        self.ctx = ScriptContext(master)
        self.ns = None
        self.load()

    @classmethod
    def parse_command(cls, command):
        if not command or not command.strip():
            raise ScriptError("Empty script command.")
        if os.name == "nt":  # Windows: escape all backslashes in the path.
            backslashes = shlex.split(command, posix=False)[0].count("\\")
            command = command.replace("\\", "\\\\", backslashes)
        args = shlex.split(command)
        args[0] = os.path.expanduser(args[0])
        if not os.path.exists(args[0]):
            raise ScriptError(
                ("Script file not found: %s.\r\n"
                 "If your script path contains spaces, "
                 "make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") %
                args[0])
        elif os.path.isdir(args[0]):
            raise ScriptError("Not a file: %s" % args[0])
        return args

    def load(self):
        """
            Loads an inline script.

            Returns:
                The return value of self.run("start", ...)

            Raises:
                ScriptError on failure
        """
        if self.ns is not None:
            self.unload()
        script_dir = os.path.dirname(os.path.abspath(self.args[0]))
        ns = {'__file__': os.path.abspath(self.args[0])}
        sys.path.append(script_dir)
        try:
            execfile(self.args[0], ns, ns)
        except Exception as e:
            # Python 3: use exception chaining, https://www.python.org/dev/peps/pep-3134/
            raise ScriptError(traceback.format_exc(e))
        sys.path.pop()
        self.ns = ns
        return self.run("start", self.args)

    def unload(self):
        ret = self.run("done")
        self.ns = None
        return ret

    def run(self, name, *args, **kwargs):
        """
            Runs an inline script hook.

            Returns:
                The return value of the method.
                None, if the script does not provide the method.

            Raises:
                ScriptError if there was an exception.
        """
        f = self.ns.get(name)
        if f:
            try:
                return f(self.ctx, *args, **kwargs)
            except Exception as e:
                raise ScriptError(traceback.format_exc(e))
        else:
            return None

class ReplyProxy(object):
    def __init__(self, original_reply, script_thread):
        self.original_reply = original_reply
        self.script_thread = script_thread
        self._ignore_call = True
        self.lock = threading.Lock()

    def __call__(self, *args, **kwargs):
        with self.lock:
            if self._ignore_call:
                self.script_thread.start()
                self._ignore_call = False
                return
        self.original_reply(*args, **kwargs)

    def __getattr__(self, k):
        return getattr(self.original_reply, k)

def _handle_concurrent_reply(fn, o, *args, **kwargs):
    # Make first call to o.reply a no op and start the script thread.
    # We must not start the script thread before, as this may lead to a nasty race condition
    # where the script thread replies a different response before the normal reply, which then gets swallowed.

    def run():
        fn(*args, **kwargs)
        # If the script did not call .reply(), we have to do it now.
        reply_proxy()

    script_thread = ScriptThread(target=run)

    reply_proxy = ReplyProxy(o.reply, script_thread)
    o.reply = reply_proxy

class ScriptThread(threading.Thread):
    name = "ScriptThread"

def concurrent(fn):
    if fn.func_name in (
            "request",
            "response",
            "error",
            "clientconnect",
            "serverconnect",
            "clientdisconnect",
            "next_layer"):
        def _concurrent(ctx, obj):
            _handle_concurrent_reply(fn, obj, ctx, obj)

        return _concurrent
    raise NotImplementedError(
        "Concurrent decorator not supported for '%s' method." % fn.func_name)


@@ -19,15 +19,10 @@
 #  DEALINGS IN THE SOFTWARE.
 """Login Command Module."""

-import fcntl
 import os
-import signal
-import struct
-import sys
-import termios
+from subprocess import run

 import click
-import pexpect

 from molecule import logger, scenarios, util
 from molecule.command import base

#  Copyright (c) 2015-2018 Cisco Systems, Inc.
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to
#  deal in the Software without restriction, including without limitation the
#  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
#  sell copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.
"""Login Command Module."""

import fcntl
import os
import signal
import struct
import sys
import termios

import click
import pexpect

from molecule import logger, scenarios, util
from molecule.command import base

LOG = logger.get_logger(__name__)

class Login(base.Base):
    """
    Login Command Class.

    .. program:: molecule login

    .. option:: molecule login

        Target the default scenario.

    .. program:: molecule login --scenario-name foo

    .. option:: molecule login --scenario-name foo

        Targeting a specific scenario.

    .. program:: molecule login --host hostname

    .. option:: molecule login --host hostname

        Targeting a specific running host.

    .. program:: molecule login --host hostname --scenario-name foo

    .. option:: molecule login --host hostname --scenario-name foo

        Targeting a specific running host and scenario.

    .. program:: molecule --debug login

    .. option:: molecule --debug login

        Executing with `debug`.

    .. program:: molecule --base-config base.yml login

    .. option:: molecule --base-config base.yml login

        Executing with a `base-config`.

    .. program:: molecule --env-file foo.yml login

    .. option:: molecule --env-file foo.yml login

        Load an env file to read variables from when rendering
        molecule.yml.
    """

    def __init__(self, c):
        """Construct Login."""
        super(Login, self).__init__(c)
        self._pt = None

    def execute(self):
        """
        Execute the actions necessary to perform a `molecule login` and \
        returns None.

        :return: None
        """
        c = self._config
        if (not c.state.created) and c.driver.managed:
            msg = "Instances not created.  Please create instances first."
            util.sysexit_with_message(msg)

        hosts = [d["name"] for d in self._config.platforms.instances]
        hostname = self._get_hostname(hosts)
        self._get_login(hostname)

    def _get_hostname(self, hosts):
        hostname = self._config.command_args.get("host")
        if hostname is None:
            if len(hosts) == 1:
                hostname = hosts[0]
            else:
                msg = (
                    "There are {} running hosts. Please specify "
                    "which with --host.\n\n"
                    "Available hosts:\n{}".format(len(hosts), "\n".join(sorted(hosts)))
                )
                util.sysexit_with_message(msg)
        match = [x for x in hosts if x.startswith(hostname)]
        if len(match) == 0:
            msg = (
                "There are no hosts that match '{}'.  You "
                "can only login to valid hosts."
            ).format(hostname)
            util.sysexit_with_message(msg)
        elif len(match) != 1:
            # If there are multiple matches, but one of them is an exact string
            # match, assume this is the one they're looking for and use it.
            if hostname in match:
                match = [hostname]
            else:
                msg = (
                    "There are {} hosts that match '{}'. You "
                    "can only login to one at a time.\n\n"
                    "Available hosts:\n{}".format(
                        len(match), hostname, "\n".join(sorted(hosts))
                    )
                )
                util.sysexit_with_message(msg)

        return match[0]

    def _get_login(self, hostname):  # pragma: no cover
        lines, columns = os.popen("stty size", "r").read().split()
        login_options = self._config.driver.login_options(hostname)
        login_options["columns"] = columns
        login_options["lines"] = lines
        login_cmd = self._config.driver.login_cmd_template.format(**login_options)

        dimensions = (int(lines), int(columns))
        cmd = "/usr/bin/env {}".format(login_cmd)
        self._pt = pexpect.spawn(cmd, dimensions=dimensions)
        signal.signal(signal.SIGWINCH, self._sigwinch_passthrough)
        self._pt.interact()

    def _sigwinch_passthrough(self, sig, data):  # pragma: no cover
        tiocgwinsz = 1074295912  # assume
        if "TIOCGWINSZ" in dir(termios):
            tiocgwinsz = termios.TIOCGWINSZ
        s = struct.pack("HHHH", 0, 0, 0, 0)
        a = struct.unpack("HHHH", fcntl.ioctl(sys.stdout.fileno(), tiocgwinsz, s))
        self._pt.setwinsize(a[0], a[1])

@base.click_command_ex()
@click.pass_context
@click.option("--host", "-h", help="Host to access.")
@click.option(
    "--scenario-name",
    "-s",
    default=base.MOLECULE_DEFAULT_SCENARIO_NAME,
    help="Name of the scenario to target. ({})".format(
        base.MOLECULE_DEFAULT_SCENARIO_NAME
    ),
)
def login(ctx, host, scenario_name):  # pragma: no cover
    """Log in to one instance."""
    args = ctx.obj.get("args")
    subcommand = base._get_subcommand(__name__)
    command_args = {"subcommand": subcommand, "host": host}

    s = scenarios.Scenarios(base.get_configs(args, command_args), scenario_name)
    for scenario in s.all:
        base.execute_subcommand(scenario.config, subcommand)


@@ -1,11 +1,12 @@
 # -*- coding: UTF-8 -*-
 #A part of NonVisual Desktop Access (NVDA)
-#Copyright (C) 2016 NV Access Limited
+#Copyright (C) 2016-2018 NV Access Limited
 #This file is covered by the GNU General Public License.
 #See the file COPYING for more details.

 import wx
 from wx.lib.mixins import listctrl as listmix
+import winUser

 class AutoWidthColumnListCtrl(wx.ListCtrl, listmix.ListCtrlAutoWidthMixin):
 	"""

# -*- coding: UTF-8 -*-
#A part of NonVisual Desktop Access (NVDA)
#Copyright (C) 2016 NV Access Limited
#This file is covered by the GNU General Public License.
#See the file COPYING for more details.

import wx
from wx.lib.mixins import listctrl as listmix

class AutoWidthColumnListCtrl(wx.ListCtrl, listmix.ListCtrlAutoWidthMixin):
	"""
	A list control that allows you to specify a column to resize to take up the remaining width of a wx.ListCtrl
	"""
	def __init__(self, parent, id=wx.ID_ANY, autoSizeColumnIndex="LAST", pos=wx.DefaultPosition, size=wx.DefaultSize, style=0):
		""" initialiser
			Takes the same parameter as a wx.ListCtrl with the following additions:
			autoSizeColumnIndex - defaults to "LAST" which results in the last column being resized. Pass the index of
			the column to be resized.
		"""
		wx.ListCtrl.__init__(self, parent, id, pos, size, style)
		listmix.ListCtrlAutoWidthMixin.__init__(self)
		self.setResizeColumn(autoSizeColumnIndex)

class SelectOnFocusSpinCtrl(wx.SpinCtrl):
	"""
	A spin control that automatically selects the value when the control gains focus.
	This makes editing the values quicker.
	"""
	def __init__(self, parent, id=wx.ID_ANY, value=wx.EmptyString, pos=wx.DefaultPosition, size=wx.DefaultSize, style=wx.SP_ARROW_KEYS|wx.ALIGN_RIGHT, min=0, max=100, initial=0, name=wx.SpinCtrlNameStr):
		""" initialiser - Takes the same parameters as a wx.SpinCtrl.
		"""
		wx.SpinCtrl.__init__(self, parent, id, value, pos, size, style, min, max, initial, name)
		self.Bind(wx.EVT_SET_FOCUS, self.OnSetFocus)

	def OnSetFocus(self, evt):
		numChars = len(str(self.GetValue()))
		self.SetSelection(0, numChars)
		evt.Skip()


@@ -113,4 +113,11 @@ class QuadMeshPlot(RasterPlot):
         if self.invert_axes:
             y, x = 'x', 'y'
             zdata = zdata.T
+
+        plot_opts = element.opts.get('plot', 'plotly')
+        nodata = plot_opts.kwargs.get('nodata')
+        if nodata is not None and (zdata.dtype.kind  == 'i'):
+            zdata = zdata.astype(np.float64)
+            zdata[zdata == nodata] = np.NaN
+
         return [{x: xc, y: yc, 'z': zdata}]

from __future__ import absolute_import, division, unicode_literals

import numpy as np
import param

from ...core.options import SkipRendering
from ...element import Image, Raster
from ..mixins import HeatMapMixin
from .element import ColorbarPlot

class RasterPlot(ColorbarPlot):

    padding = param.ClassSelector(default=0, class_=(int, float, tuple))

    style_opts = ['visible', 'cmap', 'alpha']

    @classmethod
    def trace_kwargs(cls, is_geo=False, **kwargs):
        return {'type': 'heatmap'}

    def graph_options(self, element, ranges, style, **kwargs):
        opts = super(RasterPlot, self).graph_options(element, ranges, style, **kwargs)
        copts = self.get_color_opts(element.vdims[0], element, ranges, style)
        opts['zmin'] = copts.pop('cmin')
        opts['zmax'] = copts.pop('cmax')
        opts['zauto'] = copts.pop('cauto')
        return dict(opts, **copts)

    def get_data(self, element, ranges, style, **kwargs):
        if isinstance(element, Image):
            l, b, r, t = element.bounds.lbrt()
        else:
            l, b, r, t = element.extents
        array = element.dimension_values(2, flat=False)
        if type(element) is Raster:
            array=array.T[::-1,...]
        ny, nx = array.shape
        dx, dy = float(r-l)/nx, float(t-b)/ny
        x0, y0 = l+dx/2., b+dy/2.
        if self.invert_axes:
            x0, y0, dx, dy = y0, x0, dy, dx
            array = array.T
        return [dict(x0=x0, y0=y0, dx=dx, dy=dy, z=array)]

class HeatMapPlot(HeatMapMixin, RasterPlot):

    def init_layout(self, key, element, ranges, **kwargs):
        layout = super(HeatMapPlot, self).init_layout(key, element, ranges)
        gridded = element.gridded
        xdim, ydim = gridded.dimensions()[:2]

        if self.invert_axes:
            xaxis, yaxis = ('yaxis', 'xaxis')
        else:
            xaxis, yaxis = ('xaxis', 'yaxis')

        shape = gridded.interface.shape(gridded, gridded=True)

        xtype = gridded.interface.dtype(gridded, xdim)
        if xtype.kind in 'SUO':
            layout[xaxis]['tickvals'] = np.arange(shape[1])
            layout[xaxis]['ticktext'] = gridded.dimension_values(0, expanded=False)

        ytype = gridded.interface.dtype(gridded, ydim)
        if ytype.kind in 'SUO':
            layout[yaxis]['tickvals'] = np.arange(shape[0])
            layout[yaxis]['ticktext'] = gridded.dimension_values(1, expanded=False)
        return layout

    def get_data(self, element, ranges, style, **kwargs):
        if not element._unique:
            self.param.warning('HeatMap element index is not unique,  ensure you '
                               'aggregate the data before displaying it, e.g. '
                               'using heatmap.aggregate(function=np.mean). '
                               'Duplicate index values have been dropped.')

        gridded = element.gridded
        xdim, ydim = gridded.dimensions()[:2]
        data = gridded.dimension_values(2, flat=False)

        xtype = gridded.interface.dtype(gridded, xdim)
        if xtype.kind in 'SUO':
            xvals = np.arange(data.shape[1]+1)-0.5
        else:
            xvals = gridded.interface.coords(gridded, xdim, edges=True, ordered=True)

        ytype = gridded.interface.dtype(gridded, ydim)
        if ytype.kind in 'SUO':
            yvals = np.arange(data.shape[0]+1)-0.5
        else:
            yvals = gridded.interface.coords(gridded, ydim, edges=True, ordered=True)

        if self.invert_axes:
            xvals, yvals = yvals, xvals
            data = data.T

        return [dict(x=xvals, y=yvals, z=data)]

class QuadMeshPlot(RasterPlot):

    def get_data(self, element, ranges, style, **kwargs):
        x, y, z = element.dimensions()[:3]
        irregular = element.interface.irregular(element, x)
        if irregular:
            raise SkipRendering("Plotly QuadMeshPlot only supports rectilinear meshes")
        xc, yc = (element.interface.coords(element, x, edges=True, ordered=True),
                  element.interface.coords(element, y, edges=True, ordered=True))
        zdata = element.dimension_values(z, flat=False)
        x, y = ('x', 'y')
        if self.invert_axes:
            y, x = 'x', 'y'
            zdata = zdata.T
        return [{x: xc, y: yc, 'z': zdata}]


@@ -91,7 +91,6 @@ def init_error_handlers(app):
         return error.render_error()

-
 class InvalidAPIUsage(Exception):
     """ General error class for the API_compat to render errors in multiple formats """
     def __init__(self, api_error, status_code=500, output_format="xml"):

from flask import render_template, make_response
from yattag import Doc
import yattag
import ujson
import collections

LastFMError = collections.namedtuple('LastFMError', ['code', 'message'])

# List of errors compatible with LastFM messages for API_compat.
class CompatError(object):
    DOES_NOT_EXIST           = LastFMError(code = 1, message = "This error does not exist")
    INVALID_SERVICE          = LastFMError(code = 2, message = "Invalid service -This service does not exist")
    INVALID_METHOD           = LastFMError(code = 3, message = "Invalid Method - No method with that name in this package")
    INVALID_TOKEN            = LastFMError(code = 4, message = "Invalid Token - Invalid authentication token supplied")
    INVALID_FORMAT           = LastFMError(code = 5, message = "Invalid format - This service doesn't exist in that format")
    INVALID_PARAMETERS       = LastFMError(code = 6, message = "Invalid parameters - " \
                                                               "Your request is missing a required parameter")
    INVALID_RESOURCE         = LastFMError(code = 7, message = "Invalid resource specified")
    OP_FAILED                = LastFMError(code = 8, message = "Operation failed - Most likely the backend service failed. " \
                                                               "Please try again.")
    INVALID_SESSION_KEY      = LastFMError(code = 9, message = "Invalid session key - Please re-authenticate")
    INVALID_API_KEY          = LastFMError(code = 10, message = "Invalid API key - You must be granted a valid key by last.fm")
    SERVICE_OFFLINE          = LastFMError(code = 11, message = "Service Offline - This service is temporarily offline. " \
                                                                "Try again later.")
    SUBSCRIBERS_ONLY         = LastFMError(code = 12, message = "Subscribers Only - This station is only available to " \
                                                                "paid last.fm subscribers")
    INVALID_METHOD_SIGNATURE = LastFMError(code = 13, message = "Invalid method signature supplied")
    UNAUTHORIZED_TOKEN       = LastFMError(code = 14, message = "Unauthorized Token - This token has not been authorized")
    TOKEN_EXPIRED            = LastFMError(code = 15, message = "This token has expired")
    SERVICE_UNAVAILABLE      = LastFMError(code = 16, message = "The service is temporarily unavailable, please try again.")
    NEED_LOGIN               = LastFMError(code = 17, message = "Login: User requires to be logged in")
    TRIAL_EXPIRED            = LastFMError(code = 18, message = "Trial Expired - This user has no free radio plays left. " \
                                                                "Subscription required.")
    DOES_NOT_EXIST_19        = LastFMError(code = 19, message = "This error does not exist")
    NOT_ENOUGH_CONTENT       = LastFMError(code = 20, message = "Not Enough Content - There is not enough content to " \
                                                                "play this station")
    NOT_ENOUGH_MEMBERS       = LastFMError(code = 21, message = "Not Enough Members - This group does not have enough members " \
                                                                "for radio")
    NOT_ENOUGH_FANS          = LastFMError(code = 22, message = "Not Enough Fans - This artist does not have enough fans " \
                                                                "for radio")
    NOT_ENOUGH_NEIGHBOURS    = LastFMError(code = 23, message = "Not Enough Neighbours - There are not enough neighbours " \
                                                                "for radio")
    NO_PEAK_RADIO            = LastFMError(code = 24, message = "No Peak Radio - This user is not allowed to listen to " \
                                                                "radio during peak usage")
    RADIO_NOT_FOUND          = LastFMError(code = 25, message = "Radio Not Found - Radio station not found")
    API_KEY_SUSPENDED        = LastFMError(code = 26, message = "API Key Suspended - This application is not allowed to make "
                                                                "requests to the web services")
    DEPRECATED               = LastFMError(code = 27, message = "Deprecated - This type of request is no longer supported")
    RATE_LIMIT_EXCEEDED      = LastFMError(code = 29, message = "Rate Limit Exceded - Your IP has made too many requests in " \
                                                                "exceeding our API guidelines")

def init_error_handlers(app):

    def error_wrapper(template, error, code):
        resp = make_response(render_template(template, error=error))
        resp.headers['Access-Control-Allow-Origin'] = '*'
        return resp, code

    @app.errorhandler(400)
    def bad_request(error):
        return error_wrapper('errors/400.html', error, 400)

    @app.errorhandler(401)
    def unauthorized(error):
        return error_wrapper('errors/401.html', error, 401)

    @app.errorhandler(403)
    def forbidden(error):
        return error_wrapper('errors/403.html', error, 403)

    @app.errorhandler(404)
    def not_found(error):
        return error_wrapper('errors/404.html', error, 404)

    @app.errorhandler(413)
    def file_size_too_large(error):
        return error_wrapper('errors/413.html', error, 413)

    @app.errorhandler(500)
    def internal_server_error(error):
        return error_wrapper('errors/500.html', error, 500)

    @app.errorhandler(503)
    def service_unavailable(error):
        return error_wrapper('errors/503.html', error, 503)

    # Handle error of API_compat
    @app.errorhandler(InvalidAPIUsage)
    def handle_api_compat_error(error):
        return error.render_error()

class InvalidAPIUsage(Exception):
    """ General error class for the API_compat to render errors in multiple formats """
    def __init__(self, api_error, status_code=500, output_format="xml"):
        Exception.__init__(self)
        self.api_error = api_error
        self.status_code = status_code
        self.output_format = output_format

    def render_error(self):
        return {
            "json": self.to_json,
            "xml": self.to_xml
        }.get(self.output_format, self.to_xml)()

    def to_json(self):
        return ujson.dumps({
            "error": self.api_error.code,
            "message": self.api_error.message
        }, indent=4)

    def to_xml(self):
        doc, tag, text = Doc().tagtext()
        with tag('lfm', status="failed"):
            with tag('error', code=self.api_error.code):
                text(self.api_error.message)
        return '<?xml version="1.0" encoding="utf-8"?>\n' + yattag.indent(doc.getvalue())


@@ -67,6 +67,6 @@ t = TLSClientAutomaton(client_hello=ch,
                        resumption_master_secret=args.res_master,
                        session_ticket_file_in=args.session_ticket_file_in,
                        session_ticket_file_out=args.session_ticket_file_out,
-                      )
+                       debug=args.debug)
 t.run()

#!/usr/bin/env python

## This file is part of Scapy
## This program is published under a GPLv2 license

"""
Basic TLS client. A ciphersuite may be commanded via a first argument.
Default protocol version is TLS 1.3.
"""

import os
import sys

basedir = os.path.abspath(os.path.join(os.path.dirname(__file__),"../../"))
sys.path=[basedir]+sys.path

from scapy.layers.tls.automaton_cli import TLSClientAutomaton
from scapy.layers.tls.basefields import _tls_version_options
from scapy.layers.tls.handshake import TLSClientHello, TLS13ClientHello

from argparse import ArgumentParser

psk = None
parser = ArgumentParser(description='Simple TLS Client')
parser.add_argument("--psk",
                    help="External PSK for symmetric authentication (for TLS 1.3)")  # noqa: E501
parser.add_argument("--no_pfs", action="store_true",
                    help="Disable (EC)DHE exchange with PFS")
parser.add_argument("--ciphersuite", help="Ciphersuite preference")
parser.add_argument("--version", help="TLS Version", default="tls13")
parser.add_argument("--ticket_in", dest='session_ticket_file_in',
                    help="File to read a ticket from (for TLS 1.3)")
parser.add_argument("--ticket_out", dest='session_ticket_file_out',
                    help="File to write a ticket to (for TLS 1.3)")
parser.add_argument("--res_master",
                    help="Resumption master secret (for TLS 1.3)")

args = parser.parse_args()

# By default, PFS is set
if args.no_pfs:
    psk_mode = "psk_ke"
else:
    psk_mode = "psk_dhe_ke"

v = _tls_version_options.get(args.version, None)
if not v:
    sys.exit("Unrecognized TLS version option.")

if args.ciphersuite:
    ciphers = int(args.ciphersuite, 16)
    if ciphers not in list(range(0x1301, 0x1306)):
        ch = TLSClientHello(ciphers=ciphers)
    else:
        ch = TLS13ClientHello(ciphers=ciphers)
else:
    ch = None

t = TLSClientAutomaton(client_hello=ch,
                       version=args.version,
                       mycert=basedir+"/test/tls/pki/cli_cert.pem",
                       mykey=basedir+"/test/tls/pki/cli_key.pem",
                       psk=args.psk,
                       psk_mode=psk_mode,
                       resumption_master_secret=args.res_master,
                       session_ticket_file_in=args.session_ticket_file_in,
                       session_ticket_file_out=args.session_ticket_file_out,
                      )
t.run()


@@ -24,7 +24,11 @@ from databricks.koalas.dask.compatibility import string_types

 class Metadata(object):
     """
-    Manages column names and index information
+    Manages column names and index information.
+
+    :ivar _column_fields: list of the Spark field names to be seen as columns in Koalas DataFrame.
+    :ivar _index_info: list of pair holding the Spark field names for indexes,
+                       and the index name to be seen in Koalas DataFrame.
     """

     def __init__(self, column_fields, index_info=None):

#
# Copyright (C) 2019 Databricks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
A metadata to manage indexes.
"""
import pandas as pd

from databricks.koalas.dask.compatibility import string_types

class Metadata(object):
    """
    Manages column names and index information
    """

    def __init__(self, column_fields, index_info=None):
        """ Create a new metadata to manage column fields and index fields and names.

        :param column_fields: list of string
                              Field names to appear as columns.
        :param index_info: list of string pair
                           Each pair holds the index field name which exists in Spark fields,
                           and the index name.
        """
        assert all(isinstance(col, string_types) for col in column_fields)
        assert index_info is None \
            or all(isinstance(index_field, string_types)
                   and (index_name is None or isinstance(index_name, string_types))
                   for index_field, index_name in index_info)
        self._column_fields = column_fields
        self._index_info = index_info or []

    @property
    def column_fields(self):
        return self._column_fields

    @property
    def index_info(self):
        return self._index_info

    @property
    def index_fields(self):
        return [index_field for index_field, _ in self._index_info]

    @property
    def index_names(self):
        return [name for _, name in self._index_info]

    @property
    def all_fields(self):
        index_fields = self.index_fields
        return index_fields + [field for field in self._column_fields
                               if field not in index_fields]

    def copy(self, column_fields=None, index_info=None):
        if column_fields is None:
            column_fields = self._column_fields
        if index_info is None:
            index_info = self._index_info
        return Metadata(column_fields=column_fields.copy(), index_info=index_info.copy())

    @staticmethod
    def from_pandas(pdf):
        column_fields = [str(col) for col in pdf.columns]
        index = pdf.index
        if isinstance(index, pd.MultiIndex):
            if index.names is None:
                index_info = [('__index_level_{}__'.format(i), None)
                              for i in range(len(index.levels))]
            else:
                index_info = [('__index_level_{}__'.format(i) if name is None else name, name)
                              for i, name in enumerate(index.names)]
        else:
            index_info = [(index.name
                          if index.name is not None else '__index_level_0__', index.name)]

        return Metadata(column_fields=column_fields, index_info=index_info)


@@ -63,13 +63,13 @@ def test_no_cache():
                    names=["A", "B", "C"])
     assert dt0.names == ("A", "B", "C")
     assert dt0.ltypes == (dt.ltype.real, dt.ltype.str, dt.ltype.int)
-    assert dt0.stypes == (dt.float64, dt.str32, dt.int8)
+    assert dt0.stypes == (dt.float64, dt.str32, dt.int32)
     assert dt0.colindex("B") == 1
     frame_integrity_check(dt0)
     dt0.key = "C"
     assert dt0.names == ("C", "A", "B")
     assert dt0.ltypes == (dt.ltype.int, dt.ltype.real, dt.ltype.str)
-    assert dt0.stypes == (dt.int8, dt.float64, dt.str32)
+    assert dt0.stypes == (dt.int32, dt.float64, dt.str32)
     assert dt0.colindex("B") == 2
     frame_integrity_check(dt0)

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# Copyright 2018 H2O.ai
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#-------------------------------------------------------------------------------
import datatable as dt
import pytest
import random
from datatable.internal import frame_integrity_check
from tests import isview

def test_keys_simple():
    dt0 = dt.Frame([["Joe", "Mary", "Leslie", "Adam", "Alice"],
                    [1, 5, 15, 12, 8],
                    [3.6, 9.78, 2.01, -4.23, 5.3819]],
                   names=["name", "sex", "avg"])
    assert dt0.key == tuple()
    dt0.key = "name"
    frame_integrity_check(dt0)
    assert dt0.key == ("name",)
    assert dt0.shape == (5, 3)
    assert dt0.names == ("name", "sex", "avg")
    assert dt0.to_list() == [["Adam", "Alice", "Joe", "Leslie", "Mary"],
                             [12, 8, 1, 15, 5],
                             [-4.23, 5.3819, 3.6, 2.01, 9.78]]
    dt0.key = "sex"
    frame_integrity_check(dt0)
    assert not isview(dt0)
    assert dt0.key == ("sex",)
    assert dt0.shape == (5, 3)
    assert dt0.names == ("sex", "name", "avg")
    assert dt0.to_list() == [[1, 5, 8, 12, 15],
                             ["Joe", "Mary", "Alice", "Adam", "Leslie"],
                             [3.6, 9.78, 5.3819, -4.23, 2.01]]
    dt0.key = None
    assert dt0.key == tuple()

def test_no_cache():
    # Check that assigning a key properly disposes of potentially cached
    # types / names of the Frame
    dt0 = dt.Frame([[1.1] * 4, list("ABCD"), [3, 5, 2, 1]],
                   names=["A", "B", "C"])
    assert dt0.names == ("A", "B", "C")
    assert dt0.ltypes == (dt.ltype.real, dt.ltype.str, dt.ltype.int)
    assert dt0.stypes == (dt.float64, dt.str32, dt.int8)
    assert dt0.colindex("B") == 1
    frame_integrity_check(dt0)
    dt0.key = "C"
    assert dt0.names == ("C", "A", "B")
    assert dt0.ltypes == (dt.ltype.int, dt.ltype.real, dt.ltype.str)
    assert dt0.stypes == (dt.int8, dt.float64, dt.str32)
    assert dt0.colindex("B") == 2
    frame_integrity_check(dt0)

def test_multi_key():
    dt0 = dt.Frame(D=range(6), A=[3, 7, 5, 2, 2, 3], B=[1, 2, 2, 3, 4, 4])
    dt0.key = ["A", "B"]
    frame_integrity_check(dt0)
    assert dt0.key == ("A", "B")
    assert dt0.names == ("A", "B", "D")
    assert dt0.to_list() == [[2, 2, 3, 3, 5, 7],
                             [3, 4, 1, 4, 2, 2],
                             [3, 4, 0, 5, 2, 1]]

def test_key_invalid1():
    dt0 = dt.Frame(A=range(5), B=[3] * 5)
    with pytest.raises(TypeError) as e:
        dt0.key = 0
    assert ("Key should be a column name, or a list/tuple of column names"
            in str(e.value))
    with pytest.raises(TypeError) as e:
        dt0.key = ["A", None]
    assert ("Key should be a list/tuple of column names, instead element 1 "
            "was a <class 'NoneType'>" == str(e.value))

def test_key_invalid2():
    dt0 = dt.Frame([["Joe", "Mary", "Leslie", "Adam", "Alice"],
                    [7, 9, 2, 2, 7],
                    [3, 4, 5, 3, 4]],
                   names=["name", "A", "B"])
    with pytest.raises(ValueError) as e:
        dt0.key = "A"
    assert ("the values are not unique" in str(e.value))

    dt0.key = ["A", "B"]
    assert dt0.key == ("A", "B")
    assert dt0.names == ("A", "B", "name")
    assert dt0.to_list() == [[2, 2, 7, 7, 9],
                             [3, 5, 3, 4, 4],
                             ["Adam", "Leslie", "Joe", "Alice", "Mary"]]

    with pytest.raises(ValueError) as e:
        dt0.key = "B"
    assert ("the values are not unique" in str(e.value))
    # Check that the columns don't get reordered during a bad key assignment
    assert dt0.key == ("A", "B")
    assert dt0.names == ("A", "B", "name")

def test_key_duplicate():
    dt0 = dt.Frame(A=range(5))
    with pytest.raises(ValueError) as e:
        dt0.key = ("A", "A")
    assert ("Column `A` is specified multiple times within the key" ==
            str(e.value))

def test_set_empty_key():
    dt0 = dt.Frame([range(5), [None] * 5], names=["A", "B"])
    dt0.key = []
    assert dt0.key == tuple()
    dt0.key = "A"
    assert dt0.key == ("A",)
    dt0.key = []
    frame_integrity_check(dt0)
    assert dt0.key == tuple()
    assert dt0.names == ("A", "B")

def test_key_save(tempfile):
    dt0 = dt.Frame(D=range(6), A=[3, 7, 5, 2, 2, 3], B=[1, 2, 2, 3, 4, 4])
    dt0.key = ["A", "B"]
    frame_integrity_check(dt0)
    dt0.to_jay(tempfile)
    dt1 = dt.open(tempfile)
    assert dt1.key == ("A", "B")
    frame_integrity_check(dt1)

def test_key_after_group():
    n = 1000
    DT = dt.Frame(A=[random.choice("abcd") for _ in range(n)])
    tmp = DT[:, dt.count(), dt.by(0)]
    frame_integrity_check(tmp)
    tmp.key = "A"
    assert tmp.to_list()[0] == ["a", "b", "c", "d"]
    assert sum(tmp.to_list()[1]) == n


@@ -117,6 +117,7 @@ class MaskTestMixin(object):
             segm_result = [[] for _ in range(self.mask_head.num_classes - 1)]
         else:
             aug_masks = []
+            mask_roi_extractor = self.mask_roi_extractor
             for x, img_meta in zip(feats, img_metas):
                 img_shape = img_meta[0]['img_shape']
                 scale_factor = img_meta[0]['scale_factor']

from mmdet.core import (bbox2roi, bbox_mapping, merge_aug_proposals,
                        merge_aug_bboxes, merge_aug_masks, multiclass_nms)

class RPNTestMixin(object):

    def simple_test_rpn(self, x, img_meta, rpn_test_cfg):
        rpn_outs = self.rpn_head(x)
        proposal_inputs = rpn_outs + (img_meta, rpn_test_cfg)
        proposal_list = self.rpn_head.get_proposals(*proposal_inputs)
        return proposal_list

    def aug_test_rpn(self, feats, img_metas, rpn_test_cfg):
        imgs_per_gpu = len(img_metas[0])
        aug_proposals = [[] for _ in range(imgs_per_gpu)]
        for x, img_meta in zip(feats, img_metas):
            proposal_list = self.simple_test_rpn(x, img_meta, rpn_test_cfg)
            for i, proposals in enumerate(proposal_list):
                aug_proposals[i].append(proposals)
        # after merging, proposals will be rescaled to the original image size
        merged_proposals = [
            merge_aug_proposals(proposals, img_meta, rpn_test_cfg)
            for proposals, img_meta in zip(aug_proposals, img_metas)
        ]
        return merged_proposals

class BBoxTestMixin(object):

    def simple_test_bboxes(self,
                           x,
                           img_meta,
                           proposals,
                           rcnn_test_cfg,
                           rescale=False):
        """Test only det bboxes without augmentation."""
        rois = bbox2roi(proposals)
        roi_feats = self.bbox_roi_extractor(
            x[:len(self.bbox_roi_extractor.featmap_strides)], rois)
        cls_score, bbox_pred = self.bbox_head(roi_feats)
        img_shape = img_meta[0]['img_shape']
        scale_factor = img_meta[0]['scale_factor']
        det_bboxes, det_labels = self.bbox_head.get_det_bboxes(
            rois,
            cls_score,
            bbox_pred,
            img_shape,
            scale_factor,
            rescale=rescale,
            nms_cfg=rcnn_test_cfg)
        return det_bboxes, det_labels

    def aug_test_bboxes(self, feats, img_metas, proposal_list, rcnn_test_cfg):
        aug_bboxes = []
        aug_scores = []
        for x, img_meta in zip(feats, img_metas):
            # only one image in the batch
            img_shape = img_meta[0]['img_shape']
            scale_factor = img_meta[0]['scale_factor']
            flip = img_meta[0]['flip']
            # TODO more flexible
            proposals = bbox_mapping(proposal_list[0][:, :4], img_shape,
                                     scale_factor, flip)
            rois = bbox2roi([proposals])
            # recompute feature maps to save GPU memory
            roi_feats = self.bbox_roi_extractor(
                x[:len(self.bbox_roi_extractor.featmap_strides)], rois)
            cls_score, bbox_pred = self.bbox_head(roi_feats)
            bboxes, scores = self.bbox_head.get_det_bboxes(
                rois,
                cls_score,
                bbox_pred,
                img_shape,
                scale_factor,
                rescale=False,
                nms_cfg=None)
            aug_bboxes.append(bboxes)
            aug_scores.append(scores)
        # after merging, bboxes will be rescaled to the original image size
        merged_bboxes, merged_scores = merge_aug_bboxes(
            aug_bboxes, aug_scores, img_metas, self.test_cfg.rcnn)
        det_bboxes, det_labels = multiclass_nms(
            merged_bboxes, merged_scores, self.test_cfg.rcnn.score_thr,
            self.test_cfg.rcnn.nms_thr, self.test_cfg.rcnn.max_per_img)
        return det_bboxes, det_labels

class MaskTestMixin(object):

    def simple_test_mask(self,
                         x,
                         img_meta,
                         det_bboxes,
                         det_labels,
                         rescale=False):
        # image shape of the first image in the batch (only one)
        ori_shape = img_meta[0]['ori_shape']
        scale_factor = img_meta[0]['scale_factor']
        if det_bboxes.shape[0] == 0:
            segm_result = [[] for _ in range(self.mask_head.num_classes - 1)]
        else:
            # if det_bboxes is rescaled to the original image size, we need to
            # rescale it back to the testing scale to obtain RoIs.
            _bboxes = (det_bboxes[:, :4] * scale_factor
                       if rescale else det_bboxes)
            mask_rois = bbox2roi([_bboxes])
            mask_feats = self.mask_roi_extractor(
                x[:len(self.mask_roi_extractor.featmap_strides)], mask_rois)
            mask_pred = self.mask_head(mask_feats)
            segm_result = self.mask_head.get_seg_masks(
                mask_pred, _bboxes, det_labels, self.test_cfg.rcnn, ori_shape,
                scale_factor, rescale)
        return segm_result

    def aug_test_mask(self, feats, img_metas, det_bboxes, det_labels):
        if det_bboxes.shape[0] == 0:
            segm_result = [[] for _ in range(self.mask_head.num_classes - 1)]
        else:
            aug_masks = []
            for x, img_meta in zip(feats, img_metas):
                img_shape = img_meta[0]['img_shape']
                scale_factor = img_meta[0]['scale_factor']
                flip = img_meta[0]['flip']
                _bboxes = bbox_mapping(det_bboxes[:, :4], img_shape,
                                       scale_factor, flip)
                mask_rois = bbox2roi([_bboxes])
                mask_feats = self.mask_roi_extractor(
                    x[:len(self.mask_roi_extractor.featmap_strides)],
                    mask_rois)
                mask_pred = self.mask_head(mask_feats)
                # convert to numpy array to save memory
                aug_masks.append(mask_pred.sigmoid().cpu().numpy())
            merged_masks = merge_aug_masks(aug_masks, img_metas,
                                           self.test_cfg.rcnn)

            ori_shape = img_metas[0][0]['ori_shape']
            segm_result = self.mask_head.get_seg_masks(
                merged_masks,
                det_bboxes,
                det_labels,
                self.test_cfg.rcnn,
                ori_shape,
                scale_factor=1.0,
                rescale=False)
        return segm_result


@@ -83,6 +83,12 @@ class GenericBaseModel(CloudFormationModel):
         """Return template configurations used to create the final API requests (implemented by subclasses)."""
         pass

+    # TODO: rework to normal instance method when resources aren't mutated in different place anymore
+    @staticmethod
+    def add_defaults(resource, stack_name: str):
+        """Set any defaults required, including auto-generating names. Must be called before deploying the resource"""
+        pass
+
     # ----------------------
     # GENERIC BASE METHODS
     # ----------------------

import logging

# TODO: remove
from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
from moto.core.models import CloudFormationModel

from localstack.utils.aws import aws_stack
from localstack.utils.common import camel_to_snake_case

LOG = logging.getLogger(__name__)

# dict key used to store the deployment state of a resource
KEY_RESOURCE_STATE = "_state_"

# ref attribute definitions
REF_ATTRS = ["PhysicalResourceId", "Ref"]
REF_ID_ATTRS = REF_ATTRS + ["Id"]
REF_ARN_ATTRS = ["Ref", "Arn"]

class DependencyNotYetSatisfied(Exception):
    """Exception indicating that a resource dependency is not (yet) deployed/available."""

    def __init__(self, resource_ids, message=None):
        message = message or "Unresolved dependencies: %s" % resource_ids
        super(DependencyNotYetSatisfied, self).__init__(message)
        resource_ids = resource_ids if isinstance(resource_ids, list) else [resource_ids]
        self.resource_ids = resource_ids

class GenericBaseModel(CloudFormationModel):
    """Abstract base class representing a resource model class in LocalStack.
    This class keeps references to a combination of (1) the CF resource
    properties (as defined in the template), and (2) the current deployment
    state of a resource.

    Concrete subclasses will implement convenience methods to manage resources,
    e.g., fetching the latest deployment state, getting the resource name, etc.
    """

    def __init__(self, resource_json, region_name=None, **params):
        self.region_name = region_name or aws_stack.get_region()
        self.resource_json = resource_json
        self.resource_type = resource_json["Type"]
        # Properties, as defined in the resource template
        self.properties = resource_json["Properties"] = resource_json.get("Properties") or {}
        # State, as determined from the deployed resource; use a special dict key here to keep
        # track of state changes within resource_json (this way we encapsulate all state details
        # in `resource_json` and the changes will survive creation of multiple instances of this class)
        self.state = resource_json[KEY_RESOURCE_STATE] = resource_json.get(KEY_RESOURCE_STATE) or {}

    # ----------------------
    # ABSTRACT BASE METHODS
    # ----------------------

    def get_resource_name(self):
        """Return the name of this resource, based on its properties (to be overwritten by subclasses)"""
        return None

    def get_physical_resource_id(self, attribute=None, **kwargs):
        """Determine the physical resource ID (Ref) of this resource (to be overwritten by subclasses)"""
        return None

    # TODO: change the signature to pass in a Stack instance (instead of stack_name and resources)
    def fetch_state(self, stack_name, resources):
        """Fetch the latest deployment state of this resource, or return None if not currently deployed."""
        return None

    # TODO: change the signature to pass in a Stack instance (instead of stack_name and resources)
    def update_resource(self, new_resource, stack_name, resources):
        """Update the deployment of this resource, using the updated properties (implemented by subclasses)."""
        # TODO: evaluate if we can add a generic implementation here, using "update" parameters from
        # get_deploy_templates() responses, and based on checking whether resource attributes have changed
        pass

    @classmethod
    def cloudformation_type(cls):
        """Return the CloudFormation resource type name, e.g., "AWS::S3::Bucket" (implemented by subclasses)."""
        return super(GenericBaseModel, cls).cloudformation_type()

    @staticmethod
    def get_deploy_templates():
        """Return template configurations used to create the final API requests (implemented by subclasses)."""
        pass

    # ----------------------
    # GENERIC BASE METHODS
    # ----------------------

    def get_cfn_attribute(self, attribute_name):
        """Retrieve the given CF attribute for this resource (inherited from moto's CloudFormationModel)"""
        if attribute_name in REF_ARN_ATTRS and hasattr(self, "arn"):
            return self.arn
        if attribute_name in REF_ATTRS:
            result = self.get_physical_resource_id(attribute=attribute_name)
            if result:
                return result
        props = self.props
        if attribute_name in props:
            return props.get(attribute_name)

        raise UnformattedGetAttTemplateException()

    # ---------------------
    # GENERIC UTIL METHODS
    # ---------------------

    def fetch_and_update_state(self, *args, **kwargs):
        from localstack.utils.cloudformation import template_deployer

        try:
            state = self.fetch_state(*args, **kwargs)
            self.update_state(state)
            return state
        except Exception as e:
            if not template_deployer.check_not_found_exception(
                e, self.resource_type, self.properties
            ):
                LOG.debug("Unable to fetch state for resource %s: %s" % (self, e))

    def fetch_state_if_missing(self, *args, **kwargs):
        if not self.state:
            self.fetch_and_update_state(*args, **kwargs)
        return self.state

    def set_resource_state(self, state):
        """Set the deployment state of this resource."""
        self.state = state or {}

    def update_state(self, details):
        """Update the deployment state of this resource (existing attributes will be overwritten)."""
        details = details or {}
        self.state.update(details)
        return self.props

    @property
    def physical_resource_id(self):
        """Return the (cached) physical resource ID."""
        return self.resource_json.get("PhysicalResourceId")

    @property
    def logical_resource_id(self):
        """Return the logical resource ID."""
        return self.resource_json.get("LogicalResourceId")

    @property
    def props(self):
        """Return a copy of (1) the resource properties (from the template), combined with
        (2) the current deployment state properties of the resource."""
        result = dict(self.properties)
        result.update(self.state or {})
        return result

    @property
    def resource_id(self):
        """Return the logical resource ID of this resource (i.e., the ref. name within the stack's resources)."""
        return self.resource_json["LogicalResourceId"]

    @classmethod
    def update_from_cloudformation_json(
        cls, original_resource, new_resource_name, cloudformation_json, region_name
    ):
        props = cloudformation_json.get("Properties", {})
        for key, val in props.items():
            snake_key = camel_to_snake_case(key)
            lower_key = key.lower()
            for candidate in [key, lower_key, snake_key]:
                if hasattr(original_resource, candidate) or candidate == snake_key:
                    setattr(original_resource, candidate, val)
                    break
        return original_resource

    @classmethod
    def create_from_cloudformation_json(cls, resource_name, resource_json, region_name):
        return cls(
            resource_name=resource_name,
            resource_json=resource_json,
            region_name=region_name,
        )

    @classmethod
    def resolve_refs_recursively(cls, stack_name, value, resources):
        # TODO: restructure code to avoid circular import here
        from localstack.utils.cloudformation.template_deployer import resolve_refs_recursively

        return resolve_refs_recursively(stack_name, value, resources)


@@ -41,6 +41,18 @@ class WorkflowValidator:
         """
         self.config = config
         self.physical_key = physical_key
+        self.loaded_schemas_by_id = {}
+        self.loaded_schemas = {}
+
+    @staticmethod
+    def get_config_data_version_str(data: dict) -> str:
+        version_obj = data['version']
+        assert isinstance(version_obj, (str, dict))
+        return version_obj if isinstance(version_obj, str) else version_obj['base']
+
+    @classmethod
+    def is_supported_config_data_version(cls, version: ConfigDataVersion):
+        return cls.CONFIG_DATA_VERSION >= version

     @classmethod
     def load(cls, pk: util.PhysicalKey):

import functools
import json

import botocore.exceptions
import jsonschema
import pkg_resources
import yaml

from quilt3.data_transfer import get_bytes_and_effective_pk

from .. import util
from ..backends import PackageRegistry

@functools.lru_cache(maxsize=None)
def _get_conf_validator():
    schema = json.loads(pkg_resources.resource_string(__name__, 'config-1.schema.json'))
    return jsonschema.Draft7Validator(schema).validate

SUPPORTED_META_SCHEMAS = {
    'http://json-schema.org/draft-07/schema#': jsonschema.Draft7Validator,
}

def _schema_load_object_hook(o):
    if '$ref' in o:
        raise util.QuiltException("Currently we don't support $ref in schema.")
    return o

_load_schema_json = json.JSONDecoder(object_hook=_schema_load_object_hook).decode

class WorkflowValidator:
    def __init__(self, config: dict, physical_key: util.PhysicalKey):
        """
        Args:
            config: validated workflow config or `None` if there is no config
            physical_key: from where config was loaded
        """
        self.config = config
        self.physical_key = physical_key

    @classmethod
    def load(cls, pk: util.PhysicalKey):
        data = None
        try:
            data, pk = get_bytes_and_effective_pk(pk)
        except FileNotFoundError:
            pass
        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] != 'NoSuchKey':
                raise util.QuiltException(f"Couldn't load workflows config. {e}.")
        if data is None:
            return

        try:
            # TODO: raise if objects contain duplicate properties
            config = yaml.safe_load(data.decode())
        except yaml.YAMLError as e:
            raise util.QuiltException("Couldn't parse workflows config as YAML.") from e
        conf_validator = _get_conf_validator()
        try:
            conf_validator(config)
        except jsonschema.ValidationError as e:
            raise util.QuiltException(f'Workflows config failed validation: {e.message}.') from e

        return cls(config, pk)

    def validate(self, workflow, meta, message):
        if workflow is ...:
            workflow = self.config.get('default_workflow')

        result = {
            'id': workflow,
            'config': str(self.physical_key),
        }
        if workflow is None:
            if self.config.get('is_workflow_required', True):
                raise util.QuiltException('Workflow required, but none specified.')
            return result

        workflows_data = self.config['workflows']
        if workflow not in workflows_data:
            raise util.QuiltException(f'There is no {workflow!r} workflow in config.')
        workflow_data = workflows_data[workflow]
        metadata_schema_id = workflow_data.get('metadata_schema')
        if metadata_schema_id:
            schemas = self.config.get('schemas', {})
            if metadata_schema_id not in schemas:
                raise util.QuiltException(f'There is no {metadata_schema_id!r} in schemas.')
            schema_url = schemas[metadata_schema_id]['url']
            try:
                schema_pk = util.PhysicalKey.from_url(schema_url)
            except util.URLParseError as e:
                raise util.QuiltException(f"Couldn't parse URL {schema_url!r}. {e}.")
            if schema_pk.is_local() and not self.physical_key.is_local():
                raise util.QuiltException(f"Local schema {str(schema_pk)!r} can't be used on the remote registry.")

            handled_exception = (OSError if schema_pk.is_local() else botocore.exceptions.ClientError)
            try:
                schema_data, schema_pk_to_store = get_bytes_and_effective_pk(schema_pk)
            except handled_exception as e:
                raise util.QuiltException(f"Couldn't load schema at {schema_pk}. {e}.")
            try:
                schema = _load_schema_json(schema_data.decode())
            except json.JSONDecodeError as e:
                raise util.QuiltException(f"Couldn't parse {schema_pk} as JSON. {e}.")

            validator_cls = jsonschema.Draft7Validator
            if isinstance(schema, dict) and '$schema' in schema:
                meta_schema = schema['$schema']
                if not isinstance(meta_schema, str):
                    raise util.QuiltException('$schema must be a string.')
                validator_cls = SUPPORTED_META_SCHEMAS.get(meta_schema)
                if validator_cls is None:
                    raise util.QuiltException(f"Unsupported meta-schema: {meta_schema}.")
            try:
                jsonschema.validate(meta, schema, cls=validator_cls)
            except jsonschema.ValidationError as e:
                raise util.QuiltException(f"Metadata failed validation: {e.message}.")
            result['schemas'] = {metadata_schema_id: str(schema_pk_to_store)}
        if workflow_data.get('is_message_required', False) and not message:
            raise util.QuiltException('Commit message is required by workflow, but none was provided.')

        return result

def validate(*, registry: PackageRegistry, workflow, meta, message):
    # workflow is ... => no workflow provided by user;
    # workflow is None => don't use any workflow.
    if not (workflow in (None, ...) or isinstance(workflow, str)):
        raise TypeError

    workflow_validator = registry.get_workflow_validator()
    if workflow_validator is None:
        if workflow is ...:
            return
        raise util.QuiltException(f'{workflow!r} workflow is specified, but no workflows config exist.')

    return workflow_validator.validate(workflow, meta, message)


@@ -22,12 +22,7 @@
 from invenio.ext.assets import Bundle

 js = Bundle(
-    "vendors/jquery/dist/jquery.js",
-    "vendors/bootstrap/dist/js/bootstrap.js",
     "vendors/jquery-ui/jquery-ui.js",
-    "vendors/jquery.jeditable/index.js",
-    "vendors/jquery.hotkeys/jquery.hotkeys.js",
-    "vendors/json2/json2.js",
     "js/editor/refextract.js",
     "js/editor/display.js",
     "js/editor/engine.js",

# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""Editor bundles."""

from invenio.ext.assets import Bundle

js = Bundle(
    "vendors/jquery/dist/jquery.js",
    "vendors/bootstrap/dist/js/bootstrap.js",
    "vendors/jquery-ui/jquery-ui.js",
    "vendors/jquery.jeditable/index.js",
    "vendors/jquery.hotkeys/jquery.hotkeys.js",
    "vendors/json2/json2.js",
    "js/editor/refextract.js",
    "js/editor/display.js",
    "js/editor/engine.js",
    "js/editor/keys.js",
    "js/editor/menu.js",
    "js/editor/holdingpen.js",
    "js/editor/marcxml.js",
    "js/editor/clipboard.js",
    output="editor.js",
    weight=51,
)

styles = Bundle(
    "css/editor/base.css",
    "vendors/jquery-ui/themes/redmond/jquery-ui.css",
    output="editor.css",
    filters="cleancss",
    weight=51
)


@@ -60,24 +60,23 @@ def test_generate_thumbnail(
         body=input_file.read_bytes(),
         status=200
     )
-
     # Create the lambda request event
-    event = _make_event({"url": url, "size": thumb_size})
-
+    event = _make_event({"url": url, **params})
     # Get the response
     response = lambda_handler(event, None)
-
     # Assert the request was handled with no errors
     assert response["statusCode"] == 200
-
     # Parse the body / the returned thumbnail
     body = json.loads(read_body(response))
-
-    # Assert basic metadata was fill properly
-    assert body["info"]["original_size"] == expected_original_size
+    # Assert basic metadata was filled properly
+    if expected_original_size:  # PDFs don't have an expected size
+        assert body["info"]["original_size"] == expected_original_size
     assert body["info"]["thumbnail_size"] == expected_thumb_size
-
     # Assert the produced image is the same as the expected
-    actual = AICSImage(base64.b64decode(body['thumbnail'])).reader.data
-    expected = AICSImage(data_dir / expected_thumb).reader.data
+    if params.get('input') == 'pdf':
+        actual = Image.open(BytesIO(base64.b64decode(body['thumbnail'])))
+        expected = Image.open(data_dir / expected_thumb)
+    else:
+        actual = AICSImage(base64.b64decode(body['thumbnail'])).reader.data
+        expected = AICSImage(data_dir / expected_thumb).reader.data
     assert np.array_equal(actual, expected)

import base64
import json
from pathlib import Path

import numpy as np
import pytest
import responses
from aicsimageio import AICSImage

from t4_lambda_shared.utils import read_body

from ..index import lambda_handler

@pytest.fixture
def data_dir():
    return Path(__file__).parent / 'data'

def _make_event(query, headers=None):
    return {
        'httpMethod': 'POST',
        'path': '/foo',
        'pathParameters': {},
        'queryStringParameters': query or None,
        'headers': headers or None,
        'body': None,
        'isBase64Encoded': False,
    }

@responses.activate
@pytest.mark.parametrize("input_file, thumb_size, expected_thumb, expected_original_size, expected_thumb_size", [
    ("penguin.jpg", "w256h256", "penguin-256.jpg", [1526, 1290, 3], [217, 256]),
    ("cell.tiff", "w640h480", "cell-480.png", [15, 1, 158, 100], [514, 480]),
    ("cell.png", "w64h64", "cell-64.png", [168, 104, 3], [39, 64]),
    ("sat_greyscale.tiff", "w640h480", "sat_greyscale-480.png", [512, 512], [480, 480]),
    ("generated.ome.tiff", "w256h256", "generated-256.png", [6, 36, 76, 68], [224, 167]),
    ("sat_rgb.tiff", "w256h256", "sat_rgb-256.png", [256, 256, 4], [256, 256]),
    ("single_cell.ome.tiff", "w256h256", "single_cell.png", [6, 40, 152, 126], [256, 205]),
    # Test for statusCode error
    pytest.param("cell.png", "w1h1", None, None, None, marks=pytest.mark.raises(exception=AssertionError))
])
def test_generate_thumbnail(
    data_dir,
    input_file,
    thumb_size,
    expected_thumb,
    expected_original_size,
    expected_thumb_size
):
    # Resolve the input file path
    input_file = data_dir / input_file

    # Mock the request
    url = f"https://example.com/{input_file}"
    responses.add(
        responses.GET,
        url=url,
        body=input_file.read_bytes(),
        status=200
    )

    # Create the lambda request event
    event = _make_event({"url": url, "size": thumb_size})

    # Get the response
    response = lambda_handler(event, None)

    # Assert the request was handled with no errors
    assert response["statusCode"] == 200

    # Parse the body / the returned thumbnail
    body = json.loads(read_body(response))

    # Assert basic metadata was fill properly
    assert body["info"]["original_size"] == expected_original_size
    assert body["info"]["thumbnail_size"] == expected_thumb_size

    # Assert the produced image is the same as the expected
    actual = AICSImage(base64.b64decode(body['thumbnail'])).reader.data
    expected = AICSImage(data_dir / expected_thumb).reader.data
    assert np.array_equal(actual, expected)


@@ -95,17 +95,29 @@ class Chooser(urwid.WidgetWrap, layoutwidget.LayoutWidget):
         self.master = master
         self.choices = choices
         self.callback = callback
+        shortcutwidth = 3
         choicewidth = max([len(i) for i in choices])
-        self.width = max(choicewidth, len(title)) + 5
+        self.width = max(shortcutwidth+choicewidth, len(title)) + 5
+        self.possible_shortcuts = "123456789abcdefghijklmnopqrstuvwxyz"
+        self.shortcuts = self.get_shortcuts(choices)
+
+        shortcuts_walker = urwid.SimpleListWalker([
+            urwid.Text([("key", s), ")"], align="left") for s in self.shortcuts
+        ])
+        shortcuts_listbox = urwid.ListBox(shortcuts_walker)
+        shortcuts_listbox._selectable = False  # We don't want to focus on it
         self.walker = ChooserListWalker(choices, current)
+        content = urwid.Columns([(shortcutwidth, shortcuts_listbox),
+                                 (choicewidth+3, urwid.ListBox(self.walker))],
+                                focus_column=1)
         super().__init__(
             urwid.AttrWrap(
                 urwid.LineBox(
                     urwid.BoxAdapter(
-                        urwid.ListBox(self.walker),
+                        content,
                         len(choices)
                     ),
-                    title= title
+                    title=title
                 ),
                 "background"
             )

import math

import urwid

from mitmproxy.tools.console import signals
from mitmproxy.tools.console import grideditor
from mitmproxy.tools.console import layoutwidget
from mitmproxy.tools.console import keymap

class SimpleOverlay(urwid.Overlay, layoutwidget.LayoutWidget):

    def __init__(self, master, widget, parent, width, valign="middle"):
        self.widget = widget
        self.master = master
        super().__init__(
            widget,
            parent,
            align="center",
            width=width,
            valign=valign,
            height="pack"
        )

    @property
    def keyctx(self):
        return getattr(self.widget, "keyctx")

    def key_responder(self):
        return self.widget.key_responder()

    def focus_changed(self):
        return self.widget.focus_changed()

    def view_changed(self):
        return self.widget.view_changed()

    def layout_popping(self):
        return self.widget.layout_popping()

class Choice(urwid.WidgetWrap):
    def __init__(self, txt, focus, current):
        if current:
            s = "option_active_selected" if focus else "option_active"
        else:
            s = "option_selected" if focus else "text"
        return super().__init__(
            urwid.AttrWrap(
                urwid.Padding(urwid.Text(txt)),
                s,
            )
        )

    def selectable(self):
        return True

    def keypress(self, size, key):
        return key

class ChooserListWalker(urwid.ListWalker):
    def __init__(self, choices, current):
        self.index = 0
        self.choices = choices
        self.current = current

    def _get(self, idx, focus):
        c = self.choices[idx]
        return Choice(c, focus, c == self.current)

    def set_focus(self, index):
        self.index = index

    def get_focus(self):
        return self._get(self.index, True), self.index

    def get_next(self, pos):
        if pos >= len(self.choices) - 1:
            return None, None
        pos = pos + 1
        return self._get(pos, False), pos

    def get_prev(self, pos):
        pos = pos - 1
        if pos < 0:
            return None, None
        return self._get(pos, False), pos

class Chooser(urwid.WidgetWrap, layoutwidget.LayoutWidget):
    keyctx = "chooser"

    def __init__(self, master, title, choices, current, callback):
        self.master = master
        self.choices = choices
        self.callback = callback
        choicewidth = max([len(i) for i in choices])
        self.width = max(choicewidth, len(title)) + 5
        self.walker = ChooserListWalker(choices, current)
        super().__init__(
            urwid.AttrWrap(
                urwid.LineBox(
                    urwid.BoxAdapter(
                        urwid.ListBox(self.walker),
                        len(choices)
                    ),
                    title= title
                ),
                "background"
            )
        )

    def selectable(self):
        return True

    def keypress(self, size, key):
        key = self.master.keymap.handle_only("chooser", key)
        if key == "m_select":
            self.callback(self.choices[self.walker.index])
            signals.pop_view_state.send(self)
            return
        elif key == "esc":
            signals.pop_view_state.send(self)
            return

        binding = self.master.keymap.get("global", key)
        # This is extremely awkward. We need a better way to match nav keys only.
        if binding and binding.command.startswith("console.nav"):
            self.master.keymap.handle("global", key)
        elif key in keymap.navkeys:
            return super().keypress(size, key)

class OptionsOverlay(urwid.WidgetWrap, layoutwidget.LayoutWidget):
    keyctx = "grideditor"

    def __init__(self, master, name, vals, vspace):
        """
            vspace: how much vertical space to keep clear
        """
        cols, rows = master.ui.get_cols_rows()
        self.ge = grideditor.OptionsEditor(master, name, vals)
        super().__init__(
            urwid.AttrWrap(
                urwid.LineBox(
                    urwid.BoxAdapter(self.ge, rows - vspace),
                    title=name
                ),
                "background"
            )
        )
        self.width = math.ceil(cols * 0.8)

    def key_responder(self):
        return self.ge.key_responder()

    def layout_popping(self):
        return self.ge.layout_popping()

class DataViewerOverlay(urwid.WidgetWrap, layoutwidget.LayoutWidget):
    keyctx = "grideditor"

    def __init__(self, master, vals):
        """
            vspace: how much vertical space to keep clear
        """
        cols, rows = master.ui.get_cols_rows()
        self.ge = grideditor.DataViewer(master, vals)
        super().__init__(
            urwid.AttrWrap(
                urwid.LineBox(
                    urwid.BoxAdapter(self.ge, rows - 5),
                    title="Data viewer"
                ),
                "background"
            )
        )
        self.width = math.ceil(cols * 0.8)

    def key_responder(self):
        return self.ge.key_responder()

    def layout_popping(self):
        return self.ge.layout_popping()


@@ -56,3 +56,19 @@ class TestModel:
         model_roles = (cloudsql_connection.execute(query).fetchone())
         assert model_roles
         assert model_roles[0] > 0
+
+
+    @pytest.mark.client
+    @pytest.mark.e2e
+    @pytest.mark.model
+    def test_model_use(self, forseti_cli: ForsetiCli, forseti_model_readonly):
+        # Arrange
+        model_name, handle, _ = forseti_model_readonly
+
+        # Act
+        forseti_cli.model_use(model_name)
+        result = forseti_cli.config_show()
+
+        # Assert
+        assert handle
+        assert re.search(fr'{handle}', str(result.stdout))

# Copyright 2020 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Model end-to-end tests"""

import pytest
import re
from endtoend_tests.helpers.forseti_cli import ForsetiCli
from sqlalchemy.sql import text

class TestModel:
    """Model tests

    Execute the basic model functionality such as: create, get, use, etc.
    """

    @pytest.mark.client
    @pytest.mark.e2e
    @pytest.mark.model
    def test_model_use(self, forseti_cli: ForsetiCli, forseti_model_readonly):
        # Arrange
        model_name, handle, _ = forseti_model_readonly

        # Act
        forseti_cli.model_use(model_name)
        result = forseti_cli.config_show()

        # Assert
        assert handle
        assert re.search(fr'{handle}', str(result.stdout))

    @pytest.mark.e2e
    @pytest.mark.model
    @pytest.mark.server
    def test_model_roles(self, cloudsql_connection, forseti_model_readonly):
        # Arrange/Act
        model_name, handle, _ = forseti_model_readonly

        # Assert
        table_name = f'forseti_security.{handle}_roles'
        query = text('SELECT '
                     'COUNT(*) '
                     f'FROM {table_name}')
        model_roles = (cloudsql_connection.execute(query).fetchone())
        assert model_roles
        assert model_roles[0] > 0