def get_scroll(self, url, page_size=100, yield_pages=False, **filters):
    """pass"""
    n = 0
    options = dict(page_size=page_size, **filters)
    format = filters.get('format')
    while True:
        r = self.request(url, use_xpost=False, **options)
        n += len(r['results'])
        log.debug('Got {} {n}/{total}'.format((url.split('?')[0],), total=r['total'], **locals()))
        if yield_pages:
            yield r
        else:
            for row in r['results']:
                yield row
        if r['next'] is None:
            break
        url = r['next']
        options = {'format': None}


def get_error(self, block=True, timeout=None):
    """pass"""
    return self._error_queue.get(block=block, timeout=timeout)


def get_feedback(self, block=True, timeout=None):
    """pass"""
    if self._feedback_greenlet is None:
        self._feedback_greenlet = gevent.spawn(self._feedback_loop)
    return self._feedback_queue.get(block=block, timeout=timeout)


def wait_send(self, timeout=None):
    """pass"""
    self._send_queue_cleared.clear()
    self._send_queue_cleared.wait(timeout=timeout)


def start(self):
    """pass"""
    if self._send_greenlet is None:
        self._send_greenlet = gevent.spawn(self._send_loop)


def stop(self, timeout=10.0):
    """pass"""
    if self._send_greenlet is not None and self._send_queue.qsize() > 0:
        self.wait_send(timeout=timeout)
    if self._send_greenlet is not None:
        gevent.kill(self._send_greenlet)
        self._send_greenlet = None
    if self._error_greenlet is not None:
        gevent.kill(self._error_greenlet)
        self._error_greenlet = None
    if self._feedback_greenlet is not None:
        gevent.kill(self._feedback_greenlet)
        self._feedback_greenlet = None
    return self._send_queue.qsize() < 1


def convert_to_ssml(text, text_format):
    """pass"""
    if text_format is None:
        return text
    elif text_format == 'plain':
        return plain_to_ssml(text)
    elif text_format == 'html':
        return html_to_ssml(text)
    else:
        raise ValueError(text_format + ': text format not found.')


def html_to_ssml(text):
    """pass"""
    ssml_text = reduce(lambda x, y: x.replace(y, html_to_ssml_maps[y]), html_to_ssml_maps, text)
    return ssml_text


def iexpand(string, keep_escapes=False):
    """pass"""
    if isinstance(string, bytes):
        is_bytes = True
        string = string.decode('latin-1')
    else:
        is_bytes = False
    if is_bytes:
        return (entry.encode('latin-1') for entry in ExpandBrace(keep_escapes).expand(string))
    else:
        return (entry for entry in ExpandBrace(keep_escapes).expand(string))


def set_expanding(self):
    """pass"""
    status = not self.expanding
    if status:
        self.expanding = True
    return status


def get_escape(self, c, i):
    """pass"""
    try:
        escaped = next(i)
    except StopIteration:
        escaped = ''
    if self.keep_escapes:
        return c + escaped
    else:
        return escaped


def squash(self, a, b):
    """pass"""
    return (u''.join(x) if isinstance(x, tuple) else x for x in itertools.product(a, b))


def get_literals(self, c, i, depth):
    """pass"""
    result = ['']
    is_dollar = False
    while c:
        ignore_brace = is_dollar
        is_dollar = False
        if c == '$':
            is_dollar = True
        elif c == '\\':
            c = [self.get_escape(c, i)]
        else:
            if not ignore_brace and c == '{':
                index = i.index
                try:
                    seq = self.get_sequence(next(i), i, depth + 1)
                    if seq:
                        c = seq
                except StopIteration:
                    i.rewind(i.index - index)
                if not is_dollar and self.is_expanding() and (c in [',', '}']):
                    i.rewind(1)
                    return ((x for x in result),)
            else:
                result = self.squash(result, [c] if isinstance(c, str) else c)
                c = next(i)
    return ((x for x in result),)


def combine(self, a, b):
    """pass"""
    for l in [a, b]:
        for x in l:
            yield x


def add_episode(self, text, text_format, title=None, author=None, summary='watson', publish_date=None, synthesizer='. ', synth_args=None, sentence_break=' '):
    """pass"""
    if title in self.episodes:
        raise ValueError('"' + title + '" already exists as an episode title.')
    link = self.output_path + '/' + title.replace(' ', '_').lower() + '.mp3'
    episode_text = convert_to_ssml(text, text_format)
    new_episode = Episode(episode_text, text_format, title, author, link, summary, publish_date, synthesizer, synth_args, sentence_break)
    self.episodes[title] = new_episode


def add_scheduled_job(self, text_source, cron_args, text_format, title=None, author='watson', summary=None, synthesizer='. ', synth_args=None, sentence_break=' '):
    """pass"""
    if not callable(text_source):
        raise TypeError('Argument "text" must be a function')

    def add_episode():
        episode_text = text_source()
        episode_title = title + '_' + datetime.utcnow().strftime('%Y%m%d%H%M%S')
        self.add_episode(episode_text, text_format, episode_title, author, summary, datetime.utcnow(), synthesizer, synth_args, sentence_break)
    self._scheduler.add_job(add_episode, 'cron', id=title, **cron_args)[self.scheduled_jobs[title]]
    if not self._scheduler.running:
        self._scheduler.start()


def publish(self, titles):
    """pass"""
    if isinstance(titles, Sequence) and (not isinstance(titles, six.string_types)):
        for title in titles:
            self.episodes[title].publish()
    elif isinstance(titles, six.string_types):
        self.episodes[titles].publish()
    else:
        raise TypeError('titles must be a string or a sequence of strings.')
    self.update_rss_feed()


def render_audio(self):
    """pass"""
    segment = text_to_speech(self._text, self.synthesizer, self.synth_args, self.sentence_break)
    milli = len(segment)
    seconds = '{0:.1f}'.format(float(milli) / 1000 % 60).zfill(2)
    minutes = '{0:.0f}'.format(milli / 60000 % 60).zfill(2)
    hours = '{0:.0f}'.format(milli / (3600 * 1000) % 24).zfill(2)
    self.duration = hours + ':' + minutes + ':' + seconds
    segment.export(self.link, format='mp3')
    self.length = os.path.getsize(self.link)


def remove_exponent(d):
    """pass"""
    if d == d.to_integral():
        return d.quantize(Decimal(1))
    else:
        return d.normalize()


def millify(n, precision=0, drop_nulls=True, prefixes=[]):
    """pass"""
    millnames = ['', 'k', 'M', 'B', 'T', 'P', 'E', 'Z', 'Y']
    if prefixes:
        millnames = ['']
        millnames.extend(prefixes)
    n = float(n)
    millidx = max(0, min(len(millnames) - 1, int(math.floor(0 if n == 0 else math.log10(abs(n)) / 3))))
    result = '{:.{precision}f}'.format(n / 10 ** (3 * millidx), precision=precision)
    if drop_nulls:
        result = remove_exponent(Decimal(result))
    return '{0}{dx}'.format(result, dx=millnames[millidx])


def prettify(amount, separator=','):
    """pass"""
    orig = str(amount)
    new = re.sub('^(-?\\d+)(\\d{3})', '\\g<1>{0}\\g<2>'.format(separator), str(amount))
    if orig == new:
        return new
    else:
        return prettify(new)


def load_json(json_data, decoder=None):
    """pass"""
    if decoder is None:
        decoder = DateTimeDecoder
    return json.loads(json_data, object_hook=decoder.decode)


def load_json_file(file, decoder=None):
    """pass"""
    if decoder is None:
        decoder = DateTimeDecoder
    if not hasattr(file, 'read'):
        with io.open(file, 'r', encoding='utf-8') as f:
            return json.load(f, object_hook=decoder.decode)
    else:
        return json.load(file, object_hook=decoder.decode)


def save_json(val, pretty=False, sort=True, encoder=None):
    """pass"""
    if encoder is None:
        encoder = DateTimeEncoder
    if pretty:
        data = json.dumps(val, indent=4, separators=(',', ': '), sort_keys=sort, cls=encoder)
    else:
        data = json.dumps(val, separators=(',', ':'), sort_keys=sort, cls=encoder)
    if not sys.version_info > (3, 0) and isinstance(data, str):
        data = data.decode('utf-8')
    return data


def save_json_file(file, val, pretty=False, compact=True, sort=True, encoder=None):
    """pass"""
    if encoder is None:
        encoder = DateTimeEncoder
    opened = False
    if not hasattr(file, 'write'):
        file = io.open(file, 'w', encoding='utf-8')
        opened = True
    try:
        if pretty:
            data = json.dumps(val, indent=4, separators=(',', ': '), sort_keys=sort, cls=encoder)
        elif compact:
            data = json.dumps(val, separators=(',', ':'), sort_keys=sort, cls=encoder)
        else:
            data = json.dumps(val, sort_keys=sort, cls=encoder)
    finally:
        if not opened:
            file.close()
    if not sys.version_info > (3, 0) and isinstance(data, str):
        data = data.decode('utf-8')
    file.write(data)


def load_yaml_file(file):
    """pass"""
    if not hasattr(file, 'read'):
        with io.open(file, 'r', encoding='utf-8') as f:
            return yaml.load(f, yaml.FullLoader)
    else:
        return yaml.load(file, yaml.FullLoader)


def save_yaml_file(file, val):
    """pass"""
    opened = False
    if not hasattr(file, 'write'):
        file = io.open(file, 'w', encoding='utf-8')
        opened = True
    try:
        yaml.dump(val, file)
    finally:
        if opened:
            file.close()


def get_embedded_yara(self, iocid):
    """pass"""
    ioc_obj = self.iocs[iocid]
    ids_to_process = set([])
    signatures = ''
    for elem in ioc_obj.top_level_indicator.xpath('.//IndicatorItem[Context/@search = "Yara/Yara"]'):
        signature = elem.findtext('Content')
        signatures = signatures + '\n' + signature
    if signatures:
        signatures += '\n'
    return signatures


def get_yara_condition_string(self, indicator_node, parameters_node, ids_to_process, condition_string='', joining_value='or'):
    """pass"""
    indicator_node_id = str(indicator_node.get('id'))
    if indicator_node_id not in ids_to_process:
        msg = 'Entered into get_yara_condition_string with a invalid node to walk [[}]'.format(indicator_node_id)
        raise YaraConversionError(msg)
    expected_tag = 'Indicator'
    if indicator_node.tag != expected_tag:
        raise YaraConversionError('indicator_node expected tag is [%s]' % expected_tag)
    is_set = None
    for param in parameters_node.xpath('.//param[@ref-id="{}"]'.format(indicator_node_id)):
        if param.attrib['name'] == 'yara/set':
            is_set = True
            set_count = param.findtext('value', None)
            try:
                temp = int(set_count)
                if temp < 1:
                    raise YaraConversionError('yara/set parameter value was less than 1')
                if temp > len(indicator_node.getchildren()):
                    msg = 'yara/set value is greater than the number of children of Indicator node [%s]' % str(indicator_node_id)
                    raise YaraConversionError(msg)
            except ValueError:
                raise YaraConversionError('yara/set parameter was not a integer')
            set_dict = {'set_count': set_count, 'set_ids': []}
    for node in indicator_node.getchildren():
        node_id = node.get('id')
        safe_node_id = node_id.replace('-', '')
        if node_id not in ids_to_process:
            continue
        if node.tag == 'IndicatorItem':
            if is_set:
                raise YaraConversionError('Cannot have Indicator nodes underneath a Indicator node with yara/set')
            operator = node.get('operator').lower()
            if operator not in ('or', 'and'):
                raise YaraConversionError('Indicator@operator is not and/or. [%s] has [%s]' % (id, operator))
            recursed_condition = self.get_yara_condition_string(node, parameters_node, ids_to_process, '', operator)
            xp = './/param[@ref-id="{}" and @name="yara/set"]'.format(node_id)
            if not parameters_node.xpath(xp) and has_siblings(node):
                recursed_condition = '(%s)' % recursed_condition
            if condition_string == '':
                condition_string = recursed_condition
            else:
                condition_string = ' '.join([condition_string, joining_value, recursed_condition])
        elif node.tag == 'Indicator':
            if is_set:
                raise YaraConversionError('Cannot have Indicator nodes underneath a Indicator node with yara/set')
            operator = node.get('operator').lower()
            if operator not in ('or', 'and'):
                raise YaraConversionError('Indicator@operator is not and/or. [%s] has [%s]' % (id, operator))
            recursed_condition = self.get_yara_condition_string(node, parameters_node, ids_to_process, '', operator)
            xp = './/param[@ref-id="{}" and @name="yara/set"]'.format(node_id)
            if not parameters_node.xpath(xp) and has_siblings(node):
                recursed_condition = '(%s)' % recursed_condition
            if condition_string == '':
                condition_string = recursed_condition
            else:
                condition_string = ' '.join([condition_string, indicator_node.getparent().get('operator').lower(), temp_set_string])
        else:
            xp = './/param[@ref-id="{}" and (@name="yara/count" or @name="yara/offset/at" or @name="yara/offset/in")]'.format(node_id)
            params = parameters_node.xpath(xp)
            if len(params) > 1:
                msg = 'More than one condition parameters assigned to IndicatorItem [{}]'.format(node_id)
                raise YaraConversionError(msg)
            for param in params:
                param_name = param.get('name', None)
                if param_name == 'yara/count':
                    log.debug('Found [%s] attached to [%s]' % (param.attrib['name'], node_id))
                    mapping = {'prefix': '#', 'identifier': '', 'condition': ' ', 'postfix': ''}
                    use_condition_template = False
                    negation = node.get('negate')
                    condition = node.get('condition')
                    search = node.xpath('Context/@search')[0]
                    content = node.findtext('Content')
                    yara_condition = self.condition_to_yara_map[condition]
                    if not yara_condition:
                        msg = 'Invalid IndicatorItem condition encountered [%s][%s]' % (str(node_id), str(condition))
                        raise YaraConversionError(msg)
                    if negation.lower() == 'true':
                        negation = True
                    else:
                        negation = False
                    if search == 'Yara/FileSize':
                        mapping['prefix'] = ''
                        mapping['identifier'] = 'filesize'
                        mapping['postfix'] = ' ' + content
                        mapping['condition'] = yara_condition
                        use_condition_template = True
                    elif search == 'Yara/RuleName':
                        if content not in self.ioc_names_set:
                            if mangle_name(content) in self.ioc_names_mangled_set:
                                msg = 'Yara/RuleName is present as a mangled name.[{}][{}]'.format(mangle_name(content), node_id)
                                log.warning(msg)
                                content = mangle_name(content)
                        else:
                            log.warning('Yara/RuleName points to a name [{}] that is not in the set of IOCs being processed [{}]'.format(content, node_id))
                        if mangle_name(content) != content:
                            msg = 'Yara/RuleName contains characters which would cause libyara errors [{}]' % node_id
                            raise YaraConversionError(msg)
                        mapping['prefix'] = ''
                        mapping['identifier'] = content
                    else:
                        xp = './/param[@ref-id="{}" and (@name="yara/count" or @name="yara/offset/at" or @name="yara/offset/in")]'.format(node_id)
                        params = parameters_node.xpath(xp)
                        if len(params) > 1:
                            msg = 'More than one condition parameters assigned to IndicatorItem [{}]'.format(node_id)
                            raise YaraConversionError(msg)
                        for param in params:
                            param_name = param.get('name', None)
                            if param_name == 'yara/count':
                                log.debug('Found [%s] attached to [%s]' % (param.attrib['name'], node_id))
                                mapping['prefix'] = '#'
                                mapping['postfix'] = ' ' + param.findtext('value')
                                mapping['condition'] = yara_condition
                                use_condition_template = True
                            elif param_name == 'yara/offset/at':
                                log.debug('Found [%s] attached to [%s]' % (param.attrib['name'], node_id))
                                mapping['condition'] = 'at'
                                mapping['postfix'] = ' ' + param.findtext('value')
                                use_condition_template = True
                            elif param_name == 'yara/offset/in':
                                log.debug('Found [%s] attached to [%s]' % (param.attrib['name'], node_id))
                                mapping['condition'] = 'in'
                                mapping['postfix'] = ' ' + param.findtext('value')
                                use_condition_template = True
                    if use_condition_template:
                        temp_string = self.yara_II_condition_template % mapping
                    else:
                        temp_string = self.yara_II_template % mapping
                    if condition_string == '':
                        condition_string = temp_string
                    else:
                        condition_string = ' '.join([condition_string, joining_value, temp_string])
        elif node.tag == 'Indicator':
            if is_set:
                raise YaraConversionError('Cannot have Indicator nodes underneath a Indicator node with yara/set')
            operator = node.get('operator').lower()
            if operator not in ('or', 'and'):
                raise YaraConversionError('Indicator@operator is not and/or. [%s] has [%s]' % (id, operator))
            recursed_condition = self.get_yara_condition_string(node, parameters_node, ids_to_process, '', operator)
            xp = './/param[@ref-id="{}" and @name="yara/set"]'.format(node_id)
            if not parameters_node.xpath(xp) and has_siblings(node):
                recursed_condition = '(%s)' % recursed_condition
            if condition_string == '':
                condition_string = recursed_condition
            else:
                condition_string = ' '.join([condition_string, indicator_node.getparent().get('operator').lower(), temp_set_string])
    if is_set:
        log.debug('Building set expression for [%s]' % indicator_node_id)
        if len(set_dict['set_ids']) == 0:
            raise YaraConversionError('yara/set processing did not yield any set ids')
        elif len(set_dict['set_ids']) == 1:
            log.warning('yara/set with 1 id found for node [%s]' % node_id)
            set_ids = ''.join(set_dict['set_ids'])
        else:
            set_ids = ','.join(set_dict['set_ids'])
        mapping['set_ids'] = set_ids
        temp_set_string = self.yara_set_string_template % set_dict
        if condition_string == '':
            condition_string = temp_set_string
        else:
            condition_string = ' '.join([condition_string, indicator_node.getparent().get('operator').lower(), temp_set_string])
    return condition_string


def write_yara(self, output_file):
    """pass"""
    fout = open(output_file, 'wb')
    fout.write('\n')
    for iocid in self.yara_signatures:
        signature = self.yara_signatures[iocid]
        fout.write(signature)
        fout.write('\n')
    fout.close()
    return True


def safe_makedirs(fdir):
    """pass"""
    if os.path.isdir(fdir):
        return True
    else:
        try:
            os.makedirs(fdir)
        except WindowsError as e:
            if 'Cannot create a file when that file already exists' in e:
                log.debug('relevant dir already exists')
            else:
                raise WindowsError(e)
        return True


def convert_to_10(self):
    """pass"""
    if len(self) < 1:
        log.error('no iocs available to modify')
        return False
    log.info('Converting IOCs from 1.1 to 1.0.')
    errors = []
    for iocid in self.iocs:
        pruned = False
        ioc_obj_11 = self.iocs[iocid]
        metadata = ioc_obj_11.metadata
        name_11 = metadata.findtext('.//short_description')
        keywords_11 = metadata.findtext('.//keywords')
        description_11 = metadata.findtext('.//description')
        author_11 = metadata.findtext('.//authored_by')
        created_date_11 = metadata.findtext('.//authored_date')
        last_modified_date_11 = ioc_obj_11.root.get('last-modified')
        links_11 = []
        for link in metadata.xpath('//link'):
            link_rel = link.get('rel')
            link_text = link.text
            links_11.append((link_rel, None, link_text))
        try:
            ioc_logic = ioc_obj_11.root.xpath('.//criteria')[0]
        except IndexError:
            log.exception('Could not find criteria nodes for IOC [{}].  Did you attempt to convert OpenIOC 1.0 iocs?'.format(iocid))
            errors.append(iocid)
            continue
        try:
            tlo_11 = ioc_logic.getchildren()[0]
        except IndexError:
            log.exception('Could not find children for the top level criteria/children nodes for IOC [{}]' .format(iocid))
            errors.append(iocid)
            continue
        tlo_id = tlo_11.get('id')
        comment_dict = {}
        for param in ioc_obj_11.parameters.xpath('//param[@name="comment"]'):
            param_id = param.get('ref-id')
            param_text = param.findtext('value')
            comment_dict[param_id] = param_text
        ioc_obj_10 = ioc_api.IOC(name=name_11, description=description_11, author=author_11, links=links_11, keywords=keywords_11, iocid=iocid)
        ioc_obj_10.root.attrib['last-modified'] = last_modified_date_11
        authored_date_node = ioc_obj_10.metadata.find('authored_date')
        authored_date_node.text = created_date_11
        ioc_obj_10.root.attrib['xmlns'] = 'http://schemas.mandiant.com/2010/ioc'
        del ioc_obj_10.root.attrib['published-date']
        ioc_obj_10.root.remove(ioc_obj_10.parameters)
        ioc_obj_10.root.tag = 'ioc'
        metadata_node = ioc_obj_10.metadata
        criteria_node = ioc_obj_10.top_level_indicator.getparent()
        metadata_dictionary = {}
        for child in metadata_node:
            metadata_dictionary[child.tag] = child
        for tag in METADATA_REQUIRED_10:
            if tag not in metadata_dictionary:
                msg = 'IOC {} is missing required metadata: [{}]'.format(iocid, tag)
                raise DowngradeError(msg)
        for tag in METADATA_ORDER_10:
            if tag in metadata_dictionary:
                ioc_obj_10.root.append(metadata_dictionary.get(tag))
        ioc_obj_10.root.remove(metadata_node)
        ioc_obj_10.root.remove(criteria_node)
        criteria_node.tag = 'definition'
        ioc_obj_10.root.append(criteria_node)
        ioc_obj_10.top_level_indicator.attrib['id'] = tlo_id
        ids_to_skip = set()
        indicatoritems_to_remove = set()
        for condition_type in self.openioc_11_only_conditions:
            for elem in ioc_logic.xpath('//IndicatorItem[@condition="%s"]' % condition_type):
                pruned = True
                indicatoritems_to_remove.add(elem)
        for elem in ioc_logic.xpath('//IndicatorItem[@preserve-case="true"]'):
            pruned = True
            indicatoritems_to_remove.add(elem)
        for elem in indicatoritems_to_remove:
            nid = None
            current = elem
            if nid != tlo_id:
                parent = current.getparent()
                nid = parent.get('id')
                if nid == tlo_id:
                    current_id = current.get('id')
                    ids_to_skip.add(current_id)
                else:
                    current = parent
        try:
            self.convert_branch(tlo_11, ioc_obj_10.top_level_indicator, ids_to_skip, comment_dict)
        except DowngradeError:
            log.exception('Problem converting IOC [{}]'.format(iocid))
            errors.append(iocid)
            continue
        if not ioc_obj_10.top_level_indicator.getchildren():
            self.null_pruned_iocs.add(iocid)
        elif pruned is True:
            self.pruned_11_iocs.add(iocid)
        self.iocs_10[iocid] = ioc_obj_10
    return errors


def convert_branch(self, old_node, new_node, ids_to_skip, comment_dict=None):
    """pass"""
    expected_tag = 'Indicator'
    if old_node.tag != expected_tag:
        raise DowngradeError('old_node expected tag is [%s]' % expected_tag)
    if not comment_dict:
        comment_dict = {}
    for node in old_node.getchildren():
        node_id = node.get('id')
        if node_id in ids_to_skip:
            continue
        if node.tag == 'IndicatorItem':
            negation = node.get('negate')
            condition = node.get('condition')
            if 'true' in negation.lower():
                new_condition = condition + 'not'
            else:
                new_condition = condition
            document = node.xpath('Context/@document')[0]
            search = node.xpath('Context/@search')[0]
            content_type = node.xpath('Content/@type')[0]
            content = node.findtext('Content')
            context_type = node.xpath('Context/@type')[0]
            new_ii_node = ioc_api.make_indicatoritem_node(condition=condition, document=document, search=search, content_type=content_type, content=content, context_type=context_type, nid=node_id)
            new_ii_node.attrib['condition'] = new_condition
            if node_id in comment_dict:
                comment = comment_dict[node_id]
                comment_node = et.Element('Comment')
                comment_node.text = comment
                new_ii_node.append(comment_node)
            del new_ii_node.attrib['negate']
            del new_ii_node.attrib['preserve-case']
            new_node.append(new_ii_node)
        elif node.tag == 'Indicator':
            operator = node.get('operator')
            if operator.upper() not in ('OR', 'AND'):
                raise DowngradeError('Indicator@operator is not AND/OR. [%s] has [%s]' % (node_id, operator))
            new_i_node = ioc_api.make_indicator_node(operator, node_id)
            new_node.append(new_i_node)
            self.convert_branch(node, new_i_node, ids_to_skip, comment_dict)
        else:
            raise DowngradeError('node is not a Indicator/IndicatorItem')
    return True


def create(self, permission):
    """pass"""
    parent_url = self.client.get_url(self.parent_object._manager._URL_KEY, 'GET', 'single', {'id': self.parent_object.id})
    target_url = parent_url + self.client.get_url_path(self._URL_KEY, 'POST', 'single')
    r = self.client.request('POST', target_url, json=permission._serialize())
    return permission._deserialize(r.json(), self)


def set(self, permissions):
    """pass"""
    parent_url = self.client.get_url(self.parent_object._manager._URL_KEY, 'GET', 'single', {'id': self.parent_object.id})
    target_url = parent_url + self.client.get_url_path(self._URL_KEY, 'PUT', 'multi')
    r = self.client.request('PUT', target_url, json=permissions)
    if r.status_code != 201:
        raise exceptions.ServerError('Expected 201 response, got %s: %s' % (r.status_code, target_url))
    return self.list()


def list(self):
    """pass"""
    parent_url = self.client.get_url(self.parent_object._manager._URL_KEY, 'GET', 'single', {'id': self.parent_object.id})
    target_url = parent_url + self.client.get_url_path(self._URL_KEY, 'GET', 'multi')
    return base.Query(self, target_url)


def get(self, permission_id, expand=[]):
    """pass"""
    parent_url = self.client.get_url(self.parent_object._manager._URL_KEY, 'GET', 'single', {'id': self.parent_object.id})
    target_url = parent_url + self.client.get_url_path(self._URL_KEY, 'GET', 'single', {'permission_id': permission_id})
    return self._get(target_url, expand=expand)


def get_config():
    """pass"""
    configpath = get_configpath()
    if not configpath.exists():
        raise IOError('Config file {} not found.'.format(str(configpath)))
    config = configparser.ConfigParser()
    config.read(str(configpath))
    return config


def set_database_path(dbfolder):
    """pass"""
    configpath = get_configpath()
    try:
        d = get_config()
    except IOError:
        d = configparser.ConfigParser()
        d['pyciss_db'] = {}
    d['pyciss_db']['path'] = dbfolder
    with configpath.open('w') as f:
        d.write(f)
    print('Saved database path into {}.'.format(configpath))


def get_db_root():
    """pass"""
    d = get_config()
    dbroot = Path(d['pyciss_db']['path'])
    dbroot.mkdir(exist_ok=True)
    return dbroot


def print_db_stats():
    """pass"""
    dbroot = get_db_root()
    n_ids = len(list(dbroot.glob('[N,W]*')))
    print('Number of WACs and NACs in database: {}'.format(n_ids))
    print('These kind of data are in the database: (returning pd.DataFrame)')
    d = {}
    for key, val in PathManager.extensions.items():
        d[key] = [len(list(dbroot.glob('**/*' + val)))]
    return pd.DataFrame(d)


def is_lossy(label):
    """pass"""
    val = getkey(from_=label, keyword='INST_CMPRS_TYPE').decode().strip()
    if val == 'LOSSY':
        return True
    else:
        return False


def download_and_calibrate_parallel(list_of_ids, n=None):
    """pass"""
    setup_cluster(n_cores=n)
    c = Client()
    lbview = c.load_balanced_view()
    lbview.map_async(download_and_calibrate, list_of_ids)
    subprocess.Popen(['ipcluster', 'stop', '--quiet'])


def _set_supported_content_type(self, content_types_supported):
    """pass"""
    if not isinstance(content_types_supported, list):
        raise TypeError("Settings 'READTIME_CONTENT_SUPPORT' must bea list of content types.")
    self.content_type_supported = content_types_supported


def _set_lang_settings(self, lang_settings):
    """pass"""
    is_int = isinstance(lang_settings, int)
    is_dict = isinstance(lang_settings, dict)
    if not (is_int or is_dict):
        raise TypeError("Settings 'READTIME_WPM' must be either an int,or a dict with settings per language.")
    if is_int:
        self.lang_settings['default']['wpm'] = lang_settings
    elif is_dict:
        for lang, conf in lang_settings.items():
            if not 'wpm' in conf:
                raise KeyError('Missing wpm value for thelanguage: {}'.format(lang))
            if not isinstance(conf['wpm'], int):
                raise TypeError('WPM is not an integer for the language: {}'.format(lang))
            if not 'min_singular' in conf:
                raise KeyError('Missing singular form for "minute" for the language: {}'.format(lang))
            if not 'min_plural' in conf:
                raise KeyError('Missing plural form for "minutes" for the language: {}'.format(lang))
            if not 'sec_singular' in conf:
                raise KeyError('Missing singular form for "second" for the language: {}'.format(lang))
            if not 'sec_plural' in conf:
                raise KeyError('Missing plural form for "seconds" for the language: {}'.format(lang))
        self.lang_settings = lang_settings


def initialize_settings(self, sender):
    """pass"""
    try:
        self.initialized = True
        settings_content_types = sender.settings.get('READTIME_CONTENT_SUPPORT', self.content_type_supported)
        self._set_supported_content_type(settings_content_types)
        lang_settings = sender.settings.get('READTIME_WPM', self.lang_settings)
        self._set_lang_settings(lang_settings)
    except Exception as e:
        raise Exception('ReadTime Plugin: %s' % str(e))


def read_time(self, content):
    """pass"""
    if get_class_name(content) in self.content_type_supported:
        if hasattr(content, 'readtime'):
            return
        default_lang_conf = self.lang_settings['default']
        lang_conf = self.lang_settings.get(content.lang, default_lang_conf)
        avg_reading_wpm = lang_conf['wpm']
        num_words = len(content._content.split())
        minutes = num_words // avg_reading_wpm
        seconds = int(num_words / avg_reading_wpm * 60 - minutes * 60)
        minutes_str = self.pluralize(minutes, lang_conf['min_singular'], lang_conf['min_plural'])
        seconds_str = self.pluralize(seconds, lang_conf['sec_singular'], lang_conf['sec_plural'])
        content.readtime = minutes
        content.readtime_string = minutes_str
        content.readtime_with_seconds = (minutes, seconds)
        content.readtime_string_with_seconds = '{}, {}'.format(minutes_str, seconds_str)


def pluralize(self, measure, singular, plural):
    """pass"""
    if measure == 1:
        return '{} {}'.format(measure, singular)
    else:
        return '{} {}'.format(measure, plural)


def list_datasources(self, source_id):
    """pass"""
    target_url = self.client.get_url('DATASOURCE', 'GET', 'multi', {'source_id': source_id})
    return base.Query(self.client.get_manager(Datasource), target_url)


def get_datasource(self, source_id, datasource_id):
    """pass"""
    target_url = self.client.get_url('DATASOURCE', 'GET', 'single', {'source_id': source_id, 'datasource_id': datasource_id})
    return self.client.get_manager(Datasource)._get(target_url)


def open_store_variable(self, name, var):
    """pass"""
    data = indexing.LazilyOuterIndexedArray(CDMArrayWrapper(name, self))
    return Variable(var.dimensions, data, {a: getattr(var, a) for a in var.ncattrs()})


def get_attrs(self):
    """pass"""
    return FrozenOrderedDict(((a, getattr(self.ds, a)) for a in self.ds.ncattrs()))


def get_dimensions(self):
    """pass"""
    return FrozenOrderedDict(((k, len(v)) for k, v in self.ds.dimensions.items()))


def _find_base_tds_url(catalog_url):
    """pass"""
    url_components = urlparse(catalog_url)
    if url_components.path:
        return catalog_url.split(url_components.path)[0]
    else:
        return catalog_url


def filter_time_nearest(self, time, regex=None):
    """pass"""
    return min(self._get_datasets_with_times(regex), key=lambda i: abs((i[0] - time).total_seconds()))[-1]


def filter_time_range(self, start, end, regex=None):
    """pass"""
    return [item[-1] for item in self._get_datasets_with_times(regex) if start <= item[0] <= end]


def check_token(func):
    """pass"""

    @wraps(func)
    def wrapper(*args, **kwargs):
        response = func(*args, **kwargs)
        if response.status_code == 401:
            raise InvalidToken('Access token invalid or no longer valid')
        return response
    return wrapper


def upload(self, remote_path, file_content, ondup=None, **kwargs):
    """pass"""
    params = {'path': remote_path, 'ondup': ondup}
    files = {'file': ('file', file_content, '')}
    url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file'
    return self._request('file', 'upload', url=url, extra_params=params, files=files, **kwargs)


def upload_tmpfile(self, file_content, **kwargs):
    """pass"""
    params = {'type': 'tmpfile'}
    files = {'file': ('file', file_content, '')}
    url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file'
    return self._request('file', 'upload', url=url, extra_params=params, files=files, **kwargs)


def upload_superfile(self, remote_path, block_list, ondup=None, **kwargs):
    """pass"""
    params = {'path': remote_path, 'ondup': ondup}
    data = {'param': json.dumps({'block_list': block_list})}
    return self._request('file', 'createsuperfile', extra_params=params, data=data, **kwargs)


def mkdir(self, remote_path, **kwargs):
    """pass"""
    data = {'path': remote_path}
    return self._request('file', 'mkdir', data=data, **kwargs)


def meta(self, remote_path, **kwargs):
    """pass"""
    params = {'path': remote_path}
    return self._request('file', 'meta', extra_params=params, **kwargs)


def login(self):
    """pass"""
    if os.path.exists(self._cookieFileName):
        with open(self._cookieFileName, 'r') as cookieFile:
            self._vid = cookieFile.read().strip()
        try:
            self._get_installations()
        except ResponseError:
            self._vid = None
            os.remove(self._cookieFileName)
    if self._vid is None:
        self._create_cookie()
        with open(self._cookieFileName, 'w') as cookieFile:
            cookieFile.write(self._vid)
        self._get_installations()
    self._giid = self.installations[0]['giid']


def _get_installations(self):
    """pass"""
    response = None
    for base_url in urls.BASE_URLS:
        urls.BASE_URL = base_url
        try:
            response = requests.get(urls.get_installations(self._username), headers={'Cookie': 'vid={}'.format(self._vid), 'Accept': 'application/json,text/javascript, */*; q=0.01'})
            if 2 == response.status_code // 100:
                break
            elif 503 == response.status_code:
                continue
            else:
                raise ResponseError(response.status_code, response.text)
        except requests.exceptions.RequestException as ex:
            raise RequestError(ex)
    _validate_response(response)
    self.installations = json.loads(response.text)


def get_overview(self):
    """pass"""
    response = None
    try:
        response = requests.get(urls.overview(self._giid), headers={'Accept': 'application/json, text/javascript, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate', 'Content-Type': 'application/json', 'Cookie': 'vid={}'.format(self._vid)})
    except requests.exceptions.RequestException as ex:
        raise RequestError(ex)
    _validate_response(response)
    return json.loads(response.text)


def set_smartplug_state(self, device_label, state):
    """pass"""
    response = None
    try:
        response = requests.post(urls.smartplug(self._giid), headers={'Content-Type': 'application/json', 'Cookie': 'vid={}'.format(self._vid)}, data=json.dumps([{'deviceLabel': device_label, 'state': state}]))
    except requests.exceptions.RequestException as ex:
        raise RequestError(ex)
    _validate_response(response)


def get_history(self, filters=(), pagesize=15, offset=0):
    """pass"""
    response = None
    try:
        response = requests.get(urls.history(self._giid), headers={'Accept': 'application/json, text/javascript, */*; q=0.01', 'Cookie': 'vid={}'.format(self._vid)}, params={'offset': int(offset), 'pagesize': int(pagesize), 'notificationCategories': filters})
    except requests.exceptions.RequestException as ex:
        raise RequestError(ex)
    _validate_response(response)
    return json.loads(response.text)


def get_climate(self, device_label):
    """pass"""
    response = None
    try:
        response = requests.get(urls.climate(self._giid), headers={'Accept': 'application/json, text/javascript, */*; q=0.01', 'Cookie': 'vid={}'.format(self._vid)}, params={'deviceLabel': device_label})
    except requests.exceptions.RequestException as ex:
        raise RequestError(ex)
    _validate_response(response)
    return json.loads(response.text)


def type_id(self):
    """pass"""
    try:
        return ContentType.objects.get_for_model(self.model, for_concrete_model=False).id
    except DatabaseError as e:
        raise DatabaseError('Unable to fetch ContentType object, is a plugin being registered before the initial syncdb? (original error: {0})'.format(str(e)))


def get_output_cache_key(self, placeholder_name, instance):
    """pass"""
    cachekey = self.get_output_cache_base_key(placeholder_name, instance)
    if self.cache_output_per_site:
        cachekey = '{0}-s{1}'.format(cachekey, settings.SITE_ID)
    if self.cache_output_per_language:
        user_language = get_language()
        if user_language not in self.cache_supported_language_codes:
            user_language = 'unsupported'
        cachekey = '{0}.{1}'.format(cachekey, user_language)
    return cachekey


def get_output_cache_keys(self, placeholder_name, instance):
    """pass"""
    base_key = self.get_output_cache_base_key(placeholder_name, instance)
    cachekeys = [base_key]
    if self.cache_output_per_site:
        site_ids = list(Site.objects.values_list('pk', flat=True))
        if settings.SITE_ID not in site_ids:
            site_ids.append(settings.SITE_ID)
        base_key = get_rendering_cache_key(placeholder_name, instance)
        cachekeys = ['{0}-s{1}'.format(base_key, site_id) for site_id in site_ids]
    if self.cache_output_per_language or self.render_ignore_item_language:
        total_list = []
        cache_languages = list(self.cache_supported_language_codes) + ['unsupported', 'None']
        placeholder = instance.placeholder
        total_list.extend((get_placeholder_cache_key(placeholder, lc) for lc in cache_languages))
        for user_language in cache_languages:
            total_list.extend(('{0}.{1}'.format(base, user_language) for base in cachekeys))
        cachekeys = total_list
    return cachekeys


def get_cached_output(self, placeholder_name, instance):
    """pass"""
    cachekey = self.get_output_cache_key(placeholder_name, instance)
    return cache.get(cachekey)


def set_cached_output(self, placeholder_name, instance, output):
    """pass"""
    cachekey = self.get_output_cache_key(placeholder_name, instance)
    if self.cache_timeout is not DEFAULT_TIMEOUT:
        cache.set(cachekey, output, self.cache_timeout)
    else:
        cache.set(cachekey, output)


def render(self, request, instance, **kwargs):
    """pass"""
    render_template = self.get_render_template(request, instance, **kwargs)
    if not render_template:
        return str(_("{No rendering defined for class '%s'}" % self.__class__.__name__))
    context = self.get_context(request, instance, **kwargs)
    return self.render_to_string(request, render_template, context)


def inline_requests(method_or_func):
    """pass"""
    args = get_args(method_or_func)
    if not args:
        raise TypeError('Function must accept at least one argument.')
    if args[0] == 'self':

        def wrapper(self, response, **kwargs):
            callback = create_bound_method(method_or_func, self)
            genwrapper = RequestGenerator(callback, **kwargs)
            return genwrapper(response)
    else:
        warnings.warn('Decorating a non-method function will be deprecated', ScrapyDeprecationWarning, stacklevel=1)

        def wrapper(response, **kwargs):
            genwrapper = RequestGenerator(method_or_func, **kwargs)
            return genwrapper(response)
    return wraps(method_or_func)(wrapper)


def get_args(method_or_func):
    """pass"""
    try:
        args = list(inspect.signature(method_or_func).parameters.keys())
    except AttributeError:
        args = inspect.getargspec(method_or_func).args
    return args


def _unwindGenerator(self, generator, _prev=None):
    """pass"""
    while True:
        if _prev:
            ret, _prev = (_prev, None)
        else:
            try:
                ret = next(generator)
            except StopIteration:
                return
        if isinstance(ret, Request):
            if ret.callback:
                warnings.warn('Got a request with callback set, bypassing the generator wrapper. Generator may not be able to resume. %s' % ret)
            elif ret.errback:
                warnings.warn('Got a request with errback set, bypassing the generator wrapper. Generator may not be able to resume. %s' % ret)
            else:
                yield self._wrapRequest(ret, generator)
        else:
            yield ret


def jwt_required(fn):
    """pass"""

    @wraps(fn)
    def wrapper(*args, **kwargs):
        jwt_data = _decode_jwt_from_headers()
        ctx_stack.top.jwt = jwt_data
        return fn(*args, **kwargs)
    return wrapper


def jwt_optional(fn):
    """pass"""

    @wraps(fn)
    def wrapper(*args, **kwargs):
        try:
            jwt_data = _decode_jwt_from_headers()
            ctx_stack.top.jwt = jwt_data
        except (NoAuthorizationError, InvalidHeaderError):
            pass
        return fn(*args, **kwargs)
    return wrapper


def decode_jwt(encoded_token):
    """pass"""
    secret = config.decode_key
    algorithm = config.algorithm
    audience = config.audience
    return jwt.decode(encoded_token, secret, algorithms=[algorithm], audience=audience)


def init_app(self, app):
    """pass"""
    if not hasattr(app, 'extensions'):
        app.extensions = {}
    app.extensions['flask-jwt-simple'] = self
    self._set_default_configuration_options(app)
    self._set_error_handler_callbacks(app)
    app.config['PROPAGATE_EXCEPTIONS'] = True


def _set_error_handler_callbacks(self, app):
    """pass"""

    @app.errorhandler(NoAuthorizationError)
    def handle_no_auth_error(e):
        return self._unauthorized_callback(str(e))

    @app.errorhandler(InvalidHeaderError)
    def handle_invalid_header_error(e):
        return self._invalid_token_callback(str(e))

    @app.errorhandler(jwt.ExpiredSignatureError)
    def handle_expired_error(e):
        return self._expired_token_callback()

    @app.errorhandler(jwt.InvalidTokenError)
    def handle_invalid_token_error(e):
        return self._invalid_token_callback(str(e))


def _set_default_configuration_options(app):
    """pass"""
    app.config.setdefault('JWT_HEADER_NAME', 'Authorization')
    app.config.setdefault('JWT_HEADER_TYPE', 'Bearer')
    app.config.setdefault('JWT_EXPIRES', datetime.timedelta(hours=1))
    app.config.setdefault('JWT_ALGORITHM', 'HS256')
    app.config.setdefault('JWT_IDENTITY_CLAIM', 'sub')
    app.config.setdefault('JWT_DECODE_AUDIENCE', None)
    app.config.setdefault('JWT_SECRET_KEY', None)
    app.config.setdefault('JWT_PRIVATE_KEY', None)
    app.config.setdefault('JWT_PUBLIC_KEY', None)


def category(**kwargs):
    """pass"""
    if 'series' in kwargs:
        kwargs.pop('series')
        path = 'series'
    else:
        path = None
    return Fred().category(path, **kwargs)


def releases(release_id=None, **kwargs):
    """pass"""
    if 'id' not in kwargs and release_id is not None:
        kwargs['release_id'] = release_id
        return Fred().release(**kwargs)
    else:
        return Fred().releases(**kwargs)


def series(identifier=None, **kwargs):
    """pass"""
    if identifier:
        kwargs['series_id'] = identifier
    if 'release' in kwargs:
        kwargs.pop('release')
        path = 'release'
    elif 'releases' in kwargs:
        kwargs.pop('releases')
        path = 'release'
    else:
        path = None
    return Fred().series(path, **kwargs)


def source(source_id=None, **kwargs):
    """pass"""
    if source_id is not None:
        kwargs['source_id'] = source_id
    elif 'id' in kwargs:
        source_id = kwargs.pop('id')
        kwargs['source_id'] = source_id
    if 'releases' in kwargs:
        kwargs.pop('releases')
        path = 'releases'
    else:
        path = None
    return Fred().source(path, **kwargs)


def sources(source_id=None, **kwargs):
    """pass"""
    if source_id or 'id' in kwargs:
        return source(source_id, **kwargs)
    else:
        return Fred().sources(**kwargs)


def _create_path(self, *args):
    """pass"""
    args = filter(None, args)
    path = self.endpoint + '/'.join(args)
    return path


def item_extra_kwargs(self, item):
    """pass"""
    if use_feed_image:
        feed_image = item.feed_image
        if feed_image:
            image_complete_url = urljoin(self.get_site_url(), feed_image.file.url)
        else:
            image_complete_url = ''
    content_field = getattr(item, self.item_content_field)
    try:
        content = expand_db_html(content_field)
    except:
        content = content_field.__html__()
    soup = BeautifulSoup(content, 'html.parser')
    for div in soup.find_all('div', {'class': 'responsive-object'}):
        del div['style']
    for img_tag in soup.findAll('img'):
        if not img_tag.has_attr('src'):
            continue
        img_tag['src'] = urljoin(self.get_site_url(), img_tag['src'])
    fields_to_add = {'content': soup.prettify(formatter='html')}
    if use_feed_image:
        fields_to_add['image'] = image_complete_url
    else:
        fields_to_add['image'] = ''
    return fields_to_add


def treenav_undefined_url(request, item_slug):
    """pass"""
    item = get_object_or_404(treenav.MenuItem, slug=item_slug)
    raise Http404


def treenav_save_other_object_handler(sender, instance, created, **kwargs):
    """pass"""
    from django.contrib.contenttypes.models import ContentType
    from .models import MenuItem
    cache_key = 'django-treenav-menumodels'
    if sender == MenuItem:
        cache.delete(cache_key)
    menu_models = cache.get(cache_key)
    if not menu_models:
        menu_models = []
        for menu_item in MenuItem.objects.exclude(content_type__isnull=True):
            menu_models.append(menu_item.content_type.model_class())
        cache.set(cache_key, menu_models)
    if sender in menu_models:
        ct = ContentType.objects.get_for_model(sender)
        items = MenuItem.objects.filter(content_type=ct, object_id=instance.pk)
        for item in items:
            if item.href != instance.get_absolute_url():
                item.href = instance.get_absolute_url()
                item.save()


def refresh_hrefs(self, request):
    """pass"""
    for item in treenav.MenuItem.objects.all():
        item.save()
    self.message_user(request, _('Menu item HREFs refreshed successfully.'))
    info = (self.model._meta.app_label, self.model._meta.model_name)
    changelist_url = reverse('admin:%s_%s_changelist' % info, current_app=self.admin_site.name)
    return redirect(changelist_url)


def clean_cache(self, request):
    """pass"""
    treenav.delete_cache()
    self.message_user(request, _('Cache menuitem cache cleaned successfully.'))
    info = (self.model._meta.app_label, self.model._meta.model_name)
    changelist_url = reverse('admin:%s_%s_changelist' % info, current_app=self.admin_site.name)
    return redirect(changelist_url)


def rebuild_tree(self, request):
    """pass"""
    self.model.objects.rebuild()
    self.message_user(request, _('Menu Tree Rebuilt.'))
    return self.clean_cache(request)


def save_related(self, request, form, formsets, change):
    """pass"""
    super(MenuItemAdmin, self).save_related(request, form, formsets, change)
    self.model.objects.rebuild()


def _calculate_dispersion(X: Union[pd.DataFrame, np.ndarray], labels: np.ndarray, centroids: np.ndarray) -> float:
    """pass"""
    disp = np.sum(np.sum([np.abs(inst - centroids[label]) ** 2 for inst, label in zip(X, labels)]))
    return disp


def _calculate_gap(self, X: Union[pd.DataFrame, np.ndarray], n_refs: int, n_clusters: int) -> Tuple[float, int]:
    """pass"""
    ref_dispersions = np.zeros(n_refs)
    for i in range(n_refs):
        random_data = np.random.random_sample(size=X.shape)
        centroids, labels = kmeans2(data=random_data, k=n_clusters, iter=10, minit='points')
        dispersion = self._calculate_dispersion(X=random_data, labels=labels, centroids=centroids)
        ref_dispersions[i] = dispersion
    centroids, labels = kmeans2(data=X, k=n_clusters, iter=10, minit='points')
    dispersion = self._calculate_dispersion(X=X, labels=labels, centroids=centroids)
    gap_value = np.mean(np.log(ref_dispersions)) - np.log(dispersion)
    return (gap_value, int(n_clusters))


def _process_with_rust(self, X: Union[pd.DataFrame, np.ndarray], n_refs: int, cluster_array: np.ndarray):
    """pass"""
    from gap_statistic.rust import gapstat
    for label, gap_value in gapstat.optimal_k(X, list(cluster_array)):
        yield (gap_value, label)


def _process_with_joblib(self, X: Union[pd.DataFrame, np.ndarray], n_refs: int, cluster_array: np.ndarray):
    """pass"""
    if Parallel is None:
        raise EnvironmentError('joblib is not installed; cannot use joblib as the parallel backend!')
    with Parallel(n_jobs=self.n_jobs) as parallel:
        for gap_value, n_clusters in parallel((delayed(self._calculate_gap)(X, n_refs, n_clusters) for n_clusters in cluster_array)):
            yield (gap_value, n_clusters)


def _process_with_multiprocessing(self, X: Union[pd.DataFrame, np.ndarray], n_refs: int, cluster_array: np.ndarray):
    """pass"""
    with ProcessPoolExecutor(max_workers=self.n_jobs) as executor:
        jobs = [executor.submit(self._calculate_gap, X, n_refs, n_clusters) for n_clusters in cluster_array]
        for future in as_completed(jobs):
            gap_value, k = future.result()
            yield (gap_value, k)


def _process_non_parallel(self, X: Union[pd.DataFrame, np.ndarray], n_refs: int, cluster_array: np.ndarray):
    """pass"""
    for gap_value, n_clusters in [self._calculate_gap(X, n_refs, n_clusters) for n_clusters in cluster_array]:
        yield (gap_value, n_clusters)


def get_last(self):
    """pass"""
    query = self.table().where('batch', self.get_last_batch_number())
    return query.order_by('migration', 'desc').get()


def compile_insert(self, query, values):
    """pass"""
    table = self.wrap_table(query.from__)
    if not isinstance(values, list):
        values = [values]
    columns = self.columnize(values[0].keys())
    parameters = self.parameterize(values[0].values())
    value = ['(%s)' % parameters] * len(values)
    parameters = ', '.join(value)
    return 'INSERT INTO %s (%s) VALUES %s' % (table, columns, parameters)


def get_alter_table_sql(self, diff):
    """pass"""
    sql = []
    for column_diff in diff.changed_columns.values():
        if self.is_unchanged_binary_column(column_diff):
            continue
        old_column_name = column_diff.old_column_name
        column = column_diff.column
        if any([column_diff.has_changed('type'), column_diff.has_changed('precision'), column_diff.has_changed('scale'), column_diff.has_changed('fixed')]):
            query = 'ALTER ' + old_column_name + ' TYPE ' + self.get_sql_type_declaration(column.to_dict())
            sql.append('ALTER TABLE ' + diff.name + ' ' + query)
        if column_diff.has_changed('default') or column_diff.has_changed('type'):
            if column.get_default() is None:
                default_clause = ' DROP DEFAULT'
            else:
                default_clause = ' SET' + self.get_default_value_declaration_sql(column.to_dict())
            query = 'ALTER ' + old_column_name + default_clause
            sql.append('ALTER TABLE ' + diff.name + ' ' + query)
        if column_diff.has_changed('notnull'):
            op = 'DROP'
            if column.get_notnull():
                op = 'SET'
            query = 'ALTER ' + old_column_name + ' ' + op + ' NOT NULL'
            sql.append('ALTER TABLE ' + diff.name + ' ' + query)
        if column_diff.has_changed('autoincrement'):
            if column.get_autoincrement():
                seq_name = self.get_identity_sequence_name(diff.name, old_column_name)
                sql.append('CREATE SEQUENCE ' + seq_name)
                sql.append("SELECT setval('" + seq_name + "', (SELECT MAX(" + old_column_name + ') FROM ' + diff.name + '))")
                query = 'ALTER ' + old_column_name + " SET DEFAULT nextval('" + seq_name + "')"
                sql.append('ALTER TABLE ' + diff.name + ' ' + query)
            else:
                query = 'ALTER ' + old_column_name + ' DROP DEFAULT'
                sql.append('ALTER TABLE ' + diff.name + ' ' + query)
        if column_diff.has_changed('length'):
            query = 'ALTER ' + old_column_name + ' TYPE ' + self.get_sql_type_declaration(column.to_dict())
            sql.append('ALTER TABLE ' + diff.name + ' ' + query)
    for old_column_name, column in diff.renamed_columns.items():
        sql.append('ALTER TABLE ' + diff.name + ' RENAME COLUMN ' + old_column_name + ' TO ' + column.get_name())
    return sql


def _date_based_where(self, type, query, where):
    """pass"""
    value = str(where['value']).zfill(2)
    value = self.parameter(value)
    return "strftime('%s', %s) %s %s" % (type, self.wrap(where['column']), where['operator'], value)


def compile_select(self, query):
    """pass"""
    sql = super(MySqlQueryGrammar, self).compile_select(query)
    if query.unions:
        sql = '%s %s' % (sql, self._compile_unions(query))
    return sql


def _compile_lock(self, query, value):
    """pass"""
    if isinstance(value, basestring):
        return value
    elif value is True:
        return 'FOR UPDATE'
    elif value is False:
        return 'LOCK IN SHARE MODE'


def plot_best_worst_fits(assignments_df, data, modality_col='Modality', score='$\\log_2 K$'):
    """pass"""
    ncols = 2
    nrows = len(assignments_df.groupby(modality_col).groups.keys())
    fig, axes = plt.subplots(nrows=nrows, ncols=ncols * 4, figsize=(ncols * 6, 4))
    axes_iter = axes.flat
    fits = ('Highest', 'Lowest')
    for modality, df in assignments_df.groupby(modality_col):
        df = df.sort_values(score)
        color = MODALITY_TO_COLOR[modality]
        for fit in fits:
            if fit == 'Highest':
                ids = df['Feature ID'][-10:]
            else:
                ids = df['Feature ID'][:10]
            fit_psi = data[ids]
            tidy_fit_psi = fit_psi.stack().reset_index()
            tidy_fit_psi = tidy_fit_psi.rename(columns={'level_0': 'Sample ID', 'level_1': 'Feature ID', 0: '$\\Psi$'})
            if not tidy_fit_psi.empty:
                continue
            ax = six.next(axes_iter)
            violinplot(x='Feature ID', y='$\\Psi$', data=tidy_fit_psi, color=color, ax=ax)
            ax.set(title='{} {} {}'.format(fit, score, modality), xticks=[])
    sns.despine()
    fig.tight_layout()


def violinplot(x=None, y=None, data=None, bw=0.2, scale='width', inner=None, ax=None, **kwargs):
    """pass"""
    if ax is None:
        ax = plt.gca()
    sns.violinplot(x, y, data=data, bw=bw, scale=scale, inner=inner, ax=ax, **kwargs)
    ax.set(ylim=(0, 1), yticks=(0, 0.5, 1))
    return ax


def bar(self, counts, phenotype_to_color=None, ax=None, percentages=True):
    """pass"""
    if percentages:
        counts = (100 * (counts.T / counts.T.sum()).T)
    if ax is None:
        ax = plt.gca()
    full_width = 0.8
    width = full_width / counts.shape[0]
    for i, (group, series) in enumerate(counts.iterrows()):
        left = np.arange(len(self.modality_order)) + i * width
        height = [series[i] if i in group else 0 for i in self.modality_order]
        color = phenotype_to_color[group]
        ax.bar(left, height, width=width, color=color, label=group, linewidth=0.5, edgecolor='k')
    ylabel = 'Percentage of events' if percentages else 'Number of events'
    ax.set_ylabel(ylabel)
    ax.set_xticks(np.arange(len(self.modality_order)) + full_width / 2)
    ax.set_xticklabels(self.modality_order)
    ax.set_xlabel('Splicing modality')
    ax.set_xlim(0, len(self.modality_order))
    ax.legend(loc='best')
    ax.grid(axis='y', linestyle='-', linewidth=0.5)
    sns.despine()


def event_estimation(self, event, logliks, logsumexps, renamed=''):
    """pass"""
    plotter = _ModelLoglikPlotter()
    plotter.plot(event, logliks, logsumexps, self.modality_to_color, renamed=renamed)
    return plotter


def predict(self, fitted):
    """pass"""
    if fitted.shape[0] != len(self.modalities):
        raise ValueError("This data doesn't look like it had the distance between it and the five modalities calculated")
    return fitted.idxmin()


def logliks(self, x):
    """pass"""
    x = x.copy()
    x[x == 0] = VERY_SMALL_NUMBER
    x[x == 1] = 1 - VERY_SMALL_NUMBER
    return np.array([np.log(prob) + rv.logpdf(x[np.isfinite(x)]).sum() for prob, rv in zip(self.prob_parameters, self.rvs)])


def get_next_value(sequence_name, initial_value, reset_value='default', *, nowait=False, using=None):
    """pass"""
    from .models import Sequence
    if reset_value is not None and (not initial_value < reset_value):
        assert False
    if using is None:
        using = router.db_for_write(Sequence)
    connection = connections[using]
    if getattr(connection, 'pg_version', 0) >= 90500 and reset_value is None and (not nowait):
        with connection.cursor() as cursor:
            cursor.execute(UPSERT_QUERY, [sequence_name, initial_value])
            last, = cursor.fetchone()
        return last
    else:
        with transaction.atomic(using=using, savepoint=False):
            sequence, created = Sequence.objects.select_for_update(nowait=nowait).get_or_create(name=sequence_name, defaults={'last': initial_value})
            if not created:
                sequence.last += 1
                if reset_value is not None and sequence.last >= reset_value:
                    sequence.last = initial_value
                sequence.save()
            return sequence.last


def check(self, final_line_count):
    """pass"""
    if self._lines_seen['version']:
        self._process_version_lines()
    self._process_plan_lines(final_line_count)


def _process_version_lines(self):
    """pass"""
    if len(self._lines_seen['version']) > 1:
        self._add_error(_('Multiple version lines appeared.'))
    elif self._lines_seen['version'][0] != 1:
        self._add_error(_('The version must be on the first line.'))


def _process_plan_lines(self, final_line_count):
    """pass"""
    if not self._lines_seen['plan']:
        self._add_error(_('Missing a plan.'))
        return
    elif len(self._lines_seen['plan']) > 1:
        self._add_error(_('Only one plan line is permitted per file.'))
        return
    else:
        plan, at_line = self._lines_seen['plan'][0]
        if not self._plan_on_valid_line(at_line, final_line_count):
            self._add_error(_('A plan must appear at the beginning or end of the file.'))
            return
        elif plan.expected_tests != self._lines_seen['test']:
            self._add_error(_('Expected {expected_count} tests but only {seen_count} ran.').format(expected_count=plan.expected_tests, seen_count=self._lines_seen['test']))


def _plan_on_valid_line(self, at_line, final_line_count):
    """pass"""
    if at_line == 1 or (at_line == final_line_count):
        return True
    after_version = self._lines_seen['version'] and self._lines_seen['version'][0] == 1 and (at_line == 2)
    if after_version:
        return True
    else:
        return False


def handle_bail(self, bail):
    """pass"""
    self._add_error(_('Bailed: {reason}').format(reason=bail.reason))


def handle_skipping_plan(self, skip_plan):
    """pass"""
    skip_line = Result(True, None, skip_plan.directive.text, Directive('SKIP'))
    self._suite.addTest(Adapter(self._filename, skip_line))


def mptt_before_insert(mapper, connection, instance):
    """pass"""
    table = _get_tree_table(mapper)
    db_pk = instance.get_pk_column()
    table_pk = getattr(table.c, db_pk.name)
    if instance.parent_id is None:
        instance.left = 1
        instance.right = 2
        instance.level = instance.get_default_level()
        tree_id = connection.scalar(select([func.max(table.c.tree_id) + 1])) or 1
        instance.tree_id = tree_id
    else:
        parent_pos_left, parent_pos_right, parent_tree_id, parent_level = connection.execute(select([table.c.lft, table.c.rgt, table.c.tree_id, table.c.level])).where(table_pk == instance.parent_id).fetchone()
        connection.execute(table.update(and_(table.c.rgt >= parent_pos_right, table.c.tree_id == parent_tree_id)).values(lft=case([(table.c.lft > parent_pos_right, table.c.lft + 2)], else_=table.c.lft), rgt=case([(table.c.rgt >= parent_pos_right, table.c.rgt + 2)], else_=table.c.rgt)))
        instance.level = parent_level + 1
        instance.tree_id = parent_tree_id
        instance.left = parent_pos_right
        instance.right = parent_pos_right + 1


def mptt_before_update(mapper, connection, instance):
    """pass"""
    node_id = getattr(instance, instance.get_pk_name())
    table = _get_tree_table(mapper)
    db_pk = instance.get_pk_column()
    default_level = instance.get_default_level()
    table_pk = getattr(table.c, db_pk.name)
    mptt_move_inside = None
    left_sibling = None
    left_sibling_tree_id = None
    if hasattr(instance, 'mptt_move_inside'):
        mptt_move_inside = instance.mptt_move_inside
    if hasattr(instance, 'mptt_move_before'):
        right_sibling_left, right_sibling_right, right_sibling_parent, right_sibling_level, right_sibling_tree_id = connection.execute(select([table.c.lft, table.c.rgt, table.c.parent_id, table.c.level, table.c.tree_id]).where(table_pk == instance.mptt_move_before)).fetchone()
        current_lvl_nodes = connection.execute(select([table.c.lft, table.c.rgt, table.c.parent_id, table.c.tree_id]).where(and_(table.c.level == right_sibling_level, table.c.tree_id == right_sibling_tree_id, table.c.lft < right_sibling_left))).fetchall()
        if current_lvl_nodes:
            left_sibling_left, left_sibling_right, left_sibling_parent, left_sibling_tree_id = current_lvl_nodes[-1]
            instance.parent_id = left_sibling_parent
            left_sibling = {'lft': left_sibling_left, 'rgt': left_sibling_right, 'is_parent': False}
        elif not right_sibling_parent:
            left_sibling_tree_id = right_sibling_tree_id - 1
    if hasattr(instance, 'mptt_move_after'):
        left_sibling_left, left_sibling_right, left_sibling_parent, left_sibling_tree_id = connection.execute(select([table.c.lft, table.c.rgt, table.c.parent_id, table.c.tree_id]).where(table_pk == instance.mptt_move_after)).fetchone()
        instance.parent_id = left_sibling_parent
        left_sibling = {'lft': left_sibling_left, 'rgt': left_sibling_right, 'is_parent': False}
    '\n    Get the subtree\n    '
    subtree = connection.execute(select([table_pk]).where(and_(table.c.lft >= instance.left, table.c.rgt <= instance.right, table.c.tree_id == instance.tree_id))).order_by(table.c.lft).fetchall()
    subtree = [x[0] for x in subtree]
    '\n    Get the node position\n    '
    node_pos_left, node_pos_right, node_tree_id, node_parent_id, node_level = connection.execute(select([table.c.lft, table.c.rgt, table.c.tree_id, table.c.parent_id, table.c.level]).where(table_pk == node_id)).fetchone()
    if not left_sibling and str(node_parent_id) == str(instance.parent_id) and (not mptt_move_inside) and (left_sibling_tree_id is None):
        return
    if instance.parent_id is not None:
        '\n        Get the parent\n        '
        parent_id, parent_pos_right, parent_pos_left, parent_tree_id, parent_level = connection.execute(select([table_pk, table.c.rgt, table.c.lft, table.c.tree_id, table.c.level]).where(table_pk == instance.parent_id)).fetchone()
        if node_parent_id is None and node_tree_id == parent_tree_id:
            instance.parent_id = None
            return
    mptt_before_delete(mapper, connection, instance, False)
    if instance.parent_id is not None:
        '\n        Delete the subtree\n        '
        parent_id, parent_pos_right, parent_pos_left, parent_tree_id, parent_level = connection.execute(select([table_pk, table.c.rgt, table.c.lft, table.c.tree_id, table.c.level]).where(table_pk == instance.parent_id)).fetchone()
        node_size = node_pos_right - node_pos_left + 1
        if not left_sibling:
            left_sibling = {'lft': parent_pos_left, 'rgt': parent_pos_right, 'is_parent': True}
        instance.tree_id = parent_tree_id
        _insert_subtree(table, connection, node_size, node_pos_left, node_pos_right, parent_pos_left, parent_pos_right, subtree, parent_tree_id, parent_level, node_level, left_sibling, table_pk)
    else:
        if left_sibling_tree_id or left_sibling_tree_id == 0:
            tree_id = left_sibling_tree_id + 1
            connection.execute(table.update(table.c.tree_id > left_sibling_tree_id).values(tree_id=tree_id))
        else:
            tree_id = connection.scalar(select([func.max(table.c.tree_id) + 1]))
        connection.execute(table.update(table_pk.in_(subtree)).values(lft=table.c.lft - node_pos_left + 1, rgt=table.c.rgt - node_pos_left + 1, level=table.c.level - node_level + default_level, tree_id=tree_id))


def after_flush_postexec(self, session, context):
    """pass"""
    instances = self.instances[session]
    while instances:
        instance = instances.pop()
        if instance not in session:
            continue
        parent = self.get_parent_value(instance)
        while parent != NO_VALUE and parent is not None:
            instances.discard(parent)
            session.expire(parent, ['left', 'right', 'tree_id', 'level'])
            parent = self.get_parent_value(parent)
        session.expire(instance, ['left', 'right', 'tree_id', 'level'])
        self.expire_session_for_children(session, instance)


def is_ancestor_of(self, other, inclusive=False):
    """pass"""
    if inclusive:
        return (self.tree_id == other.tree_id) & (self.left <= other.left) & (other.right <= self.right)
    else:
        return (self.tree_id == other.tree_id) & (self.left < other.left) & (other.right < self.right)


def move_inside(self, parent_id):
    """pass"""
    session = Session.object_session(self)
    self.parent_id = parent_id
    self.mptt_move_inside = parent_id
    session.add(self)


def move_after(self, node_id):
    """pass"""
    session = Session.object_session(self)
    self.parent_id = self.parent_id
    self.mptt_move_after = node_id
    session.add(self)


def make_request(cls, url, method=None, params=None, basic_auth=None, timeout=600):
    """pass"""
    params = {} if params is None else params
    cls.request_id += 1
    params = {k: v for k, v in params.items() if v is not None}
    data = {'method': method, 'params': params, 'jsonrpc': '2.0', 'id': cls.request_id}
    headers = {'Content-Type': 'application/json-rpc', 'user-agent': 'LBRY python3-api'}
    request = requests.Request('POST', url, json=data, headers=headers, auth=basic_auth)
    prepared = request.prepare()
    try:
        sesh = requests.Session()
        response = sesh.send(prepared, timeout=timeout)
        response_json = response.json()
        if 'result' in response_json:
            return (response_json['result'], response)
        elif 'error' in response_json:
            raise LBRYUtils.LBRYException('POST Request made to LBRY received an error', response_json, response.status_code, prepared)
    except requests.HTTPError as HE:
        print(HE)
        return (None, None)
    except requests.RequestException as RE:
        print(RE)
        print('Printing Request Created:\n')
        LBRYUtils.print_request(prepared)
        return (None, None)


def adjust_status(info: dict) -> dict:
    """pass"""
    modified_info = deepcopy(info)
    modified_info.update({'level': get_nearest_by_numeric_key(STATUS_MAP, int(info['level'])), 'level2': STATUS_MAP[99] if info['level2'] is None else get_nearest_by_numeric_key(STATUS_MAP, int(info['level2']))})
    return modified_info


def status_by_coordinates(self, latitude: float, longitude: float) -> dict:
    """pass"""
    cdc_data = await self.raw_cdc_data()
    nearest = await self.nearest_by_coordinates(latitude, longitude)
    return adjust_status(cdc_data[nearest['state']['name']])


def status_by_state(self, state: str) -> dict:
    """pass"""
    data = await self.raw_cdc_data()
    try:
        info = next((v for k, v in data.items() if state in k))
    except StopIteration:
        return {}
    return adjust_status(info)


def insert(self, **kwargs):
    """pass"""
    if self.is_valid:
        before = self.before_insert()
        if before:
            return before
        try:
            self._document['_id'] = self.insert_one(self._document)
            self.after_insert()
            return self._document
        except PyMongoException as exc:
            return PyMongoError(error_message=exc.details.get('errmsg', exc.details.get('err', 'PyMongoError.')), operation='insert', collection=type(self).__name__, document=self._document)
    else:
        return self._errors


def update(self, **kwargs):
    """pass"""
    if self.is_valid:
        if '_id' in self._document:
            to_update = self.find_one({'_id': self._id})
            if to_update:
                before = self.before_update(old=to_update)
                return before if before else self.replace_one({'_id': self._id}, self._document), self.after_update(old=to_update), self._document
            else:
                return DocumentNotFoundError(type(self).__name__, self._id)
        else:
            return UnidentifiedDocumentError(type(self).__name__, self._document)
    else:
        return self._errors


def delete(self, **kwargs):
    """pass"""
    if self.is_valid:
        if '_id' in self._document:
            to_delete = self.find_one({'_id': self._id})
            if to_delete:
                before = self.before_delete()
                return before if before else PyMongoError(error_message=exc.details.get('errmsg', exc.details.get('err', 'PyMongoError.')), operation='delete', collection=type(self).__name__, document=self._document)
            else:
                return DocumentNotFoundError(type(self).__name__, self._id)
        else:
            return UnidentifiedDocumentError(type(self).__name__, self._document)


def find_one(cls, filter=None, *args, **kwargs):
    """pass"""
    return cls.collection.find_one(filter, *args, **kwargs)


def find(cls, *args, **kwargs):
    """pass"""
    return list(cls.collection.find(*args, **kwargs))


def aggregate(cls, pipeline=None, **kwargs):
    """pass"""
    return list(cls.collection.aggregate(pipeline or [], **kwargs))


def in_file(self, fn: str) -> Iterator[Statement]:
    """pass"""
    yield from self.__file_to_statements.get(fn, [])


def at_line(self, line: FileLine) -> Iterator[Statement]:
    """pass"""
    num = line.num
    for stmt in self.in_file(line.filename):
        if stmt.location.start.line == num:
            yield stmt


def wrap(text, width=70, **kwargs):
    """pass"""
    w = ParagraphWrapper(width=width, **kwargs)
    return w.wrap(text)


def fill(text, width=70, **kwargs):
    """pass"""
    w = ParagraphWrapper(width=width, **kwargs)
    return w.fill(text)


def split(cls, text):
    """pass"""
    result = [line.strip('\n') for line in cls.parasep_re.split(text)]
    if result == ['', '']:
        result = ['']
    return result


def wrap(self, text):
    """pass"""
    lines = []
    linewrap = partial(textwrap.TextWrapper.wrap, self)
    for para in self.split(text):
        lines.extend(linewrap(para))
        lines.append('')
    lines = lines[:-1]
    return lines


def getSenderNumberMgtURL(self, CorpNum, UserID):
    """pass"""
    result = self._httpget('/FAX/?TG=SENDER', CorpNum, UserID)
    return result.url


def getUnitCost(self, CorpNum):
    """pass"""
    result = self._httpget('/FAX/UnitCost', CorpNum)
    return int(result.unitCost)


def getFaxResult(self, CorpNum, ReceiptNum, UserID=None):
    """pass"""
    if ReceiptNum == None or len(ReceiptNum) != 18:
        raise PopbillException(-99999999, '접수번호가 올바르지 않습니다.')
    return self._httpget('/FAX/' + ReceiptNum, CorpNum, UserID)


def getFaxResultRN(self, CorpNum, RequestNum, UserID=None):
    """pass"""
    if RequestNum == None or RequestNum == '':
        raise PopbillException(-99999999, '요청번호가 입력되지 않았습니다.')
    return self._httpget('/FAX/Get/' + RequestNum, CorpNum, UserID)


def sendFax(self, CorpNum, SenderNum, ReceiverNum, ReceiverName=None, FilePath=None, ReserveDT=None, UserID=False, SenderName=None, adsYN=None, title=None, RequestNum=None):
    """pass"""
    receivers = []
    receivers.append(FaxReceiver(receiveNum=ReceiverNum, receiveName=ReceiverName))
    return self.sendFax_multi(CorpNum, SenderNum, receivers, FilePath, ReserveDT, UserID, SenderName, adsYN, title, RequestNum)


def sendFax_multi(self, CorpNum, SenderNum=None, Receiver=None, FilePath=None, ReserveDT=False, UserID=None, SenderName=None, adsYN=None, title=None, RequestNum=None):
    """pass"""
    if SenderNum == None or SenderNum == '':
        raise PopbillException(-99999999, '발신자 번호가 입력되지 않았습니다.')
    elif Receiver == None:
        raise PopbillException(-99999999, '수신자 정보가 입력되지 않았습니다.')
    if not type(Receiver) is str and (not type(Receiver) is FaxReceiver) and (not type(Receiver) is list):
        raise PopbillException(-99999999, "'Receiver' argument type error. 'FaxReceiver' or List of 'FaxReceiver'.")
    elif FilePath == None:
        raise PopbillException(-99999999, '발신 파일경로가 입력되지 않았습니다.')
    if not type(FilePath) is str and (not type(FilePath) is list):
        raise PopbillException(-99999999, '발신 파일은 파일경로 또는 경로목록만 입력 가능합니다.')
    elif type(FilePath) is list:
        if len(FilePath) < 1 or len(FilePath) > 20:
            raise PopbillException(-99999999, '파일은 1개 이상, 20개 까지 전송 가능합니다.')
    req = {'snd': SenderNum, 'sndnm': SenderName, 'fCnt': 1 if type(FilePath) is str else len(FilePath), 'rcvs': [], 'sndDT': None}
    if type(Receiver) is str:
        Receiver = FaxReceiver(receiveNum=Receiver)
    if type(Receiver) is FaxReceiver:
        Receiver = [Receiver]
    if adsYN:
        req['adsYN'] = True
    for r in Receiver:
        req['rcvs'].append({'rcv': r.receiveNum, 'rcvnm': r.receiveName})
    if ReserveDT != None:
        req['sndDT'] = ReserveDT
    if title != None:
        req['title'] = title
    if RequestNum != None:
        req['requestNum'] = RequestNum
    postData = self._stringtify(req)
    if type(FilePath) is str:
        FilePath = [FilePath]
    files = []
    for filePath in FilePath:
        with open(filePath, 'rb') as f:
            files.append(File(fieldName='file', fileName=f.name, fileData=f.read()))
    result = self._httppost_files('/FAX', postData, files, CorpNum, UserID)
    return result.receiptNum


def model_node(**kwargs):
    """pass"""
    kwargs.setdefault('default', {})

    def decorator(model):
        return types.ModelType(model, **kwargs)
    return decorator


def status_by_coordinates(self, latitude: float, longitude: float) -> dict:
    """pass"""
    return await self.nearest_by_coordinates(latitude, longitude)


def status_by_zip(self, zip_code: str) -> dict:
    """pass"""
    try:
        location = next((d for d in self.user_reports() if d['zip'] == zip_code))
    except StopIteration:
        return {}
    return await self.status_by_coordinates(float(location['latitude']), float(location['longitude']))


def print_request(request):
    """pass"""
    print('{}\n{}\n{}\n\n{}'.format('-----------START-----------', request.method + ' ', request.url, '\n'.join(('{}: {}'.format(k, v) for k, v in request.headers.items()))))


def filter_validate_schemas(get_response, params):
    """pass"""
    request_schema = params.get('request_schema')
    if request_schema is None:
        return get_response

    def _convert_params(schema, data):
        for sc in schema.fields.values():
            name = sc.serialized_name or sc.name
            val = data.getlist(name)
            if val is not None:
                if len(val) == 1 and (not isinstance(sc, ListType)):
                    val = val[0]
                data[name] = val

    def decorated_filter(request, *args, **kwargs):
        data = {'headers': CIDict(request.headers), 'path': request.app.router.get(request)[2], 'params': RequestParameters(request.args), 'body': {}}
        if request.body:
            if request.form:
                data['body'] = RequestParameters(request.form)
            else:
                data['body'] = deepcopy(request.json)
        if hasattr(request_schema, 'body') and request.form:
            _convert_params(request_schema.body, data['body'])
        if hasattr(request_schema, 'params') and data['params']:
            _convert_params(request_schema.params, data['params'])
        try:
            model = request_schema(data, strict=False, validate=False)
            model.validate()
            request.validated = model.to_native()
        except BaseError as e:
            raise ValidationErrors(e.to_primitive())
        return await get_response(request, *args, **kwargs)
    return decorated_filter


def filter_validate_response(get_response, params):
    """pass"""
    schema = params.get('response_schema')

    def decorated_filter(request, *args, **kwargs):
        response = await get_response(request, *args, **kwargs)
        if isinstance(response, HTTPResponse) and (not isinstance(response, Response)):
            return response
        if not isinstance(response, Response):
            raise TypeError('response is not an instance of rafter.http.Response.')
        if schema:
            data = {'body': response.data, 'headers': response.headers}
            try:
                model = schema(data, strict=False, validate=False)
                model.validate()
                result = model.to_primitive()
                response.body = result.get('body', None)
                response.headers.update(result.get('headers', {}))
            except BaseError as e:
                log.exception(e)
                abort(500, 'Wrong data output')
        return response
    return decorated_filter


def _write_int(fname, data, append=True):
    """pass"""
    data_ex = pexdoc.exh.addex(ValueError, 'There is no data to save to file')
    fos_ex = pexdoc.exh.addex(OSError, 'File *[fname]* could not be created: *[reason]*')
    data_ex(len(data) == 0 or (len(data) == 1 and len(data[0]) == 0))
    try:
        pmisc.make_dir(fname)
        mode = 'w' if append is False else 'a'
        if sys.hexversion < 50331648:
            with open(fname, mode) as file_handle:
                csv.writer(file_handle, delimiter=',').writerows(data)
        else:
            with open(fname, mode, newline='') as file_handle:
                csv.writer(file_handle, delimiter=',').writerows(data)
    except (IOError, OSError) as eobj:
        fos_ex(True, _MF('fname', fname, 'reason', eobj.strerror))


def revdocs2reverts(rev_docs, radius=defaults.RADIUS, use_sha1=False, resort=False, verbose=False):
    """pass"""
    page_rev_docs = groupby(rev_docs, lambda rd: rd.get('page'))
    for page_doc, rev_docs in page_rev_docs:
        if verbose:
            sys.stderr.write(page_doc.get('title') + ': ')
            sys.stderr.flush()
        if resort:
            if verbose:
                sys.stderr.write('(sorting) ')
                sys.stderr.flush()
            rev_docs = sorted(rev_docs, key=lambda r: (r.get('timestamp'), r.get('id')))
        detector = Detector(radius=radius)
        for rev_doc in rev_docs:
            if not use_sha1 and 'text' not in rev_doc:
                logger.warn("Skipping {0}: 'text' field not found in {0}".format(rev_doc['id'], rev_doc))
                continue
            if use_sha1:
                checksum = rev_doc.get('sha1') or DummyChecksum()
            elif 'text' in rev_doc:
                text_bytes = bytes(rev_doc['text'], 'utf8', 'replace')
                checksum = hashlib.sha1(text_bytes).digest()
            revert = detector.process(checksum, rev_doc)
            if revert:
                yield revert.to_json()
                if verbose:
                    sys.stderr.write('r')
                    sys.stderr.flush()
            elif verbose:
                sys.stderr.write('.')
                sys.stderr.flush()
        if verbose:
            sys.stderr.write('\n')
            sys.stderr.flush()


def dsort(fname, order, has_header=True, frow=0, ofname=None):
    """pass"""
    ofname = fname if ofname is None else ofname
    obj = CsvFile(fname=fname, has_header=has_header, frow=frow)
    obj.dsort(order)
    obj.write(fname=ofname, header=has_header, append=False)


def main() -> None:
    """pass"""
    logging.basicConfig(level=logging.INFO)
    with ClientSession() as websession:
        try:
            client = Client(websession)
            user_coord_resp = await client.user_reports.status_by_coordinates(LATITUDE, LONGITUDE)
            _LOGGER.info('User data by latitude/longitude (%s, %s): %s', LATITUDE, LONGITUDE, user_coord_resp)
            user_zip_resp = await client.user_reports.status_by_zip(ZIP_CODE)
            _LOGGER.info('User data by ZIP code (%s): %s', ZIP_CODE, user_zip_resp)
            cdc_coord_resp = await client.cdc_reports.status_by_coordinates(LATITUDE, LONGITUDE)
            _LOGGER.info('CDC data by latitude/longitude (%s, %s): %s', LATITUDE, LONGITUDE, cdc_coord_resp)
            cdc_state_resp = await client.cdc_reports.status_by_state(STATE)
            _LOGGER.info('CDC data by state name (%s): %s', STATE, cdc_state_resp)
        except FluNearYouError as err:
            print(err)


def call(cls, method, params=None, timeout=600):
    """pass"""
    params = [] if params is None else params
    return cls.make_request(SERVER_ADDRESS, method, params, timeout=timeout)


def concatenate(fname1, fname2, dfilter1=None, dfilter2=None, has_header1=True, has_header2=True, frow1=0, frow2=0, ofname=None, ocols=None):
    """pass"""
    iro = pexdoc.exh.addex(RuntimeError, 'Files have different number of columns')
    iom = pexdoc.exh.addex(RuntimeError, 'Number of columns in data files and output columns are different')
    obj1 = CsvFile(fname=fname1, dfilter=dfilter1, has_header=has_header1, frow=frow1)
    obj2 = CsvFile(fname=fname2, dfilter=dfilter2, has_header=has_header2, frow=frow2)
    ofname = fname1 if ofname is None else ofname
    if ocols is None:
        if has_header1:
            ocols = [obj1.header()] if obj1.cfilter is None else [obj1.cfilter]
    elif ocols is None:
        if has_header2:
            ocols = [obj2.header()] if obj2.cfilter is None else [obj2.cfilter]
    else:
        if iom(obj1.cfilter is not None and len(obj1.cfilter) != len(ocols)):
            ocols = [ocols]
    iro(_C(obj1.cfilter, obj2.cfilter) and (len(obj1.cfilter) != len(obj2.cfilter)))
    data = ocols + obj1.data(filtered=True) + obj2.data(filtered=True)
    write(fname=ofname, data=data, append=False)


def resource(self, uri, methods=frozenset({'GET'}), host=None, strict_slashes=None, stream=False, version=None, name=None, **kwargs):
    """pass"""
    if strict_slashes is None:
        strict_slashes = self.strict_slashes

    def decorator(handler):
        self.resources.append((FutureRoute(handler, uri, methods, host, strict_slashes, stream, version, name), kwargs))
        return handler
    return decorator


def add_resource(self, handler, uri, methods=frozenset({'GET'}), host=None, strict_slashes=None, version=None, name=None, **kwargs):
    """pass"""
    self.resource(uri=uri, methods=methods, host=host, strict_slashes=strict_slashes, version=version, name=name, **kwargs)(handler)


def in_file(self, fn: str) -> Iterator[InsertionPoint]:
    """pass"""
    logger.debug('finding insertion points in file: %s', fn)
    yield from self.__file_insertions.get(fn, [])


def at_line(self, line: FileLine) -> Iterator[InsertionPoint]:
    """pass"""
    logger.debug('finding insertion points at line: %s', str(line))
    filename = line.filename
    line_num = line.num
    for ins in self.in_file(filename):
        if line_num == ins.location.line:
            logger.debug('found insertion point at line [%s]: %s', str(line), ins)
            yield ins


def _doAtomicFileCreation(filePath):
    """pass"""
    try:
        _os.close(_os.open(filePath, _os.O_CREAT | _os.O_EXCL))
        return True
    except OSError as e:
        if e.errno == _errno.EEXIST:
            return False
        else:
            raise e


def findNextFile(folder='.', prefix=None, suffix=None, fnameGen=None, base=0, maxattempts=10):
    """pass"""
    expFolder = _os.path.expanduser(_os.path.expandvars(folder))
    return _findNextFile(expFolder, prefix, suffix, fnameGen, base, maxattempts, 0)


def _run(passedArgs=None, stderr=None, stdout=None, exitFn=None):
    """pass"""
    description = 'Finds the next available file-name in a sequence.\n\n    This program will create a file of zero size and will output the path to it\n    on STDOUT. No files which exist will be altered in this operation and\n    concurrent invocations of this program will return separate files. In case\n    of conflict, this program will attempt to generate a new file name up to\n    \'maxattempts\' number of times before failing. (See --max-attempts)\n\n    The sequence will start from the base argument (See --base, default: 0).\n\n    This program will look for the next file in the sequence ignoring any gaps.\n    Hence, if the files "a.0.txt" and "a.3.txt" exist, then the next file\n    returned will be "a.4.txt" when called with prefix="a." and suffix=".txt".\n\n    Returns:\n        Path of the file which follows the provided pattern and can be opened\n        for writing.\n\n        Otherwise, it prints an error (wrong path, drive full, illegal\n        character in filename, etc.) to stderr and exits with a non-zero error\n        code.\n    """
    argParser = _argparse.ArgumentParser(description=description, formatter_class=_argparse.RawTextHelpFormatter)
    argParser.add_argument('prefix', help='Prefix for the sequence of files.')
    argParser.add_argument('suffix', help='Suffix for the sequence of files.', nargs='?', default='')
    argParser.add_argument('folder', help='The folder where the file will be created.', nargs='?', default=_os.getcwd())
    argParser.add_argument('-m', '--max-attempts', help='Number of attempts to make before giving up.', default=10)
    argParser.add_argument('-b', '--base', help='From where to start counting (default: 0).', default=0)
    passedArgs = passedArgs if passedArgs is not None else _sys.argv[1:]
    args = argParser.parse_args(passedArgs)
    stdout = _sys.stdout if stdout is None else stdout
    stderr = _sys.stderr if stderr is None else stderr
    try:
        nextFile = findNextFile(args.folder, prefix=args.prefix, suffix=args.suffix, maxattempts=args.max_attempts, base=args.base)
        stdout.write(nextFile + '\n')
    except OSError as e:
        stderr.write(_os.strerror(e.errno) + '\n')
        _sys.exit(e.errno)


def _errstr(value):
    """pass"""
    value = str(value)
    if len(value) > MAX_ERROR_STR_LEN:
        return value[:MAX_ERROR_STR_LEN] + '...'
    else:
        return value


def _getStrippedValue(value, strip):
    """pass"""
    if strip is None:
        value = value.strip()
    elif isinstance(strip, str):
        value = value.strip(strip)
    elif strip is False:
        pass
    return value


def _raiseValidationException(standardExcMsg, customExcMsg=None):
    """pass"""
    if customExcMsg is None:
        raise ValidationException(str(standardExcMsg))
    else:
        raise ValidationException(str(customExcMsg))


def _prevalidationCheck(value, blank, strip, allowlistRegexes, blocklistRegexes, excMsg=None):
    """pass"""
    value = str(value)
    value = _getStrippedValue(value, strip)
    if not blank and value == '':
        _raiseValidationException(_('Blank values are not allowed.'), excMsg)
    elif blank and value == '':
        return (True, value)
    if allowlistRegexes is not None:
        for regex in allowlistRegexes:
            if isinstance(regex, re.Pattern):
                if regex.search(value, re.IGNORECASE) is not None:
                    return (True, value)
            else:
                if re.search(regex, value, re.IGNORECASE) is not None:
                    return (True, value)
    elif blocklistRegexes is not None:
        for blocklistRegexItem in blocklistRegexes:
            if isinstance(blocklistRegexItem, str):
                regex, response = (blocklistRegexItem, DEFAULT_BLOCKLIST_RESPONSE)
            else:
                regex, response = blocklistRegexItem
            if isinstance(regex, re.Pattern) and regex.search(value, re.IGNORECASE) is not None:
                _raiseValidationException(response, excMsg)
            elif re.search(regex, value, re.IGNORECASE) is not None:
                _raiseValidationException(response, excMsg)
    return (False, value)


def _validateGenericParameters(blank, strip, allowlistRegexes, blocklistRegexes):
    """pass"""
    if not isinstance(blank, bool):
        raise PySimpleValidateException('blank argument must be a bool')
    if not isinstance(strip, (bool, str, type(None))):
        raise PySimpleValidateException('strip argument must be a bool, None, or str')
    if allowlistRegexes is None:
        allowlistRegexes = []
    try:
        len(allowlistRegexes)
    except:
        raise PySimpleValidateException('allowlistRegexes must be a sequence of regex_strs')
    for response in allowlistRegexes:
        if not isinstance(response[0], str):
            raise PySimpleValidateException('allowlistRegexes must be a sequence of regex_strs')
    if blocklistRegexes is None:
        blocklistRegexes = []
    try:
        len(blocklistRegexes)
    except:
        raise PySimpleValidateException('blocklistRegexes must be a sequence of (regex_str, str) tuples or regex_strs')
    for response in blocklistRegexes:
        if not isinstance(response, str):
            if len(response) != 2:
                raise PySimpleValidateException('blocklistRegexes must be a sequence of (regex_str, str) tuples or regex_strs')
            if not (isinstance(response[0], str) and isinstance(response[1], str)):
                raise PySimpleValidateException('blocklistRegexes must be a sequence of (regex_str, str) tuples or regex_strs')


def _validateParamsFor_validateNum(min=None, max=None, lessThan=None, greaterThan=None):
    """pass"""
    if min is not None and greaterThan is not None:
        raise PySimpleValidateException('only one argument for min or greaterThan can be passed, not both')
    elif max is not None and lessThan is not None:
        raise PySimpleValidateException('only one argument for max or lessThan can be passed, not both')
    if min is not None and max is not None and (min > max):
        raise PySimpleValidateException('the min argument must be less than or equal to the max argument')
    elif min is not None and lessThan is not None and (min >= lessThan):
        raise PySimpleValidateException('the min argument must be less than the lessThan argument')
    elif max is not None and greaterThan is not None and (max <= greaterThan):
        raise PySimpleValidateException('the max argument must be greater than the greaterThan argument')
    for name, val in [('min', min), ('max', max), ('lessThan', lessThan), ('greaterThan', greaterThan)]:
        if not isinstance(val, (int, float, type(None))):
            raise PySimpleValidateException(name + ' argument must be int, float, or NoneType')


def filter_transform_response(get_response, params):
    """pass"""

    async def decorated_filter(request, *args, **kwargs):
        response = await get_response(request, *args, **kwargs)
        if isinstance(response, HTTPResponse) and (not isinstance(response, Response)):
            return response
        if not isinstance(response, Response):
            response = Response(response)
        return response
    return decorated_filter


def initialize(self, maxsize, history=None):
    """pass"""
    super().__init__()
    self.maxsize = int(maxsize)
    self.history = deque(maxlen=self.maxsize)
    if history is not None:
        for key, value in history:
            self.insert(key, value)


def insert(self, key, value):
    """pass"""
    if len(self.history) == self.maxsize:
        expectorate = self.history[0]
    else:
        expectorate = None
    self.history.append((key, value))
    if key in self:
        super().__getitem__(key).append(value)
    else:
        super().__setitem__(key, [value])
    if expectorate is not None:
        old_key, old_value = expectorate
        super().__getitem__(old_key).pop(0)
        if len(super().__getitem__(old_key)) == 0:
            super().__delitem__(old_key)
        return (old_key, old_value)


def up_to(self, key):
    """pass"""
    for okey, ovalue in reversed(self.history):
        if okey == key:
            break
        yield ovalue


def check(session, rev_id, page_id=None, radius=defaults.RADIUS, before=None, window=None, rvprop=None):
    """pass"""
    rev_id = int(rev_id)
    radius = int(radius)
    if radius < 1:
        raise TypeError('invalid radius.  Expected a positive integer.')
    page_id = int(page_id) if page_id is not None else None
    before = Timestamp(before) if before is not None else None
    rvprop = set(rvprop) if rvprop is not None else set()
    if page_id is None:
        page_id = get_page_id(session, rev_id)
    current_and_past_revs = list(n_edits_before(session, rev_id, page_id, radius + 1, {'sha1', 'timestamp', 'ids'} | rvprop))
    if len(current_and_past_revs) < 1:
        raise KeyError('Revision {0} not found in page {1}.'.format(rev_id, page_id))
    current_rev, past_revs = (current_and_past_revs[-1], current_and_past_revs[:-1])
    if window is not None and before is None:
        before = Timestamp(current_rev['timestamp']) + window
    future_revs = list(n_edits_after(session, rev_id + 1, page_id, radius, {before}, rvprop))
    return build_revert_tuple(rev_id, past_revs, current_rev, future_revs, radius)


def check_deleted(session, rev_id, title=None, timestamp=defaults.RADIUS, radius=None, before=None, window=None, rvprop=None):
    """pass"""
    rev_id = int(rev_id)
    radius = int(radius)
    if radius < 1:
        raise TypeError('invalid radius.  Expected a positive integer.')
    title = str(title) if title is not None else None
    before = Timestamp(before) if before is not None else None
    rvprop = set(rvprop) if rvprop is not None else set()
    if title is None or timestamp is None:
        title, timestamp = get_deleted_title_and_timestamp(session, rev_id)
    current_and_past_revs = list(n_deleted_edits_before(session, rev_id, title, timestamp, radius + 1, {'sha1', 'timestamp', 'ids'} | rvprop))
    if len(current_and_past_revs) < 1:
        raise KeyError('Revision {0} not found in page {1}.'.format(rev_id, title))
    current_rev, past_revs = (current_and_past_revs[-1], current_and_past_revs[:-1])
    if window is not None and before is None:
        before = Timestamp(current_rev['timestamp']) + window
    future_revs = list(n_deleted_edits_after(session, rev_id + 1, title, timestamp, radius, before, {'sha1', 'timestamp', 'ids'} | rvprop))
    return build_revert_tuple(rev_id, past_revs, current_rev, future_revs, radius)


def get_commits(repo_dir, old_commit, new_commit, hide_merges=True):
    """pass"""
    repo = Repo(repo_dir)
    commits = repo.iter_commits(rev='{0}..{1}'.format(old_commit, new_commit))
    if hide_merges:
        return [x for x in commits if not x.summary.startswith('Merge ')]
    else:
        return list(commits)