"""Various computers of job posting properties. Each class is generally a generic algorithm (such as skill extraction or occupation classification) paired with enough configuration to run on its own"""
from . import JobPostingComputedProperty, ComputedPropertyColumn

from skills_ml.algorithms.string_cleaners.nlp import title_phase_one
from skills_ml.algorithms.jobtitle_cleaner.clean import JobTitleStringClean
from skills_ml.algorithms.occupation_classifiers.classifiers import \
    SocClassifier
from skills_ml.job_postings.corpora import SimpleCorpusCreator
import logging
import statistics

NUMERIC_AGGREGATION_FUNCTION_PATHS = {
    'numpy.mean': 'Arithemetic mean',
    'numpy.median': 'Median (middle value)',
    'numpy.std': 'Sample standard deviation',
    'numpy.var': 'Sample variance',
}

class TitleCleanPhaseOne(JobPostingComputedProperty):
    """Perform one phase of job title cleaning: lowercase/remove punctuation"""
    property_name = 'title_clean_phase_one'
    property_columns = [
        ComputedPropertyColumn(
            name='title_clean_phase_one',
            description='Job title, cleaned by lowercasing and removing punctuation'
        )
    ]

    def _compute_func_on_one(self):
        title_func = title_phase_one
        return lambda job_posting: title_func(job_posting['title'])

class TitleCleanPhaseTwo(JobPostingComputedProperty):
    """Perform two phases of job title cleaning:

    1. lowercase/remove punctuation
    2. Remove geography information
    """
    property_name = 'title_clean_phase_two'
    property_columns = [
        ComputedPropertyColumn(
            name='title_clean_phase_two',
            description='Job title, cleaned by lowercasing, removing punctuation, and removing geography information',
        )
    ]

    def _compute_func_on_one(self):
        return lambda job_posting: JobTitleStringClean().clean_title(title_phase_one(job_posting['title']))

class Geography(JobPostingComputedProperty):
    """Produce a geography by querying a given JobGeographyQuerier

    Args:
        geo_querier
    """
    def __init__(self, geo_querier, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.geo_querier = geo_querier

    @property
    def property_name(self):
        return self.geo_querier.name

    @property
    def property_columns(self):
        return [
            ComputedPropertyColumn(name=name, description=description)
            for name, description
            in self.geo_querier.output_columns
        ]

    def _compute_func_on_one(self):
        return lambda job_posting: self.geo_querier.query(job_posting)

class SOCClassifyProperty(JobPostingComputedProperty):
    """Classify the SOC code from a trained classifier

    Args:
        classifier_obj (object, optional) An object to use as a classifier.
            If not sent one will be downloaded from s3
    """
    def __init__(self, classifier_obj, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.classifier = SocClassifier(classifier_obj)

    def _compute_func_on_one(self):

        common_classifier = self.classifier
        corpus_creator = SimpleCorpusCreator()

        def func(job_posting):
            return common_classifier.predict_soc(corpus_creator._transform(job_posting))

        return func

    @property
    def property_name(self):
        return self.classifier.name

    @property
    def property_description(self):
        return self.classifier.description

    @property
    def property_columns(self):
        property_columns = [
            ComputedPropertyColumn(
                name=self.property_name,
                description=self.property_description,
                compatible_aggregate_function_paths={
                    'skills_ml.job_postings.aggregate.pandas.n_most_common': 'Most common'
                }
            )
        ]
        return property_columns

class GivenSOC(JobPostingComputedProperty):
    """Assign the SOC code given by the partner"""
    property_name = 'soc_given'
    property_columns = [
        ComputedPropertyColumn(
            name='soc_given',
            description='SOC code given by partner',
            compatible_aggregate_function_paths={
                'skills_ml.job_postings.aggregate.pandas.n_most_common': 'Most common'
            }
        )
    ]

    def _compute_func_on_one(self):
        def func(job_posting):
            return job_posting.get('onet_soc_code', '99-9999.00')
        return func

class PayMixin(object):
    def salary_from_job_posting(self, job_posting):
        minSalary = None
        maxSalary = None
        try:
            minSalary = float(job_posting['baseSalary']['minValue'])
        except ValueError:
            logging.warning('Could not cast minValue string %s to float', job_posting['baseSalary']['minValue'])

        try:
            maxSalary = float(job_posting['baseSalary']['maxValue'])
        except ValueError:
            logging.warning('Could not cast maxValue string %s to float', job_posting['baseSalary']['maxValue'])

        if not maxSalary and not minSalary:
            logging.warning('Neither minSalary nor maxSalary could be converted to float, no extraction possible')
            return None

        if not maxSalary:
            return minSalary

        if not minSalary:
            return maxSalary

        return statistics.mean([minSalary, maxSalary])

class HourlyPay(JobPostingComputedProperty, PayMixin):
    """The pay given in the baseSalary field if salaryFrequency is hourly"""
    property_name = 'hourly_pay'
    property_columns = [
        ComputedPropertyColumn(
            name='pay_hourly',
            description='Pay given in baseSalary field if salaryFrequency is hourly',
            compatible_aggregate_function_paths=NUMERIC_AGGREGATION_FUNCTION_PATHS,
        )
    ]

    def _compute_func_on_one(self):
        def func(job_posting):
            if job_posting.get('baseSalary', {}).get('salaryFrequency', None) != 'hourly':
                return None
            return self.salary_from_job_posting(job_posting)
        return func

class YearlyPay(JobPostingComputedProperty, PayMixin):
    """The pay given in the baseSalary field if salaryFrequency is yearly"""
    property_name = 'yearly_pay'
    property_columns = [
        ComputedPropertyColumn(
            name='pay_yearly',
            description='Pay given in baseSalary field if salaryFrequency is yearly',
            compatible_aggregate_function_paths=NUMERIC_AGGREGATION_FUNCTION_PATHS,
        )
    ]

    def _compute_func_on_one(self):
        def func(job_posting):
            if job_posting.get('baseSalary', {}).get('salaryFrequency', None) != 'yearly':
                return None
            return self.salary_from_job_posting(job_posting)
        return func

class SkillCounts(JobPostingComputedProperty):
    """Adding top skill counts from a skill extractor

    Args: (skills_ml.algorithms.skill_extractors.base.SkillExtractorBase) A skill extractor object
    """
    def __init__(self, skill_extractor, *args, **kwargs):
        self.skill_extractor = skill_extractor
        super().__init__(*args, **kwargs)

    @property
    def property_name(self):
        return f'skill_counts_{self.skill_extractor.name}'

    @property
    def property_columns(self):
        return [ComputedPropertyColumn(
            name=self.property_name,
            description=self.skill_extractor.description,
            compatible_aggregate_function_paths={
                'skills_ml.job_postings.aggregate.pandas.listy_n_most_common': 'Most common'
            }
        )]

    def _compute_func_on_one(self):
        def func(job_posting):
            count_dict = self.skill_extractor.document_skill_counts(job_posting)
            count_lists = [[k] * v for k, v in count_dict.items()]
            flattened = [count for countlist in count_lists for count in countlist]
            return {self.property_name: flattened}
        return func

class PostingIdPresent(JobPostingComputedProperty):
    """Records job posting ids. Used for counting job postings"""
    property_name = 'posting_id_present'
    property_columns = [
        ComputedPropertyColumn(
            name='posting_id_present',
            description='Job postings',
            compatible_aggregate_function_paths={'numpy.sum': 'Count of'}
        )
    ]

    def _compute_func_on_one(self):
        return lambda job_posting: 1


<filename>tacoma/drawing.py
# -*- coding: utf-8 -*-
"""
This module contains functions for drawing temporal networks
in different ways. It depends on three packages not being installed
during installation with ``pip``, which are

- matplotlib
- networkx
- python-louvain

If you want to use this module, please install the dependencies
listed above.
"""
from __future__ import print_function

try:
    import matplotlib.pyplot as pl
    from matplotlib.collections import LineCollection
except ImportError as e:
    print("\033[1m tacoma does not install `matplotlib` as a dependency. Please install it manually. \033[0m")
    raise e

try:
    import networkx as nx
except ImportError as e:
    print("\033[1m tacoma does not install `networkx` as a dependency. Please install it manually. \033[0m")
    raise e

try:
    import community
except ImportError as e:
    print("\033[1m tacoma does not install `python-louvain`, neccessary for `community`,  as a dependency. Please install it manually. \033[0m")
    raise e

import numpy as np

from scipy.optimize import curve_fit

import tacoma as tc

_layout_function = 'graphviz'

def _draw_edge_lists(L):
    """This draws a force-directed layout for each snapshot of a temporal network given in :mod:`_tacoma.edge_lists` format and hence should be used with caution."""
    from rocsNWL.drawing import draw
    from rocsNWL.drawing import get_pos

    G = nx.Graph()
    G.add_nodes_from(range(L.N))
    G.add_edges_from(L.edges[0])

    pos, _ = get_pos(G, layout_function=_layout_function)

    fig, ax = pl.subplots(1, len(L.edges), figsize=(len(L.edges)*3, 4))
    ax = ax.flatten()

    draw(G, pos=pos, ax=ax[0], layout_function=_layout_function)

    for i in range(1, L.N):
        G = nx.Graph()
        G.add_nodes_from(range(L.N))
        G.add_edges_from(L.edges[i])

        draw(G, pos=pos, ax=ax[i], layout_function=_layout_function)

def fit_function(x, alpha, scale, fac, intervals_to_discard_for_fit):
    r"""
    A fit function for the number of uniquely observed edges over
    time, following the assumption that edge activity rates follow a gamma
    distribution.

    .. math::
        f(x) = \frac{\lambda^\alpha}{\Gamma(\lambda)} x^{\alpha-1}\\exp(-\lambda x)

    The fit function is

    .. math::
        y(x) = \phi\times \left[ 1 - \left(\frac{\lambda}{\lambda+x}\right)^\alpha\right]

    Parameters
    ----------
    x : numpy.ndarray
        Data on the x-axis, typically time
    alpha : float
        exponent in gamma distribution, has to be alpha > 0
    scale : float
        scale :math:`\\lambda` in gamma distribution, has to be scale > 0
    fac : float
        prefactor, typically :math:`\\phi=N(N-1)/2`.
    intervals_to_discard_for_fit : list of tuple of float
        a list of time intervals which have to be discarded for the fit

    Returns
    -------

    y : numpy.ndarray
        value of the function
    """

    x_ = x.copy()

    offset = 0.0

    for interval in intervals_to_discard_for_fit:
        t0, t1 = interval
        x_[np.logical_and(x >= t0, x < t1)] = t0 - offset

        x_[x >= t1] -= t1 - t0
        offset += t1 - t0

    return fac * (1.0 - (scale/(scale+x_))**(alpha))

def draw_edges(traj,
               time_normalization_factor=1.,
               time_unit=None,
               ax=None,
               fit=False,
               edge_order=None,
               color=None,
               alpha=0.5,
               linewidth=1.0,
               intervals_to_discard_for_fit=[],
               fit_color='k',
               return_fit_params=False,
               ):
    """
    Draw edges according to an edge activity plot.

    Parameters
    ----------
    traj : list of :class:`_tacoma.edge_trajectory_entry`
        The result of :func:`tacoma.api.get_edge_trajectories`.
    time_normalization_factor, float, default : 1.0
        Rescale time by this factor.
    time_unit : str, default : None
        Unit of time to put on the axis.
    ax : matplotlib.Axes, default : None
        Axis to draw on, will create new one if none provided.
    fit : bool, default : False
        Fit a curve to the number of uniquely observed edges.
    edge_order : list of int, default : None
        Reorder the edges according to this list before drawing.
    color : a matplotlib color, default : None
        Color in which to draw the edges in
    alpha : float, default : 0.5
        Line opacity of edges
    linewidth : float, default : 1.0
        Line width of edges
    intervals_to_discard_for_fit : list of tuple of float
        a list of time intervals which have to be discarded for the fit
    fit_color : a matplotlib color, default : 'k'
        color in which to draw the fit in
    return_fit_params : bool, default : False
        Switch this on if you want to obtain the fit parameters.

    Returns
    -------
    fig : matplotlib.Figure
        If an axes was provided, this is `None`.
    ax : matplotlib.Axes
        The axes the plot was drawn on.
    popt : tuple of float
        Fit parameters, will only be returned if return_fit_params is `True`.
    """

    if ax is None:
        fig, ax = pl.subplots(1, 1)
    else:
        fig = None

    if color is None:
        color = 'k'

    lines = []
    max_i = len(traj)
    all_t_max = []
    all_t_min = []
    max_node = []
    for i, entry in enumerate(traj):
        all_t_max.append(entry.time_pairs[-1][-1] * time_normalization_factor)
        all_t_min.append(entry.time_pairs[0][0] * time_normalization_factor)
        max_node.extend(entry.edge)
        for t_ in entry.time_pairs:
            t_ = np.array(t_) * time_normalization_factor

            if edge_order is not None:
                y = edge_order[i]
            else:
                y = i

            lines.append([t_, [y, y]])

    # if intervals_to_discard_for_fit is not None:
    #    fit_x = []
    #
    #    for t_ in all_t_min:
    #
    # else:
    fit_x = np.array(all_t_min)

    fit_y = np.arange(len(traj), dtype=float)

    lines = [list(zip(x, y)) for x, y in lines]
    colors = [color for _ in range(len(lines))]

    ax.add_collection(LineCollection(lines, colors=colors,
                                     alpha=alpha, linewidth=linewidth))

    t0 = min(all_t_min)
    ax.set_ylim(-1, max_i)
    ax.set_xlim(t0, max(all_t_max))

    xlabel = 'time'
    if time_unit is not None:
        xlabel += ' ['+time_unit+']'

    ylabel = 'edge id'
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)

    if fit:
        N = max(max_node) + 1
        fac = N*(N-1)/2.

        def fit_f(x, alpha, scale):
            return fit_function(x, alpha, scale, fac, intervals_to_discard_for_fit)
        #popt, pcov = curve_fit(fit, fit_x, fit_y,[1./fac,fac,10.0],maxfev=10000)
        #popt, pcov = curve_fit(fit, fit_x, fit_y,[2,fac,10.0],maxfev=10000)
        popt, pcov = curve_fit(fit_f, fit_x, fit_y, [0.5, 10.0], maxfev=10000)

        #print (popt)

        ax.plot(fit_x, fit(fit_x, *popt), 'r')

        #log_y = np.log(fit_y) - 1.
        #log_x = np.log(fit_x) - 1.

    if not return_fit_params:
        return fig, ax
    else:
        return fig, ax, popt

def edge_activity_plot(temporal_network,
                       time_normalization_factor=1.,
                       time_unit=None,
                       ax=None,
                       fit=False,
                       edge_order=None,
                       color=None,
                       alpha=0.5,
                       linewidth=1,
                       intervals_to_discard_for_fit=[],
                       fit_color=None,
                       return_fit_params=False,
                       ):
    """
    Draw an edge activity plot for the given temporal network.
    This is a wrapper for :func:`tacoma.drawing.draw_edges`.

    Parameters
    ----------
    temporal_network : :class:`_tacoma.edge_lists` or :class:`_tacoma.edge_changes`.
        A temporal network.
    time_normalization_factor, float, default : 1.0
        Rescale time by this factor.
    time_unit : str, default : None
        Unit of time to put on the axis.
    ax : matplotlib.Axes, default : None
        Axis to draw an, will create new one if none provided.
    fit : bool, default : False
        Fit a curve to the number of uniquely observed edges.
    edge_order : list of int, default : None
        Reorder the edges according to this list before drawing.
    color : a matplotlib color, default : None
        Color in which to draw the edges in
    alpha : float, default : 0.5
        Line opacity of edges
    linewidth : float, default : 1.0
        Line width of edges
    intervals_to_discard_for_fit : list of tuple of float
        a list of time intervals which have to be discarded for the fit
    fit_color : a matplotlib color, default : 'k'
        color in which to draw the fit in
    return_fit_params : bool, default : False
        Switch this on if you want to obtain the fit parameters.

    Returns
    -------
    fig : matplotlib.Figure
        If an axes was provided, this is `None`.
    ax : matplotlib.Axes
        The axes the plot was drawn on.
    popt : tuple of float
        Fit parameters, will only be returned if `return_fit_params` is `True`.
    """

    traj = tc.get_edge_trajectories(temporal_network)
    return draw_edges(
        traj,
        time_normalization_factor=time_normalization_factor,
        time_unit=time_unit,
        ax=ax,
        fit=fit,
        edge_order=edge_order,
        color=color,
        alpha=alpha,
        linewidth=linewidth,
        intervals_to_discard_for_fit=intervals_to_discard_for_fit,
        fit_color=fit_color,
        return_fit_params=return_fit_params,
    )

def get_edge_order(edge_traj, edge_sim, threshold=0.):
    """
    Create an edge order by performing a Louvain-clustering
    on the thresholded edge similarity graph.

    Parameters
    ----------
    edge_traj : list of :class:`_tacoma.edge_trajectory_entry`
        Edge trajectories, first result of :func:`tacoma.api.get_edge_trajectories`,
        or entry ``trajectories`` of :class`_tacoma.edge_trajectories`.
    edge_sim : dict where key is a tuple of int and value is a float
        Edge similarities, tuple of int denoting the pair of edges,
        similarity is in dimension of time.
        2nd result of :func:`tacoma.api.get_edge_trajectories`,
        or entry ``edge_similarities`` of :class`_tacoma.edge_trajectories`.
    threshold : float
        Ignore similarities below this threshold (minimum time spent together,
        where spent together refers to edges connected to the same node
        at the same time).

    Returns
    -------
    edge_order : list of int
        Edge indices ordered in clusters.
    """

    # get nx graph
    G = get_edge_graph(edge_traj, edge_sim, threshold=0.)

    # find best partition using Louvain clustering
    partition = community.best_partition(G)
    N_comm = max([v for v in partition.values()]) + 1

    comm = [[] for i in range(N_comm)]

    for k, v in partition.items():
        comm[v].append(k)

    order = []
    for module in comm:
        order.extend(module)

    order = np.argsort(order)

    return order

def get_edge_graph(edge_traj, edge_sim, threshold=0.):
    """
    Construct a thresholded edge similarity graph.

    Parameters
    ----------
    edge_traj : list of :class:`_tacoma.edge_trajectory_entry`
        Edge trajectories, first result of :func:`tacoma.api.get_edge_trajectories`,
        or entry ``trajectories`` of :class`_tacoma.edge_trajectories`.
    edge_sim : dict where key is a tuple of int and value is a float
        Edge similarities, tuple of int denoting the pair of edges,
        similarity is in dimension of time.
        2nd result of :func:`tacoma.api.get_edge_trajectories`,
        or entry ``edge_similarities`` of :class`_tacoma.edge_trajectories`.
    threshold : float
        Ignore similarities below this threshold (minimum time spent together,
        where spent together refers to edges connected to the same node
        at the same time).

    Returns
    -------
    G : nx.Graph
        An undirected, unweighted graph where nodes are edges in the temporal network
        and edges mean their similarity is above the threshold.
    """

    N_edges = len(edge_traj)
    G = nx.Graph()
    G.add_nodes_from(range(N_edges))
    G.add_edges_from([(u, v) for u, v, val in edge_sim if val > threshold])

    return G

if __name__ == "__main__":
    import time
    import _tacoma

    L = tc.edge_lists()

    L.N = 3
    L.t = [0.0, 1.0, 2.0]
    L.tmax = 3.0
    L.edges = [
        [
            (0, 1)
        ],
        [
            (1, 2), (0, 2)
        ],
        [
            (0, 1)
        ],
    ]

    L = _tacoma.dynamic_RGG(100, 100, mean_link_duration=10)
    #F = tc.flockwork_P_varying_rates([],100,[0.5],100,[(0.0,1.0)],tmax=100)
    F = L
    FBIN = tc.bin(F, dt=1)
    # draw_rows(FBIN)

    start = time.time()
    traj, similarities = tc.get_edge_trajectories(
        FBIN, return_edge_similarities=True)
    end = time.time()
    print(similarities)

    print("needed ", end-start, "seconds")
    draw_edges(traj, fit=True)

    start = time.time()
    result = tc.get_edge_trajectories(F)
    end = time.time()
    print("needed ", end-start, "seconds")
    draw_edges(traj)

    # draw_edge_lists(L)

    pl.show()


"""

Experimental enrichment implemented over solr.

Currently the strategy implemented here does not scale due to the need for large OR clauses (alternatively: iterative queries)

The most efficient strategy may be to pre-load associations and compute in-memory

"""

from ontobio.golr.golr_associations import search_associations, GolrFields
import scipy.stats # TODO - move
import scipy as sp # TODO - move

M=GolrFields()

def get_counts(entities=[],
               object_category=None,
               min_count=1,
               **kwargs):
    """
    given a set of entities (genes, diseases, etc), finds the number of entities associated with each descriptor in a given category.

    The result is a tuple (cmap, results), where cmap is a dict of TERM:COUNT

    """
    results = search_associations(subjects=entities,
                                  subject_direct=True,
                                  rows=0,
                                  facet_fields=[M.IS_DEFINED_BY, M.SUBJECT_TAXON, M.SUBJECT_CATEGORY],
                                  object_category=object_category,
                                  facet_mincount=3, # TODO
                                  facet_limit=-1,
                                  json_facet={
                                      'categories':{
                                          'limit':-1,
                                          'type': 'terms',
                                          'field' : M.OBJECT_CLOSURE,
                                          'facet' : {
                                              'uniq_subject': "unique(subject)"
                                          }
                                      }
                                  },
                                  **kwargs)
    buckets = results['facets']['categories']['buckets']
    cmap = {}
    for bucket in buckets:
        if bucket['uniq_subject'] >= min_count:
            cmap[bucket['val']] = bucket['uniq_subject']
    return (cmap, results)

def get_background(objects, taxon, object_category, **kwargs):
    results = search_associations(objects=objects,
                                  subject_taxon=taxon,
                                  object_category=object_category,
                                  rows=0,
                                  facet_fields=[M.SUBJECT],
                                  facet_mincount=3, # TODO
                                  facet_limit=-1,
                                  **kwargs)
    return results['facet_counts'][M.SUBJECT].keys()

# TODO: refactor this - fetch compact associations
def find_enriched(sample_entities=[],
                  background_entities=None,
                  object_category=None,
                  **kwargs):

    """
    Given a sample set of sample_entities (e.g. overexpressed genes) and a background set (e.g. all genes assayed), and a category of descriptor (e.g. phenotype, function),
    return enriched descriptors/classes
    """

    (sample_counts, sample_results) = get_counts(entities=sample_entities,
                                                 object_category=object_category,
                                                 min_count=2,
                                                 **kwargs)
    print(str(sample_counts))

    sample_fcs = sample_results['facet_counts']
    taxon_count_dict = sample_fcs[M.SUBJECT_TAXON]

    taxon=None
    for (t,tc) in taxon_count_dict.items():
        # TODO - throw error if multiple taxa
        taxon = t

    if background_entities is None:
        objects = list(sample_counts.keys())
        print("OBJECTS="+str(objects))
        background_entities = get_background(objects, taxon, object_category)

    # TODO: consider caching
    (bg_counts,_) = get_counts(entities=background_entities,
                               object_category=object_category,
                               **kwargs)

    sample_n = len(sample_entities) # TODO - annotated only?
    pop_n = len(background_entities)
    # adapted from goatools
    for (sample_termid,sample_count) in sample_counts.items():
        pop_count = bg_counts[sample_termid]

        # https://en.wikipedia.org/wiki/Fisher's_exact_test
        #              Cls  NotCls
        # study/sample [a,      b]
        # rest of ref  [c,      d]
        #
        a = sample_count
        b = sample_n - sample_count
        c = pop_count - sample_count
        d = pop_n - pop_count - b
        print("ABCD="+str((sample_termid,a,b,c,d,sample_n)))
        _, p_uncorrected = sp.stats.fisher_exact( [[a, b], [c, d]])
        print("P="+str(p_uncorrected))
        # TODO: construct into object


<filename>dl/data/generator/synthtext.py<gh_stars>1-10
import os, logging, re, shutil, sys, csv
from scipy import io as sio
from lxml import etree
import cv2
import numpy as np

class _Skip(Exception):
    pass

def VOCGenerator(basedir, imagedirname='SynthText', skip_missing=False, encoding='utf-8'):

    def xmlgenerator(annodir, imagedir, cbb, wBB, imname, txts, **kwargs):
        imgpath = os.path.join(imagedir, imname)

        if not os.path.exists(imgpath):
            if not skip_missing:
                raise FileNotFoundError('{} was not found'.format(imgpath))
            else:
                logging.warning('Missing image: {}'.format(imgpath))
                raise _Skip()

        root = etree.Element('annotation')

        # folder
        folderET = etree.SubElement(root, 'folder')
        folder = os.path.dirname(imname)
        folderET.text = folder
        # filename
        filenameET = etree.SubElement(root, 'filename')
        filename = os.path.basename(imname)
        filenameET.text = filename

        # read image to get height, width, channel
        img = cv2.imread(imgpath)
        h, w, c = img.shape

        # size
        sizeET = etree.SubElement(root, 'size')

        # width
        widthET = etree.SubElement(sizeET, 'width')
        widthET.text = str(w)
        # height
        heightET = etree.SubElement(sizeET, 'height')
        heightET.text = str(h)
        # depth
        depthET = etree.SubElement(sizeET, 'depth')
        depthET.text = str(c)

        # convert txts to list of str
        # I don't know why texts is
        # ['Lines:\nI lost\nKevin ', 'will                ', 'line\nand            ',
        # 'and\nthe             ', '(and                ', 'the\nout             ',
        # 'you                 ', "don't\n pkg          "]
        # there is strange blank and the length of txts is different from the one of wBB
        txts = ' '.join(txts.tolist()).split()
        text_num = len(txts)

        if wBB.ndim == 2:
            # convert shape=(2, 4,) to (2, 4, 1)
            wBB = np.expand_dims(wBB, 2)

        assert text_num == wBB.shape[2], 'The length of text and wordBB must be same, but got {} and {}'.format(
            text_num, wBB.shape[2])
        for b in range(text_num):
            # object
            objectET = etree.SubElement(root, 'object')

            # difficult
            difficultET = etree.SubElement(objectET, 'difficult')
            difficultET.text = '0'
            # content
            contentET = etree.SubElement(objectET, 'content')
            contentET.text = '###'
            # name
            nameET = etree.SubElement(objectET, 'name')
            nameET.text = txts[b]
            # bndbox
            bndboxET = etree.SubElement(objectET, 'bndbox')

            # quad
            for q in range(4):
                xET = etree.SubElement(bndboxET, 'x{}'.format(q + 1))
                xET.text = str(wBB[0, q, b])
                yET = etree.SubElement(bndboxET, 'y{}'.format(q + 1))
                yET.text = str(wBB[1, q, b])

            # corner
            xminET = etree.SubElement(bndboxET, 'xmin')
            xminET.text = str(np.min(wBB[0, :, b]))
            yminET = etree.SubElement(bndboxET, 'ymin')
            yminET.text = str(np.min(wBB[1, :, b]))
            xmaxET = etree.SubElement(bndboxET, 'xmax')
            xmaxET.text = str(np.max(wBB[0, :, b]))
            ymaxET = etree.SubElement(bndboxET, 'ymax')
            ymaxET.text = str(np.max(wBB[1, :, b]))

        xmlstr = etree.tostring(root, pretty_print=True, encoding=encoding)
        dstpath = os.path.join(annodir, folder, os.path.splitext(filename)[0] + '.xml')

        if not os.path.isdir(os.path.dirname(dstpath)):
            os.mkdir(os.path.dirname(dstpath))

        with open(dstpath, 'wb') as f:
            f.write(xmlstr)

    _gtmatRecognizer(xmlgenerator, basedir, imagedirname)

def TextRecogCSVGenerator(basedir, imagedirname='SynthText', skip_missing=False, encoding='utf-8'):

    lines = [['folder', 'filename', 'text', 'xmin', 'ymin', 'xmax', 'ymax',
             'x1', 'y1', 'x2', 'y2', 'x3', 'y3', 'x4', 'y4']]

    def csvgenerator(annodir, imagedir, cbb, wBB, imname, txts, **kwargs):
        lines = kwargs.get('lines')

        imgpath = os.path.join(imagedir, imname)

        if not os.path.exists(imgpath):
            if not skip_missing:
                raise FileNotFoundError('{} was not found'.format(imgpath))
            else:
                logging.warning('Missing image: {}'.format(imgpath))
                raise _Skip()

        folder = os.path.dirname(imname)
        filename = os.path.basename(imname)

        # convert txts to list of str
        # I don't know why txts is
        # ['Lines:\nI lost\nKevin ', 'will                ', 'line\nand            ',
        # 'and\nthe             ', '(and                ', 'the\nout             ',
        # 'you                 ', "don't\n pkg          "]
        # there is strange blank and the length of txts is different from the one of wBB
        txts = ' '.join(txts.tolist()).split()
        text_num = len(txts)

        if wBB.ndim == 2:
            # convert shape=(2, 4,) to (2, 4, 1)
            wBB = np.expand_dims(wBB, 2)

        assert text_num == wBB.shape[2], 'The length of text and wordBB must be same, but got {} and {}'.format(
            text_num, wBB.shape[2])
        for b in range(text_num):
            text = txts[b]

            # quad
            quad = []
            for q in range(4):
                quad += [str(wBB[0, q, b]), str(wBB[1, q, b])]

            # corner
            corner = [str(np.min(wBB[0, :, b])), str(np.min(wBB[1, :, b])),
                      str(np.max(wBB[0, :, b])), str(np.max(wBB[1, :, b]))]

            lines += [[folder, filename, text, *corner, *quad]]

    _gtmatRecognizer(csvgenerator, basedir, imagedirname, lines=lines)

    annodir = os.path.join(basedir, 'Annotations')
    with open(os.path.join(annodir, 'gt.csv'), 'w') as f:
        writer = csv.writer(f)
        writer.writerows(lines)

def TextRecogOnlyAlphabetNumberCSVGenerator(basedir, imagedirname='SynthText', skip_missing=False, encoding='utf-8'):

    lines = [['folder', 'filename', 'text', 'xmin', 'ymin', 'xmax', 'ymax',
             'x1', 'y1', 'x2', 'y2', 'x3', 'y3', 'x4', 'y4']]

    def csvgenerator(annodir, imagedir, cbb, wBB, imname, txts, **kwargs):
        lines = kwargs.get('lines')

        imgpath = os.path.join(imagedir, imname)

        img = cv2.imread(imgpath)
        h, w, _ = img.shape
        if not os.path.exists(imgpath):
            if not skip_missing:
                raise FileNotFoundError('{} was not found'.format(imgpath))
            else:
                logging.warning('Missing image: {}'.format(imgpath))
                raise _Skip()

        folder = os.path.dirname(imname)
        filename = os.path.basename(imname)

        # convert txts to list of str
        # I don't know why txts is
        # ['Lines:\nI lost\nKevin ', 'will                ', 'line\nand            ',
        # 'and\nthe             ', '(and                ', 'the\nout             ',
        # 'you                 ', "don't\n pkg          "]
        # there is strange blank and the length of txts is different from the one of wBB
        txts = ' '.join(txts.tolist()).split()
        text_num = len(txts)

        if wBB.ndim == 2:
            # convert shape=(2, 4,) to (2, 4, 1)
            wBB = np.expand_dims(wBB, 2)

        assert text_num == wBB.shape[2], 'The length of text and wordBB must be same, but got {} and {}'.format(
            text_num, wBB.shape[2])

        charind = 0
        # replace non-alphanumeric characters with *
        alltexts_asterisk = ''.join([re.sub(r'[^A-Za-z0-9]', '*', text) for text in txts])
        assert len(alltexts_asterisk) == cbb.shape[2], 'The length of characters and cbb must be same, but got {} and {}'.format(
            len(alltexts_asterisk), cbb.shape[2])
        for b in range(text_num):
            text = txts[b]

            alphanumerictext = re.findall(r'[A-Za-z0-9]+', text)

            for ant in alphanumerictext:
                charind = alltexts_asterisk.index(ant, charind)

                # quad
                quad = [cbb[0, 0, charind], cbb[1, 0, charind], # top-left
                        cbb[0, 1, charind+len(ant)-1], cbb[1, 1, charind+len(ant)-1],
                        cbb[0, 2, charind+len(ant)-1], cbb[1, 2, charind+len(ant)-1],
                        cbb[0, 3, charind], cbb[1, 3, charind]]

                # corner
                xmin, ymin, xmax, ymax = max(np.min(quad[0::2]), 0), max(np.min(quad[1::2]), 0), min(np.max(quad[0::2]), w), min(np.max(quad[1::2]), h)
                _h, _w, _ = img[int(ymin):int(ymax), int(xmin):int(xmax)].shape
                if _h == 0 or _w == 0:
                    charind += len(ant)
                    continue
                corner = [xmin, ymin, xmax, ymax]

                quad = list(map(str, quad))
                corner = list(map(str, corner))
                lines += [[folder, filename, ant, *corner, *quad]]

                charind += len(ant)

    _gtmatRecognizer(csvgenerator, basedir, imagedirname, lines=lines)

    annodir = os.path.join(basedir, 'Annotations')
    with open(os.path.join(annodir, 'gt_alphanumeric.csv'), 'w') as f:
        writer = csv.writer(f)
        writer.writerows(lines)

def get_characters(basedir, imagedirname='SynthText', skip_missing=False):

    class Symbols:
        def __init__(self):
            self.symbols = set()

        def update(self, data):
            self.symbols = self.symbols.union(data)

        def __len__(self):
            return len(self.symbols)

        def __str__(self):
            return ''.join(self.symbols)

    symbols = Symbols()

    def csvgenerator(annodir, imagedir, cbb, wBB, imname, txts, symbols, **kwargs):
        image_num = kwargs.get('image_num')
        i = kwargs.get('i')

        imgpath = os.path.join(imagedir, imname)

        img = cv2.imread(imgpath)
        h, w, _ = img.shape
        if not os.path.exists(imgpath):
            if not skip_missing:
                raise FileNotFoundError('{} was not found'.format(imgpath))
            else:
                logging.warning('Missing image: {}'.format(imgpath))
                raise _Skip()

        # convert txts to list of str
        # I don't know why txts is
        # ['Lines:\nI lost\nKevin ', 'will                ', 'line\nand            ',
        # 'and\nthe             ', '(and                ', 'the\nout             ',
        # 'you                 ', "don't\n pkg          "]
        # there is strange blank and the length of txts is different from the one of wBB
        txts = ' '.join(txts.tolist()).split()
        text_num = len(txts)

        if wBB.ndim == 2:
            # convert shape=(2, 4,) to (2, 4, 1)
            wBB = np.expand_dims(wBB, 2)

        assert text_num == wBB.shape[2], 'The length of text and wordBB must be same, but got {} and {}'.format(
            text_num, wBB.shape[2])

        # replace non-alphanumeric characters with *
        alltexts_asterisk = ''.join([re.sub(r'[^A-Za-z0-9]', '*', text) for text in txts])
        assert len(alltexts_asterisk) == cbb.shape[
            2], 'The length of characters and cbb must be same, but got {} and {}'.format(
            len(alltexts_asterisk), cbb.shape[2])
        for b in range(text_num):
            text = txts[b]

            symboltext = re.sub(r'[A-Za-z0-9]+', '', text)

            symbols.update(symboltext)

        sys.stdout.write('\r{}, and number is {}...{:0.1f}% ({}/{})'.format(symbols, len(symbols), 100 * (float(i + 1) / image_num), i + 1, image_num))
        sys.stdout.flush()

    _gtmatRecognizer(csvgenerator, basedir, imagedirname, customLog=True, symbols=symbols)

    print()
    print('symbols are {}, and number is {}'.format(symbols, len(symbols)))

def _gtmatRecognizer(generator, basedir, imagedirname='SynthText', customLog=False, **kwargs):
    """
        convert gt.mat to https://github.com/MhLiao/TextBoxes_plusplus/blob/master/data/example.xml

        <annotation>
            <folder>train_images</folder>
            <filename>img_10.jpg</filename>
            <size>
                <width>1280</width>
                <height>720</height>
                <depth>3</depth>
            </size>
            <object>
                <difficult>1</difficult>
                <content>###</content>
                <name>text</name>
                <bndbox>
                    <x1>1011</x1>
                    <y1>157</y1>
                    <x2>1079</x2>
                    <y2>160</y2>
                    <x3>1076</x3>
                    <y3>173</y3>
                    <x4>1011</x4>
                    <y4>170</y4>
                    <xmin>1011</xmin>
                    <ymin>157</ymin>
                    <xmax>1079</xmax>
                    <ymax>173</ymax>
                </bndbox>
            </object>
            .
            .
            .

        </annotation>

        :param basedir: str, directory path under \'SynthText\'(, \'licence.txt\')
        :param imagedirname: (Optional) str, image directory name including \'gt.mat\
        :return:
        """
    logging.basicConfig(level=logging.INFO)

    imagedir = os.path.join(basedir, imagedirname)
    gtpath = os.path.join(imagedir, 'gt.mat')

    annodir = os.path.join(basedir, 'Annotations')

    if not os.path.exists(gtpath):
        raise FileNotFoundError('{} was not found'.format(gtpath))

    if not os.path.exists(annodir):
        # create Annotations directory
        os.mkdir(annodir)

    """
    ref: http://www.robots.ox.ac.uk/~vgg/data/scenetext/readme.txt
    gts = dict;
        __header__: bytes
        __version__: str
        __globals__: list
        charBB: object ndarray, shape = (1, image num).
                Character level bounding box. shape = (2=(x,y), 4=(top left,...: clockwise), BBox word num)
        wordBB: object ndarray, shape = (1, image num).
                Word level bounding box. shape = (2=(x,y), 4=(top left,...: clockwise), BBox char num)
        imnames: object ndarray, shape = (1, image num, 1).
        txt: object ndarray, shape = (i, image num).
             Text. shape = (word num)
    """
    logging.info('Loading {} now.\nIt may take a while.'.format(gtpath))
    gts = sio.loadmat(gtpath)
    logging.info('Loaded\n'.format(gtpath))

    charBB = gts['charBB'][0]
    wordBB = gts['wordBB'][0]
    imnames = gts['imnames'][0]
    texts = gts['txt'][0]

    image_num = imnames.size

    for i, (cbb, wBB, imname, txts) in enumerate(zip(charBB, wordBB, imnames, texts)):
        imname = imname[0]

        try:
            generator(annodir, imagedir, cbb, wBB, imname, txts, i=i, image_num=image_num, **kwargs)
        except _Skip:
            pass

        if not customLog:
            sys.stdout.write('\rGenerating... {:0.1f}% ({}/{})'.format(100 * (float(i + 1) / image_num), i + 1, image_num))
        sys.stdout.flush()

    print()
    logging.info('Finished!!!')


<filename>autoPyTorch/datasets/base_dataset.py<gh_stars>1-10
import os
import uuid
from abc import ABCMeta
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, cast

import numpy as np

from scipy.sparse import issparse

from sklearn.utils.multiclass import type_of_target

from torch.utils.data import Dataset, Subset

import torchvision

from autoPyTorch.constants import CLASSIFICATION_OUTPUTS, STRING_TO_OUTPUT_TYPES
from autoPyTorch.datasets.resampling_strategy import (
    CrossValFunc,
    CrossValFuncs,
    CrossValTypes,
    DEFAULT_RESAMPLING_PARAMETERS,
    HoldOutFunc,
    HoldOutFuncs,
    HoldoutValTypes
)
from autoPyTorch.utils.common import FitRequirement

BaseDatasetInputType = Union[Tuple[np.ndarray, np.ndarray], Dataset]
BaseDatasetPropertiesType = Union[int, float, str, List, bool]

def check_valid_data(data: Any) -> None:
    if not all(hasattr(data, attr) for attr in ['__getitem__', '__len__']):
        raise ValueError(
            'The specified Data for Dataset must have both __getitem__ and __len__ attribute.')

def type_check(train_tensors: BaseDatasetInputType,
               val_tensors: Optional[BaseDatasetInputType] = None) -> None:
    """To avoid unexpected behavior, we use loops over indices."""
    for i in range(len(train_tensors)):
        check_valid_data(train_tensors[i])
    if val_tensors is not None:
        for i in range(len(val_tensors)):
            check_valid_data(val_tensors[i])

class TransformSubset(Subset):
    """Wrapper of BaseDataset for splitted datasets

    Since the BaseDataset contains all the data points (train/val/test),
    we require different transformation for each data point.
    This class helps to take the subset of the dataset
    with either training or validation transformation.
    The TransformSubset allows to add train flags
    while indexing the main dataset towards this goal.

    Attributes:
        dataset (BaseDataset/Dataset): Dataset to sample the subset
        indices names (Sequence[int]): Indices to sample from the dataset
        train (bool): If we apply train or validation transformation

    """

    def __init__(self, dataset: Dataset, indices: Sequence[int], train: bool) -> None:
        self.dataset = dataset
        self.indices = indices
        self.train = train

    def __getitem__(self, idx: int) -> np.ndarray:
        return self.dataset.__getitem__(self.indices[idx], self.train)

class BaseDataset(Dataset, metaclass=ABCMeta):
    def __init__(
        self,
        train_tensors: BaseDatasetInputType,
        dataset_name: Optional[str] = None,
        val_tensors: Optional[BaseDatasetInputType] = None,
        test_tensors: Optional[BaseDatasetInputType] = None,
        resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
        resampling_strategy_args: Optional[Dict[str, Any]] = None,
        shuffle: Optional[bool] = True,
        seed: Optional[int] = 42,
        train_transforms: Optional[torchvision.transforms.Compose] = None,
        val_transforms: Optional[torchvision.transforms.Compose] = None,
    ):
        """
        Base class for datasets used in AutoPyTorch
        Args:
            train_tensors (A tuple of objects that have a __len__ and a __getitem__ attribute):
                training data
            dataset_name (str): name of the dataset, used as experiment name.
            val_tensors (An optional tuple of objects that have a __len__ and a __getitem__ attribute):
                validation data
            test_tensors (An optional tuple of objects that have a __len__ and a __getitem__ attribute):
                test data
            resampling_strategy (Union[CrossValTypes, HoldoutValTypes]),
                (default=HoldoutValTypes.holdout_validation):
                strategy to split the training data.
            resampling_strategy_args (Optional[Dict[str, Any]]): arguments
                required for the chosen resampling strategy. If None, uses
                the default values provided in DEFAULT_RESAMPLING_PARAMETERS
                in ```datasets/resampling_strategy.py```.
            shuffle:  Whether to shuffle the data before performing splits
            seed (int), (default=1): seed to be used for reproducibility.
            train_transforms (Optional[torchvision.transforms.Compose]):
                Additional Transforms to be applied to the training data
            val_transforms (Optional[torchvision.transforms.Compose]):
                Additional Transforms to be applied to the validation/test data
        """
        self.dataset_name = dataset_name

        if self.dataset_name is None:
            self.dataset_name = str(uuid.uuid1(clock_seq=os.getpid()))

        if not hasattr(train_tensors[0], 'shape'):
            type_check(train_tensors, val_tensors)
        self.train_tensors, self.val_tensors, self.test_tensors = train_tensors, val_tensors, test_tensors
        self.cross_validators: Dict[str, CrossValFunc] = {}
        self.holdout_validators: Dict[str, HoldOutFunc] = {}
        self.random_state = np.random.RandomState(seed=seed)
        self.shuffle = shuffle
        self.resampling_strategy = resampling_strategy
        self.resampling_strategy_args = resampling_strategy_args
        self.task_type: Optional[str] = None
        self.issparse: bool = issparse(self.train_tensors[0])
        self.input_shape: Tuple[int] = self.train_tensors[0].shape[1:]
        if len(self.train_tensors) == 2 and self.train_tensors[1] is not None:
            self.output_type: str = type_of_target(self.train_tensors[1])

            if (
                self.output_type in STRING_TO_OUTPUT_TYPES
                and STRING_TO_OUTPUT_TYPES[self.output_type] in CLASSIFICATION_OUTPUTS
            ):
                self.output_shape = len(np.unique(self.train_tensors[1]))
            else:
                self.output_shape = self.train_tensors[1].shape[-1] if self.train_tensors[1].ndim > 1 else 1

        # TODO: Look for a criteria to define small enough to preprocess
        self.is_small_preprocess = True

        # Make sure cross validation splits are created once
        self.cross_validators = CrossValFuncs.get_cross_validators(*CrossValTypes)
        self.holdout_validators = HoldOutFuncs.get_holdout_validators(*HoldoutValTypes)
        self.splits = self.get_splits_from_resampling_strategy()

        # We also need to be able to transform the data, be it for pre-processing
        # or for augmentation
        self.train_transform = train_transforms
        self.val_transform = val_transforms

    def update_transform(self, transform: Optional[torchvision.transforms.Compose],
                         train: bool = True) -> 'BaseDataset':
        """
        During the pipeline execution, the pipeline object might propose transformations
        as a product of the current pipeline configuration being tested.

        This utility allows to return self with the updated transformation, so that
        a dataloader can yield this dataset with the desired transformations

        Args:
            transform (torchvision.transforms.Compose):
                The transformations proposed by the current pipeline
            train (bool):
                Whether to update the train or validation transform

        Returns:
            self: A copy of the update pipeline
        """
        if train:
            self.train_transform = transform
        else:
            self.val_transform = transform
        return self

    def __getitem__(self, index: int, train: bool = True) -> Tuple[np.ndarray, ...]:
        """
        The base dataset uses a Subset of the data. Nevertheless, the base dataset expects
        both validation and test data to be present in the same dataset, which motivates
        the need to dynamically give train/test data with the __getitem__ command.

        This method yields a datapoint of the whole data (after a Subset has selected a given
        item, based on the resampling strategy) and applies a train/testing transformation, if any.

        Args:
            index (int): what element to yield from all the train/test tensors
            train (bool): Whether to apply a train or test transformation, if any

        Returns:
            A transformed single point prediction
        """

        X = self.train_tensors[0].iloc[[index]] if hasattr(self.train_tensors[0], 'loc') \
            else self.train_tensors[0][index]

        if self.train_transform is not None and train:
            X = self.train_transform(X)
        elif self.val_transform is not None and not train:
            X = self.val_transform(X)

        # In case of prediction, the targets are not provided
        Y = self.train_tensors[1][index] if self.train_tensors[1] is not None else None

        return X, Y

    def __len__(self) -> int:
        return int(self.train_tensors[0].shape[0])

    def _get_indices(self) -> np.ndarray:
        return self.random_state.permutation(len(self)) if self.shuffle else np.arange(len(self))

    def get_splits_from_resampling_strategy(self) -> List[Tuple[List[int], List[int]]]:
        """
        Creates a set of splits based on a resampling strategy provided

        Returns
            (List[Tuple[List[int], List[int]]]): splits in the [train_indices, val_indices] format
        """
        splits = []
        if isinstance(self.resampling_strategy, HoldoutValTypes):
            val_share = DEFAULT_RESAMPLING_PARAMETERS[self.resampling_strategy].get(
                'val_share', None)
            if self.resampling_strategy_args is not None:
                val_share = self.resampling_strategy_args.get('val_share', val_share)
            splits.append(
                self.create_holdout_val_split(
                    holdout_val_type=self.resampling_strategy,
                    val_share=val_share,
                )
            )
        elif isinstance(self.resampling_strategy, CrossValTypes):
            num_splits = DEFAULT_RESAMPLING_PARAMETERS[self.resampling_strategy].get(
                'num_splits', None)
            if self.resampling_strategy_args is not None:
                num_splits = self.resampling_strategy_args.get('num_splits', num_splits)
            # Create the split if it was not created before
            splits.extend(
                self.create_cross_val_splits(
                    cross_val_type=self.resampling_strategy,
                    num_splits=cast(int, num_splits),
                )
            )
        else:
            raise ValueError(f"Unsupported resampling strategy={self.resampling_strategy}")
        return splits

    def create_cross_val_splits(
        self,
        cross_val_type: CrossValTypes,
        num_splits: int
    ) -> List[Tuple[Union[List[int], np.ndarray], Union[List[int], np.ndarray]]]:
        """
        This function creates the cross validation split for the given task.

        It is done once per dataset to have comparable results among pipelines
        Args:
            cross_val_type (CrossValTypes):
            num_splits (int): number of splits to be created

        Returns:
            (List[Tuple[Union[List[int], np.ndarray], Union[List[int], np.ndarray]]]):
                list containing 'num_splits' splits.
        """
        # Create just the split once
        # This is gonna be called multiple times, because the current dataset
        # is being used for multiple pipelines. That is, to be efficient with memory
        # we dump the dataset to memory and read it on a need basis. So this function
        # should be robust against multiple calls, and it does so by remembering the splits
        if not isinstance(cross_val_type, CrossValTypes):
            raise NotImplementedError(f'The selected `cross_val_type` "{cross_val_type}" is not implemented.')
        kwargs = {}
        if cross_val_type.is_stratified():
            # we need additional information about the data for stratification
            kwargs["stratify"] = self.train_tensors[-1]
        splits = self.cross_validators[cross_val_type.name](
            self.random_state, num_splits, self._get_indices(), **kwargs)
        return splits

    def create_holdout_val_split(
        self,
        holdout_val_type: HoldoutValTypes,
        val_share: float,
    ) -> Tuple[np.ndarray, np.ndarray]:
        """
        This function creates the holdout split for the given task.

        It is done once per dataset to have comparable results among pipelines
        Args:
            holdout_val_type (HoldoutValTypes):
            val_share (float): share of the validation data

        Returns:
            (Tuple[np.ndarray, np.ndarray]): Tuple containing (train_indices, val_indices)
        """
        if holdout_val_type is None:
            raise ValueError(
                '`val_share` specified, but `holdout_val_type` not specified.'
            )
        if self.val_tensors is not None:
            raise ValueError(
                '`val_share` specified, but the Dataset was a given a pre-defined split at initialization already.')
        if val_share < 0 or val_share > 1:
            raise ValueError(f"`val_share` must be between 0 and 1, got {val_share}.")
        if not isinstance(holdout_val_type, HoldoutValTypes):
            raise NotImplementedError(f'The specified `holdout_val_type` "{holdout_val_type}" is not supported.')
        kwargs = {}
        if holdout_val_type.is_stratified():
            # we need additional information about the data for stratification
            kwargs["stratify"] = self.train_tensors[-1]
        train, val = self.holdout_validators[holdout_val_type.name](
            self.random_state, val_share, self._get_indices(), **kwargs)
        return train, val

    def get_dataset_for_training(self, split_id: int) -> Tuple[Dataset, Dataset]:
        """
        The above split methods employ the Subset to internally subsample the whole dataset.

        During training, we need access to one of those splits. This is a handy function
        to provide training data to fit a pipeline

        Args:
            split (int): The desired subset of the dataset to split and use

        Returns:
            Dataset: the reduced dataset to be used for testing
        """
        # Subset creates a dataset. Splits is a (train_indices, test_indices) tuple
        return (TransformSubset(self, self.splits[split_id][0], train=True),
                TransformSubset(self, self.splits[split_id][1], train=False))

    def replace_data(self, X_train: BaseDatasetInputType,
                     X_test: Optional[BaseDatasetInputType]) -> 'BaseDataset':
        """
        To speed up the training of small dataset, early pre-processing of the data
        can be made on the fly by the pipeline.

        In this case, we replace the original train/test tensors by this pre-processed version

        Args:
            X_train (np.ndarray): the pre-processed (imputation/encoding/...) train data
            X_test (np.ndarray): the pre-processed (imputation/encoding/...) test data

        Returns:
            self
        """
        self.train_tensors = (X_train, self.train_tensors[1])
        if X_test is not None and self.test_tensors is not None:
            self.test_tensors = (X_test, self.test_tensors[1])
        return self

    def get_dataset_properties(
        self, dataset_requirements: List[FitRequirement]
    ) -> Dict[str, BaseDatasetPropertiesType]:
        """
        Gets the dataset properties required in the fit dictionary.
        This depends on the components that are active in the
        pipeline and returns the properties they need about the dataset.
        Information of the required properties of each component
        can be found in their documentation.
        Args:
            dataset_requirements (List[FitRequirement]): List of
                fit requirements that the dataset properties must
                contain. This is created using the `get_dataset_requirements
                function in
                <https://github.com/automl/Auto-PyTorch/blob/refactor_development/autoPyTorch/utils/pipeline.py#L25>`

        Returns:
            dataset_properties (Dict[str, BaseDatasetPropertiesType]):
                Dict of the dataset properties.
        """
        dataset_properties = dict()
        for dataset_requirement in dataset_requirements:
            dataset_properties[dataset_requirement.name] = getattr(self, dataset_requirement.name)

        # Add the required dataset info to dataset properties as
        # they might not be a dataset requirement in the pipeline
        dataset_properties.update(self.get_required_dataset_info())
        return dataset_properties

    def get_required_dataset_info(self) -> Dict[str, BaseDatasetPropertiesType]:
        """
        Returns a dictionary containing required dataset
        properties to instantiate a pipeline.
        """
        info: Dict[str, BaseDatasetPropertiesType] = {'output_type': self.output_type,
                                                      'issparse': self.issparse}
        return info