"""Xiaomi Mi BLE monitor integration."""
from datetime import timedelta
import logging
import os
import statistics as sts
import struct
import subprocess
import sys
import tempfile
import voluptuous as vol

from homeassistant.const import (
    DEVICE_CLASS_TEMPERATURE,
    DEVICE_CLASS_HUMIDITY,
    TEMP_CELSIUS,
    ATTR_BATTERY_LEVEL,
)
from homeassistant.components.sensor import PLATFORM_SCHEMA
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity import Entity
from homeassistant.helpers.event import track_point_in_utc_time
import homeassistant.util.dt as dt_util

from .const import (
    DEFAULT_ROUNDING,
    DEFAULT_DECIMALS,
    DEFAULT_PERIOD,
    DEFAULT_LOG_SPIKES,
    DEFAULT_USE_MEDIAN,
    DEFAULT_HCITOOL_ACTIVE,
    DEFAULT_IFACE,
    CONF_ROUNDING,
    CONF_DECIMALS,
    CONF_PERIOD,
    CONF_LOG_SPIKES,
    CONF_USE_MEDIAN,
    CONF_HCITOOL_ACTIVE,
    CONF_IFACE,
    CONF_TMIN,
    CONF_TMAX,
    CONF_HMIN,
    CONF_HMAX,
    XIAOMI_TYPE_DICT,
)

_LOGGER = logging.getLogger(__name__)

PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
    {
        vol.Optional(CONF_ROUNDING, default=DEFAULT_ROUNDING): cv.boolean,
        vol.Optional(CONF_DECIMALS, default=DEFAULT_DECIMALS): cv.positive_int,
        vol.Optional(CONF_PERIOD, default=DEFAULT_PERIOD): cv.positive_int,
        vol.Optional(CONF_LOG_SPIKES, default=DEFAULT_LOG_SPIKES): cv.boolean,
        vol.Optional(CONF_USE_MEDIAN, default=DEFAULT_USE_MEDIAN): cv.boolean,
        vol.Optional(
            CONF_HCITOOL_ACTIVE, default=DEFAULT_HCITOOL_ACTIVE
        ): cv.boolean,
        vol.Optional(CONF_IFACE, default=DEFAULT_IFACE): cv.positive_int,
    }
)

# Structured objects for data conversions
TH_STRUCT = struct.Struct("<hH")
H_STRUCT = struct.Struct("<H")
T_STRUCT = struct.Struct("<h")
CND_STRUCT = struct.Struct("<H")
ILL_STRUCT = struct.Struct("<I")

def reverse_mac(rmac):
    """Change LE order to BE."""
    if len(rmac) != 12:
        return None
    return (
        rmac[10:12]
        + rmac[8:10]
        + rmac[6:8]
        + rmac[4:6]
        + rmac[2:4]
        + rmac[0:2]
    )

def parse_xiomi_value(hexvalue, typecode):
    """Convert value depending on its type."""
    vlength = len(hexvalue) / 2
    if vlength == 4:
        if typecode == "0D":
            temp, humi = TH_STRUCT.unpack(bytes.fromhex(hexvalue))
            return {"temperature": temp / 10, "humidity": humi / 10}
    if vlength == 2:
        if typecode == "06":
            humi, = H_STRUCT.unpack(bytes.fromhex(hexvalue))
            return {"humidity": humi / 10}
        if typecode == "04":
            temp, = T_STRUCT.unpack(bytes.fromhex(hexvalue))
            return {"temperature": temp / 10}
        if typecode == "09":
            cond, = CND_STRUCT.unpack(bytes.fromhex(hexvalue))
            return {"conductivity": cond}
    if vlength == 1:
        if typecode == "0A":
            return {"battery": int(hexvalue, 16)}
        if typecode == "08":
            return {"moisture": int(hexvalue, 16)}
    if vlength == 3:
        if typecode == "07":
            illum, = ILL_STRUCT.unpack(bytes.fromhex(hexvalue + "00"))
            return {"illuminance": illum}
    return {}

def parse_raw_message(data):
    """Parse the raw data."""
    if data is None:
        return None

    # check for Xiaomi service data
    xiaomi_index = data.find("1695FE", 33)
    if xiaomi_index == -1:
        return None

    # check for no BR/EDR + LE General discoverable mode flags
    adv_index = data.find("020106", 28, 34)
    if adv_index == -1:
        return None

    # check for BTLE msg size
    msg_length = int(data[4:6], 16) * 2 + 6
    if msg_length != len(data):
        return None

    # check for MAC presence in message and in service data
    xiaomi_mac_reversed = data[xiaomi_index + 16:xiaomi_index + 28]
    source_mac_reversed = data[adv_index - 14:adv_index - 2]
    if xiaomi_mac_reversed != source_mac_reversed:
        return None

    # check if RSSI is valid
    rssi, = struct.unpack(
        "<b", bytes.fromhex(data[msg_length - 2:msg_length])
    )
    if not 0 >= rssi >= -127:
        return None

    try:
        sensor_type, toffset = XIAOMI_TYPE_DICT[
            data[xiaomi_index + 8:xiaomi_index + 14]
        ]
    except KeyError:
        _LOGGER.debug(
            "Unknown sensor type: %s",
            data[xiaomi_index + 8:xiaomi_index + 14],
        )
        return None

    # xiaomi data length = message length
    #     -all bytes before XiaomiUUID
    #     -3 bytes Xiaomi UUID + ADtype
    #     -1 byte rssi
    #     -3+1 bytes sensor type
    #     -1 byte packet_id
    #     -6 bytes MAC
    #     - sensortype offset
    xdata_length = msg_length - xiaomi_index - 30 - toffset * 2
    if xdata_length < 8:
        return None
    xdata_point = xiaomi_index + (14 + toffset) * 2
    xnext_point = xdata_point + 6
    # check if xiaomi data start and length is valid
    if xdata_length != len(data[xdata_point:-2]):
        return None

    packet_id = int(data[xiaomi_index + 14:xiaomi_index + 16], 16)
    result = {
        "rssi": rssi,
        "mac": reverse_mac(xiaomi_mac_reversed),
        "type": sensor_type,
        "packet": packet_id,
    }

    # loop through xiaomi payload
    # assume that the data may have several values of different types,
    # although I did not notice this behavior with my LYWSDCGQ sensors
    while True:
        xvalue_typecode = data[xdata_point:xdata_point + 2]
        try:
            xvalue_length = int(data[xdata_point + 4:xdata_point + 6], 16)
        except ValueError as error:
            _LOGGER.error("xvalue_length conv. error: %s", error)
            result = {}
            break
        xnext_point = xdata_point + 6 + xvalue_length * 2
        xvalue = data[xdata_point + 6:xnext_point]
        res = parse_xiomi_value(xvalue, xvalue_typecode)
        if res:
            result.update(res)
        if xnext_point > msg_length - 6:
            break
        xdata_point = xnext_point
    return result

class BLEScanner:
    """BLE scanner."""

    hcitool = None
    hcidump = None
    tempf = tempfile.TemporaryFile(mode="w+b")
    devnull = (
        subprocess.DEVNULL
        if sys.version_info > (3, 0)
        else open(os.devnull, "wb")
    )

    def start(self, config):
        """Start receiving broadcasts."""
        hcitool_active = config[CONF_HCITOOL_ACTIVE]
        iface = config[CONF_IFACE]
        _LOGGER.debug("Start receiving broadcasts")
        hcitoolcmd = ["hcitool", "lescan", "--duplicates", "--passive"]
        if hcitool_active:
            hcitoolcmd = ["hcitool", "lescan", "--duplicates"]
        if iface != 0:
            hcitoolcmd.append("-i hci" + iface)
        self.hcitool = subprocess.Popen(
            hcitoolcmd, stdout=self.devnull, stderr=self.devnull
        )
        hcidumpcmd = ["hcidump", "--raw"]
        if iface != 0:
            hcidumpcmd.append("-i hci" + iface)
        hcidumpcmd.append("hci")
        self.hcidump = subprocess.Popen(
            hcidumpcmd, stdout=self.tempf, stderr=self.devnull
        )

    def stop(self):
        """Stop receiving broadcasts."""
        _LOGGER.debug("Stop receiving broadcasts")
        self.hcidump.terminate()
        self.hcidump.communicate()
        self.hcitool.terminate()
        self.hcitool.communicate()

    def shutdown_handler(self, event):
        """Run homeassistant_stop event handler."""
        _LOGGER.debug("Running homeassistant_stop event handler: %s", event)
        self.hcidump.kill()
        self.hcidump.communicate()
        self.hcitool.kill()
        self.hcitool.communicate()
        self.tempf.close()

    def messages(self):
        """Get data from hcidump."""
        data = ""
        try:
            _LOGGER.debug("reading hcidump...")
            self.tempf.flush()
            self.tempf.seek(0)
            for line in self.tempf:
                try:
                    sline = line.decode()
                except AttributeError:
                    _LOGGER.debug("Error decoding line: %s", line)
                # _LOGGER.debug(line)
                if sline.startswith("> "):
                    yield data
                    data = sline[2:].strip().replace(" ", "")
                elif sline.startswith("< "):
                    yield data
                    data = ""
                else:
                    data += sline.strip().replace(" ", "")
        except RuntimeError as error:
            _LOGGER.error("Error during reading of hcidump: %s", error)
            data = ""
        self.tempf.seek(0)
        self.tempf.truncate(0)
        yield data

def setup_platform(hass, config, add_entities, discovery_info=None):
    """Set up the sensor platform."""
    _LOGGER.debug("Starting")
    scanner = BLEScanner()
    hass.bus.listen("homeassistant_stop", scanner.shutdown_handler)
    scanner.start(config)

    sensors_by_mac = {}

    def discover_ble_devices(config):
        """Discover Bluetooth LE devices."""
        _LOGGER.debug("Discovering Bluetooth LE devices")
        rounding = config[CONF_ROUNDING]
        decimals = config[CONF_DECIMALS]
        log_spikes = config[CONF_LOG_SPIKES]
        use_median = config[CONF_USE_MEDIAN]

        _LOGGER.debug("Stopping")
        scanner.stop()

        _LOGGER.debug("Analyzing")
        stype = {}
        hum_m_data = {}
        temp_m_data = {}
        illum_m_data = {}
        moist_m_data = {}
        cond_m_data = {}
        batt = {}  # battery
        lpacket = {}  # last packet number
        rssi = {}
        macs = {}  # all found macs
        for msg in scanner.messages():
            data = parse_raw_message(msg)
            if data and "mac" in data:
                # ignore duplicated message
                packet = int(data["packet"])
                if data["mac"] in lpacket:
                    prev_packet = lpacket[data["mac"]]
                else:
                    prev_packet = None
                if prev_packet == packet:
                    _LOGGER.debug("DUPLICATE: %s, IGNORING!", data)
                    continue
                _LOGGER.debug("NEW DATA: %s", data)
                lpacket[data["mac"]] = packet
                # store found readings per device
                if "temperature" in data:
                    if CONF_TMAX >= data["temperature"] >= CONF_TMIN:
                        if data["mac"] not in temp_m_data:
                            temp_m_data[data["mac"]] = []
                        temp_m_data[data["mac"]].append(data["temperature"])
                        macs[data["mac"]] = data["mac"]
                    elif log_spikes:
                        _LOGGER.error(
                            "Temperature spike: %s (%s)",
                            data["temperature"],
                            data["mac"],
                        )
                if "humidity" in data:
                    if CONF_HMAX >= data["humidity"] >= CONF_HMIN:
                        if data["mac"] not in hum_m_data:
                            hum_m_data[data["mac"]] = []
                        hum_m_data[data["mac"]].append(data["humidity"])
                        macs[data["mac"]] = data["mac"]
                    elif log_spikes:
                        _LOGGER.error(
                            "Humidity spike: %s (%s)",
                            data["humidity"],
                            data["mac"],
                        )
                if "conductivity" in data:
                    if data["mac"] not in cond_m_data:
                        cond_m_data[data["mac"]] = []
                    cond_m_data[data["mac"]].append(data["conductivity"])
                    macs[data["mac"]] = data["mac"]
                if "moisture" in data:
                    if data["mac"] not in moist_m_data:
                        moist_m_data[data["mac"]] = []
                    moist_m_data[data["mac"]].append(data["moisture"])
                    macs[data["mac"]] = data["mac"]
                if "illuminance" in data:
                    if data["mac"] not in illum_m_data:
                        illum_m_data[data["mac"]] = []
                    illum_m_data[data["mac"]].append(data["illuminance"])
                    macs[data["mac"]] = data["mac"]
                if "battery" in data:
                    batt[data["mac"]] = int(data["battery"])
                    macs[data["mac"]] = data["mac"]
                if data["mac"] not in rssi:
                    rssi[data["mac"]] = []
                rssi[data["mac"]].append(int(data["rssi"]))
                stype[data["mac"]] = data["type"]

        # for every seen device
        for mac in macs:
            if mac in sensors_by_mac:
                sensors = sensors_by_mac[mac]
            else:
                if stype[mac] == "HHCCJCY01":
                    sensors = [
                        TemperatureSensor(mac),
                        MoistureSensor(mac),
                        ConductivitySensor(mac),
                        IlluminanceSensor(mac),
                    ]
                else:
                    sensors = [TemperatureSensor(mac), HumiditySensor(mac)]
                sensors_by_mac[mac] = sensors
                add_entities(sensors)
            for sensor in sensors:
                getattr(sensor, "_device_state_attributes")[
                    "last packet id"
                ] = lpacket[mac]
                getattr(sensor, "_device_state_attributes")["rssi"] = round(
                    sts.mean(rssi[mac])
                )
                getattr(sensor, "_device_state_attributes")[
                    "sensor type"
                ] = stype[mac]
                if mac in batt:
                    getattr(sensor, "_device_state_attributes")[
                        ATTR_BATTERY_LEVEL
                    ] = batt[mac]
            # averaging and states updating
            tempstate_mean = None
            humstate_mean = None
            illumstate_mean = None
            moiststate_mean = None
            condstate_mean = None
            tempstate_median = None
            humstate_median = None
            illumstate_median = None
            moiststate_median = None
            condstate_median = None
            if use_median:
                textattr = "last median of"
            else:
                textattr = "last mean of"
            if mac in temp_m_data:
                try:
                    if rounding:
                        tempstate_median = round(
                            sts.median(temp_m_data[mac]), decimals
                        )
                        tempstate_mean = round(
                            sts.mean(temp_m_data[mac]), decimals
                        )
                    else:
                        tempstate_median = sts.median(temp_m_data[mac])
                        tempstate_mean = sts.mean(temp_m_data[mac])
                    if use_median:
                        setattr(sensors[0], "_state", tempstate_median)
                    else:
                        setattr(sensors[0], "_state", tempstate_mean)
                    getattr(sensors[0], "_device_state_attributes")[
                        textattr
                    ] = len(temp_m_data[mac])
                    getattr(sensors[0], "_device_state_attributes")[
                        "median"
                    ] = tempstate_median
                    getattr(sensors[0], "_device_state_attributes")[
                        "mean"
                    ] = tempstate_mean
                    sensors[0].async_schedule_update_ha_state()
                except AttributeError:
                    _LOGGER.info("Sensor %s not yet ready for update", mac)
                except ZeroDivisionError:
                    _LOGGER.error(
                        "Division by zero while temperature averaging!"
                    )
                    continue
                except IndexError as error:
                    _LOGGER.error(
                        "Sensor %s (%s, temp.) update error:", mac, stype[mac]
                    )
                    _LOGGER.error("%s. Index is 0!", error)
                    _LOGGER.error("sensors list size: %i", len(sensors))
            if mac in hum_m_data:
                try:
                    if rounding:
                        humstate_median = round(
                            sts.median(hum_m_data[mac]), decimals
                        )
                        humstate_mean = round(
                            sts.mean(hum_m_data[mac]), decimals
                        )
                    else:
                        humstate_median = sts.median(hum_m_data[mac])
                        humstate_mean = sts.mean(hum_m_data[mac])
                    if use_median:
                        setattr(sensors[1], "_state", humstate_median)
                    else:
                        setattr(sensors[1], "_state", humstate_mean)
                    getattr(sensors[1], "_device_state_attributes")[
                        textattr
                    ] = len(hum_m_data[mac])
                    getattr(sensors[1], "_device_state_attributes")[
                        "median"
                    ] = humstate_median
                    getattr(sensors[1], "_device_state_attributes")[
                        "mean"
                    ] = humstate_mean
                    sensors[1].async_schedule_update_ha_state()
                except AttributeError:
                    _LOGGER.info("Sensor %s not yet ready for update", mac)
                except ZeroDivisionError:
                    _LOGGER.error("Division by zero while humidity averaging!")
                    continue
                except IndexError as error:
                    _LOGGER.error(
                        "Sensor %s (%s, hum.) update error:", mac, stype[mac]
                    )
                    _LOGGER.error("%s. Index is 1!", error)
                    _LOGGER.error("sensors list size: %i", len(sensors))
            if mac in moist_m_data:
                try:
                    if rounding:
                        moiststate_median = round(
                            sts.median(moist_m_data[mac]), decimals
                        )
                        moiststate_mean = round(
                            sts.mean(moist_m_data[mac]), decimals
                        )
                    else:
                        moiststate_median = sts.median(moist_m_data[mac])
                        moiststate_mean = sts.mean(moist_m_data[mac])
                    if use_median:
                        setattr(sensors[1], "_state", moiststate_median)
                    else:
                        setattr(sensors[1], "_state", moiststate_mean)
                    getattr(sensors[1], "_device_state_attributes")[
                        textattr
                    ] = len(moist_m_data[mac])
                    getattr(sensors[1], "_device_state_attributes")[
                        "median"
                    ] = moiststate_median
                    getattr(sensors[1], "_device_state_attributes")[
                        "mean"
                    ] = moiststate_mean
                    sensors[1].async_schedule_update_ha_state()
                except AttributeError:
                    _LOGGER.info("Sensor %s not yet ready for update", mac)
                except ZeroDivisionError:
                    _LOGGER.error("Division by zero while moisture averaging!")
                    continue
                except IndexError as error:
                    _LOGGER.error(
                        "Sensor %s (%s, moist.) update error:", mac, stype[mac]
                    )
                    _LOGGER.error("%s. Index is 1!", error)
                    _LOGGER.error("sensors list size: %i", len(sensors))
            if mac in cond_m_data:
                try:
                    if rounding:
                        condstate_median = round(
                            sts.median(cond_m_data[mac]), decimals
                        )
                        condstate_mean = round(
                            sts.mean(cond_m_data[mac]), decimals
                        )
                    else:
                        condstate_median = sts.median(cond_m_data[mac])
                        condstate_mean = sts.mean(cond_m_data[mac])
                    if use_median:
                        setattr(sensors[2], "_state", condstate_median)
                    else:
                        setattr(sensors[2], "_state", condstate_mean)
                    getattr(sensors[2], "_device_state_attributes")[
                        textattr
                    ] = len(cond_m_data[mac])
                    getattr(sensors[2], "_device_state_attributes")[
                        "median"
                    ] = condstate_median
                    getattr(sensors[2], "_device_state_attributes")[
                        "mean"
                    ] = condstate_mean
                    sensors[2].async_schedule_update_ha_state()
                except AttributeError:
                    _LOGGER.info("Sensor %s not yet ready for update", mac)
                except ZeroDivisionError:
                    _LOGGER.error("Division by zero while humidity averaging!")
                    continue
                except IndexError as error:
                    _LOGGER.error(
                        "Sensor %s (%s, cond.) update error:", mac, stype[mac]
                    )
                    _LOGGER.error("%s. Index is 2!", error)
                    _LOGGER.error("sensors list size: %i", len(sensors))
            if mac in illum_m_data:
                try:
                    if rounding:
                        illumstate_median = round(
                            sts.median(illum_m_data[mac]), decimals
                        )
                        illumstate_mean = round(
                            sts.mean(illum_m_data[mac]), decimals
                        )
                    else:
                        illumstate_median = sts.median(illum_m_data[mac])
                        illumstate_mean = sts.mean(illum_m_data[mac])
                    if use_median:
                        setattr(sensors[3], "_state", illumstate_median)
                    else:
                        setattr(sensors[3], "_state", illumstate_mean)
                    getattr(sensors[3], "_device_state_attributes")[
                        textattr
                    ] = len(illum_m_data[mac])
                    getattr(sensors[3], "_device_state_attributes")[
                        "median"
                    ] = illumstate_median
                    getattr(sensors[3], "_device_state_attributes")[
                        "mean"
                    ] = illumstate_mean
                    sensors[3].async_schedule_update_ha_state()
                except AttributeError:
                    _LOGGER.info("Sensor %s not yet ready for update", mac)
                except ZeroDivisionError:
                    _LOGGER.error(
                        "Division by zero while illuminance averaging!"
                    )
                    continue
                except IndexError as error:
                    _LOGGER.error(
                        "Sensor %s (%s, illum.) update error:", mac, stype[mac]
                    )
                    _LOGGER.error("%s. Index is 3!", error)
                    _LOGGER.error("sensors list size: %i", len(sensors))
        scanner.start(config)
        return []

    def update_ble(now):
        """Lookup Bluetooth LE devices and update status."""
        period = config[CONF_PERIOD]
        _LOGGER.debug("update_ble called")

        try:
            discover_ble_devices(config)
        except RuntimeError as error:
            _LOGGER.error("Error during Bluetooth LE scan: %s", error)

        track_point_in_utc_time(
            hass, update_ble, dt_util.utcnow() + timedelta(seconds=period)
        )

    update_ble(dt_util.utcnow())

class TemperatureSensor(Entity):
    """Representation of a sensor."""

    def __init__(self, mac):
        """Initialize the sensor."""
        self._state = None
        self._battery = None
        self._unique_id = "t_" + mac
        self._device_state_attributes = {}

    @property
    def name(self):
        """Return the name of the sensor."""
        return "mi {}".format(self._unique_id)

    @property
    def state(self):
        """Return the state of the sensor."""
        return self._state

    @property
    def unit_of_measurement(self):
        """Return the unit of measurement."""
        return TEMP_CELSIUS

    @property
    def device_class(self):
        """Return the unit of measurement."""
        return DEVICE_CLASS_TEMPERATURE

    @property
    def should_poll(self):
        """No polling needed."""
        return False

    @property
    def device_state_attributes(self):
        """Return the state attributes."""
        return self._device_state_attributes

    @property
    def unique_id(self) -> str:
        """Return a unique ID."""
        return self._unique_id

    @property
    def force_update(self):
        """Force update."""
        return True

class HumiditySensor(Entity):
    """Representation of a Sensor."""

    def __init__(self, mac):
        """Initialize the sensor."""
        self._state = None
        self._battery = None
        self._unique_id = "h_" + mac
        self._device_state_attributes = {}

    @property
    def name(self):
        """Return the name of the sensor."""
        return "mi {}".format(self._unique_id)

    @property
    def state(self):
        """Return the state of the sensor."""
        return self._state

    @property
    def unit_of_measurement(self):
        """Return the unit of measurement."""
        return "%"

    @property
    def device_class(self):
        """Return the unit of measurement."""
        return DEVICE_CLASS_HUMIDITY

    @property
    def should_poll(self):
        """No polling needed."""
        return False

    @property
    def device_state_attributes(self):
        """Return the state attributes."""
        return self._device_state_attributes

    @property
    def unique_id(self) -> str:
        """Return a unique ID."""
        return self._unique_id

    @property
    def force_update(self):
        """Force update."""
        return True

class MoistureSensor(Entity):
    """Representation of a Sensor."""

    def __init__(self, mac):
        """Initialize the sensor."""
        self._state = None
        self._battery = None
        self._unique_id = "m_" + mac
        self._device_state_attributes = {}

    @property
    def name(self):
        """Return the name of the sensor."""
        return "mi {}".format(self._unique_id)

    @property
    def state(self):
        """Return the state of the sensor."""
        return self._state

    @property
    def unit_of_measurement(self):
        """Return the unit of measurement."""
        return "%"

    @property
    def device_class(self):
        """Return the unit of measurement."""
        return DEVICE_CLASS_HUMIDITY

    @property
    def should_poll(self):
        """No polling needed."""
        return False

    @property
    def device_state_attributes(self):
        """Return the state attributes."""
        return self._device_state_attributes

    @property
    def unique_id(self) -> str:
        """Return a unique ID."""
        return self._unique_id

    @property
    def force_update(self):
        """Force update."""
        return True

class ConductivitySensor(Entity):
    """Representation of a Sensor."""

    def __init__(self, mac):
        """Initialize the sensor."""
        self._state = None
        self._battery = None
        self._unique_id = "c_" + mac
        self._device_state_attributes = {}

    @property
    def name(self):
        """Return the name of the sensor."""
        return "mi {}".format(self._unique_id)

    @property
    def state(self):
        """Return the state of the sensor."""
        return self._state

    @property
    def unit_of_measurement(self):
        """Return the unit of measurement."""
        return "µS/cm"

    @property
    def icon(self):
        """Return the icon of the sensor."""
        return "mdi:flash-circle"

    @property
    def should_poll(self):
        """No polling needed."""
        return False

    @property
    def device_state_attributes(self):
        """Return the state attributes."""
        return self._device_state_attributes

    @property
    def unique_id(self) -> str:
        """Return a unique ID."""
        return self._unique_id

    @property
    def force_update(self):
        """Force update."""
        return True

class IlluminanceSensor(Entity):
    """Representation of a Sensor."""

    def __init__(self, mac):
        """Initialize the sensor."""
        self._state = None
        self._battery = None
        self._unique_id = "l_" + mac
        self._device_state_attributes = {}

    @property
    def name(self):
        """Return the name of the sensor."""
        return "mi {}".format(self._unique_id)

    @property
    def state(self):
        """Return the state of the sensor."""
        return self._state

    @property
    def unit_of_measurement(self):
        """Return the unit of measurement."""
        return "lx"

    @property
    def icon(self):
        """Return the icon of the sensor."""
        return "mdi:white-balance-sunny"

    @property
    def should_poll(self):
        """No polling needed."""
        return False

    @property
    def device_state_attributes(self):
        """Return the state attributes."""
        return self._device_state_attributes

    @property
    def unique_id(self) -> str:
        """Return a unique ID."""
        return self._unique_id

    @property
    def force_update(self):
        """Force update."""
        return True


<reponame>dankovacek/dkhydrotech
import numpy as np
import scipy.stats as st

def calculate_Q(data):
    mean = np.mean(data)
    stdev = np.std(data)

    return np.power(10,mean + K * stdev)

def calculate_skew(data):
    """
    Calculate weighted skew.
    Reference: CFA2 Manual.
    Based on (Wallis, Matalas, and Slack)
    K_n is 'standard normal deviate' which I'm assuming means standard deviation
    """

    g = abs(np.log10(st.skew(data)))

    mu = np.mean(np.log10(data))
    s = np.std(np.log10(data))
    alpha = 4 / g**2
    beta = 0.5 * s * g
    epsilon = mu - 2 * s / g

    return K, MSE_g

def calculate_K_weighted(data):
    g = abs(np.log10(st.skew(data)))

    if g == 0:
        print('Skew cannot equal zero.')
        return None

    if g <= 0.9:
        A = -0.33 + 0.08 * g
    else:
        A = -0.52 + 0.3 * g

    if g <= 1.5:
        B = 0.94 - 0.26 * g
    else:
        B = 0.55

    # assuming 'standard normal deviate'
    K_n = np.std(data)

    K = (2 / g) * (((K_n - g/6)*g/6 + 1)**3 - 1)

    # MSE_g = (A - B * (np.log10(len(data)/10)))
    return K


<gh_stars>0
"""
Script used to run most of the results used in the paper.
Compring different PCEs, uncertainty measures due to factor fixing.
"""

import pandas as pd
import numpy as np
from SALib.util import read_param_file
import pyapprox as pya
from pyapprox.approximate import approximate
from pyapprox.indexing import compute_hyperbolic_indices
from pyapprox.multivariate_polynomials import PolynomialChaosExpansion
from pyapprox.multivariate_polynomials import \
        define_poly_options_from_variable_transformation, AffineRandomVariableTransformation
from pyapprox.variables import IndependentMultivariateRandomVariable

from basic.boots_pya import fun
from basic.utils import to_df, adjust_sampling
from basic.group_fix import group_fix, uncond_cal
from basic.read_data import file_settings, read_specify

##==============================##============================##
##==============================##============================##
# apply_pya to produce the sensitivities of parameters for different PCEs
from apply_pya import run_pya, pce_22
outpath = file_settings()[0]
seed=222
# check whether the mean and variance from 'exact' is correct
def pmf_check(product_uniform = False):
    variable, _ = read_specify('parameter', 'reduced', product_uniform, num_vars=11)
    samples, values = read_specify('model', 'reduced', product_uniform, num_vars=11)

    approx_list_all = {}
    mean_list = {}
    variance_list = {}
    n_strat, n_end, n_step = [156, 252, 13]
    for i in range(n_strat, n_end+1, n_step):
    # for i in n_list:
        print(i)
        if (n_end - i)  < n_step:
            i = n_end
        np.random.seed(seed)
        approx_list_all[f'nsample_{i}'] = fun(variable, samples[:, :i], values[:i], product_uniform, nboot=1)
    for key, pce in approx_list_all.items():
        mean_list[key], variance_list[key] = pce.mean(), pce.variance()
    pmf_stat = pd.concat([pd.DataFrame.from_dict(mean_list).T, \
        pd.DataFrame.from_dict(variance_list).T], axis=1)

    return pmf_stat

pmf_stat = pmf_check(product_uniform = 'exact')
pmf_stat.to_csv('mean_var.csv')

"""
Script used to run most of the results used in the paper.
Compring different PCEs, uncertainty measures due to factor fixing.
"""

import pandas as pd
import numpy as np
import pyapprox as pya
from pyapprox.approximate import approximate
from pyapprox.indexing import compute_hyperbolic_indices
from pyapprox.multivariate_polynomials import PolynomialChaosExpansion
from pyapprox.multivariate_polynomials import \
        define_poly_options_from_variable_transformation, AffineRandomVariableTransformation
from pyapprox.variables import IndependentMultivariateRandomVariable

##==============================##============================##
##===============RUN test with a analytic function=============##
def fun_prod(x1, x2, x3, x4, x5, x6):
    """
    x: array-like
    """
    y = x1*x2 + x3 + x4 * x5 * x6
    return y

from scipy.stats import uniform, norm
univariate_variables = [uniform(0, 1)] * 6
# the dist. of the first and last vars are not correct but will be overwriten using the product
re_univariable = [uniform(0, 1.5), uniform(0, 1), uniform(0.2, 1)]
re_variable = IndependentMultivariateRandomVariable(re_univariable)
re_var_trans = AffineRandomVariableTransformation(re_variable)

variable = IndependentMultivariateRandomVariable(univariate_variables)
var_trans = AffineRandomVariableTransformation(variable)

# generate samples and values for training and comparison
samples = pya.generate_independent_random_samples(variable, 10000)
values = np.array([[fun_prod(*ii)] for ii in (samples.T)])
index_product = [[0, 1], [3, 4, 5]]
samples_adjust = np.zeros((3, samples.shape[1]))
samples_adjust[0, :] = np.prod(samples[index_product[0], :], axis=0)
samples_adjust[1, :] = samples[2, :]
samples_adjust[2, :] = np.prod(samples[index_product[1], :], axis=0)

# the following use the product of uniforms to define basis
from pyapprox.variables import get_distribution_info
from pyapprox.univariate_quadrature import gauss_jacobi_pts_wts_1D
from pyapprox.utilities import total_degree_space_dimension

def identity_fun(x):
    return x

degree = 3
poly = PolynomialChaosExpansion()
basis_opts = dict()
identity_map_indices = []
cnt = 0
for ii in range(re_variable.nunique_vars):
    rv = re_variable.unique_variables[ii]
    name, scales, shapes = get_distribution_info(rv)
    if ii not in [0, 2]:
        opts = {'rv_type': name, 'shapes': shapes,
                'var_nums': re_variable.unique_variable_indices[ii]}
        basis_opts['basis%d' % ii] = opts
        continue

    #identity_map_indices += re_variable.unique_variable_indices[ii] # wrong
    identity_map_indices += list(re_variable.unique_variable_indices[ii]) # right

    quad_rules = []
    inds = index_product[cnt]
    nquad_samples_1d = 50

    for jj in inds:
        a, b = variable.all_variables()[jj].interval(1)
        x, w = gauss_jacobi_pts_wts_1D(nquad_samples_1d, 0, 0)
        x = (x+1)/2 # map to [0, 1]
        x = (b-a)*x+a # map to [a,b]
        quad_rules.append((x, w))
    funs = [identity_fun]*len(inds)
    basis_opts['basis%d' % ii] = {'poly_type': 'product_indpnt_vars',
                                    'var_nums': [ii], 'funs': funs,
                                    'quad_rules': quad_rules}
    cnt += 1

poly_opts = {'var_trans': re_var_trans}
poly_opts['poly_types'] = basis_opts
#var_trans.set_identity_maps(identity_map_indices) #wrong
re_var_trans.set_identity_maps(identity_map_indices) #right

indices = compute_hyperbolic_indices(re_variable.num_vars(), degree)
nterms = total_degree_space_dimension(samples_adjust.shape[0], degree)
options = {'basis_type': 'fixed', 'variable': re_variable,
            'poly_opts': poly_opts,
            'options': {'linear_solver_options': dict(),
                        'indices': indices, 'solver_type': 'lstsq'}}

approx_res = approximate(samples_adjust[:, 0:(2 * nterms)], values[0:(2 * nterms)], 'polynomial_chaos', options).approx
y_hat = approx_res(samples_adjust[:, 2 * nterms:])
print((y_hat - values[2 * nterms:]).mean())
print(f'Mean of samples: {values.mean()}')
print(f'Mean of pce: {approx_res.mean()}')


<reponame>njoel-ethz/saliency-rl<filename>run_train.py
import sys
import os
import random
import numpy as np
import cv2
import time
from datetime import timedelta
import torch
from model import TASED_v2
from loss import KLDLoss
from dataset import DHF1KDataset, InfiniteDataLoader
from itertools import islice
import matplotlib.pyplot as plt
from PIL import Image
from scipy.ndimage import gaussian_filter
import csv

def main():
    ''' concise script for training '''
    # optional two command-line arguments
    path_indata = 'Atari_dataset'
    path_output = 'output'
    shuffle_data = True

    if len(sys.argv) > 1:
        path_indata = sys.argv[1]
        if len(sys.argv) > 2:
            path_output = sys.argv[2]

    # we checked that using only 2 gpus is enough to produce similar results
    num_gpu = 1 #2 on Server
    pile = 5
    batch_size = 1 #6 on Server
    num_iters = 1000
    len_temporal = 32
    file_weight = 'TASED_updated.pt'
    path_output = os.path.join(path_output, time.strftime("%m-%d_%H-%M-%S"))
    if not os.path.isdir(path_output):
        os.makedirs(path_output)
    if shuffle_data or not os.path.isfile('Atari_num_frame_train.csv'):
        split_train_test_set()

    model = TASED_v2()

    # load the weight file and copy the parameters
    if os.path.isfile(file_weight):
        print ('loading weight file')
        weight_dict = torch.load(file_weight)
        model_dict = model.state_dict()
        for name, param in weight_dict.items():
            if 'module' in name:
                name = '.'.join(name.split('.')[1:])
            if 'base.' in name:
                bn = int(name.split('.')[1])
                sn_list = [0, 5, 8, 14]
                sn = sn_list[0]
                if bn >= sn_list[1] and bn < sn_list[2]:
                    sn = sn_list[1]
                elif bn >= sn_list[2] and bn < sn_list[3]:
                    sn = sn_list[2]
                elif bn >= sn_list[3]:
                    sn = sn_list[3]
                name = '.'.join(name.split('.')[2:])
                name = 'base%d.%d.'%(sn_list.index(sn)+1, bn-sn)+name
            if name in model_dict:
                if param.size() == model_dict[name].size():
                    model_dict[name].copy_(param)
                else:
                    print (' size? ' + name, param.size(), model_dict[name].size())
            else:
                print (' name? ' + name)

        print (' loaded')
    else:
        print ('weight file?')

    # parameter setting for fine-tuning
    params = []
    lr1 = 0.002 #0.0001 David
    lr2 = 0.05 #0.01 David
    for key, value in dict(model.named_parameters()).items():
        if 'convtsp' in key:
            params += [{'params':[value], 'key':key+'(new)'}]
        else:
            params += [{'params':[value], 'lr':lr1, 'key':key}] #0.001

    optimizer = torch.optim.SGD(params, lr=lr2, momentum=0.9, weight_decay=2e-7) #lr = 0.1
    lr_adaption = 0.1
    criterion = KLDLoss()

    model = model.cuda()
    model = torch.nn.DataParallel(model, device_ids=range(num_gpu))
    torch.backends.cudnn.benchmark = False
    model.train()

    train_loader = InfiniteDataLoader(DHF1KDataset(path_indata, len_temporal), batch_size=batch_size, shuffle=True, num_workers=0) #was 24, 0 means that data is loaded in the main process

    loss_statistic = []
    averaged_loss_statistic = []
    index_statistic = []

    i, step = 0, 0
    loss_sum = 0
    start_time = time.time()
    for clip, annt, (file_name, picture_name) in islice(train_loader, num_iters*pile):
        with torch.set_grad_enabled(True):
            output = model(clip.cuda())
            loss = criterion(output, annt.cuda())

        loss_sum += loss.item()
        loss.backward()
        if (i+1) % pile == 0:
            optimizer.step()
            optimizer.zero_grad()
            step += 1

            # whole process takes less than 3 hours
            print ('iteration: [%4d/%4d], loss: %.4f, %s' % (step, num_iters, loss_sum/pile, timedelta(seconds=int(time.time()-start_time))), flush=True)

            loss_statistic.append(loss_sum/pile)

            visualize(output, path_indata, file_name[0], picture_name[0], step)
            plt.ylabel('Loss')
            plt.xlabel(path_indata + ', weights: ' + file_weight + ', lr/optim = ' + str((lr1, lr2)))
            plt.plot(loss_statistic, color='b')
            if step==1:
                plt.show(block=False)
            else:
                plt.draw()
            plt.pause(0.00001)

            if step%10==0:
                averaged_loss_statistic.append(sum(loss_statistic[step-10:step])/10)
                index_statistic.append(step)

            loss_sum = 0
            # adjust learning rate
            if step in [250, 750, 950]:
                for opt in optimizer.param_groups:
                    if 'new' in opt['key']:
                        opt['lr'] *= lr_adaption   #0.1

            if step % 25 == 0:
                try:
                    if not os.path.exists(os.path.join(path_output, 'weights')):
                        os.makedirs(os.path.join(path_output, 'weights'))
                except OSError:
                    print('Error: Creating directory of data')
                torch.save(model.state_dict(), os.path.join(path_output, 'weights', 'iter_%04d.pt' % step))

        i += 1
    torch.save(model.state_dict(), os.path.join(path_indata, 'produced_weight_file.pt'))

    print('plotten')
    #plt.plot(loss_statistic)
    plt.savefig(os.path.join(path_indata, "loss.png"))

    plt.plot(index_statistic, averaged_loss_statistic, color='r')
    plt.ylabel('Averaged loss')
    plt.xlabel(path_indata + ', weights: ' + file_weight + ', lr/optim = ' + str((lr1, lr2)))
    plt.savefig(os.path.join(path_indata, "averaged_loss.png"))

    os.system('python run_saliency_metrics.py')

def split_train_test_set():
    path_full = 'Atari_num_frame_FullData.csv'
    path_train = 'Atari_num_frame_train.csv'
    path_test = 'Atari_num_frame_testing.csv'

    list_num_frame = [int(row[0]) for row in csv.reader(open(path_full, 'r'))]
    total_len = len(list_num_frame)
    half_len = int(np.floor(total_len/2)) #for 50/50 split
    idx = range(1, total_len+1)
    z = list(zip(list_num_frame, idx))

    #comment this out for training on ~ highscore data
    random.shuffle(z)

    list_num_frame, idx = zip(*z)

    test_list, test_idx = list_num_frame[:half_len], idx[:half_len]
    train_list, train_idx = list_num_frame[half_len:], idx[half_len:]
    train_strings = []
    test_strings = []

    for i in range(len(train_list)):
        train_strings.append(str(train_list[i])+',%04d'%train_idx[i])
    for i in range(len(test_list)):
        test_strings.append(str(test_list[i])+',%04d'%test_idx[i])

    with open(path_train, 'w') as file:
        for line in train_strings:
            file.write(line)
            file.write('\n')
    with open(path_test, 'w') as file:
        for line in test_strings:
            file.write(line)
            file.write('\n')

    return 0

def visualize(output, path_indata, file_name, picture_name, step):
    # print(output.shape)  ---> torch.Size([1, 224, 384])
    # print(output.cpu().data.numpy())
    np_array = (output.cpu().data[0].numpy()*255.).astype(np.int)/255.
    # print(np_array.shape)    ---> (224, 384)
    np_array = gaussian_filter(np_array, sigma=7)
    np_array = (np_array/np.max(np_array)*255.).astype(np.uint8)
    mask = Image.fromarray(np_array)  # gives a 384x224 Image object
    path_to_clip = os.path.join(path_indata, 'video', file_name, picture_name)
    video_img = cv2.resize(cv2.imread(path_to_clip, cv2.IMREAD_COLOR), (384, 224))
    video_img = Image.fromarray(video_img)
    red_img = Image.new('RGB', (384, 224), (0, 0, 255))
    video_img.paste(red_img, mask=mask)

    open_cv_image = cv2.resize(np.array(video_img), (160, 210))

    output_path = os.path.join(path_indata, 'training_visualized')
    try:
        if not os.path.exists(output_path):
            os.makedirs(output_path)
    except OSError:
        print('Error: Creating directory of data')
    cv2.imwrite(os.path.join(output_path, '%06d.png' % (step)), open_cv_image)
    """cv2.imshow('step', open_cv_image)
    cv2.waitKey(3000)
    cv2.destroyAllWindows()"""
    return 0

if __name__ == '__main__':
    main()


import sys
sys.path.insert(0, "/home/hardik/dl_exp/nglod/sdf-net/lib/submodules/libigl/python/")

import pyigl as igl
from decimal import *
import iglhelpers
# import miniball
import math

import numpy as np

from matplotlib import pyplot
from mpl_toolkits.mplot3d import Axes3D

from scipy.stats import gaussian_kde

import matplotlib.cm as cm

#constants used for sampling box AND miniball normalization
BOUNDING_SPHERE_RADIUS = 0.9
SAMPLE_SPHERE_RADIUS = 1.0

class CubeMarcher():
    def __init__(self):
        self._rV = igl.eigen.MatrixXd()
        self._rF = igl.eigen.MatrixXi()
        self.hasMarched = False

    def march(self, pts, sdf):
        res = round(pts.shape[0]**(1/3))
        v = iglhelpers.p2e(pts)
        s = iglhelpers.p2e(sdf)
        igl.copyleft.marching_cubes(s, v, res, res, res, self._rV, self._rF)
        self.hasMarched = True

    def createGrid(self, res):
        K = np.linspace(
            -SAMPLE_SPHERE_RADIUS,
            SAMPLE_SPHERE_RADIUS,
            res
        )
        grid = [[x,y,z] for x in K for y in K for z in K]
        return np.array(grid)

    def getMesh(self, viewer = None):
        if self.hasMarched:
            V = self._rV
            F = self._rF
        else:
            raise("Mesh has not been marched!")

        if viewer is None:
            return Mesh(V=V.copy(), F=F.copy(), doNormalize=False)

        return Mesh(V=V.copy(), F=F.copy(), doNormalize=False, viewer=viewer)

# class PointSampler():
#     def __init__(self, mesh, ratio = 0.0, std=0.0, verticeSampling=False, importanceSampling=False):
#         self._V = iglhelpers.e2p(mesh.V())
#         self._F = iglhelpers.e2p(mesh.F())
#         self._sampleVertices = verticeSampling

#         if ratio < 0 or ratio > 1:
#             raise(ValueError("Ratio must be [0,1]"))

#         self._ratio = ratio

#         if std < 0 or std > 1:
#             raise(ValueError("Normal deviation must be [0,1]"))

#         self._std = std

#         self._calculateFaceBins()

#     def _calculateFaceBins(self):
#         """Calculates and saves face area bins for sampling against"""
#         vc = np.cross(
#             self._V[self._F[:, 0], :] - self._V[self._F[:, 2], :],
#             self._V[self._F[:, 1], :] - self._V[self._F[:, 2], :])

#         A = np.sqrt(np.sum(vc ** 2, 1))
#         FA = A / np.sum(A)
#         self._faceBins = np.concatenate(([0],np.cumsum(FA)))

#     def _surfaceSamples(self,n):
#         """Returns n points uniformly sampled from surface of mesh"""
#         R = np.random.rand(n)   #generate number between [0,1]
#         sampleFaceIdxs = np.array(np.digitize(R,self._faceBins)) -1

#         #barycentric coordinates for each face for each sample :)
#         #random point within face for each sample
#         r = np.random.rand(n, 2)
#         A = self._V[self._F[sampleFaceIdxs, 0], :]
#         B = self._V[self._F[sampleFaceIdxs, 1], :]
#         C = self._V[self._F[sampleFaceIdxs, 2], :]
#         P = (1 - np.sqrt(r[:,0:1])) * A \
#                 + np.sqrt(r[:,0:1]) * (1 - r[:,1:]) * B \
#                 + np.sqrt(r[:,0:1]) * r[:,1:] * C

#         return P

#     def _verticeSamples(self, n):
#         """Returns n random vertices of mesh"""
#         verts = np.random.choice(len(self._V), n)
#         return self._V[verts]

#     def _normalDist(self, V):
#         """Returns normal distribution about each point V"""
#         if self._std > 0.0:
#             return np.random.normal(loc = V,scale = self._std)

#         return V

#     def _randomSamples(self, n):
#         """Returns n random points in unit sphere"""
#         # we want to return points in unit sphere, could do using spherical coords
#         #   but rejection method is easier and arguably faster :)
#         points = np.array([])
#         while points.shape[0] < n:
#             remainingPoints = n - points.shape[0]
#             p = (np.random.rand(remainingPoints,3) - 0.5)*2
#             #p = p[np.linalg.norm(p, axis=1) <= SAMPLE_SPHERE_RADIUS]

#             if points.size == 0:
#                 points = p
#             else:
#                 points = np.concatenate((points, p))
#         return points

#     def sample(self,n):
#         """Returns n points according to point sampler settings"""

#         nRandom = round(Decimal(n)*Decimal(self._ratio))
#         nSurface = n - nRandom

#         xRandom = self._randomSamples(nRandom)

#         if nSurface > 0:
#             if self._sampleVertices:
#                 # for comparison later :)
#                 xSurface = self._verticeSamples(nSurface)
#             else:
#                 xSurface = self._surfaceSamples(nSurface)

#             xSurface = self._normalDist(xSurface)
#             if nRandom > 0:
#                 x = np.concatenate((xSurface,xRandom))
#             else:
#                 x = xSurface
#         else:
#             x = xRandom

#         np.random.shuffle(x)    #remove bias on order

#         return x

# class ImportanceSampler():
#     # M, initital uniform set size, N subset size.
#     def __init__(self, mesh, M, W):
#         self.M = M # uniform sample set size
#         self.W = W # sample weight...

#         if (not mesh is None):
#             #if mesh given, we can create our own uniform sampler
#             self.uniformSampler = PointSampler(mesh, ratio=1.0) # uniform sampling
#             self.sdf = SDF(mesh)
#         else:
#             # otherwise we assume uniform samples (and the sdf val) will be passed in.
#             self.uniformSampler = None
#             self.sdf = None

#     def _subsample(self, s, N):

#         # weighted by exp distance to surface
#         w = np.exp(-self.W*np.abs(s))
#         # probabilities to choose each
#         pU = w / np.sum(w)
#         # exclusive sum
#         C = np.concatenate(([0],np.cumsum(pU)))
#         C = C[0:-1]

#         # choose N random buckets
#         R = np.random.rand(N)

#         # histc
#         I = np.array(np.digitize(R,C)) - 1

#         return I

#     ''' importance sample a given mesh, M uniform samples, N subset based on importance'''
#     def sample(self, N):
#         if (self.uniformSampler is None):
#             raise("No mesh supplied, cannot run importance sampling...")

#         #uniform samples
#         U = self.uniformSampler.sample(self.M)
#         s = self.sdf.query(U)
#         I = self._subsample(s, N)

#         R = np.random.choice(len(U), int(N*0.1))
#         S = U[I,:]#np.concatenate((U[I,:],U[R, :]), axis=0)
#         return S

#     ''' sampling against a supplied U set, where s is sdf at each U'''
#     def sampleU(self, N, U, s):
#         I = self._subsample(s, N)
#         q = U[I,:]
#         d = s[I]
#         return U[I,:], s[I]

# class SDF():
#     _V = igl.eigen.MatrixXd()
#     _F = igl.eigen.MatrixXi()

#     def __init__(self, mesh, signType = 'fast_winding_number', doPrecompute=True):
#         self._V = mesh.V()
#         self._F = mesh.F()
#         self._precomputed = False

#         if signType == 'fast_winding_number':
#             self._signType = igl.SIGNED_DISTANCE_TYPE_FAST_WINDING_NUMBER
#             if doPrecompute:
#                 self._tree = igl.AABB()
#                 self._fwn_bvh = igl.FastWindingNumberBVH()
#                 print("[INFO] Precomuting bvh trees...")
#                 self._tree.init(self._V,self._F)
#                 igl.fast_winding_number(self._V,self._F,2,self._fwn_bvh)
#                 print("[INFO] Done precomputing")
#                 self._precomputed = True
#         elif signType == 'pseudonormal':
#             self._signType = igl.SIGNED_DISTANCE_TYPE_PSEUDONORMAL
#         else:
#             raise("Invalid signing type given")

#     def query(self, queries):
#         """Returns numpy array of SDF values for each point in queries"""
#         queryV = iglhelpers.p2e(queries)

#         S = igl.eigen.MatrixXd()
#         B = igl.eigen.MatrixXd()
#         I = igl.eigen.MatrixXi()
#         C = igl.eigen.MatrixXd()
#         N = igl.eigen.MatrixXd()

#         if self._precomputed and self._signType == igl.SIGNED_DISTANCE_TYPE_FAST_WINDING_NUMBER:
#             # generate samples from precomputed bvh's
#             print("[INFO] Generating SDFs")
#             igl.signed_distance_fast_winding_number(
#                 queryV,
#                 self._V,
#                 self._F,
#                 self._tree,
#                 self._fwn_bvh,
#                 S
#             )
#             print("[INFO] SDFs done")
#         else:
#             igl.signed_distance(
#                 queryV,
#                 self._V,
#                 self._F,
#                 self._signType,
#                 S,
#                 I,
#                 C,
#                 N
#             )

#         return iglhelpers.e2p(S)

class Mesh():
    _V = igl.eigen.MatrixXd()
    _F = igl.eigen.MatrixXi()
    _normalized = False

    def __init__(
        self,
        meshPath=None,
        V=None,
        F=None,
        viewer = None,
        doNormalize = True):

        if meshPath == None:
            if V == None or F == None:
                raise("Mesh path or Mesh data must be given")
            else:
                self._V = V
                self._F = F
                # self._normalizeMesh()
        else:
            self._loadMesh(meshPath,doNormalize)

        self._viewer = viewer

    def _loadMesh(self, fp, doNormalize):
        #load mesh
        igl.read_triangle_mesh(fp, self._V, self._F)

        if doNormalize:
            self._normalizeMesh()

    def V(self):
        return self._V.copy()

    def F(self):
        return self._F.copy()

    # def _normalizeMesh(self):
    #     mb = miniball.Miniball(iglhelpers.e2p(self._V))
    #     scale = BOUNDING_SPHERE_RADIUS / math.sqrt(mb.squared_radius())

    #     T = igl.eigen.Affine3d()
    #     T.setIdentity()
    #     T.translate(
    #         igl.eigen.MatrixXd(
    #             [
    #                 -mb.center()[0] * scale,
    #                 -mb.center()[1] * scale,
    #                 -mb.center()[2] * scale
    #             ]
    #         )
    #     )
    #     print("[INFO] scaled down by", scale)
    #     Vscale = T.matrix().block(0,0,3,3).transpose()
    #     Vtrans = igl.eigen.MatrixXd(self._V.rows(), self._V.cols())
    #     Vtrans.rowwiseSet(T.matrix().block(0,3,3,1).transpose())

    #     self._V = (self._V*Vscale)*scale + Vtrans

    def show(self, doLaunch = False):
        if self._viewer == None:
            self._viewer = igl.glfw.Viewer()

        self._viewer.data(0).set_mesh(self._V,self._F)

        if doLaunch:
            self._viewer.launch()

    def save(self, fp):
        igl.writeOBJ(fp,self._V,self._F)

# def normSDF(S, minVal=None,maxVal=None):
#     if minVal is None:
#         minVal = np.min(S)
#         maxVal = np.max(S)

#     # we don't shift. Keep 0 at 0.
#     S = np.array([item for sublist in S for item in sublist])

#     #S[S<0] = -(S[S<0] / minVal)
#     #S[S>0] = (S[S>0] / maxVal)
#     #S = (S + 1)/2

#     S[S<0] = -0.8
#     S[S>0] = 0.8

#     return S

# def createAx(idx):
#     subplot = pyplot.subplot(idx, projection='3d')
#     subplot.set_xlim((-1,1))
#     subplot.set_ylim((-1,1))
#     subplot.set_zlim((-1,1))
#     subplot.view_init(elev=10, azim=100)
#     subplot.axis('off')
#     subplot.dist = 8
#     return subplot

# def createAx2d(idx):
#     subplot = pyplot.subplot(idx)
#     subplot.set_xlim((-1,1))
#     subplot.set_ylim((-1,1))
#     subplot.axis('off')
#     return subplot

# def plotCube(ax):
#     # draw cube
#     r = [-1, 1]

#     from itertools import product, combinations
#     for s, e in combinations(np.array(list(product(r, r, r))), 2):
#         if np.sum(np.abs(s-e)) == r[1]-r[0]:
#             ax.plot3D(*zip(s, e), color="black")

# def density(U):
#     c = gaussian_kde(np.transpose(U))(np.transpose(U))
#     return c

# def plotMesh(ax, mesh, N=10000):
#     surfaceSampler = PointSampler(mesh, ratio=0.0, std=0.0)
#     surfaceSamples = surfaceSampler.sample(N)
#     x,y,z = np.hsplit(surfaceSamples,3)
#     ax.scatter(x,z,y, c='black', marker='.')

# def plotSamples(ax, U, c, vmin = -1, is2d = False):
#     x,y,z = np.hsplit(U,3)
#     ax.scatter(x,y,z,c=c, marker='.',cmap='coolwarm', norm=None, vmin=vmin, vmax=1)

# def importanceSamplingComparisonPlot(mesh,sdf):
#     fig = pyplot.figure(figsize=(30,10))
#     axUniform = createAx(131)
#     axSurface = createAx(132)
#     axImportance = createAx(133)

#     plotCube(axUniform)
#     plotCube(axSurface)
#     plotCube(axImportance)

#     #plotMesh(axUniform,mesh)
#     #plotMesh(axImportance,mesh)
#     #plotMesh(axSurface,mesh)

#     # plot uniform sampled
#     uniformSampler = PointSampler(mesh, ratio = 1.0)
#     U = uniformSampler.sample(10000)
#     SU = sdf.query(U)
#     c = normSDF(SU)
#     plotSamples(axUniform, U,c)

#     # plot surface + noise sampling
#     sampler = PointSampler(mesh, ratio = 0.1, std = 0.01, verticeSampling=False)
#     p = sampler.sample(10000)
#     S = sdf.query(p)
#     c = normSDF(S, np.min(SU), np.max(SU))
#     plotSamples(axSurface, p,c)

#     # plot importance
#     importanceSampler = ImportanceSampler(mesh, 100000, 20)
#     p = importanceSampler.sample(10000)
#     S = sdf.query(p)
#     c = normSDF(S, np.min(SU), np.max(SU))

#     plotSamples(axImportance, p,c)

#     fig.patch.set_visible(False)

#     pyplot.axis('off')
#     pyplot.show()

# def beforeAndAfterPlot(mesh,sdf):
#     fig = pyplot.figure(figsize=(10,10))
#     fig.patch.set_visible(False)
#     axBefore = createAx(111)

#     pyplot.axis('off')
#     #plotCube(axBefore)
#     #plotCube(axAfter)

#     # plot importance
#     importanceSampler = ImportanceSampler(mesh, 100000, 20)
#     p = importanceSampler.sample(10000)
#     plotSamples(axBefore, p,'grey')
#     pyplot.savefig('before.png', dpi=300, transparent=True)

#     fig = pyplot.figure(figsize=(10,10))
#     fig.patch.set_visible(False)
#     axAfter = createAx(111)
#     S = sdf.query(p)
#     c = normSDF(S)
#     plotSamples(axAfter, p,c)
#     pyplot.savefig('after.png', dpi=300, transparent=True)

# def importanceMotivationPlot(mesh,sdf):

#     fig = pyplot.figure(figsize=(10,10))
#     axSurface = createAx(111)

#     #surface sampling
#     sampler = PointSampler(mesh, ratio = 0.0, std = 0.01, verticeSampling=False)
#     p = sampler.sample(10000)
#     c = density(p)
#     maxDensity = np.max(c)
#     c = c/maxDensity
#     plotSamples(axSurface, p,c, vmin=0)
#     #pyplot.show()
#     pyplot.savefig('surface.png', dpi=300, transparent=True)

#     #vertex sampling
#     fig = pyplot.figure(figsize=(10,10))
#     axVertex = createAx(111)
#     sampler = PointSampler(mesh, ratio = 0.0, std = 0.1, verticeSampling=True)
#     p = sampler.sample(10000)
#     c = density(p)
#     maxDensity = np.max(c)
#     c = c/maxDensity
#     plotSamples(axVertex, p,c, vmin=0)
#     #pyplot.show()
#     pyplot.savefig('vertex.png', dpi=300, transparent=True)

#     fig = pyplot.figure(figsize=(10,10))
#     axImportance = createAx(111)

#     # importance sampling
#     importanceSampler = ImportanceSampler(mesh, 1000000, 50)
#     p = importanceSampler.sample(10000)
#     c = density(p)
#     maxDensity = np.max(c)
#     c = c/maxDensity
#     plotSamples(axImportance, p, c, vmin = 0)
#     #pyplot.show()
#     pyplot.savefig('importance.png', dpi=300, transparent=True)

# if __name__ == '__main__':
#     def main():
#         import argparse
#         import h5py
#         import os
#         parser = argparse.ArgumentParser(description='Train model to predict sdf of a given mesh, by default visualizes reconstructed mesh to you, and plots loss.')
#         parser.add_argument('input_mesh', help='path to input mesh')

#         args = parser.parse_args()

#         mesh = Mesh(meshPath = args.input_mesh)

#         # first test mesh is loaded correctly
#         mesh.show()

#         # test sdf sampling mesh
#         sdf = SDF(mesh)

#         cubeMarcher = CubeMarcher()
#         grid = cubeMarcher.createGrid(64)

#         S = sdf.query(grid)

#         cubeMarcher.march(grid, S)
#         marchedMesh = cubeMarcher.getMesh()

#         marchedMesh.show()

#         #importanceSamplingComparisonPlot(mesh, sdf)
#         #beforeAndAfterPlot(mesh,sdf)
#         #importanceMotivationPlot(mesh,sdf)

#     main()


r"""
Array expressions are expressions representing N-dimensional arrays, without
evaluating them. These expressions represent in a certain way abstract syntax
trees of operations on N-dimensional arrays.

Every N-dimensional array operator has a corresponding array expression object.

Table of correspondences:

=============================== =============================
         Array operator           Array expression operator
=============================== =============================
        tensorproduct                 ArrayTensorProduct
        tensorcontraction             ArrayContraction
        tensordiagonal                ArrayDiagonal
        permutedims                   PermuteDims
=============================== =============================

Examples
========

``ArraySymbol`` objects are the N-dimensional equivalent of ``MatrixSymbol``
objects in the matrix module:

>>> from sympy.tensor.array.expressions import ArraySymbol
>>> from sympy.abc import i, j, k
>>> A = ArraySymbol("A", (3, 2, 4))
>>> A.shape
(3, 2, 4)
>>> A[i, j, k]
A[i, j, k]
>>> A.as_explicit()
[[[A[0, 0, 0], A[0, 0, 1], A[0, 0, 2], A[0, 0, 3]],
  [A[0, 1, 0], A[0, 1, 1], A[0, 1, 2], A[0, 1, 3]]],
 [[A[1, 0, 0], A[1, 0, 1], A[1, 0, 2], A[1, 0, 3]],
  [A[1, 1, 0], A[1, 1, 1], A[1, 1, 2], A[1, 1, 3]]],
 [[A[2, 0, 0], A[2, 0, 1], A[2, 0, 2], A[2, 0, 3]],
  [A[2, 1, 0], A[2, 1, 1], A[2, 1, 2], A[2, 1, 3]]]]

Component-explicit arrays can be added inside array expressions:

>>> from sympy import Array
>>> from sympy import tensorproduct
>>> from sympy.tensor.array.expressions import ArrayTensorProduct
>>> a = Array([1, 2, 3])
>>> b = Array([i, j, k])
>>> expr = ArrayTensorProduct(a, b, b)
>>> expr
ArrayTensorProduct([1, 2, 3], [i, j, k], [i, j, k])
>>> expr.as_explicit() == tensorproduct(a, b, b)
True

Constructing array expressions from index-explicit forms
--------------------------------------------------------

Array expressions are index-implicit. This means they do not use any indices to
represent array operations. The function ``convert_indexed_to_array( ... )``
may be used to convert index-explicit expressions to array expressions.
It takes as input two parameters: the index-explicit expression and the order
of the indices:

>>> from sympy.tensor.array.expressions import convert_indexed_to_array
>>> from sympy import Sum
>>> A = ArraySymbol("A", (3, 3))
>>> B = ArraySymbol("B", (3, 3))
>>> convert_indexed_to_array(A[i, j], [i, j])
A
>>> convert_indexed_to_array(A[i, j], [j, i])
PermuteDims(A, (0 1))
>>> convert_indexed_to_array(A[i, j] + B[j, i], [i, j])
ArrayAdd(A, PermuteDims(B, (0 1)))
>>> convert_indexed_to_array(Sum(A[i, j]*B[j, k], (j, 0, 2)), [i, k])
ArrayContraction(ArrayTensorProduct(A, B), (1, 2))

The diagonal of a matrix in the array expression form:

>>> convert_indexed_to_array(A[i, i], [i])
ArrayDiagonal(A, (0, 1))

The trace of a matrix in the array expression form:

>>> convert_indexed_to_array(Sum(A[i, i], (i, 0, 2)), [i])
ArrayContraction(A, (0, 1))

Compatibility with matrices
---------------------------

Array expressions can be mixed with objects from the matrix module:

>>> from sympy import MatrixSymbol
>>> from sympy.tensor.array.expressions import ArrayContraction
>>> M = MatrixSymbol("M", 3, 3)
>>> N = MatrixSymbol("N", 3, 3)

Express the matrix product in the array expression form:

>>> from sympy.tensor.array.expressions import convert_matrix_to_array
>>> expr = convert_matrix_to_array(M*N)
>>> expr
ArrayContraction(ArrayTensorProduct(M, N), (1, 2))

The expression can be converted back to matrix form:

>>> from sympy.tensor.array.expressions import convert_array_to_matrix
>>> convert_array_to_matrix(expr)
M*N

Add a second contraction on the remaining axes in order to get the trace of `M \cdot N`:

>>> expr_tr = ArrayContraction(expr, (0, 1))
>>> expr_tr
ArrayContraction(ArrayContraction(ArrayTensorProduct(M, N), (1, 2)), (0, 1))

Flatten the expression by calling ``.doit()`` and remove the nested array contraction operations:

>>> expr_tr.doit()
ArrayContraction(ArrayTensorProduct(M, N), (0, 3), (1, 2))

Get the explicit form of the array expression:

>>> expr.as_explicit()
[[M[0, 0]*N[0, 0] + M[0, 1]*N[1, 0] + M[0, 2]*N[2, 0], M[0, 0]*N[0, 1] + M[0, 1]*N[1, 1] + M[0, 2]*N[2, 1], M[0, 0]*N[0, 2] + M[0, 1]*N[1, 2] + M[0, 2]*N[2, 2]],
 [M[1, 0]*N[0, 0] + M[1, 1]*N[1, 0] + M[1, 2]*N[2, 0], M[1, 0]*N[0, 1] + M[1, 1]*N[1, 1] + M[1, 2]*N[2, 1], M[1, 0]*N[0, 2] + M[1, 1]*N[1, 2] + M[1, 2]*N[2, 2]],
 [M[2, 0]*N[0, 0] + M[2, 1]*N[1, 0] + M[2, 2]*N[2, 0], M[2, 0]*N[0, 1] + M[2, 1]*N[1, 1] + M[2, 2]*N[2, 1], M[2, 0]*N[0, 2] + M[2, 1]*N[1, 2] + M[2, 2]*N[2, 2]]]

Express the trace of a matrix:

>>> from sympy import Trace
>>> convert_matrix_to_array(Trace(M))
ArrayContraction(M, (0, 1))
>>> convert_matrix_to_array(Trace(M*N))
ArrayContraction(ArrayTensorProduct(M, N), (0, 3), (1, 2))

Express the transposition of a matrix (will be expressed as a permutation of the axes:

>>> convert_matrix_to_array(M.T)
PermuteDims(M, (0 1))

Compute the derivative array expressions:

>>> from sympy.tensor.array.expressions import array_derive
>>> d = array_derive(M, M)
>>> d
PermuteDims(ArrayTensorProduct(I, I), (3)(1 2))

Verify that the derivative corresponds to the form computed with explicit matrices:

>>> d.as_explicit()
[[[[1, 0, 0], [0, 0, 0], [0, 0, 0]], [[0, 1, 0], [0, 0, 0], [0, 0, 0]], [[0, 0, 1], [0, 0, 0], [0, 0, 0]]], [[[0, 0, 0], [1, 0, 0], [0, 0, 0]], [[0, 0, 0], [0, 1, 0], [0, 0, 0]], [[0, 0, 0], [0, 0, 1], [0, 0, 0]]], [[[0, 0, 0], [0, 0, 0], [1, 0, 0]], [[0, 0, 0], [0, 0, 0], [0, 1, 0]], [[0, 0, 0], [0, 0, 0], [0, 0, 1]]]]
>>> Me = M.as_explicit()
>>> Me.diff(Me)
[[[[1, 0, 0], [0, 0, 0], [0, 0, 0]], [[0, 1, 0], [0, 0, 0], [0, 0, 0]], [[0, 0, 1], [0, 0, 0], [0, 0, 0]]], [[[0, 0, 0], [1, 0, 0], [0, 0, 0]], [[0, 0, 0], [0, 1, 0], [0, 0, 0]], [[0, 0, 0], [0, 0, 1], [0, 0, 0]]], [[[0, 0, 0], [0, 0, 0], [1, 0, 0]], [[0, 0, 0], [0, 0, 0], [0, 1, 0]], [[0, 0, 0], [0, 0, 0], [0, 0, 1]]]]

"""

__all__ = [
    "ArraySymbol", "ArrayElement", "ZeroArray", "OneArray",
    "ArrayTensorProduct",
    "ArrayContraction",
    "ArrayDiagonal",
    "PermuteDims",
    "ArrayAdd",
    "ArrayElementwiseApplyFunc",
    "Reshape",
    "convert_array_to_matrix",
    "convert_matrix_to_array",
    "convert_array_to_indexed",
    "convert_indexed_to_array",
    "array_derive",
]

from sympy.tensor.array.expressions.array_expressions import ArrayTensorProduct, ArrayAdd, PermuteDims, ArrayDiagonal, \
    ArrayContraction, Reshape, ArraySymbol, ArrayElement, ZeroArray, OneArray, ArrayElementwiseApplyFunc
from sympy.tensor.array.expressions.arrayexpr_derivatives import array_derive
from sympy.tensor.array.expressions.conv_array_to_indexed import convert_array_to_indexed
from sympy.tensor.array.expressions.conv_array_to_matrix import convert_array_to_matrix
from sympy.tensor.array.expressions.conv_indexed_to_array import convert_indexed_to_array
from sympy.tensor.array.expressions.conv_matrix_to_array import convert_matrix_to_array


#Auto mesh generator
#Importing libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import pygmsh as pg
from PIL import Image
from scipy import ndimage as ndi

from skimage import io, filters,feature
from skimage.measure import find_contours, approximate_polygon, \
    subdivide_polygon
import meshio
import cv2 as cv
#import os
#os.environ.pop("QT_QPA_PLATFORM_PLUGIN_PATH")
import pymesh
import matplotlib.colors as mcolors
from contour_selc import contour_selc, contour_tree
def Mesh(contours, cont_idx,mesh_holes, dim, indices, lcar, remove_low_dim_cells = True):

	"""
	Creates a mesh based on the contours given as input along with containment hierarchy.

	Parameters
	----------

	contours 	: list,
				list containing a list of contour points
	cont_idx 	: list,
				list of indices of contours to be meshed
	mesh_holes	: list,
				list containing the information regarding the child contours of each contour
	dim 		:list (int,int),
				dimension of the image file
	indices		: list of arrays,
				a list containing the subset of indices of the contours to be used in the mesh
	lcar 		: list,
				the meshing distance parameter
	remove_low_dim_cells : Bool, default True
				Removes the lower dimensional objects from the mesh after meshing

	Returns
	-------
	out:
		mesh: meshio instance

	"""

	with pg.geo.Geometry() as geom:

		m,n = dim #dimensions of the CT scan image

		b1 = geom.add_point([0,0],40) #first boundary point element in the mesh
		b2 = geom.add_point([0,m],40) #second boundary point element in the mesh
		b3 = geom.add_point([n,0],40) #third boundary point element in the mesh
		b4 = geom.add_point([n,m],40) #fourth boundary point element in the mesh
		line1 = geom.add_line(b1,b2) #we create a line element in the mesh between points b1 and b2
		line2 = geom.add_line(b2,b4) #we create a line element in the mesh between points b2 and b4
		line3 = geom.add_line(b4,b3) #we create a line element in the mesh between points b4 and b3
		line4 = geom.add_line(b3,b1) #we create a line element in the mesh between points b3 and b1

		lines = geom.add_curve_loop([line1,line2,line3,line4]) #Creating a loop in the mesh using the four lines created above

		loops = [] #empty list to which we append the loop elements created using the contours
		for k in range(len(cont_idx)):
			l = []	#empty list to append the point elements created using the points in a particular contour
			index = indices[k] #we do not use all the points of a contour but only a portion of it which we obtain using specific indices
			cont = contours[cont_idx[k]][:,0,:]	 #extracting the list of points of a contour
			#print(k,len(cont),len(index))
			for i in range(len(index)):
				p = geom.add_point(cont[index[i]],lcar[k]) #creating a point element in the mesh of the k^th contour
				l.append(p) #appending the point element to l

			s1_l = geom.add_bspline(l) #creating a spline using the points in the list l
			#creating a two point spline element with the first and last point of the contour as the add_curve_loop function requires
			#that the first and the last points of two spline elements that are to be stitched together must have the same point element
			s2_l = geom.add_spline([l[-1],l[0]])

			l_loop = geom.add_curve_loop([s1_l,s2_l]) #creating the loop element of the contour
			loops.append(l_loop) #appending this element to loops

		geom.add_plane_surface(lines, holes = loops) #creating boundary surface element that can now be meshed
		#print(loops)

		#We create surface elements in the mesh for each loop (k). Using the hierarchy structure of the loops we
		#specify the loops (i1, i2,...,ir) that are contained the loop k and they form holes in surface element
		#corresponding to loop k

		for k in range(len(cont_idx)):
			holes_idx = mesh_holes[cont_idx[k]] #loading the indices of the loops that form holes in the kth loop
			holes = [] #empty list to append the loop elements according to the indices
			for i in range(len(holes_idx)):
				for j in range(len(cont_idx)):
					if holes_idx[i] == cont_idx[j]:
						holes.append(loops[j])
			if not holes:
				geom.add_plane_surface(loops[k])
				print('surface with no holes',cont_idx[k])
			else:
				geom.add_plane_surface(loops[k], holes = holes)
				print('surface with holes',cont_idx[k])
		mesh = geom.generate_mesh() #generate the mesh element

		if remove_low_dim_cells == True:
			mesh.remove_orphaned_nodes()
			mesh.remove_lower_dimensional_cells()

		return mesh

def contrast_equ(img_filename,out_filename):
	image = cv.imread(img_filename,0)

	clahe = cv.createCLAHE(clipLimit = 5.0, tileGridSize = (8,8))
	cl1 = clahe.apply(image)
	equ = cv.equalizeHist(cl1)

	cv.imwrite(out_filename,equ)

def mesh_from_image(image_path, mesh_path, num_points, method = 'watershed', small_contour = False, default_lcar = True, plot_contours = True):

	"""
	Creates a meshio instance containing the desired mesh based on the given image.

	Parameters
	----------

	image_path	: str,
				location of the image on the device
	mesh_path	: str,
				location where the mesh should be stored with the name of the file. Must be in .vtk format, for example,
				'C:/Documents/meshautogen.vtk'
	num_points	: int,
				number of points to be used for selecting contours, must be an even number
	method		: str,
				method to use for contouring the image, default is 'watershed'
	small_contour : Bool,
				If True, all the contours which have lower than 10 points are removed
	default_lcar : Bool or list,
				If True uses the in-built meshing parameters for the mesh, feed this as a list otherwise
	plot_contours : Bool,
				If True plots the selected contours on top of the contoured image, default is False

	Returns
	-------

	"""

	cont_idx,contours,hierarchy, dim = contour_selc(image_path,method,num_points)

	mesh_holes = contour_tree(hierarchy)

	if small_contour == False:
		del_list = []
		for k in range(len(cont_idx)):
			if len(contours[cont_idx[k]]) < 10:
				del_list.append(k)

		cont_idx = np.delete(cont_idx,del_list)

	if default_lcar == True:
		indices = []
		lcar = []
		for i in range(len(cont_idx)):
			if len(contours[cont_idx[i]]) <= 25:
				index = np.arange(0,len(contours[cont_idx[i]]),1)
				lcar_val = 10
			elif 25 < len(contours[cont_idx[i]]) <= 50:
				index = np.arange(0,len(contours[cont_idx[i]]),2)
				lcar_val = 10
			elif 50 < len(contours[cont_idx[i]]) <= 200:
				index = np.arange(0,len(contours[cont_idx[i]]),5)
				lcar_val = 20
			elif 200 < len(contours[cont_idx[i]]) <= 1000:
				index = np.arange(0,len(contours[cont_idx[i]]),10)
				lcar_val = 25
			else:
				index = np.arange(0,len(contours[cont_idx[i]]),15)
				lcar_val = 40
			indices.append(index)
			lcar.append(lcar_val)
	else:
		lcar = default_lcar

	for i in range(len(cont_idx)):
		plt.scatter(contours[cont_idx[i]][:,0,0],contours[cont_idx[i]][:,0,1], label = '{}'.format(cont_idx[i]))
	plt.legend()
	plt.show()

	plt.close()

	mesh = Mesh(contours,cont_idx,mesh_holes,dim, indices,lcar,remove_low_dim_cells = True)
	#help(mesh.write)

	mesh.write(mesh_path)

contrast_equ('G:\\HiWi\\Prostate_1.png','G:\\HiWi\\Prostate_1_conteq.png')

mesh_from_image('G:\\HiWi\\Prostate_1_conteq.png','G:\\HiWi\\autogentest.vtk',4)


from pysb import *
from pysb.macros import *
from scipy.constants import N_A

Model()

VOL = 1e-20
NA_V = N_A*VOL

Parameter('k1', 0.015*NA_V)
# Parameter('k1', 0.015)

Parameter('k2', 0)

Parameter('k3', 200/NA_V)
# Parameter('k3', 200)

Parameter('k4', 2*180/NA_V/NA_V)
# Parameter('k4', 2*180)

Parameter('kp4', 0.018)
Parameter('k5', 0)
Parameter('k6', 1.0)
Parameter('k7', 0.6)
Parameter('k8', 1e6) #1e12)
Parameter('k9', 1e3) #1e6)

Monomer('cyclin', ['Y', 'b'], {'Y': ['U','P']})
Monomer('cdc2', ['Y', 'b'], {'Y': ['U','P']})

# Rule 1
synthesize(cyclin(Y='U', b=None), k1)

# Rule 2
#degrade(cyclin(Y='U', b=None), k2)

# Rule 3
Rule('BindingAndPhosphoylation', cyclin(Y='U', b=None) + cdc2(Y='P', b=None) >> cyclin(Y='P', b=1) % cdc2(Y='P', b=1), k3)

# Rule 4
Rule('Activation', cyclin(Y='P', b=1) % cdc2(Y='P', b=1) >> cyclin(Y='P', b=1) % cdc2(Y='U', b=1), kp4)

# Rule 4'
Rule('Autocatalytic', cyclin(Y='P', b=1) % cdc2(Y='P', b=1) + cyclin(Y='P', b=2) % cdc2(Y='U', b=2) + cyclin(Y='P', b=2) % cdc2(Y='U', b=2) >>
cyclin(Y='P', b=1) % cdc2(Y='U', b=1) + cyclin(Y='P', b=2) % cdc2(Y='U', b=2) + cyclin(Y='P', b=2) % cdc2(Y='U', b=2), k4)

# Rule 5
#Rule('Opposed', cyclin(Y='P', b=1) % cdc2(Y='U', b=1) >> cyclin(Y='P', b=1) % cdc2(Y='P', b=1), k5)

# Rule 6
#Rule('Dissociation', cyclin(Y='P', b=1) % cdc2(Y='U', b=1) >> cyclin(Y='P', b=None) + cdc2(Y='U', b=None), k6)
Rule('Dissociation', cyclin(Y='P', b=1) % cdc2(Y='U', b=1) >> cdc2(Y='U', b=None), k6)

# Rule 7
#degrade(cyclin(Y='P', b=None), k7)

# Rules 8 and 9
equilibrate(cdc2(Y='U', b=None), cdc2(Y='P', b=None), [k8, k9])

Observable("YT", cyclin()) # Total Cyclin
Observable("CT", cdc2()) # Total CDC2
Observable("M", cyclin(Y='P', b=1) % cdc2(Y='U', b=1) ) # Active Complex

Observable("Y1", cdc2(Y='U', b=None))
Observable("Y2", cdc2(Y='P', b=None))
Observable("Y3", cdc2(Y='U', b=1) % cyclin(Y='P', b=1))
Observable("Y4", cdc2(Y='P', b=1) % cyclin(Y='P', b=1))
Observable("Y5", cyclin(Y='U', b=None))

#Observable("CYCLIN_P", cyclin(Y='P', b=None))
# [C2] in Tyson

Parameter("cdc0", 1*NA_V)
# Parameter("cdc0", 1.0)
Initial(cdc2(Y='P', b=None), cdc0)

# [Y] in Tyson
Parameter('cyc0', 0.25*NA_V)
# Parameter('cyc0', 0.25)
Initial(cyclin(Y='U', b=None), cyc0)


# -*- coding: utf-8 -*-
"""project.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1X5s8AO6MC_VvVxfA6r-1j0R5SRpeN_rT
"""

import wave
import numpy as np
from scipy.io import wavfile
import matplotlib.pyplot as plot

FREQ = 1000
THRESHOLD = 128
WPM = 5
NOISE_FACTOR = 0.01
#AUDIO_FILENAME = '/content/Hello Ahmed 5 wpm freq 1000.wav'
#AUDIO_FILENAME = '/content/Hello Ahmed 0 wpm freq 500.wav'
AUDIO_FILENAME = '/content/hello this is sara 10 wpm.wav'
letter_to_morse = {
	"a" : ".-",	"b" : "-...",	"c" : "-.-.",
	"d" : "-..",	"e" : ".",	"f" : "..-.",
	"g" : "--.",	"h" : "....",	"i" : "..",
	"j" : ".---",	"k" : "-.-",	"l" : ".-..",
	"m" : "--",	"n" : "-.",	"o" : "---",
	"p" : ".--.",	"q" : "--.-",	"r" : ".-.",
	"s" : "...",	"t" : "-",	"u" : "..-",
	"v" : "...-",	"w" : ".--",	"x" : "-..-",
	"y" : "-.--",	"z" : "--..",	"1" : ".----",
	"2" : "..---",	"3" : "...--",	"4" : "....-",
	"5" : ".....", 	"6" : "-....",	"7" : "--...",
	"8" : "---..",	"9" : "----.",	"0" : "-----",
	" " : " "}
morse_to_letters = dict([(value, key) for key, value in letter_to_morse.items()])

def digitize(snd_data):
  sampled = np.asarray(snd_data).copy()
  if sampled[0] == THRESHOLD:
    sampled[0] = 0
  else: sampled[0] = 1
  for i in range(1, len(snd_data)-1):
    if snd_data[i]>THRESHOLD + 4 or snd_data[i]<THRESHOLD-4:
      sampled[i] = 1
    elif snd_data[i-1]<THRESHOLD+4 and snd_data[i-1]>THRESHOLD-4 and snd_data[i+1]<THRESHOLD+4 and snd_data[i+1]>THRESHOLD-4:
      sampled[i] = 0
    else:
      sampled[i] = 1
  sampled[-1] = sampled[-2]
  return np.asarray(sampled)

def preprocess(AUDIO_FILENAME):
  wav_file = wave.open(AUDIO_FILENAME)
  fs, data = wavfile.read(AUDIO_FILENAME)
  data = data
  data = digitize(data) #convert to zeros and ones
  length = data.shape[0] / fs
  time = np.linspace(0., length, data.shape[0])
  plot.plot(data[:3000])
  return time, data

def tone_separator(AUDIO_FILENAME): # returns list of tuples (duration,high/low)
  time, data = preprocess(AUDIO_FILENAME)
  c1 = 0
  c2 = 0
  c3 = 0
  c4 = 0
  tone_sep = []
  rising_edge = 0
  falling_edge = 0
  for i in range(1, len(time)):
    #rising edge 0->1
    if data[i] == 1:
      if data[i-1] == 0:
        c1 = time[i]
        c4 = time[i] #why c4 and c3 ??
        rising_edge = 1
    #falling edge 1->0
    elif data[i] == 0:
      if data[i-1] == 1:
        c2 = time[i]
        c3 = time[i]
        falling_edge = 1
    if c1 != 0 and c2 != 0:
      tone_sep.append((c2-c1, falling_edge))
      rising_edge = 0
      falling_edge = 0
      c1 = 0
      c2 = 0
    if c3 != 0 and c4 != 0 and c4>c3:
      tone_sep.append((c4-c3, falling_edge))
      c3 = 0
      c4 = 0
  tone_sep = filter_tones(tone_sep)
  tone_sep = np.asarray(tone_sep)
  tone_sep = np.round(tone_sep, 2)
  print(tone_sep)
  return tone_sep

def filter_tones(tone_list):
  tone_list = [s for s in tone_list if s[0]>NOISE_FACTOR]
  return tone_list

def normalize_time(tone_list):  ##normaloze the values of the tones, 1  = '.', 2= '-', 3 = letter separation, 4= word separation
  normalized = []
  dot_len= min(tone_list, key = lambda t: t[0]) [0]
  dash_len= dot_len * 3
  word_space = dot_len * 7
  for x in tone_list:
    if x[1] == 1:
      if x[0] == dot_len:
        normalized.append((1, x[1]))
      elif x[0] == dash_len :
        normalized.append((2, x[1]))
    elif x[1]==0:
      if x[0] == word_space:
        normalized.append((4, x[1]))
      elif x[0] == dash_len:
        normalized.append((3, x[1]))
  normalized.append((3,0.0))
  #print(normalized)
  return normalized

def encode(Tones): #convert the resultant audio into text morse (dots & dashes)
  scentence=[]
  letter = ''
  for tone in Tones:
      if tone[0] == 1 and tone[1] == 1:
        letter+='.'
      elif tone[0] == 2:
        letter += '-'
      elif tone[0] == 3:
        scentence.append(letter)
        letter=''
      elif tone[0] == 4:
        scentence.append(letter)
        scentence.append(' ')
        letter = ''
  return scentence

def decode(morse): #convert the text morse into english letters
  scentence = ''
  for e in morse:
    scentence += morse_to_letters[e]
  return scentence

def Translate_Morse(AUDIO_FILENAME):
  tones = tone_separator(AUDIO_FILENAME) #correct with all
  get_types(tones)
  tones = normalize_time(tones)
  Morse = encode(tones)
  translation = decode(Morse)
  return translation

translated = Translate_Morse(AUDIO_FILENAME)
print(translated)

"""The above cell has the main code the runs all the functions and converts the audio file in 'AUDIO_FILENAME' into english
the morse has to be generated via https://www.meridianoutpost.com/resources/etools/calculators/calculator-morse-code.php?
with the following settings:
frequency = 1000
WPM = 5
further adjustments may be made to make the translation more generic

***
https://morsecode.world/international/translator.html
"""

wav_file = wave.open(AUDIO_FILENAME)
fs, data = wavfile.read(AUDIO_FILENAME)
data = data
data=digitize(data)
print(data)
length = data.shape[0] / fs
time = np.linspace(0., length, data.shape[0])
plot.figure(figsize=(20, 7))
plot.plot(time, data, label="Audio Signal")
plot.legend()
plot.xlabel("Time (sec)")
plot.ylabel("Amplitude")
plot.show()

"""to be general :  we have to get the length of a dot

1 dash = 3 dots high

space between dots and dashes of the letter = 1 dot low

space between letters = 3 dots low

space between words = 7 dots low

we can check the smallest length we detected and that would be the length of 1 dot. then, translate high and low dots to dots/dashes or spaces accordingly
"""

import math
import wave
import struct

# Audio will contain a long list of samples (i.e. floating point numbers describing the
# waveform).  If you were working with a very long sound you'd want to stream this to
# disk instead of buffering it all in memory list this.  But most sounds will fit in
# memory.
audio = []
sample_rate = 44100.0

def append_silence(duration_milliseconds=500):
    """
    Adding silence is easy - we add zeros to the end of our array
    """
    num_samples = duration_milliseconds * (sample_rate / 1000.0)

    for x in range(int(num_samples)):
        audio.append(0.0)

    return

def append_sinewave(
        freq=440.0,
        duration_milliseconds=500,
        volume=1.0):
    """
    The sine wave generated here is the standard beep.  If you want something
    more aggresive you could try a square or saw tooth waveform.   Though there
    are some rather complicated issues with making high quality square and
    sawtooth waves... which we won't address here :)
    """

    global audio # using global variables isn't cool.

    num_samples = duration_milliseconds * (sample_rate / 1000.0)

    for x in range(int(num_samples)):
        audio.append(volume * math.sin(2 * math.pi * freq * ( x / sample_rate )))

    return

def save_wav(file_name):
    # Open up a wav file
    wav_file=wave.open(file_name,"w")

    # wav params
    nchannels = 1

    sampwidth = 2

    # 44100 is the industry standard sample rate - CD quality.  If you need to
    # save on file size you can adjust it downwards. The stanard for low quality
    # is 8000 or 8kHz.
    nframes = len(audio)
    comptype = "NONE"
    compname = "not compressed"
    wav_file.setparams((nchannels, sampwidth, sample_rate, nframes, comptype, compname))

    # WAV files here are using short, 16 bit, signed integers for the
    # sample size.  So we multiply the floating point data we have by 32767, the
    # maximum value for a short integer.  NOTE: It is theortically possible to
    # use the floating point -1.0 to 1.0 data directly in a WAV file but not
    # obvious how to do that using the wave module in python.
    for sample in audio:
        wav_file.writeframes(struct.pack('h', int( sample * 32767.0 )))

    wav_file.close()

    return

append_sinewave(duration_milliseconds=500, volume=0.25)
append_silence()
append_sinewave(duration_milliseconds=1000, volume=0.5)
append_silence()
append_sinewave()
save_wav("output.wav")

wav_file = wave.open("output.wav")
fs, data = wavfile.read("output.wav")
data = data
#data=digitize(data)
print(data)
length = data.shape[0] / fs
time = np.linspace(0., length, data.shape[0])
plot.figure(figsize=(20, 7))
plot.plot(time, data, label="Audio Signal")
plot.legend()
plot.xlabel("Time (sec)")
plot.ylabel("Amplitude")
plot.show()


# Helper functions for 14 time domain features calculation

import numpy as np
from scipy import stats

EPSILON = 1e-15

def time_domain_features(a) :
    squared_a = np.square(a)
    abs_a = np.absolute(a)
    sqrt_a = np.sqrt(abs_a)
    arange_a = np.arange(1, a.shape[0]+1)

    max =  np.maximum.accumulate(a).reshape(-1, 1)
    min = np.minimum.accumulate(a).reshape(-1, 1)

    absmean = np.cumsum(abs_a) / arange_a
    absmean = absmean.reshape(-1, 1)

    mean = np.cumsum(a) / arange_a
    mean = mean.reshape(-1, 1)

    rms = np.sqrt(np.cumsum(squared_a) / arange_a )
    rms = rms.reshape(-1, 1)

    smr = np.square(np.cumsum(sqrt_a) / arange_a )
    smr = smr.reshape(-1, 1)

    peaktopeak =  np.maximum.accumulate(abs_a) * 2
    peaktopeak = peaktopeak.reshape(-1, 1)

    stddev = np.sqrt( (np.cumsum(squared_a) / arange_a) -  (np.square(np.cumsum(a) / arange_a)) )
    stddev = stddev.reshape(-1, 1)

    kurtosis = np.zeros((a.shape[0], 1))
    for i in range(a.shape[0]):
        subarr = a[0:i+1]
        kurtosis[i] = stats.kurtosis(subarr)

    # kurtosisfactor = kurtosis / (EPSILON + (stddev ** 4))

    waveformfactor =  rms / (EPSILON + absmean)

    crestfactor =  peaktopeak / (EPSILON + rms)

    impactfactor = peaktopeak / (EPSILON + absmean)

    clearancefactor =  peaktopeak / (EPSILON + rms)

    res = np.concatenate((max, min, absmean, mean, rms, smr, peaktopeak,
                          stddev, kurtosis, waveformfactor, crestfactor,
                          impactfactor, clearancefactor), axis=1)

    return res


import pathlib, sys
import numpy as np
import pressiotools.linalg as ptla
import scipy.linalg as la

np.set_printoptions(linewidth=140)

#-----------------------------------------
def pinv_run(comm):
  rank = comm.Get_rank()
  np.random.seed(312367)

  # create matrix
  A = np.asfortranarray(np.random.rand(15,4))

  BT = la.pinv(A).T
  if rank==0:
    print(BT)

  # create distributed A
  myStartRow = rank*5
  A1 = ptla.MultiVector(A[myStartRow:myStartRow+5, :])
  piO = ptla.PseudoInverse()
  piO.compute(A1)
  # view the local part of A^*T
  # remember that pressiotools.PseudoInverse stores A^*T NOT A^*
  AstarT = piO.viewTransposeLocal()
  print("rank", rank, AstarT)

  myBT = BT[myStartRow:myStartRow+5, :]
  assert(np.allclose(myBT, AstarT, atol=1e-10))

#-----------------------------------------
def pinv_apply(comm):
  print("\n")
  rank = comm.Get_rank()
  np.random.seed(312367)

  # create the matrix
  A = np.asfortranarray(np.random.rand(15,4))

  # compute scipy pseudo-inverse
  B = la.pinv(A)
  print(B.T)
  # apply to vector of ones
  d0 = np.ones(15)
  c = np.dot(B, d0)
  print(c)

  # do same using our code
  # create distributed A
  myStartRow = rank*5
  A1 = ptla.MultiVector(A[myStartRow:myStartRow+5, :])
  piO = ptla.PseudoInverse()
  piO.compute(A1)
  d1 = ptla.Vector(np.ones(5))
  c1 = piO.apply(d1)
  print(c1)

  assert(np.allclose(c, c1, atol=1e-10))

if __name__ == '__main__':
  from mpi4py import MPI
  comm = MPI.COMM_WORLD
  assert(comm.Get_size() == 3)

  pinv_run(comm)
  pinv_apply(comm)


# -*- coding: utf-8 -*-
"""
Created on Tue Oct  1 14:02:59 2019

@author: ag
"""

# -*- coding: utf-8 -*-
"""
Created on Tue Oct  1 12:36:44 2019

@author: ag
"""

import numpy as np
import re
from scipy import interpolate
import pandas as pd
import pickle

def loadfile(txt):
    df = pd.read_csv(txt,delim_whitespace=True)
    df.columns = df.columns.str.strip()

    #remove 20year means in bottom of file.
    ix = df.index[df.year.apply(lambda x: not x.isnumeric())]
    if len(ix)>0:
        ix=ix[0]
        df=df.iloc[0:ix]
    df.year=pd.to_numeric(df.year)
    df['05']=pd.to_numeric(df['05'])

    df = df.append({'year': 1995.5, '05': 0.0, '50':0.0, '95':0.0},ignore_index=True)
    df.set_index('year',inplace=True)

    add = pd.DataFrame({'year': np.arange(1996,2007), '50': np.nan})
    add.set_index('year',inplace=True)
    df= pd.concat((df,add),sort=False)
    df = df.loc[~df.index.duplicated(keep='first')]
    df.sort_index(inplace=True)
    df.interpolate(inplace=True)

#    V = np.append(V, [[1995.5,0]], axis=0) # baseline: 1986–2005

#    f = interpolate.interp1d(V[:,0], V, axis=0, fill_value='extrapolate',bounds_error=0)
#    V= f(np.arange(2000,2101)) #ensure that it has exactly this period! Some files goes to 2099
    return df

output={}

d=['ant_26','ant_45','ant_85','gmsl_26','gmsl_45','gmsl_85']
for ii,filename in enumerate(d):
    (component,scenario)=re.findall(r'(.*)_(\d+)',filename)[0]

    df = loadfile(filename)

    if scenario not in output:
        output[scenario] = {}
    if component not in output[scenario]:
        output[scenario][component] = {}
    output[scenario][component]=df

with open('SROCC.pickle', 'wb') as file:
    pickle.dump(output, file)

#-------------------JOIN WITH AR5--------------------------
ar5 = loadpickle('../../AR5datafiles/13.SM.1/AR5.pickle')

naming = {'26': 'rcp26', '45': 'rcp45', '85': 'rcp85'}
for k,v in naming.items():
    output[v] = output.pop(k)

naming = {'05': 'low', '50': 'mid', '95': 'upper'}
for scen,sitem in output.items():
    for contrib,item in sitem.items():
        for k,v in naming.items():
            item[v] = item.pop(k)

naming = {'ant': 'antnet', 'gmsl': 'sum'}
for scenario,item in output.items():
    for k,v in naming.items():
        item[v] = item.pop(k)
    item['temperature'] = ar5[scenario]['temperature']

with open('SROCC-AR5.pickle', 'wb') as file:
    pickle.dump(output, file)