# -*- coding: utf-8 -*-
"""
Created on Mon Jul  2 14:35:46 2018

@author: SilverDoe
"""

#=================== Still to verify/correct===================================

import pandas as pd
from matplotlib import pyplot as plt
from scipy.cluster.vq import kmeans, kmeans2, whiten

df = pd.read_csv('E:\\Documents\\PythonProjects\\1_Basics\\DataForFiles\\scidata.csv')
print(df.shape)
df.head()

coordinates = df.as_matrix(columns=['V1', 'V2'])
plt.figure(figsize=(10, 6), dpi=100)
plt.scatter(coordinates[:,0], coordinates[:,1], c='c', s=100)
plt.show()

N = len(coordinates)
w = whiten(coordinates)
k = 100
i = 20

cluster_centroids1, distortion = kmeans(w, k, iter=i)
cluster_centroids2, closest_centroids = kmeans2(w, k, iter=i)

plt.figure(figsize=(10, 6), dpi=100)
plt.scatter(cluster_centroids2[:,0], cluster_centroids2[:,1], c='r', s=100)
plt.show()

#==============================================================================
from numpy import vstack,array
from numpy.random import rand
import pandas as pd
from scipy.cluster import vq
from matplotlib import pyplot as plt

# data generation with three features
data = vstack((rand(100,3) + array([.5,.5,.5]),rand(100,3)))
print(data)
print('===================================================')
#The features in obs(observation) should have unit variance, which can be achieved by passing them through the whiten function
data = vq.whiten(data)
print(data)

# computing K-Means with K = 3 (2 clusters)
centroids,_ = vq.kmeans(data,3)
print(centroids)

# assign each sample to a cluster
clx,_ = vq.vq(data,centroids)

# check clusters of observation
print(clx)

plt.figure(figsize=(10, 6), dpi=100)
plt.scatter(centroids[:,0], centroids[:,1], c='r', s=100)
plt.show()

#====================== using sklearn =====================================
# example a
import numpy as np
from sklearn.cluster import KMeans
import pandas as pd
from matplotlib import pyplot as plt

df = pd.read_csv('E:\\Documents\\PythonProjects\\1_Basics\\DataForFiles\\scidata.csv')
print(df.shape)
df.head()
# whitening of data
# df = whiten(df)
f1 = df['V1'].values
f2 = df['V2'].values

X=np.matrix(zip(f1,f2))
plt.scatter(f1, f2, c='black', s=7)

kmeans = KMeans(n_clusters=2).fit(X)

kmeans = KMeans(n_clusters=4).fit(X)

# example 1 ===================================================
from sklearn.cluster import KMeans
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
plt.rcParams['figure.figsize'] = (16, 9)
plt.style.use('ggplot')

# Importing the dataset
data = pd.read_csv('E:\\Documents\\PythonProjects\\1_Basics\\DataForFiles\\scidata.csv')
print("Input Data and Shape")
print(data.shape)
data.head()

# Getting the values and plotting it
f1 = data['V1'].values
f2 = data['V2'].values
X = np.array(list(zip(f1, f2)))
#plt.scatter(f1, f2, c='black', s=7)
#====

# Number of clusters
kmeans = KMeans(n_clusters=3)
# Fitting the input data
kmeans = kmeans.fit(X)
# Getting the cluster labels
labels = kmeans.predict(X)
# Centroid values
centroids = kmeans.cluster_centers_
print(centroids) # From sci-kit learn

fig = plt.figure()
ax = Axes3D(fig)
ax.scatter(X[:, 0], X[:, 1], X[:, 2], c='y',s=100)
ax.scatter(centroids[:, 0], centroids[:, 1], centroids[:, 2], marker='*', c='#050505', s=1000)
#====================================================================
# example 2
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.cluster import KMeans
from sklearn.datasets import make_blobs

plt.rcParams['figure.figsize'] = (16, 9)

# Creating a sample dataset with 4 clusters
X, y = make_blobs(n_samples=800, n_features=3, centers=4)
y
fig = plt.figure()
ax = Axes3D(fig)
ax.scatter(X[:, 0], X[:, 1], X[:, 2])

# Initializing KMeans
kmeans = KMeans(n_clusters=4)
# Fitting with inputs
kmeans = kmeans.fit(X)
# Predicting the clusters
labels = kmeans.predict(X)
# Getting the cluster centers
C = kmeans.cluster_centers_

fig = plt.figure()
ax = Axes3D(fig)
ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=y)
ax.scatter(C[:, 0], C[:, 1], C[:, 2], marker='*', c='#050505', s=1000)


# Released under The MIT License (MIT)
# http://opensource.org/licenses/MIT
# Copyright (c) 2015 <NAME>

"""
Summary
-------
Provides functions to warp electrode layouts.
"""

import numpy as np
import scipy as sp

def warp_locations(locations, y_center=None, return_ellipsoid=False, verbose=False):
    """ Warp EEG electrode locations to spherical layout.

    EEG Electrodes are warped to a spherical layout in three steps:
        1. An ellipsoid is least-squares-fitted to the electrode locations.
        2. Electrodes are displaced to the nearest point on the ellipsoid's surface.
        3. The ellipsoid is transformed to a sphere, causing the new locations to lie exactly on a spherical surface
           with unit radius.

    This procedure intends to minimize electrode displacement in the original coordinate space. Simply projecting
    electrodes on a sphere (e.g. by normalizing the x/y/z coordinates) typically gives much larger displacements.

    Parameters
    ----------
    locations : array-like, shape = [n_electrodes, 3]
        Eeach row of `locations` corresponds to the location of an EEG electrode in cartesian x/y/z coordinates.
    y_center : float, optional
        Fix the y-coordinate of the ellipsoid's center to this value (optional). This is useful to align the ellipsoid
        with the central electrodes.
    return_ellipsoid : bool, optional
        If `true` center and radii of the ellipsoid are returned.

    Returns
    -------
    newlocs : array-like, shape = [n_electrodes, 3]
        Electrode locations on unit sphere.
    c : array-like, shape = [3], (only returned if `return_ellipsoid` evaluates to `True`)
        Center of the ellipsoid in the original location's coordinate space.
    r : array-like, shape = [3], (only returned if `return_ellipsoid` evaluates to `True`)
        Radii (x, y, z) of the ellipsoid in the original location's coordinate space.
    """
    locations = np.asarray(locations)

    if y_center is None:
        c, r = _fit_ellipsoid_full(locations)
    else:
        c, r = _fit_ellipsoid_partial(locations, y_center)

    elliptic_locations = _project_on_ellipsoid(c, r, locations)

    if verbose:
        print('Head ellipsoid center:', c)
        print('Head ellipsoid radii:', r)
        distance = np.sqrt(np.sum((locations - elliptic_locations)**2, axis=1))
        print('Minimum electrode displacement:', np.min(distance))
        print('Average electrode displacement:', np.mean(distance))
        print('Maximum electrode displacement:', np.max(distance))

    spherical_locations = (elliptic_locations - c) / r

    if return_ellipsoid:
        return spherical_locations, c, r

    return spherical_locations

def _fit_ellipsoid_full(locations):
    """identify all 6 ellipsoid parametes (center, radii)"""
    a = np.hstack([locations*2, locations**2])
    lsq = sp.linalg.lstsq(a, np.ones(locations.shape[0]))
    x = lsq[0]
    c = -x[:3] / x[3:]
    gam = 1 + np.sum(x[:3]**2 / x[3:])
    r = np.sqrt(gam / x[3:])
    return c, r

def _fit_ellipsoid_partial(locations, cy):
    """identify only 5 ellipsoid parameters (y-center determined by e.g. Cz)"""
    a = np.vstack([locations[:, 0]**2,
                   locations[:, 1]**2 - 2 * locations[:, 1] * cy,
                   locations[:, 2]**2,
                   locations[:, 0]*2,
                   locations[:, 2]*2]).T
    x = sp.linalg.lstsq(a, np.ones(locations.shape[0]))[0]
    c = [-x[3] / x[0], cy, -x[4] / x[2]]
    gam = 1 + x[3]**2 / x[0] + x[4]**2 / x[2]
    r = np.sqrt([gam / x[0], gam / x[1], gam / x[2]])
    return c, r

def _project_on_ellipsoid(c, r, locations):
    """displace locations to the nearest point on ellipsoid surface"""
    p0 = locations - c  # original locations

    l2 = 1 / np.sum(p0**2 / r**2, axis=1, keepdims=True)
    p = p0 * np.sqrt(l2)  # initial approximation (projection of points towards center of ellipsoid)

    fun = lambda x: np.sum((x.reshape(p0.shape) - p0)**2)              # minimize distance between new and old points
    con = lambda x: np.sum(x.reshape(p0.shape)**2 / r**2, axis=1) - 1  # new points constrained to surface of ellipsoid
    res = sp.optimize.minimize(fun, p, constraints={'type': 'eq', 'fun': con}, method='SLSQP')

    return res['x'].reshape(p0.shape) + c


<filename>tests/conftest.py
import os
import pytest
import numpy as np
import pandas as pd

import matplotlib
matplotlib.use("agg")
import matplotlib.pyplot as plt

from pytest import fail
from scipy.misc import imsave, imread

from abfs.data import Data
from abfs.constants import DataConfig, BAND3, RIO_REGION

TOP_LEVEL_DATA_DIR = os.path.join(os.path.dirname(__file__),
                                  './fixtures/spacenet')

@pytest.fixture
def config():
    return DataConfig(TOP_LEVEL_DATA_DIR, BAND3, RIO_REGION)

@pytest.fixture
def data(config):
    return Data(config)

@pytest.fixture
def sample_df():
    return pd.DataFrame(data={
        'group_id': [ 1,  1,  1,  2,  2,  3,  3,  4,  5,  5],
        'value':    [56, 18, 19, 51, 15, 96, 99, 95, 66, 41]
    })

@pytest.fixture
def sample_image():
    return plt.imread(os.path.join(
        TOP_LEVEL_DATA_DIR,
        'AOI_1_Rio/processedData/processedBuildingLabels/3band',
        '3band_AOI_1_RIO_img5792.tif'
    ))

@pytest.fixture
def box_mask():
    def _box_mask(shape):
        box_mask_height, box_mask_width = shape
        x1 = int(box_mask_width / 4)
        y1 = int(box_mask_height / 4)
        x2 = x1 + int(box_mask_width / 2)
        y2 = y1 + int(box_mask_height / 2)

        box_mask = np.zeros(shape, dtype='uint8')
        box_mask[y1:y2, x1:x2] = 1
        return box_mask

    return _box_mask

def compare_image(received, filename):
    approved_path = f'tests/fixtures/{filename}.approved.png'
    received_path = f'tests/fixtures/{filename}.received.png'
    imsave(received_path, received)

    if os.path.isfile(approved_path) is False:
        fail(f'No image explicitly approved. Please verify {received_path}.')
        return

    approved_image = imread(approved_path)
    received_image = imread(received_path)

    if (approved_image != received_image).any():
        fail(f'{received_path} != {approved_path}. Please verify.')
    else:
        os.remove(received_path)


import os
import sys
import numpy as np
from scipy.stats import skew
from matplotlib import pyplot as plt

# Function: get_int_input()
# Description: Gets integer from the user between min and max provided. Shows
#              prompt to user before every input. Handles erroneous input.
# Pre-Conditions: String prompt, integer minimum and maximum given to function
# Post-Conditions: Returned integer between minimum and maximum
def get_int_input(prompt, min, max):
    while True:
        answer = min - 1
        try:
            answer = int(input(prompt))
        except ValueError:
            print(f"Attempted input was not an integer. Please try again")
        if not (answer <= max and answer >= min):
            print(f"Input was not between {min} and {max}. Please try again")
        else:
            return answer

def smart_check_int_param(param_value, param_name, min_val, max_val):
    try:
        param_value = int(param_value)
    except:
        raise ValueError(f"Parameter {param_name} must be an integer.")
    if not (param_value <= max_val and param_value >= min_val):
        raise ValueError(f"Parameter {param_name} must be larger than {min_val-1} and smaller than {max_val+1}.")
    return param_value

def smart_make_dir(dirpath):
    if not os.path.isdir(dirpath):
        os.mkdir(dirpath)

def norm_dirname(dirpath, dirtype, create = False):
    if dirpath == 0:
        return dirpath
    if not os.path.isdir(dirpath):
        if create:
            os.mkdir(dirpath)
        else:
            raise ValueError(f"Provided {dirtype} directory '{dirpath}' doesn't exist")
    if dirpath[-1] == os.path.sep:
        return dirpath[:-1]
    else:
        return dirpath

# Function: get_files()
# Description: Gets all files of '.tiff' format in a folder
# Pre-Conditions: Path-like string to valid directory provided
# Post-Conditions: Returns list of path-like objects to existing tiffs within dir.
def get_files(folder):
    tiffs = []
    root_name = ""
    for root, dirs, files in os.walk(folder):
        for f in files:
            this_name = '_'.join(f.split("_")[:-1])
            if root_name == "":
                root_name = this_name
            elif this_name != root_name:
                raise ValueError(f"Cannot confirm all images in {folder} are from the same stack. Ensure tiff filenames are all 'imgname_zX.tiff'")
            if f.split('.')[-1] == 'tif' or f.split('.')[-1] == 'tiff':
                tiffs.append(os.path.join(root, f))
    if not len(tiffs):
        raise ValueError("No tiffs found!")
    return tiffs

# Function: min_max_scale()
# Description: Performs min-max scaling on img object (pixel array)
# Pre-Conditions: Img object provided
# Post-Conditions: Return img object with pixels min-max scaled.
def min_max_scale(img):
    # std_max = np.nanmean(img) + 4 * np.nanstd(img)
    # img[img > std_max] = std_max
    # plt.hist(img.flatten())
    # plt.show()

    np.seterr(all = 'raise')

    minimum = img.min()
    maximum = img.max()
    # The formula for min-max scaling:
    img = (img - minimum) / (maximum - minimum) if maximum - minimum != 0 else (img - minimum) / 1
    img *= 255
    if isinstance(img, np.ma.MaskedArray):
        img = np.ma.getdata(img)
    return img

def gauss(x, mu, sigma, A):
    out = x
    for idx, inp in enumerate(x):
        try:
            out[idx] = A * np.exp(-(inp - mu)**2 / 2 / sigma**2)
        except FloatingPointError:
            out[idx] = 0
    return out

def bimodal(x, mu1, s1, A1, mu2, s2, A2):
    return gauss(x, mu1, s1, A1) + gauss(x, mu2, s2, A2)


from autotabular.pipeline.components.base import AutotabularPreprocessingAlgorithm
from autotabular.pipeline.constants import DENSE, INPUT, SPARSE, UNSIGNED_DATA
from ConfigSpace.configuration_space import ConfigurationSpace

class Densifier(AutotabularPreprocessingAlgorithm):

    def __init__(self, random_state=None):
        pass

    def fit(self, X, y=None):
        self.fitted_ = True
        return self

    def transform(self, X):
        from scipy import sparse
        if sparse.issparse(X):
            return X.todense().getA()
        else:
            return X

    @staticmethod
    def get_properties(dataset_properties=None):
        return {
            'shortname': 'RandomTreesEmbedding',
            'name': 'Random Trees Embedding',
            'handles_regression': True,
            'handles_classification': True,
            'handles_multiclass': True,
            'handles_multilabel': True,
            'handles_multioutput': True,
            'is_deterministic': True,
            'input': (SPARSE, UNSIGNED_DATA),
            'output': (DENSE, INPUT)
        }

    @staticmethod
    def get_hyperparameter_search_space(dataset_properties=None):
        cs = ConfigurationSpace()
        return cs


'''
特征工程部分：
1.各特征转化为数值型特征
2.异常值检测
3.缺失值填充
4.特征变换
5.特征选择
最终数据集：包含做完特征变换后的特征样本值，以及未变换前的房屋总价和房屋每平米价
'''
import pymysql
import pandas as pd
import numpy as np
import category_encoders as ce
import re
import cn2an
import datetime
from scipy.stats import skew
from scipy import stats
from sklearn.preprocessing import PowerTransformer

conn = pymysql.connect(
    host = '192.168.127.12',
    user = 'root',
    passwd = '<PASSWORD>',
    db = 'house',
    port=3306,
    charset = 'utf8'
)

cursor01 = conn.cursor()
cursor01.execute(
        "select column_name, column_comment from information_schema.columns where table_schema ='house' and table_name = 'allhouses'")
all_info = list(cursor01.fetchall())
print(all_info)
df = pd.read_sql('select * from allhouses',conn)
column_names = []
for i in all_info:
    column_names.append(i[1])
df.columns = column_names
df1 = df.copy()

#有几十个房子由于页面特殊爬取错乱
df1 = df1.dropna(subset=['装修情况'])
#某些特征无用，删除
df1 = df1.drop(columns=['小区详情url','房屋年限','房子ID'],axis=1)

df1.to_excel('初始数据集.xlsx',index=None)
'''
(1)各特征转化为数值特征
1.去除单位
2.特征组合
3.特征编码
'''
#物业费用有些是1.2至100元/平米/月 有些是1.5元/平米/月的类型，对于前者取中值，对于后者只去除单位
df1.loc[df1['小区物业费用']=='暂无信息','小区物业费用'] = np.nan #对于暂无信息的值记作空值
df1.loc[df1['小区物业费用'].notnull(),'小区物业费用']=\
df1.loc[df1['小区物业费用'].notnull(),'小区物业费用'].apply(lambda x:x[:-6])
df1.loc[df1['小区物业费用'].notnull(),'小区物业费用']=\
df1.loc[df1['小区物业费用'].notnull(),'小区物业费用'].apply(lambda x: (np.double(x.split('至')[0])+np.double(x.split('至')[1]))/2 if '至' in x else x)

#对于上次交易时间与挂牌时间，做衍生指标处理，为挂牌时间-上次交易时间，单位天数
df1.loc[df['上次交易']=='暂无数据','上次交易'] = np.nan #对于暂无信息的值记作空值
today = str(datetime.datetime.now().year)+'-'+str(datetime.datetime.now().month)+'-'+str(datetime.datetime.now().day)
df1['当前时间'] = today
df1['当前时间'] = pd.to_datetime(df1['当前时间'],errors = 'coerce')
df1['上次交易'] = pd.to_datetime(df1['上次交易'],errors = 'coerce')
df1['挂牌时间'] = pd.to_datetime(df1['挂牌时间'],errors = 'coerce')
df1['挂牌时间-上次交易时间'] = ((df1['挂牌时间'] - df1['上次交易']).values/np.timedelta64(1, 'h'))/24
df1['当前时间-挂牌时间']=((df1['当前时间'] - df1['挂牌时间']).values/np.timedelta64(1, 'h'))/24
#删除原指标
df1 = df1.drop(columns = ['上次交易','挂牌时间','当前时间'])

#房屋用途，标签编码
yongtu_mapping = {
           '别墅':5,
           '商业':4,
           '商住两用':3,
           '普通住宅':2,
           '平房':1,
}
df1['房屋用途'] = df1['房屋用途'].map(yongtu_mapping)

#抵押信息，将信息改为有无抵押，暂无信息则改为NaN，后进行编码处理
df1.loc[(df1['抵押信息']!='无抵押')&(df1['抵押信息'].notnull()),'抵押信息']=\
df1.loc[(df1['抵押信息']!='无抵押')&(df1['抵押信息'].notnull()),'抵押信息'].apply(lambda x:x[:3])
df1.loc[df1['抵押信息']=='暂无数','抵押信息']=np.nan

#前3个特征为有序特征，如产权共有的房子会更受消费者的青睐，因此均作二值化处理
df1.loc[df1['产权所属']=='共有','产权所属']=1
df1.loc[df1['产权所属']=='非共有','产权所属']=0
df1.loc[df1['抵押信息']=='无抵押','抵押信息']=1
df1.loc[df1['抵押信息']=='有抵押','抵押信息']=0
df1.loc[df1['房本备件']=='已上传房本照片','房本备件']=1
df1.loc[df1['房本备件']=='未上传房本照片','房本备件']=0

#对于交易权属，使用频数编码
count_enc = ce.CountEncoder()
#Transform the features, rename the columns with the _count suffix, and join to dataframe
df1['交易权属'] = count_enc.fit_transform(df1['交易权属'])

#建筑时间
df1.loc[df1['建楼时间'].notnull(),'建楼时间']=df1.loc[df1['建楼时间'].notnull(),'建楼时间'].apply(lambda x:x[:-2])
df1.loc[df1['建楼时间']=='未知','建楼时间']=np.nan
df1['建楼时间-小区建筑年代']=np.nan
df1['建楼时间-小区建筑年代']=df1.loc[df1['建楼时间'].notnull(),'建楼时间'].astype(float)-df1.loc[df1['小区建筑年代'].notnull(),'小区建筑年代'].astype(float)
df1.loc[df1['建楼时间'].notnull(),'建楼时间']=\
float(datetime.datetime.now().year)-(df1.loc[df1['建楼时间'].notnull(),'建楼时间'].astype(float))
df1.loc[df1['小区建筑年代'].notnull(),'小区建筑年代']=\
float(datetime.datetime.now().year)-(df1.loc[df1['小区建筑年代'].notnull(),'小区建筑年代'].astype(float))
df1.rename(columns={"建楼时间":"建楼距今时长", "小区建筑年代":"小区建成距今时长"},inplace=True)

#面积特征
df1.loc[df1['套内面积']=='暂无数据','套内面积']=np.nan
df1.loc[df1['建筑面积'].notnull(),'建筑面积']=df1.loc[df1['建筑面积'].notnull(),'建筑面积'].apply(lambda x:x[:-1]).astype(float)
df1.loc[df1['套内面积'].notnull(),'套内面积']=df1.loc[df1['套内面积'].notnull(),'套内面积'].apply(lambda x:x[:-1]).astype(float)
#衍生特征，建筑面积-套内面积=公摊面积
df1['公摊面积']=np.nan
df1['公摊面积']=df1.loc[df1['建筑面积'].notnull(),'建筑面积']-df1.loc[df1['套内面积'].notnull(),'套内面积']

#梯户比例：衍生特征户数/梯数
re2 = re.compile('(.+)梯(.+)户')
def cn_extraction(x):
    if x==np.nan:
        return ['一','零']
    ret = re2.findall(x)
    if ret:
        return (re2.findall(x))[0]
    else:
        return ['一','零']
def calculate_ratio(x):
    h = cn2an.cn2an(x[1],'smart')
    t = cn2an.cn2an(x[0],'smart')
    r = h/t
    return r
df1.loc[df1['梯户比例'].notnull(),'梯户比例']=df1.loc[df1['梯户比例'].notnull(),'梯户比例'].apply(cn_extraction).apply(calculate_ratio)

#房屋户型：衍生特征：卧室数量，客厅数量，厨房数量，卫生间数量
re1 = re.compile('\d+')
temp = df1.loc[:,'房屋户型'].apply(re1.findall)
df1['卧室数量']=df1['客厅数量']=df1['厨房数量']=df1['卫生间数量']=np.nan
df1['卧室数量'] = temp.apply(lambda x:x[0])
df1['客厅数量'] = temp.apply(lambda x:x[1])
df1['厨房数量'] = temp.apply(lambda x:x[2])
df1['卫生间数量'] = temp.apply(lambda x:x[3])
df1 = df1.drop(columns = ['房屋户型'])

#所在楼层衍生特征总楼层数目
re3 = re.compile('.+共(\d+)层.+')
df1['总层数'] = df1.loc[:,'所在楼层'].apply(lambda x:re3.findall(x)[0])

# 户型分间衍生以下特征
# 1.客厅面积占比
# 2.卧室面积占比
# 3.实际使用面积
# 4.落地窗数量
# 5.有无储物间
# 6.有无入室花园
# 7.窗户数量
# 8.平均卧室面积
# 9.客厅、卧室、阳台朝向，采光等级

#朝向以等级分级
#南＞东南=西南＞东=西＞东北=西北＞北
rank_direction = {
    '南':5,
    '东南':4,
    '西南':4,
    '东':3,
    '西':3,
    '东北':2,
    '西北':2,
    '北':1,
    '无':None
}

def change_direct(x):
    for k,j in rank_direction.items():
        if k in x:
            return j
df1.loc[df1['户型分间']=='{}','户型分间']=np.nan
list1=[]
list2=[]
list3=[]
list4=[]
list5=[]
list6=[]
list7=[]
list8=[]
list9=[]
list10=[]
list11=[]
for item in df1.loc[df1['户型分间'].notnull(), '户型分间']:
    item = eval(item)  # 转换为字典
    df2 = pd.DataFrame(item)
    if '储物间' in df2.columns:
        list3.append(1)
    else:
        list3.append(0)
    if '入户花园' in df2.columns:
        list4.append(1)
    else:
        list4.append(0)
    j = 0
    k = 0
    for i in df2.iloc[2]:
        if i == '落地飘窗':
            j = j + 1
        if i != '无窗':
            k = k + 1
    list2.append(j)
    list5.append(k)
    df3 = df2.filter(regex='客厅')
    df4 = df2.filter(regex='卧室')
    df5 = df2.filter(regex='阳台')
    using_area = 0
    for p in df2.iloc[0]:
        p = p.split('平')
        p = np.double(p[0])
        using_area = using_area + p
    list6.append(using_area)
    livingroom_sum = 0
    for p in df3.iloc[0]:
        p = p.split('平')
        p = np.double(p[0])
        livingroom_sum = livingroom_sum + p
    list1.append(np.double(livingroom_sum / using_area))
    room_sum = 0
    for p in df4.iloc[0]:
        p = p.split('平')
        p = np.double(p[0])
        room_sum = room_sum + p
    list8.append(np.double(room_sum / using_area))
    m = df4.shape[1]
    if m != 0:
        room_ave = room_sum / m
    else:
        room_ave = 0
    list7.append(room_ave)

    # 房屋朝向等级
    # 客厅
    try:
        list9.append(change_direct(df3['客厅'][1]))
    except:
        list9.append(None)
    # 主卧
    try:
        x1 = df4.iloc[1, np.argmax(df4.loc[0, :].apply(lambda x: x[:-2]).astype(float))]
        list10.append(change_direct(x1))
    except:
        list10.append(None)
    # 主阳台
    try:
        x1 = df5.iloc[1, np.argmax(df5.loc[0, :].apply(lambda x: x[:-2]).astype(float))]
        list11.append(change_direct(x1))
    except:
        list11.append(None)
df1['客厅面积占比']=df1['户型分间']
df1['卧室面积占比']=df1['户型分间']
df1['平均卧室面积']=df1['户型分间']
df1['实际使用面积']=df1['户型分间']
df1['有无储物间']=df1['户型分间']
df1['有无入室花园']=df1['户型分间']
df1['窗户数量']=df1['户型分间']
df1['落地窗数量']=df1['户型分间']
df1['客厅采光程度']=df1['户型分间']
df1['主卧采光程度']=df1['户型分间']
df1['阳台采光程度']=df1['户型分间']
df1.loc[df1['客厅面积占比'].notnull(),'客厅面积占比']=list1
df1.loc[df1['落地窗数量'].notnull(),'落地窗数量']=list2
df1.loc[df1['有无储物间'].notnull(),'有无储物间']=list3
df1.loc[df1['有无入室花园'].notnull(),'有无入室花园']=list4
df1.loc[df1['窗户数量'].notnull(),'窗户数量']=list5
df1.loc[df1['实际使用面积'].notnull(),'实际使用面积']=list6
df1.loc[df1['平均卧室面积'].notnull(),'平均卧室面积']=list7
df1.loc[df1['卧室面积占比'].notnull(),'卧室面积占比']=list8
df1.loc[df1['客厅采光程度'].notnull(),'客厅采光程度']=list9
df1.loc[df1['主卧采光程度'].notnull(),'主卧采光程度']=list10
df1.loc[df1['阳台采光程度'].notnull(),'阳台采光程度']=list11
df1=df1.drop(['户型分间'],axis=1)

##重要性不大，且难处理
df1=df1.drop(['小区建筑类型'],axis=1)
df1=df1.drop(['房屋朝向'],axis=1)

#户型结构、建筑类型、所在楼层（为无序特征）：使用onehot编码
df1['户型结构']=df1['户型结构'].replace("暂无数据",np.NAN)
df1 = pd.concat([df1, pd.get_dummies(df1.loc[:,['户型结构']])], sort=False, axis=1)
df1=df1.drop('户型结构',axis=1)
df1['建筑类型']=df1['建筑类型'].replace("暂无数据",np.NAN)
df1 = pd.concat([df1, pd.get_dummies(df1.loc[:,['建筑类型']])], sort=False, axis=1)
df1=df1.drop('建筑类型',axis=1)

#所在楼层
df1['所在楼层']=df1['所在楼层'].astype(str)
def rep(x):
    pattern = re.compile(r'\s\((.*)\)')
    return pattern.sub(r'',x)
df1['所在楼层']=df1['所在楼层'].apply(rep)
df1['所在楼层']=df1['所在楼层'].replace("nan",np.NAN)
df1 = pd.concat([df1, pd.get_dummies(df1.loc[:,['所在楼层']])], sort=False, axis=1)
df1=df1.drop('所在楼层',axis=1)

#建筑结构、装修情况：定义等级后编码
df1['建筑结构']=df1['建筑结构'].replace("未知结构",np.NAN)
jiegou_mapping = {
           '钢结构': 6,
           '钢混结构':5,
           '框架结构':4,
           '混合结构':3,
           '砖混结构':2,
           '砖木结构':1}
df1['建筑结构'] = df1['建筑结构'].map(jiegou_mapping)

df1['装修情况']=df1['装修情况'].replace("其他",np.NAN)
zhuangxiu_mapping={
    '精装':3,
    '简装':2,
    '毛坯':1
}
df1['装修情况'] = df1['装修情况'].map(zhuangxiu_mapping)
#电梯二值化处理
df1['配备电梯']=df1['配备电梯'].replace("暂无数据",1)
df1['配备电梯']=df1['配备电梯'].replace("NULL",1)
df1['配备电梯']=df1['配备电梯'].replace("有",1)
df1['配备电梯']=df1['配备电梯'].replace("无",0)

#所在广州区域
list_sorted=['天河','越秀','海珠','荔湾','白云','番禺','黄埔','南沙','顺德','增城','花都','从化','南海']
mapping={'天河':12,'越秀':11,'海珠':10,'荔湾':9,'白云':8,'番禺':7,'黄埔':6,'南沙':5,'顺德':4,'增城':3,'花都':2,'从化':1,'南海':0}
df1['所在区域']=df1['所在区域'].map(mapping)

'''
(2)异常值检测及处理
有以下处理
1.特征为负数的都为异常值：小区建成距今时长、挂牌时间-上次交易时间（负数的为异常值，当作缺失值）
2.小区物业费用（小区物业费用过高或过低的为异常值，当作缺失值）
3.实际使用面积应该比建筑面积小，因此将实际面积大于建筑面积的部分视为异常值
'''
# 特征为负数的都为异常值，作为缺失值
def delete_negative(df1,label):
    if(len(df1.loc[df1[label]<0,label])>0):
        print('特征{0}的异常值数目为:{1}'.format(label,len(df1.loc[df1[label]<0,label])))
    df1.loc[df1[label]<0,label]=np.nan
for i in df1.keys():
    try:
        delete_negative(df1,i)
    except:
        continue

# 小区物业费用处理
df1['小区物业费用'] = df1['小区物业费用'].astype('float')
lower_q=df1['小区物业费用'].quantile(0.25,interpolation='lower')#下四分位数
higher_q=df1['小区物业费用'].quantile(0.75,interpolation='higher')#上四分位数
IQR=higher_q-lower_q
#取下四分位数-3*IQR为下界，取上四分位数+3*IQR为上界
Lower_fence = lower_q - (IQR * 3)
Upper_fence = higher_q + (IQR * 3)
# 小区物业费用（过高过低当作缺失值）
df1.loc[df1['小区物业费用']<Lower_fence,'小区物业费用']=np.nan
df1.loc[df1['小区物业费用']>Upper_fence,'小区物业费用']=np.nan
df1.loc[df1['实际使用面积']>df1['建筑面积'],'实际使用面积']=np.nan

'''
(4)缺失值处理
(1,2,3根据一定规则进行填充)
1.套内面积与公摊面积
2.配备电梯
3.采光程度
4.其余采用条件均值或条件众值进行填充
'''
total = df1.isnull().sum().sort_values(ascending=False)
percent = (df1.isnull().sum()/df1.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total','Percent'])

'''
相关数据填充
1.套内面积与公摊面积
建筑面积是无缺失的，套内面积缺失较多，导致公摊面积也缺失较多，三者之间有联系。 得
房率=套内面积/建筑面积。
一般来说，地下室(0) 60%，低层住宅（1-3）92%，多层住宅（4-9）得房率88%，高层住宅（>=10）80%
根据得房率×建筑面积来填充套内面积。再用建筑面积-套内面积得到缺失的公摊面积。
'''
# 套内面积、公摊面积
df1[['套内面积','建筑面积','公摊面积','总层数']]=df1[['套内面积','建筑面积','公摊面积','总层数']].astype('float')
condition= (df1['套内面积'].isnull()) & (df1['总层数']==0)
condition1= (df1['套内面积'].isnull()) & (df1['总层数']<4) & (df1['总层数']>0)
condition2=(df1['套内面积'].isnull()) & (df1['总层数']<10) & (df1['总层数']>3)
condition3=(df1['套内面积'].isnull()) & (df1['总层数']>9)
df1.loc[condition,'套内面积']=df1.loc[condition,'建筑面积']*0.6
df1.loc[condition1,'套内面积']=df1.loc[condition1,'建筑面积']*0.92
df1.loc[condition2,'套内面积']=df1.loc[condition2,'建筑面积']*0.88
df1.loc[condition3,'套内面积']=df1.loc[condition3,'建筑面积']*0.80
df1.loc[df1['公摊面积'].isnull(),'公摊面积']=df1.loc[df1['公摊面积'].isnull(),'建筑面积']-df1.loc[df1['公摊面积'].isnull(),'套内面积']

'''
2.电梯
配备电梯根据总层数来判断有无电梯，楼层大于6的视为有电梯，小于等于6的视为没有电梯。
'''
# 配备电梯
condition4=(df1['配备电梯'].isnull()) & (df1['总层数']<7)
condition5=(df1['配备电梯'].isnull()) & (df1['总层数']>6)
df1.loc[condition4,'配备电梯']=0
df1.loc[condition5,'配备电梯']=1

'''
3.采光程度
三个关于采光程度的特征，以中等程度3进行填充
'''
df1['客厅采光程度']=df1['客厅采光程度'].replace(np.NAN,3)
df1['主卧采光程度']=df1['主卧采光程度'].replace(np.NAN,3)
df1['阳台采光程度']=df1['阳台采光程度'].replace(np.NAN,3)

'''
4.其他缺失值填充
'''
missing_labels=list(df1.keys())
#离散型特征，使用条件众数填充，以小区ID为条件
df2 = df1[['所在区域', '小区ID', '建楼距今时长', '小区楼栋总数', '小区建成距今时长',
           '建筑结构', '装修情况', '配备电梯', '交易权属', '房屋用途','产权所属', '抵押信息',
           '房本备件', '挂牌时间-上次交易时间','建楼时间-小区建筑年代', '卧室数量','客厅数量',
           '厨房数量', '卫生间数量', '总层数', '有无储物间', '有无入室花园', '窗户数量',
           '落地窗数量', '客厅采光程度', '主卧采光程度', '阳台采光程度', '户型结构_复式', '户型结构_平层',
           '户型结构_跃层', '户型结构_错层', '建筑类型_塔楼', '建筑类型_平房', '建筑类型_板塔结合', '建筑类型_板楼', '所在楼层_中楼层',
           '所在楼层_低楼层', '所在楼层_地下室', '所在楼层_高楼层', '教育设施数量', '交通设施数', '购物设施数',
           '生活设施数', '娱乐设施数', '医疗设施数']]
count_data =df2.groupby('小区ID').agg(lambda x: stats.mode(x)[0][0]).reset_index()
missing_labels2=['所在区域', '小区ID', '建楼距今时长', '小区楼栋总数', '小区建成距今时长',
           '建筑结构', '装修情况', '配备电梯', '交易权属', '房屋用途','产权所属', '抵押信息',
           '房本备件', '挂牌时间-上次交易时间','建楼时间-小区建筑年代', '卧室数量','客厅数量',
           '厨房数量', '卫生间数量', '总层数', '有无储物间', '有无入室花园', '窗户数量',
           '落地窗数量', '客厅采光程度', '主卧采光程度', '阳台采光程度', '户型结构_复式', '户型结构_平层',
           '户型结构_跃层', '户型结构_错层', '建筑类型_塔楼', '建筑类型_平房', '建筑类型_板塔结合', '建筑类型_板楼', '所在楼层_中楼层',
           '所在楼层_低楼层', '所在楼层_地下室', '所在楼层_高楼层', '教育设施数量', '交通设施数', '购物设施数',
           '生活设施数', '娱乐设施数', '医疗设施数']

for i in missing_labels2:
    for j in df1.loc[df1[i].isnull(),:].index:
        df1.loc[j,i]=count_data.loc[count_data['小区ID']==df1.loc[j,'小区ID'],i].values[0]

#未填完的用全部众数填充
for i in missing_labels2:
    df1[i].fillna(df1[i].mode()[0],inplace=True)

for i in df1.columns:
    try:
        df1[i]=df1[i].astype('float')
    except:
        continue

#连续特征使用条件均值，使用同个小区的均值填充
mean_data = df1.groupby('小区ID').mean()
for i in missing_labels:
    for j in df1.loc[df1[i].isnull(),:].index:
        df1.loc[j,i]=mean_data.loc[df1.loc[j].小区ID,i]

# 未填充完的特征
pretotal=total
total2 = df1.isnull().sum().sort_values(ascending=False)
percent = (df1.isnull().sum()/df1.isnull().count()).sort_values(
        ascending=False)
missing_data = pd.concat([total2, pretotal], axis=1, keys=['Total','PreTotal'])
# 剩余特征
missing_labels=missing_data[missing_data['Total']!=0].index

#剩余特征进一步使用条件均值填充，条件为所在广州区域
mean_data2= df1.groupby('所在区域').mean()
for i in missing_labels:
    for j in df1.loc[df1[i].isnull(),:].index:
        df1.loc[j,i]=mean_data2.loc[df1.loc[j].所在区域,i]

#剩余填充特征
pretotal=total2
total3 = df1.isnull().sum().sort_values(ascending=False)
percent = (df1.isnull().sum()/df1.isnull().count()).sort_values(
        ascending=False)
missing_data = pd.concat([total3, pretotal], axis=1, keys=['Total','PreTotal'])
missing_labels=missing_data[missing_data['Total']!=0].index

# 剩余特征填完，直接用整列均值
for i in missing_labels:
    df1[i].fillna(df1[i].mean(),inplace=True)

#剩余填充特征
pretotal=total2
total3 = df1.isnull().sum().sort_values(ascending=False)
percent = (df1.isnull().sum()/df1.isnull().count()).sort_values(
        ascending=False)
missing_data = pd.concat([total3, pretotal], axis=1, keys=['Total','PreTotal'])
missing_labels=missing_data[missing_data['Total']!=0].index

df1=df1.drop(columns = ['小区ID'])

'''
(4)特征变换
'''
#对房屋总价正态变换，后面可以用inverse_transform反转
df1_1 = df1.copy()
pt = PowerTransformer('box-cox').fit(df1[['房屋总价']])
df1['房屋总价'] = pt.transform(df1[['房屋总价']])

#对房屋每平米价正态变换，后面可以用inverse_transform反转
pt = PowerTransformer('box-cox').fit(df1[['房屋每平米价']])
df1['房屋每平米价'] = pt.transform(df1[['房屋每平米价']])

#连续特征表
lianxu = ['小区物业费用', '建筑面积', '套内面积', '梯户比例','公摊面积',
          '客厅面积占比','卧室面积占比', '平均卧室面积', '实际使用面积',
          '教育设施平均距离', '交通设施平均距离', '购物设施平均距离',
          '生活设施平均距离', '娱乐设施平均距离', '医疗设施平均距离']

skewness = df1[lianxu].apply(lambda x: skew(x))
skewness.sort_values(ascending=False)
skewness = skewness[abs(skewness)>0.75]
skewness = list(skewness.index)
skewness.append('房屋总价')
skewness.append('房屋每平米价')

df_tran = df1_1.loc[:,skewness]
df_tran.to_excel('连续特征转化前数值.xlsx', index=None)

#将偏度大于0.75的变量进行正态化
for i in skewness:
    pt = PowerTransformer().fit(df1[[i]])
    df1[i] = pt.transform(df1[[i]])

'''
(5)特征选择
'''
def spearman(frame, features):
    spr = pd.DataFrame()
    spr['feature'] = features
    spr['corr'] = [frame[f].corr(frame['房屋总价'], 'spearman') for f in features]
    select_dict=dict(zip(spr['feature'],spr['corr']))
    select_dict={k:v for k,v in select_dict.items() if abs(v)>0.1}
    spr = spr.sort_values('corr')
    return select_dict
df1.keys()
df2=df1.drop('房屋每平米价',1)
# 筛选出相关系数绝对值大于0.1的特征
select_dict=spearman(df2, df2.columns)
select_dict.pop('房屋总价')
select_list=list(select_dict.keys())
select_data = df1[select_list]
select_labels=[]
corrmat = select_data.corr()
for i in select_list:
    for j in select_list:
        if abs(corrmat.loc[i,j])==1:
            continue
        if 0.8<abs(corrmat.loc[i,j]):
            select_labels.append(i)
            select_labels.append(j)
select_labels = list(set(select_labels))
max_score = 0
max_label = None
for i in select_labels:
    if select_dict[i]>max_score:
        max_score = select_dict[i]
        max_label = i
select_labels.remove(max_label)
select_list = list(set(select_dict.keys()) - set(select_labels))

select_list.append('房屋总价')
select_list.append('房屋每平米价')
if '所在区域' not in select_list:
    select_list.append('所在区域')

df_final = df1[select_list]
mid = df_final['所在区域']
df_final.drop(labels=['所在区域'], axis=1,inplace = True)
df_final.insert(0, '所在区域', mid)

with open('model1Name.txt','w', encoding='gbk') as file:
    for c in list(df_final.keys()):
        if (c == '房屋总价') or (c=='房屋每平米价'):
            continue
        file.write(c)
        file.write(',')

df_final['当前时间-挂牌时间'] = df1['当前时间-挂牌时间']
# df_j01 = df_final.loc[df_final['当前时间-挂牌时间']<365,:]
# df_j01.drop(labels=['当前时间-挂牌时间'], axis=1,inplace = True)
# df_j01.to_excel('建模数据1(一年内).xlsx',index=None)
#
# df_j02 = df_final.loc[df_final['当前时间-挂牌时间']<210,:]
# df_j02.drop(labels=['当前时间-挂牌时间'], axis=1,inplace = True)
# df_j02.to_excel('建模数据2(七个月内).xlsx',index=None)
#
# df_j03 = df_final.loc[df_final['当前时间-挂牌时间']<100,:]
# df_j03.drop(labels=['当前时间-挂牌时间'], axis=1,inplace = True)
# df_j03.to_excel('建模数据3(三个月内).xlsx',index=None)

df_final.drop(labels=['当前时间-挂牌时间'], axis=1,inplace = True)
df_final.to_excel('建模数据(全部数据).xlsx', index=None)


<reponame>prise-3d/SIN3D
# main imports
import os, sys
import argparse
import json
import numpy as np
import shutil

# PNG images
from PIL import Image

# others import
from ipfml import utils
from scipy.signal import savgol_filter

'''
Display progress information as progress bar
'''
def write_progress(progress):
    barWidth = 180

    output_str = "["
    pos = barWidth * progress
    for i in range(barWidth):
        if i < pos:
           output_str = output_str + "="
        elif i == pos:
           output_str = output_str + ">"
        else:
            output_str = output_str + " "

    output_str = output_str + "] " + str(int(progress * 100.0)) + " %\r"
    print(output_str)
    sys.stdout.write("\033[F")

def extract_index(filepath):

    return int(filepath.split('_')[-1].split('.')[0])

def extracts_linear_indices(images_path, n_expected=50, indices_step=20, start_at=20, smooth_arr=False):

    # TODO : check this part
    default_add = start_at - indices_step

    # extract variance for each image path
    var_arr = []

    n_counter = 0
    n_images = len(images_path)

    for p in sorted(images_path):
        img = Image.open(p)
        var_arr.append(np.var(img))

        n_counter += 1
        write_progress((n_counter + 1) / n_images)

    # normalize variance values
    norm_arr = np.array(utils.normalize_arr_with_range(var_arr))

    if smooth_arr:
        norm_arr = utils.normalize_arr_with_range(savgol_filter(norm_arr, 201, 3)) # window size 7, polynomial order 3

    # get expected linear step (using n_expectec output images)
    linear_steps = utils.normalize_arr_with_range((1 - (np.arange(n_expected) / n_expected)))

    # get image indices from variance convergence and linear
    # => when linear step is reached we store the index found from variance values
    indices_found = []
    for i in linear_steps:

        find_index = 0

        for index, y in enumerate(norm_arr):
            if i <= y:
                find_index = index

        indices_found.append(find_index + 1)

    indices = np.array(indices_found) * indices_step

    # add tricks to avoid same indice
    # => when index is same as previous, then add number of samples expected by step
    # Example with step of 20 : [20, 20, 20, 100, 200] => [20, 40, 60, 100, 200]
    final_indices = []
    for index, i in enumerate(indices):
        value = indices[index]
        if index > 0:
            if i <= indices[index - 1]:
                value = indices[index - 1] + indices_step
                indices[index] = value

        final_indices.append(value)

    return np.array(final_indices) + default_add

def main():
    """
    main function which is ran when launching script
    """
    parser = argparse.ArgumentParser(description="Compute new dataset scene")

    parser.add_argument('--file', type=str, help='file data extracted from `utils/extract_stats_freq_and_min.py` script', required=True)
    parser.add_argument('--png_folder', type=str, help='png dataset folder with scene', required=True)
    parser.add_argument('--users', type=int, help='min number of users required per scene', required=True, default=10)
    #parser.add_argument('--samples', type=int, help='expected samples to get for this dataset', required=True, default=10000)
    parser.add_argument('--output', type=str, help='output image folder', required=True)

    args = parser.parse_args()

    p_file   = args.file
    p_png_folder = args.png_folder
    p_users  = args.users
    #p_samples = args.samples
    p_output = args.output

    with open(p_file, 'r') as f:

        for line in f.readlines():

            data = line.split(';')

            scene = data[0]
            n_users = int(data[1])
            min_index = int(data[2])

            # remove _partX from scene name
            scene_parts = scene.split('_')
            del scene_parts[-1]
            scene_name = '_'.join(scene_parts)

            output_scene_dir = os.path.join(p_output, scene)

            if os.path.exists(output_scene_dir):
                print('Extraction of custom indices already done for', scene)
                continue

            if n_users >= p_users:
                print('Extract custom indices based on minimum index for', scene)

                png_folder_scene = os.path.join(p_png_folder, scene)

                if not os.path.exists(png_folder_scene):
                    print(png_folder_scene, 'png folder does not exist')
                else:

                    # get all rawls files
                    png_files = [ os.path.join(png_folder_scene, p) for p in sorted(os.listdir(png_folder_scene)) ]

                    # extract max samples found for this scene
                    _, filename = os.path.split(png_files[-1])

                    max_samples = extract_index(filename)

                    # extract step from these files
                    input_step = int(max_samples / len(png_files))

                    # get indices using min index
                    indices = extracts_linear_indices(png_files[int(min_index / input_step):], n_expected=50, indices_step=input_step, start_at=min_index, smooth_arr=True)

                    # print('Indices found are', indices)
                    # create output directory
                    if not os.path.exists(output_scene_dir):
                        os.makedirs(output_scene_dir)

                    # get expected png image and move it
                    for index in indices:

                        str_index = str(index)

                        while len(str_index) < 5:
                            str_index = "0" + str_index

                        image_name = scene_name + '_' + str_index + '.png'
                        png_image_path = os.path.join(png_folder_scene, image_name)

                        # create output filepath
                        output_img_filepath = os.path.join(output_scene_dir, image_name)

                        # copy expected image path
                        shutil.copy2(png_image_path, output_img_filepath)
            else:
                print('Only', n_users, 'users who passed the experiment for', scene)

            print('\n---------------------------------------------')

if __name__ == "__main__":
    main()


import pdb

import numpy as np

from scipy.optimize import (
    check_grad,
    fmin_cg,
    fmin_ncg,
    fmin_bfgs,
)

from sklearn.base import (
    BaseEstimator,
    TransformerMixin,
)

from sklearn.preprocessing import (
    StandardScaler,
)

def square_dist(x1, x2=None):
    """If x1 is NxD and x2 is MxD (default x1), return NxM square distances."""

    if x2 is None:
        x2 = x1

    return (
        np.sum(x1 * x1, 1)[:, np.newaxis] +
        np.sum(x2 * x2, 1)[np.newaxis, :] -
        np.dot(x1, (2 * x2.T))
    )

def nca_cost(A, xx, yy, reg):
    """Neighbourhood Components Analysis: cost function and gradients

        ff, gg = nca_cost(A, xx, yy)

    Evaluate a linear projection from a D-dim space to a K-dim space (K<=D).
    See Goldberger et al. (2004).

    Inputs:
        A  KxD Current linear transformation.
        xx NxD Input data
        yy Nx1 Corresponding labels, taken from any discrete set

    Outputs:
        ff 1x1 NCA cost function
        gg KxD partial derivatives of ff wrt elements of A

    Motivation: gradients in existing implementations, and as written in the
    paper, have the wrong scaling with D. This implementation should scale
    correctly for problems with many input dimensions.

    Note: this function should be passed to a MINIMIZER.

    """

    N, D = xx.shape
    assert(yy.size == N)
    assert(A.shape[1] == D)
    K = A.shape[0]

    # Cost function:
    zz = np.dot(A, xx.T)  # KxN

    # TODO Subsample part of data to compute loss on.
    # kk = np.exp(-square_dist(zz.T, zz.T[idxs]))  # Nxn
    # kk[idxs, np.arange(len(idxs))] = 0

    ss = square_dist(zz.T)
    np.fill_diagonal(ss, np.inf)
    mm = np.min(ss, axis=0)
    kk = np.exp(mm - ss)  # NxN
    np.fill_diagonal(kk, 0)
    Z_p = np.sum(kk, 0)  # N,
    p_mn = kk / Z_p[np.newaxis, :]  # P(z_m | z_n), NxN
    mask = yy[:, np.newaxis] == yy[np.newaxis, :]
    p_n = np.sum(p_mn * mask, 0)  # 1xN
    ff = - np.sum(p_n)

    # Back-propagate derivatives:
    kk_bar = - (mask - p_n[np.newaxis, :]) / Z_p[np.newaxis, :]  # NxN
    ee_bar = kk * kk_bar
    zz_bar_part = ee_bar + ee_bar.T
    zz_bar = 2 * (np.dot(zz, zz_bar_part) - (zz * np.sum(zz_bar_part, 0)))  # KxN
    gg = np.dot(zz_bar, xx)  # O(DKN)

    if reg > 0:
        ff = ff + reg * np.dot(A.ravel(), A.ravel())
        gg = gg + 2 * reg * A

    return ff, gg

def nca_cost_batch(self, A, xx, yy, idxs):

    N, D = xx.shape
    n = len(idxs)

    assert(yy.size == N)
    assert(A.shape[1] == D)

    K = A.shape[0]

    # Cost function:
    zz = np.dot(A, xx.T)  # KxN
    Z_p = np.sum(kk, 0)  # N,
    p_mn = kk / Z_p[np.newaxis, :]  # P(z_m | z_n), NxN
    mask = yy[:, np.newaxis] == yy[np.newaxis, :]
    p_n = np.sum(p_mn * mask, 0)  # 1xN
    ff = - np.sum(p_n)

    # Back-propagate derivatives:
    kk_bar = - (mask - p_n[np.newaxis, :]) / Z_p[np.newaxis, :]  # NxN
    zz_bar_part = kk * (kk_bar + kk_bar.T)
    zz_bar = 2 * (np.dot(zz, zz_bar_part) - (zz * sum(zz_bar_part, 0)))  # KxN
    gg = np.dot(zz_bar, xx)  # O(DKN)

    return ff, gg

class NCA(BaseEstimator, TransformerMixin):
    def __init__(self, reg=0, dim=None, optimizer='cg'):
        self.reg = reg
        self.K = dim
        self.standard_scaler = StandardScaler()

        if optimizer in ('cg', 'conjugate_gradients'):
            self._fit = self._fit_conjugate_gradients
        elif optimizer in ('gd', 'gradient_descent'):
            self._fit = self._fit_gradient_descent
        elif optimizer in ('mb', 'mini_batches'):
            self._fit = self._fit_mini_batches
        else:
            raise ValueError("Unknown optimizer {:s}".format(optimizer))

    def fit(self, X, y):

        N, D = X.shape

        if self.K is None:
            self.K = D

        self.A = np.random.randn(self.K, D) / np.sqrt(N)

        X = self.standard_scaler.fit_transform(X)
        return self._fit(X, y)

    def _fit_gradient_descent(self, X, y):
        # Gradient descent.
        self.learning_rate = 0.001
        self.error_tol = 0.001
        self.max_iter = 1000

        curr_error = None

        # print(check_grad(costf, costg, 0.1 * np.random.randn(self.K * D)))
        # idxs = list(sorted(random.sample(range(len(X)), 100)))

        for it in range(self.max_iter):

            f, g = nca_cost(self.A, X, y, self.reg)
            self.A -= self.learning_rate * g

            prev_error = curr_error
            curr_error = f

            print('{:4d} {:+.6f}'.format(it, curr_error))

            if prev_error and np.abs(curr_error - prev_error) < self.error_tol:
                break

        return self

    def _fit_conjugate_gradients(self, X, y):
        N, D = X.shape

        def costf(A):
            f, _ = nca_cost(A.reshape([self.K, D]), X, y, self.reg)
            return f

        def costg(A):
            _, g = nca_cost(A.reshape([self.K, D]), X, y, self.reg)
            return g.ravel()

        # print(check_grad(costf, costg, 0.1 * np.random.randn(self.K * D)))
        self.A = fmin_cg(costf, self.A.ravel(), costg, maxiter=400)
        self.A = self.A.reshape([self.K, D])
        return self

    def fit_transform(self, X, y):
        self.fit(X, y)
        return self.transform(X)

    def transform(self, X):
        return np.dot(self.standard_scaler.transform(X), self.A.T)


#!/usr/bin/env python

import numpy as np
import scipy.optimize

# https://pypi.python.org/pypi/transforms3d
import transforms3d

def make_transform(tx, ty, tz, rotz):
    ''' Creates a 4x4 rigid transform matrix with
    translation: tx, ty, tz
    rotation: rotz radians around z axis
    '''
    rot = transforms3d.axangles.axangle2mat([0, 0, 1], rotz)
    return transforms3d.affines.compose([tx, ty, tz], rot, [1, 1, 1])

def transform_points(points, tf):
    ''' Input matrix of N points (one per column) 3xN
    Outputs points in the same format '''
    points_h = np.vstack((points, np.ones((1, points.shape[1]))))
    tpoints = np.matmul(tf, points_h)
    return tpoints[0:3, :] / tpoints[3, :]

def angle_between(v1, v2):
    ''' Returns the angle in radians between vectors 'v1' and 'v2'::

            >>> angle_between((1, 0, 0), (0, 1, 0))
            1.5707963267948966
            >>> angle_between((1, 0, 0), (1, 0, 0))
            0.0
            >>> angle_between((1, 0, 0), (-1, 0, 0))
            3.141592653589793
    '''
    v1_u = transforms3d.utils.normalized_vector(v1)
    v2_u = transforms3d.utils.normalized_vector(v2)

    return np.arccos(np.clip(np.dot(v1_u, v2_u), -1.0, 1.0))

def angle_between_vectorized(v1, v2):
    ''' Vectorized version of angle_between: v1 and v2 are 2D matrices, one vector per row.
    Returns 1d array, one element per row
    '''
    v1_u = v1 / np.linalg.norm(v1, axis=1, keepdims=True)
    v2_u = v2 / np.linalg.norm(v2, axis=1, keepdims=True)
    dot = np.einsum("ij,ij->i", v1_u, v2_u) # dot product for each row
    return np.arccos(np.clip(dot, -1.0, 1.0))

def error_func(p, qc, qv, tx, ty, tz, rotz):
    ''' Transform points p using tx, ty, tz, rotz.
    For each transformed point tp, compute the angle between:
    - the direction joining qc and tp
    - the direction qv '''
    tf = make_transform(tx, ty, tz, rotz)
    tp = transform_points(p, tf)
    return list(angle_between_vectorized(qv.T, (tp - qc).T))

def estimate_pose(p, qc, qv, x0):
    ''' Given points in robot frame (p) and rays in human frame (qc, qv), find
    transformation parameters from human frame to robot frame that minimize the
    residual, using starting x0 as the initial solution '''

    def f(x):
        return np.mean(error_func(p, qc, qv, *x)) + max(0.0, np.linalg.norm(x[:3]) - 7.0)# + max(0.0, np.abs(x[2]) - 1.0)

    res = scipy.optimize.minimize(f, x0)
    maxerr = np.max(error_func(p, qc, qv, *res.x)) + max(0.0, np.linalg.norm(res.x[:3]) - 7.0)

    return res, maxerr

def estimate_pose_no_constraints(p, qc, qv, x0):
    ''' Given points in robot frame (p) and rays in human frame (qc, qv), find
    transformation parameters from human frame to robot frame that minimize the
    residual, using starting x0 as the initial solution '''

    def f(x):
        return np.mean(error_func(p, qc, qv, *x))

    res = scipy.optimize.minimize(f, x0)

    maxerr = np.max(error_func(p, qc, qv, *res.x))

    return res, maxerr


#! -*- coding: utf-8 -*-
# 用Seq2Seq做小学数学应用题
# 数据集为ape210k：https://github.com/Chenny0808/ape210k
# 介绍链接：https://kexue.fm/archives/7809

from __future__ import division
import json, re
from tqdm import tqdm
from bert4torch.models import build_transformer_model
from bert4torch.tokenizers import Tokenizer, load_vocab
from torch import nn, optim
import torch
from torch.utils.data import DataLoader
from bert4torch.snippets import sequence_padding, Callback, ListDataset
from bert4torch.snippets import AutoRegressiveDecoder
from sympy import Integer
import warnings
warnings.filterwarnings("ignore")

# 基本参数
maxlen = 192
batch_size = 16
epochs = 100

# bert配置
config_path = 'F:/Projects/pretrain_ckpt/bert/[hit_torch_base]--chinese-bert-wwm-ext/config.json'
checkpoint_path = 'F:/Projects/pretrain_ckpt/bert/[hit_torch_base]--chinese-bert-wwm-ext/pytorch_model.bin'
dict_path = 'F:/Projects/pretrain_ckpt/bert/[hit_torch_base]--chinese-bert-wwm-ext/vocab.txt'
device = 'cuda' if torch.cuda.is_available() else 'cpu'

# 加载并精简词表，建立分词器
token_dict, keep_tokens = load_vocab(
    dict_path=dict_path,
    simplified=True,
    startswith=['[PAD]', '[UNK]', '[CLS]', '[SEP]'],
)
tokenizer = Tokenizer(token_dict, do_lower_case=True)

def is_equal(a, b):
    """比较两个结果是否相等
    """
    a = round(float(a), 6)
    b = round(float(b), 6)
    return a == b

def remove_bucket(equation):
    """去掉冗余的括号
    """
    l_buckets, buckets = [], []
    for i, c in enumerate(equation):
        if c == '(':
            l_buckets.append(i)
        elif c == ')':
            buckets.append((l_buckets.pop(), i))
    eval_equation = eval(equation)
    for l, r in buckets:
        new_equation = '%s %s %s' % (equation[:l], equation[l + 1:r], equation[r + 1:])
        try:
            if is_equal(eval(new_equation.replace(' ', '')), eval_equation):
                equation = new_equation
        except:
            pass
    return equation.replace(' ', '')

# 加载数据集
class MyDataset(ListDataset):
    @staticmethod
    def load_data(filename):
        """读取训练数据，并做一些标准化，保证equation是可以eval的
        参考：https://kexue.fm/archives/7809
        """
        D = []
        for l in open(filename, 'r', encoding='utf-8'):
            l = json.loads(l)
            question, equation, answer = l['original_text'], l['equation'], l['ans']
            # 处理带分数
            question = re.sub('(\d+)\((\d+/\d+)\)', '(\\1+\\2)', question)
            equation = re.sub('(\d+)\((\d+/\d+)\)', '(\\1+\\2)', equation)
            answer = re.sub('(\d+)\((\d+/\d+)\)', '(\\1+\\2)', answer)
            equation = re.sub('(\d+)\(', '\\1+(', equation)
            answer = re.sub('(\d+)\(', '\\1+(', answer)
            # 分数去括号
            question = re.sub('\((\d+/\d+)\)', '\\1', question)
            # 处理百分数
            equation = re.sub('([\.\d]+)%', '(\\1/100)', equation)
            answer = re.sub('([\.\d]+)%', '(\\1/100)', answer)
            # 冒号转除号、剩余百分号处理
            equation = equation.replace(':', '/').replace('%', '/100')
            answer = answer.replace(':', '/').replace('%', '/100')
            if equation[:2] == 'x=':
                equation = equation[2:]
            try:
                if is_equal(eval(equation), eval(answer)):
                    D.append((question, remove_bucket(equation), answer))
            except:
                continue
        return D

def collate_fn(batch):
    batch_token_ids, batch_segment_ids = [], []
    for question, equation, answer in batch:
        token_ids, segment_ids = tokenizer.encode(question, equation, maxlen=maxlen)
        batch_token_ids.append(token_ids)
        batch_segment_ids.append(segment_ids)
    batch_token_ids = torch.tensor(sequence_padding(batch_token_ids), dtype=torch.long, device=device)
    batch_segment_ids = torch.tensor(sequence_padding(batch_segment_ids), dtype=torch.long, device=device)
    return [batch_token_ids, batch_segment_ids], [batch_token_ids, batch_segment_ids]

# 加载数据集
train_dataloader = DataLoader(MyDataset('F:/Projects/data/corpus/seq2seq/ape210k/train.ape.json'), batch_size=batch_size, shuffle=True, collate_fn=collate_fn)
valid_dataset = MyDataset('F:/Projects/data/corpus/seq2seq/ape210k/valid.ape.json')
# valid_dataloader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)
# test_dataloader = DataLoader(MyDataset('F:/Projects/data/corpus/seq2seq/ape210k/test.ape.json'), batch_size=batch_size, shuffle=True, collate_fn=collate_fn)

model = build_transformer_model(
    config_path,
    checkpoint_path,
    with_mlm='linear',
    application='unilm',
    keep_tokens=keep_tokens,  # 只保留keep_tokens中的字，精简原字表
).to(device)

class CrossEntropyLoss(nn.CrossEntropyLoss):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
    def forward(self, outputs, target):
        '''
        y_pred: [btz, seq_len, vocab_size]
        targets: y_true, y_segment
        unilm式样，需要手动把非seq2seq部分mask掉
        '''
        _, y_pred = outputs
        y_true, y_mask = target
        y_true = y_true[:, 1:]# 目标token_ids
        y_mask = y_mask[:, 1:]  # segment_ids，刚好指示了要预测的部分
        y_pred = y_pred[:, :-1, :]  # 预测序列，错开一位

        y_pred = y_pred.reshape(-1, y_pred.shape[-1])
        y_true = (y_true*y_mask).flatten()
        return super().forward(y_pred, y_true)
model.compile(loss=CrossEntropyLoss(ignore_index=0), optimizer=optim.Adam(model.parameters(), 1e-5))

class AutoSolve(AutoRegressiveDecoder):
    """seq2seq解码器
    """
    @AutoRegressiveDecoder.wraps(default_rtype='logits')
    def predict(self, inputs, output_ids, states):
        token_ids, segment_ids = inputs
        token_ids = torch.cat([token_ids, output_ids], 1)
        segment_ids = torch.cat([segment_ids, torch.ones_like(output_ids, device=device)], 1)
        _, y_pred = model.predict([token_ids, segment_ids])
        return y_pred[:, -1, :]

    def generate(self, text, topk=1):
        token_ids, segment_ids = tokenizer.encode(text, maxlen=maxlen)
        output_ids = self.beam_search([token_ids, segment_ids], topk=topk)  # 基于beam search
        return tokenizer.decode(output_ids.cpu().numpy()).replace(' ', '')

autosolve = AutoSolve(start_id=None, end_id=tokenizer._token_end_id, maxlen=64, device=device)

class Evaluator(Callback):
    """评估与保存
    """
    def __init__(self):
        self.best_acc = 0.

    def on_epoch_end(self, steps, epoch, logs=None):
        metrics = self.evaluate(valid_dataset.data[:200])  # 评测模型
        if metrics['acc'] >= self.best_acc:
            self.best_acc = metrics['acc']
            # model.save_weights('./best_model_math.pt')  # 保存模型
        metrics['best_acc'] = self.best_acc
        print('valid_data:', metrics)
        print()

    def evaluate(self, data, topk=1):
        total, right = 0.0, 0.0
        for question, equation, answer in tqdm(data, desc='Evaluate'):
            total += 1
            pred_equation = autosolve.generate(question, topk)
            try:
                right += int(is_equal(eval(pred_equation), eval(answer)))
            except:
                pass
        return {'acc': right / total}

if __name__ == '__main__':
    evaluator = Evaluator()
    model.fit(train_dataloader, steps_per_epoch=500, epochs=epochs, callbacks=[evaluator])
else:
    model.load_weights('./best_model.weights')


##
# \brief Test copula rotations
from __future__ import print_function, division
from starvine.bvcopula.pc_base import PairCopula
from starvine.bvcopula.copula_factory import Copula
import unittest
import numpy as np
import seaborn as sns
from scipy.stats import kendalltau
from six import iteritems
import os
pwd_ = os.getcwd()
dataDir = pwd_ + "/tests/data/"
np.random.seed(123)

class TestRotateGauss(unittest.TestCase):
    def testGaussRotate(self):
        np.random.seed(123)
        shapes = {0: 0.7777777, 1: -0.7777777, 2: 0.7777777, 3: -0.7777777}
        family = {'gauss': 0}
        for rotation, shapeParam in iteritems(shapes):
            gauss = Copula("gauss", 0)
            u, v = gauss.sample(10000, *(shapeParam,))
            g = sns.jointplot(u, v)
            g.savefig("gauss_sample_pdf_" + str(rotation) + ".png")
            gauss.fittedParams = (shapeParam,)
            c_kTau = gauss.kTau()
            # compute rank corr coeff from resampled data
            gauss_model = PairCopula(u, v, family=family)
            gauss_model.copulaTournament()
            print(gauss_model.copulaParams)
            self.assertTrue("gauss" in gauss_model.copulaModel.name)
            # Ensure fitted shape parameter is same as original
            self.assertAlmostEqual(shapeParam, gauss_model.copulaParams[1][0], delta=0.2)
            # Ensure kTau is nearly the same from resampled data
            self.assertAlmostEqual(c_kTau, gauss_model.copulaModel.kTau(), delta=0.02)
            # fit to resampled data
            u_model, v_model = gauss_model.copulaModel.sample(10000)
            gauss_refit = PairCopula(u_model, v_model, family=family)
            gauss_refit.copulaTournament()
            u_resample, v_resample = gauss_refit.copulaModel.sample(2000)
            self.assertAlmostEqual(c_kTau, gauss_refit.copulaModel.kTau(), delta=0.05)
            self.assertAlmostEqual(shapeParam, gauss_refit.copulaParams[1][0], delta=0.2)
            # plot resampled data
            g_resample = sns.jointplot(u_resample, v_resample)
            g_resample.savefig("gauss_resample_pdf_" + str(rotation) + ".png")


<gh_stars>10-100
from warnings import warn
import numpy as np
import pandas as pd
from scipy import sparse
from sklearn.metrics import pairwise_distances
from sklearn.utils import as_float_array
from topo.base.dists import pairwise_special_metric, SPECIAL_METRICS
from topo.base.sparse import SPARSE_SPECIAL_METRICS, sparse_named_distances

#from fastlapmap.similarities import fuzzy_simplicial_set_ann, cknn_graph, diffusion_harmonics

def LapEigenmap(W,
                n_eigs=10,
                norm_laplacian=True,
                eigen_tol=10e-4,
                return_evals=False):
    """
    Performs [Laplacian Eigenmaps](https://www2.imm.dtu.dk/projects/manifold/Papers/Laplacian.pdf) on the input data.

    ----------
    Parameters
    ----------

    `W` : numpy.ndarray, pandas.DataFrame or scipy.sparse.csr_matrix.
         Affinity or adjacency matrix.

    `n_eigs` : int (optional, default 10).
         Number of eigenvectors to decompose the graph Laplacian into.

    `norm_laplacian` : bool (optional, default True).
        Whether to renormalize the graph Laplacian.

    `return_evals` : bool (optional, default False).
        Whether to also return the eigenvalues in a tuple of eigenvectors, eigenvalues. Defaults to False.

    ----------
    Returns
    ----------
        * If return_evals is True :
            A tuple of eigenvectors and eigenvalues.
        * If return_evals is False :
            An array of ranked eigenvectors.

    """

    if isinstance(W, sparse.csr_matrix):
        pass
    elif isinstance(W, np.ndarray):
        W = sparse.csr_matrix(W)
    elif isinstance(W, pd.DataFrame):
        W = W.to_numpy()
        W = sparse.csr_matrix(W)
    else:
        return print('Data should be a numpy.ndarray,pandas.DataFrame or'
                     'a scipy.sparse.csr_matrix for obtaining approximate nearest neighbors with \'nmslib\'.')

    laplacian, dd = sparse.csgraph.laplacian(W, normed=norm_laplacian, return_diag=True)
    laplacian = _set_diag(laplacian, 1, norm_laplacian)
    laplacian *= -1
    n_eigs = n_eigs + 1
    evals, evecs = sparse.linalg.eigsh(laplacian, k=n_eigs, which='LM', sigma=1.0, tol=eigen_tol)
    evecs = evecs.T[n_eigs::-1]
    if norm_laplacian:
        # recover u = D^-1/2 x from the eigenvector output x
        evecs = evecs / dd
    evecs = evecs[1:n_eigs].T

    if return_evals:
        return evecs, evals
    else:
        return evecs

def _set_diag(laplacian, value, norm_laplacian):
    """Set the diagonal of the laplacian matrix and convert it to a
    sparse format well suited for eigenvalue decomposition.
    Parameters
    ----------
    laplacian : {ndarray, sparse matrix}
        The graph laplacian.
    value : float
        The value of the diagonal.
    norm_laplacian : bool
        Whether the value of the diagonal should be changed or not.
    Returns
    -------
    laplacian : {array, sparse matrix}
        An array of matrix in a form that is well suited to fast
        eigenvalue decomposition, depending on the band width of the
        matrix.
    """
    n_nodes = laplacian.shape[0]
    # We need all entries in the diagonal to values
    if not sparse.isspmatrix(laplacian):
        if norm_laplacian:
            laplacian.flat[:: n_nodes + 1] = value
    else:
        laplacian = laplacian.tocoo()
        if norm_laplacian:
            diag_idx = laplacian.row == laplacian.col
            laplacian.data[diag_idx] = value
        # If the matrix has a small number of diagonals (as in the
        # case of structured matrices coming from images), the
        # dia format might be best suited for matvec products:
        n_diags = np.unique(laplacian.row - laplacian.col).size
        if n_diags <= 7:
            # 3 or less outer diagonals on each side
            laplacian = laplacian.todia()
        else:
            # csr has the fastest matvec and is thus best suited to
            # arpack
            laplacian = laplacian.tocsr()
    return laplacian

def spectral_decomposition(affinity_matrix, n_eigs, expand=False):
    N = np.shape(affinity_matrix)[0]
    D, V = sparse.linalg.eigsh(affinity_matrix, n_eigs, tol=1e-4, maxiter=(N // 10))
    D = np.real(D)
    V = np.real(V)
    inds = np.argsort(D)[::-1]
    D = D[inds]
    V = V[:, inds]
    # Normalize by the first diffusion component
    for i in range(V.shape[1]):
        V[:, i] = V[:, i] / np.linalg.norm(V[:, i])
    vals = np.array(V)
    pos = np.sum(vals > 0, axis=0)
    residual = np.sum(vals < 0, axis=0)

    if expand and len(residual) < 1:
        # expand eigendecomposition
        target = n_eigs + 30
        while residual < 3:
            while target < 3 * n_eigs:
                print('Eigengap not found for determined number of components. Expanding eigendecomposition to '
                      + str(target) + 'components.')
                D, V = sparse.linalg.eigsh(affinity_matrix, target, tol=1e-4, maxiter=(N // 10))
                D = np.real(D)
                V = np.real(V)
                inds = np.argsort(D)[::-1]
                D = D[inds]
                V = V[:, inds]
                # Normalize by the first diffusion component
                vals = np.array(V)
                for i in range(V.shape[1]):
                    vals[:, i] = vals[:, i] / np.linalg.norm(vals[:, i])
                pos = np.sum(vals > 0, axis=0)
                target = int(target * 1.6)
                residual = np.sum(vals < 0, axis=0)

            if residual < 1:
                print('Could not find an eigengap! Consider increasing `n_neighbors` or `n_eigs` !'
                      ' Falling back to `eigen_expansion=False`, will not attempt')
                expand = False
    if expand:
        if len(residual) > 30:
            target = n_eigs - 15
            while len(residual) > 29:
                D, V = sparse.linalg.eigsh(affinity_matrix, target, tol=1e-4, maxiter=(N // 10))
                D = np.real(D)
                V = np.real(V)
                inds = np.argsort(D)[::-1]
                D = D[inds]
                V = V[:, inds]
                vals = np.array(V)
                for i in range(V.shape[1]):
                    vals[:, i] = vals[:, i] / np.linalg.norm(vals[:, i])
                pos = np.sum(vals > 0, axis=0)
                residual = np.sum(vals < 0, axis=0)
                if len(residual) < 15:
                    break
                else:
                    target = pos - int(residual // 2)

            if len(residual) < 1:
                print('Could not find an eigengap! Consider increasing `n_neighbors` or `n_eigs` !'
                      ' Falling back to `eigen_expansion=False`, will not attempt eigendecomposition expansion.')
                expand = False

    if not expand:
        D, V = sparse.linalg.eigsh(affinity_matrix, n_eigs, tol=1e-4, maxiter=(N // 10))
        D = np.real(D)
        V = np.real(V)
        inds = np.argsort(D)[::-1]
        D = D[inds]
        V = V[:, inds]

    # Normalize by the first eigencomponent
    for i in range(V.shape[1]):
        V[:, i] = V[:, i] / np.linalg.norm(V[:, i])

    # Normalize eigenvalues
    D = D / D.max()

    return V, D

def component_layout(
        W,
        n_components,
        component_labels,
        dim,
        norm_laplacian=True,
        eigen_tol=10e-4
):
    """Provide a layout relating the separate connected components. This is done
    by taking the centroid of each component and then performing a spectral embedding
    of the centroids.
    Parameters
    ----------
    W: numpy.ndarray, pandas.DataFrame or scipy.sparse.csr_matrix.
         Affinity or adjacency matrix.
    n_components: int
        The number of distinct components to be layed out.
    component_labels: array of shape (n_samples)
        For each vertex in the graph the label of the component to
        which the vertex belongs.
    dim: int
        The chosen embedding dimension.

    Returns
    -------
    component_embedding: array of shape (n_components, dim)
        The ``dim``-dimensional embedding of the ``n_components``-many
        connected components.
    """

    # cannot compute centroids from precomputed distances
    # instead, compute centroid distances using linkage
    distance_matrix = np.zeros((n_components, n_components), dtype=np.float64)
    linkage = np.min

    for c_i in range(n_components):
        dm_i = W[component_labels == c_i]
        for c_j in range(c_i + 1, n_components):
            dist = linkage(dm_i[:, component_labels == c_j])
            distance_matrix[c_i, c_j] = dist
            distance_matrix[c_j, c_i] = dist

    affinity_matrix = np.exp(-(distance_matrix ** 2))

    component_embedding = LapEigenmap(W=affinity_matrix, n_eigs=dim, norm_laplacian=norm_laplacian, eigen_tol=eigen_tol)
    component_embedding /= component_embedding.max()

    return component_embedding

def multi_component_layout(
    graph,
    n_components,
    component_labels,
    dim,
    random_state
):
    """Specialised layout algorithm for dealing with graphs with many connected components.
    This will first find relative positions for the components by spectrally embedding
    their centroids, then spectrally embed each individual connected component positioning
    them according to the centroid embeddings. This provides a decent embedding of each
    component while placing the components in good relative positions to one another.
    Parameters
    ----------
    graph: sparse matrix
        The adjacency matrix of the graph to be embedded.
    n_components: int
        The number of distinct components to be layed out.
    component_labels: array of shape (n_samples)
        For each vertex in the graph the label of the component to
        which the vertex belongs.
    dim: int
        The chosen embedding dimension.

    Returns
    -------
    embedding: array of shape (n_samples, dim)
        The initial embedding of ``graph``.
    """

    result = np.empty((graph.shape[0], dim), dtype=np.float32)

    if n_components > 2 * dim:
        meta_embedding = component_layout(
            graph,
            n_components,
            component_labels,
            dim,
        )
    else:
        k = int(np.ceil(n_components / 2.0))
        base = np.hstack([np.eye(k), np.zeros((k, dim - k))])
        meta_embedding = np.vstack([base, -base])[:n_components]

    for label in range(n_components):
        component_graph = graph.tocsr()[component_labels == label, :].tocsc()
        component_graph = component_graph[:, component_labels == label].tocoo()

        distances = pairwise_distances([meta_embedding[label]], meta_embedding)
        data_range = distances[distances > 0.0].min() / 2.0

        if component_graph.shape[0] < 2 * dim:
            result[component_labels == label] = (
                random_state.uniform(
                    low=-data_range,
                    high=data_range,
                    size=(component_graph.shape[0], dim),
                )
                + meta_embedding[label]
            )
            continue

        diag_data = np.asarray(component_graph.sum(axis=0))
        # standard Laplacian
        # D = scipy.sparse.spdiags(diag_data, 0, graph.shape[0], graph.shape[0])
        # L = D - graph
        # Normalized Laplacian
        I = sparse.identity(component_graph.shape[0], dtype=np.float64)
        D = sparse.spdiags(
            1.0 / (np.sqrt(diag_data)+10e-6),
            0,
            component_graph.shape[0],
            component_graph.shape[0],
        )
        L = I - D * component_graph * D
        k = dim + 1
        num_lanczos_vectors = max(2 * k + 1, int(np.sqrt(component_graph.shape[0])))
        try:
            eigenvalues, eigenvectors = sparse.linalg.eigsh(
                L,
                k,
                which="SM",
                ncv=num_lanczos_vectors,
                tol=1e-4,
                v0=np.ones(L.shape[0]),
                maxiter=graph.shape[0] * 5,
            )
            order = np.argsort(eigenvalues)[1:k]
            component_embedding = eigenvectors[:, order]
            expansion = data_range / np.max(np.abs(component_embedding))
            component_embedding *= expansion
            result[component_labels == label] = (
                component_embedding + meta_embedding[label]
            )
        except sparse.linalg.ArpackError:
            warn(
                "WARNING: spectral initialisation failed! The eigenvector solver\n"
                "failed. This is likely due to too small an eigengap. Consider\n"
                "adding some noise or jitter to your data.\n\n"
                "Falling back to random initialisation!"
            )
            result[component_labels == label] = (
                random_state.uniform(
                    low=-data_range,
                    high=data_range,
                    size=(component_graph.shape[0], dim),
                )
                + meta_embedding[label]
            )

    return result

def spectral_layout(graph, dim, random_state):
    """Given a graph compute the spectral embedding of the graph. This is
    simply the eigenvectors of the laplacian of the graph. Here we use the
    normalized laplacian.

    Parameters
    ----------
    graph: sparse matrix
        The (weighted) adjacency matrix of the graph as a sparse matrix.
    dim: int
        The dimension of the space into which to embed.
    random_state: numpy RandomState or equivalent
        A state capable being used as a numpy random state.

    Returns
    -------
    embedding: array of shape (n_vertices, dim)
        The spectral embedding of the graph.
    """
    n_components, labels = sparse.csgraph.connected_components(graph)

    if n_components > 1:
        return multi_component_layout(
            graph,
            n_components,
            labels,
            dim,
            random_state,
        )

    diag_data = np.asarray(graph.sum(axis=0))
    # standard Laplacian
    # D = scipy.sparse.spdiags(diag_data, 0, graph.shape[0], graph.shape[0])
    # L = D - graph
    # Normalized Laplacian
    I = sparse.identity(graph.shape[0], dtype=np.float64)
    D = sparse.spdiags(
        1.0 / (np.sqrt(diag_data)+10e-8), 0, graph.shape[0], graph.shape[0]
    )
    L = I - D * graph * D

    k = dim + 1
    num_lanczos_vectors = max(2 * k + 1, int(np.sqrt(graph.shape[0])))
    try:
        if L.shape[0] < 2000000:
            eigenvalues, eigenvectors = sparse.linalg.eigsh(
                L,
                k,
                which="SM",
                ncv=num_lanczos_vectors,
                tol=1e-4,
                v0=np.ones(L.shape[0]),
                maxiter=graph.shape[0] * 5,
            )
        else:
            eigenvalues, eigenvectors = sparse.linalg.lobpcg(
                L, random_state.normal(size=(L.shape[0], k)), largest=False, tol=1e-8
            )
        order = np.argsort(eigenvalues)[1:k]
        return eigenvectors[:, order]
    except sparse.linalg.ArpackError:
        warn(
            "WARNING: spectral initialisation failed! The eigenvector solver\n"
            "failed. This is likely due to too small an eigengap. Consider\n"
            "adding some noise or jitter to your data.\n\n"
            "Falling back to random initialisation!"
        )
        return random_state.uniform(low=-10.0, high=10.0, size=(graph.shape[0], dim))

def spectral_clustering(init, max_svd_restarts=30, n_iter_max=30, random_state=None, copy=True):
    """Search for a partition matrix (clustering) which is closest to the
        eigenvector embedding.

    Parameters
    ----------
    init : array-like of shape (n_samples, n_clusters)
        The embedding space of the samples.
    max_svd_restarts : int, default=30
        Maximum number of attempts to restart SVD if convergence fails
    n_iter_max : int, default=30
        Maximum number of iterations to attempt in rotation and partition
        matrix search if machine precision convergence is not reached
    random_state : int, RandomState instance, default=None
        Determines random number generation for rotation matrix initialization.
        Use an int to make the randomness deterministic.
        See :term:`Glossary <random_state>`.
    copy : bool, default=True
        Whether to copy vectors, or perform in-place normalization.
    Returns
    -------
    labels : array of integers, shape: n_samples
        The labels of the clusters.
    References
    ----------
    - Multiclass spectral clustering, 2003
      <NAME>, <NAME>
      https://www1.icsi.berkeley.edu/~stellayu/publication/doc/2003kwayICCV.pdf
    Notes
    -----
    The eigenvector embedding is used to iteratively search for the
    closest discrete partition.  First, the eigenvector embedding is
    normalized to the space of partition matrices. An optimal discrete
    partition matrix closest to this normalized embedding multiplied by
    an initial rotation is calculated.  Fixing this discrete partition
    matrix, an optimal rotation matrix is calculated.  These two
    calculations are performed until convergence.  The discrete partition
    matrix is returned as the clustering solution.  Used in spectral
    clustering, this method tends to be faster and more robust to random
    initialization than k-means.
    """

    from scipy.sparse import csc_matrix
    from scipy.linalg import LinAlgError

    vectors = as_float_array(init, copy=copy)

    eps = np.finfo(float).eps
    n_samples, n_components = vectors.shape

    # Normalize the eigenvectors to an equal length of a vector of ones.
    # Reorient the eigenvectors to point in the negative direction with respect
    # to the first element.  This may have to do with constraining the
    # eigenvectors to lie in a specific quadrant to make the discretization
    # search easier.
    norm_ones = np.sqrt(n_samples)
    for i in range(vectors.shape[1]):
        vectors[:, i] = (vectors[:, i] / np.linalg.norm(vectors[:, i])) \
                        * norm_ones
    if vectors[0, i] != 0:
        vectors[:, i] = -1 * vectors[:, i] * np.sign(vectors[0, i])

    # Normalize the rows of the eigenvectors.  Samples should lie on the unit
    # hypersphere centered at the origin.  This transforms the samples in the
    # embedding space to the space of partition matrices.
    vectors = vectors / np.sqrt((vectors ** 2).sum(axis=1))[:, np.newaxis]

    svd_restarts = 0
    has_converged = False

    # If there is an exception we try to randomize and rerun SVD again
    # do this max_svd_restarts times.

    while (svd_restarts < max_svd_restarts) and not has_converged:

        # Initialize first column of rotation matrix with a row of the
        # eigenvectors
        rotation = np.zeros((n_components, n_components))
        rotation[:, 0] = vectors[random_state.randint(n_samples), :].T

        # To initialize the rest of the rotation matrix, find the rows
        # of the eigenvectors that are as orthogonal to each other as
        # possible
        c = np.zeros(n_samples)
        for j in range(1, n_components):
            # Accumulate c to ensure row is as orthogonal as possible to
            # previous picks as well as current one
            c += np.abs(np.dot(vectors, rotation[:, j - 1]))
            rotation[:, j] = vectors[c.argmin(), :].T

        last_objective_value = 0.0
        n_iter = 0

        while not has_converged:
            n_iter += 1

            t_discrete = np.dot(vectors, rotation)

            labels = t_discrete.argmax(axis=1)
            vectors_discrete = csc_matrix(
                (np.ones(len(labels)), (np.arange(0, n_samples), labels)),
                shape=(n_samples, n_components))

            t_svd = vectors_discrete.T * vectors

            try:
                U, S, Vh = np.linalg.svd(t_svd)
                svd_restarts += 1
            except LinAlgError:
                print("SVD did not converge, randomizing and trying again")
                break

            ncut_value = 2.0 * (n_samples - S.sum())
            if ((abs(ncut_value - last_objective_value) < eps) or
                    (n_iter > n_iter_max)):
                has_converged = True
            else:
                # otherwise calculate rotation and continue
                last_objective_value = ncut_value
                rotation = np.dot(Vh.T, U.T)

    if not has_converged:
        raise LinAlgError('SVD did not converge')
    return labels


<reponame>SharmaAjay19/VideoImageColorization
import numpy as np
import os
import time
import skimage.color as color
import matplotlib.pyplot as plt
import scipy.ndimage.interpolation as sni
import caffe
import argparse
import cv2
import shutil

def parse_args():
    parser = argparse.ArgumentParser(description='iColor: deep interactive colorization')
    parser.add_argument('-vid_in',dest='vid_in',help='grayscale image to read in', type=str)
    parser.add_argument('-vid_out',dest='vid_out',help='colorized image to save off', type=str)
    parser.add_argument('--gpu', dest='gpu', help='gpu id', type=int, default=0)
    parser.add_argument('--prototxt',dest='prototxt',help='prototxt filepath', type=str, default='./models/colorization_deploy_v2.prototxt')
    parser.add_argument('--caffemodel',dest='caffemodel',help='caffemodel filepath', type=str, default='./models/colorization_release_v2.caffemodel')

    args = parser.parse_args()
    return args

def readVideoFrames(vid_path):
	vidcap = cv2.VideoCapture(vid_path)
	success,image = vidcap.read()
	print("Read video")
	tempdir = os.getcwd()+"/video_temp" + str(time.clock())
	if not os.path.exists(tempdir):
		os.makedirs(tempdir)
	count = 0
	while success:
		cv2.imwrite(tempdir + "/frame%d.jpg" % count, image)     # save frame as JPEG file
		success,image = vidcap.read()
		count += 1
	print("Read %d frames"%count)
	return count, tempdir

def createVideoFromFrames(dirpath, vid_path, nframes):
	frame = cv2.imread(os.path.join(dirpath, "frame0_1.jpg"))
	height, width, layers = frame.shape
	fourcc = cv2.VideoWriter_fourcc(*'mp4v')
	video = cv2.VideoWriter(vid_path, fourcc, 20, (width,height))
	for frame in range(nframes):
		video.write(cv2.imread(os.path.join(dirpath, "frame%d_1.jpg"%frame)))
	cv2.destroyAllWindows()
	video.release()
	shutil.rmtree(dirpath)

if __name__ == '__main__':
	args = parse_args()

	caffe.set_mode_cpu()
	#caffe.set_device(args.gpu)

	# Select desired model
	net = caffe.Net(args.prototxt, args.caffemodel, caffe.TEST)

	(H_in,W_in) = net.blobs['data_l'].data.shape[2:] # get input shape
	(H_out,W_out) = net.blobs['class8_ab'].data.shape[2:] # get output shape

	pts_in_hull = np.load('./resources/pts_in_hull.npy') # load cluster centers
	net.params['class8_ab'][0].data[:,:,0,0] = pts_in_hull.transpose((1,0)) # populate cluster centers as 1x1 convolution kernel
	# print 'Annealed-Mean Parameters populated'

	nframes, dirpath = readVideoFrames(args.vid_in)
	for frame in range(nframes):
		# load the original image
		img_rgb = caffe.io.load_image(dirpath+("/frame%d.jpg"%frame))

		img_lab = color.rgb2lab(img_rgb) # convert image to lab color space
		img_l = img_lab[:,:,0] # pull out L channel
		(H_orig,W_orig) = img_rgb.shape[:2] # original image size

		# create grayscale version of image (just for displaying)
		img_lab_bw = img_lab.copy()
		img_lab_bw[:,:,1:] = 0
		img_rgb_bw = color.lab2rgb(img_lab_bw)

		# resize image to network input size
		img_rs = caffe.io.resize_image(img_rgb,(H_in,W_in)) # resize image to network input size
		img_lab_rs = color.rgb2lab(img_rs)
		img_l_rs = img_lab_rs[:,:,0]

		net.blobs['data_l'].data[0,0,:,:] = img_l_rs-50 # subtract 50 for mean-centering
		net.forward() # run network

		ab_dec = net.blobs['class8_ab'].data[0,:,:,:].transpose((1,2,0)) # this is our result
		ab_dec_us = sni.zoom(ab_dec,(1.*H_orig/H_out,1.*W_orig/W_out,1)) # upsample to match size of original image L
		img_lab_out = np.concatenate((img_l[:,:,np.newaxis],ab_dec_us),axis=2) # concatenate with original image L
		img_rgb_out = (255*np.clip(color.lab2rgb(img_lab_out),0,1)).astype('uint8') # convert back to rgb

		plt.imsave(dirpath+("/frame%d_1.jpg"%frame), img_rgb_out)
		print("Processed %d out of %d frames" % (frame+1, nframes))
	createVideoFromFrames(dirpath, args.vid_out, nframes)