Spaces:
Running
Running
File size: 2,693 Bytes
c4ac745 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | import pandas as pd
import os
from sdv.metadata import MultiTableMetadata
def get_multi_metadata(tables, relation_order):
metadata = MultiTableMetadata()
for table_name, val in tables.items():
df = val['original_df']
metadata.detect_table_from_dataframe(
table_name,
df
)
id_cols = [col for col in df.columns if '_id' in col]
for id_col in id_cols:
metadata.update_column(
table_name=table_name,
column_name=id_col,
sdtype='id'
)
domain = tables[table_name]['domain']
for col, dom in domain.items():
if col in df.columns:
if dom['type'] == 'discrete':
metadata.update_column(
table_name=table_name,
column_name=col,
sdtype='categorical',
)
elif dom['type'] == 'continuous':
metadata.update_column(
table_name=table_name,
column_name=col,
sdtype='numerical',
)
else:
raise ValueError(f'Unknown domain type: {dom["type"]}')
metadata.set_primary_key(
table_name=table_name,
column_name=f'{table_name}_id'
)
for parent, child in relation_order:
if parent is not None:
metadata.add_relationship(
parent_table_name=parent,
child_table_name=child,
parent_primary_key=f'{parent}_id',
child_foreign_key=f'{parent}_id'
)
return metadata
def baseline_load_synthetic_data(path, tables):
syn = {}
for table, val in tables.items():
syn[table] = {}
syn[table]['df'] = pd.read_csv(os.path.join(
path,
'final',
f'{table}_synthetic.csv'
))
syn[table]['domain'] = val['domain']
return syn
def clava_load_synthetic_data(path, tables):
syn = {}
for table, val in tables.items():
syn[table] = {}
syn[table]['df'] = pd.read_csv(os.path.join(
path,
table,
'_final',
f'{table}_synthetic.csv'
))
syn[table]['domain'] = val['domain']
return syn
def sdv_load_synthetic_data(path, tables):
syn = {}
for table, val in tables.items():
syn[table] = {}
syn[table]['df'] = pd.read_csv(os.path.join(
path,
f'{table}.csv'
))
syn[table]['domain'] = val['domain']
return syn
|