55 lines
1.5 KiB
Python
55 lines
1.5 KiB
Python
import os
|
|
import pickle
|
|
|
|
from federatedscope.register import register_data
|
|
from federatedscope.core.data.utils import convert_data_mode
|
|
from federatedscope.core.auxiliaries.utils import setup_seed
|
|
|
|
|
|
def load_data_from_file(config, client_cfgs=None):
|
|
from federatedscope.core.data import DummyDataTranslator
|
|
|
|
file_path = config.data.file_path
|
|
|
|
if not os.path.exists(file_path):
|
|
raise ValueError(f'The file {file_path} does not exist.')
|
|
|
|
with open(file_path, 'br') as file:
|
|
data = pickle.load(file)
|
|
# The shape of data is expected to be:
|
|
# (1) the data consist of all participants' data:
|
|
# {
|
|
# 'client_id': {
|
|
# 'train/val/test': {
|
|
# 'x/y': np.ndarray
|
|
# }
|
|
# }
|
|
# }
|
|
# (2) isolated data
|
|
# {
|
|
# 'train/val/test': {
|
|
# 'x/y': np.ndarray
|
|
# }
|
|
# }
|
|
|
|
# translator = DummyDataTranslator(config, client_cfgs)
|
|
# data = translator(data)
|
|
|
|
# Convert `StandaloneDataDict` to `ClientData` when in distribute mode
|
|
data = convert_data_mode(data, config)
|
|
|
|
# Restore the user-specified seed after the data generation
|
|
setup_seed(config.seed)
|
|
|
|
return data, config
|
|
|
|
|
|
def call_file_data(config, client_cfgs):
|
|
if config.data.type == "file":
|
|
# All the data (clients and servers) are loaded from one unified files
|
|
data, modified_config = load_data_from_file(config, client_cfgs)
|
|
return data, modified_config
|
|
|
|
|
|
register_data("file", call_file_data)
|