127 lines
4.7 KiB
Python
127 lines
4.7 KiB
Python
import logging
|
|
|
|
from pathlib import Path
|
|
from typing import Union, Dict
|
|
|
|
import ConfigSpace as CS
|
|
import numpy as np
|
|
|
|
from base_fed_surrogate_benchmark import BaseSurrogateFedHPOBench
|
|
|
|
__version__ = '0.0.1'
|
|
|
|
logger = logging.getLogger('FEMNISTabularFed')
|
|
|
|
|
|
class FENISTSurrogateFedHPOBench(BaseSurrogateFedHPOBench):
|
|
def __init__(self,
|
|
data_path: Union[str, Path],
|
|
model_path: Union[str, Path],
|
|
rng: Union[np.random.RandomState, int, None] = None):
|
|
data_url = "https://federatedscope.oss-cn-beijing.aliyuncs.com" \
|
|
"/fedhpob_cnn_tabular.zip"
|
|
model_url = "https://federatedscope.oss-cn-beijing.aliyuncs.com" \
|
|
"/fedhpob_cnn_surrogate.zip"
|
|
triplets = ('cnn', 'femnist', 'avg')
|
|
client_num = 200
|
|
num_param = 6603902
|
|
super(FENISTSurrogateFedHPOBench,
|
|
self).__init__(data_path, model_path, data_url, model_url,
|
|
triplets, client_num, num_param, rng)
|
|
|
|
@staticmethod
|
|
def get_configuration_space(
|
|
seed: Union[int, None] = None) -> CS.ConfigurationSpace:
|
|
"""
|
|
Interface to the get_configuration_space function from the FEMNIST
|
|
Benchmark.
|
|
Parameters
|
|
----------
|
|
seed : int, None
|
|
Random seed for the configuration space.
|
|
Returns
|
|
-------
|
|
CS.ConfigurationSpace -
|
|
Containing the benchmark's hyperparameter
|
|
"""
|
|
|
|
seed = seed if seed is not None else np.random.randint(1, 100000)
|
|
cs = CS.ConfigurationSpace(seed=seed)
|
|
cs.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('lr',
|
|
lower=1e-2,
|
|
upper=1.0,
|
|
log=True))
|
|
cs.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('wd',
|
|
lower=10e-5,
|
|
upper=0.1,
|
|
log=True))
|
|
cs.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('dropout', lower=0, upper=0.5))
|
|
cs.add_hyperparameter(
|
|
CS.UniformIntegerHyperparameter('batch', lower=16, upper=64))
|
|
cs.add_hyperparameter(
|
|
CS.UniformIntegerHyperparameter('step', lower=1, upper=4))
|
|
return cs
|
|
|
|
@staticmethod
|
|
def get_fidelity_space(
|
|
seed: Union[int, None] = None) -> CS.ConfigurationSpace:
|
|
"""
|
|
Creates a ConfigSpace.ConfigurationSpace containing all fidelity
|
|
parameters for
|
|
the FCNetBaseBenchmark
|
|
Parameters
|
|
----------
|
|
seed : int, None
|
|
Fixing the seed for the ConfigSpace.ConfigurationSpace
|
|
Returns
|
|
-------
|
|
ConfigSpace.ConfigurationSpace
|
|
"""
|
|
seed = seed if seed is not None else np.random.randint(1, 100000)
|
|
fidel_space = CS.ConfigurationSpace(seed=seed)
|
|
|
|
fidel_space.add_hyperparameters([
|
|
CS.UniformIntegerHyperparameter('round',
|
|
lower=0,
|
|
upper=249,
|
|
default_value=249)
|
|
])
|
|
fidel_space.add_hyperparameters([
|
|
CS.UniformFloatHyperparameter('sample_client',
|
|
lower=0,
|
|
upper=1.0,
|
|
default_value=1.0)
|
|
])
|
|
|
|
return fidel_space
|
|
|
|
@staticmethod
|
|
def get_meta_information() -> Dict:
|
|
""" Returns the meta information for the benchmark """
|
|
return {
|
|
'name': 'Surrogate Benchmarks for FEMNIST FedAvg',
|
|
'references': [
|
|
'@article{Wang2022FedHPOBench,'
|
|
'title = {FedHPO-Bench: A Benchmark Suite for Federated '
|
|
'Hyperparameter Optimization},'
|
|
'author = {Zhen Wang and Weirui Kuang and Ce Zhang and '
|
|
'Bolin Ding and Yaliang Li},'
|
|
'journal = {arXiv preprint arXiv:2206.03966},'
|
|
'year = {2022}}', 'https://arxiv.org/pdf/2206.03966v4.pdf',
|
|
'https://github.com/alibaba/FederatedScope/tree/master'
|
|
'/benchmark/FedHPOBench'
|
|
],
|
|
'code': 'https://github.com/alibaba/FederatedScope/tree/master'
|
|
'/benchmark/FedHPOBench',
|
|
}
|
|
|
|
|
|
if __name__ == '__main__':
|
|
b = FENISTSurrogateFedHPOBench('data', 'model', 1)
|
|
config = b.get_configuration_space(seed=1).sample_configuration()
|
|
result_dict = b.objective_function(configuration=config, rng=1)
|
|
print(result_dict)
|