433 lines
22 KiB
Python
433 lines
22 KiB
Python
import ConfigSpace as CS
|
|
from federatedscope.core.configs.config import CN
|
|
from fedhpobench.benchmarks import TabularBenchmark
|
|
from fedhpobench.benchmarks import RawBenchmark
|
|
from fedhpobench.benchmarks import SurrogateBenchmark
|
|
|
|
fhb_cfg = CN()
|
|
|
|
|
|
def get_cs(dname, model, mode, alg='avg'):
|
|
# raw and surrogate are ONLY FOR NIPS2022
|
|
configuration_space = CS.ConfigurationSpace()
|
|
fidelity_space = CS.ConfigurationSpace()
|
|
# configuration_space
|
|
if dname == 'twitter':
|
|
fidelity_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('round',
|
|
choices=[x for x in range(500)]))
|
|
fidelity_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('sample_rate', choices=[0.01]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'lr', choices=[0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('wd', choices=[0.0, 0.001, 0.01,
|
|
0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('dropout', choices=[0.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1, 2, 3, 4]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('batch', choices=[64]))
|
|
|
|
elif dname in ['cora', 'citeseer', 'pubmed']:
|
|
# GNN tabular, raw and surrogate
|
|
fidelity_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('round',
|
|
choices=[x for x in range(500)]))
|
|
fidelity_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('sample_rate',
|
|
choices=[0.2, 0.4, 0.6, 0.8, 1.0]))
|
|
if mode == 'tabular':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('lr',
|
|
choices=[
|
|
0.01, 0.01668, 0.02783,
|
|
0.04642, 0.07743, 0.12915,
|
|
0.21544, 0.35938, 0.59948, 1.0
|
|
]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('wd',
|
|
choices=[0.0, 0.001, 0.01, 0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('dropout', choices=[0.0, 0.5]))
|
|
if alg == 'avg':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'step', choices=[1, 2, 3, 4, 5, 6, 7, 8]))
|
|
else:
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('lrserver',
|
|
choices=[0.1, 0.5, 1.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('momentumsserver',
|
|
choices=[0.0, 0.9]))
|
|
elif mode in ['surrogate', 'raw']:
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('lr',
|
|
lower=1e-2,
|
|
upper=1.0,
|
|
log=True))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('wd',
|
|
choices=[0.0, 0.001, 0.01, 0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('dropout', lower=.0, upper=.5))
|
|
if alg == 'avg':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'step', choices=[1, 2, 3, 4, 5, 6, 7, 8]))
|
|
else:
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('lrserver',
|
|
lower=1e-1,
|
|
upper=1.0,
|
|
log=True))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('momentumsserver',
|
|
lower=0.0,
|
|
upper=1.0))
|
|
|
|
elif dname in [
|
|
'10101@openml', '53@openml', '146818@openml', '146821@openml',
|
|
'9952@openml', '146822@openml', '31@openml', '3917@openml'
|
|
]:
|
|
# Openml tabular, raw and surrogate
|
|
fidelity_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('round',
|
|
choices=[x for x in range(250)]))
|
|
fidelity_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('sample_rate',
|
|
choices=[0.2, 0.4, 0.6, 0.8, 1.0]))
|
|
if model == 'lr':
|
|
if mode == 'tabular':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'lr', choices=[0.00001, 0.0001, 0.001, 0.01, 0.1,
|
|
1.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'wd', choices=[0.0, 0.001, 0.01, 0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'batch', choices=[8, 16, 32, 64, 128, 256]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('dropout', choices=[0.0]))
|
|
if alg == 'avg':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step',
|
|
choices=[1, 2, 3, 4]))
|
|
else:
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('lrserver',
|
|
choices=[0.1, 0.5, 1.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('momentumsserver',
|
|
choices=[0.0, 0.9]))
|
|
elif mode in ['surrogate', 'raw']:
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('lr',
|
|
lower=1e-5,
|
|
upper=1.0,
|
|
log=True))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'wd', choices=[0.0, 0.001, 0.01, 0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('dropout', choices=[0.0]))
|
|
if alg == 'avg':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'step', choices=[1, 2, 3, 4, 5, 6, 7, 8]))
|
|
else:
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('lrserver',
|
|
lower=1e-1,
|
|
upper=1.0,
|
|
log=True))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('momentumsserver',
|
|
lower=0.0,
|
|
upper=1.0))
|
|
elif model == 'mlp':
|
|
if mode == 'tabular':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'lr', choices=[0.00001, 0.0001, 0.001, 0.01, 0.1,
|
|
1.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'wd', choices=[0.0, 0.001, 0.01, 0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('batch',
|
|
choices=[32, 64, 128, 256]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('dropout', choices=[0.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('layer', choices=[2, 3, 4]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('hidden',
|
|
choices=[16, 64, 256]))
|
|
if alg == 'avg':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step',
|
|
choices=[1, 2, 3, 4]))
|
|
else:
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('lrserver',
|
|
choices=[0.1, 0.5, 1.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('momentumsserver',
|
|
choices=[0.0, 0.9]))
|
|
elif mode in ['surrogate', 'raw']:
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('lr',
|
|
lower=1e-5,
|
|
upper=1.0,
|
|
log=True))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'wd', choices=[0.0, 0.001, 0.01, 0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('dropout', choices=[0.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('layer', choices=[2, 3, 4]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('hidden',
|
|
choices=[16, 64, 256]))
|
|
if alg == 'avg':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step',
|
|
choices=[1, 2, 3, 4]))
|
|
else:
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('lrserver',
|
|
lower=1e-1,
|
|
upper=1.0,
|
|
log=True))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('momentumsserver',
|
|
lower=0.0,
|
|
upper=1.0))
|
|
elif dname in ['femnist', 'cifar10']:
|
|
# CNN tabular and surrogate
|
|
fidelity_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('round',
|
|
choices=[x for x in range(250)]))
|
|
fidelity_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('sample_rate',
|
|
choices=[0.2, 0.4, 0.6, 0.8, 1.0]))
|
|
if mode == 'tabular':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('lr',
|
|
choices=[
|
|
0.01, 0.01668, 0.02783,
|
|
0.04642, 0.07743, 0.12915,
|
|
0.21544, 0.35938, 0.59948, 1.0
|
|
]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('wd',
|
|
choices=[0.0, 0.001, 0.01, 0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('dropout', choices=[0.0, 0.5]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('batch', choices=[16, 32, 64]))
|
|
if alg == 'avg':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1, 2, 3, 4]))
|
|
elif mode in ['surrogate', 'raw']:
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('lr',
|
|
lower=1e-2,
|
|
upper=1.0,
|
|
log=True))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('wd',
|
|
choices=[0.0, 0.001, 0.01, 0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('dropout', lower=.0, upper=.5))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('batch', choices=[16, 32, 64]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1, 2, 3, 4]))
|
|
elif dname in ['sst2@huggingface_datasets', 'cola@huggingface_datasets']:
|
|
# Transformer tabular and surrogate
|
|
fidelity_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('round',
|
|
choices=[x for x in range(40)]))
|
|
fidelity_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('sample_rate',
|
|
choices=[0.2, 0.4, 0.6, 0.8, 1.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('batch', choices=[8, 16, 32, 64,
|
|
128]))
|
|
if mode == 'tabular':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('lr',
|
|
choices=[
|
|
0.01, 0.01668, 0.02783,
|
|
0.04642, 0.07743, 0.12915,
|
|
0.21544, 0.35938, 0.59948, 1.0
|
|
]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('wd',
|
|
choices=[0.0, 0.001, 0.01, 0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('dropout', choices=[0.0, 0.5]))
|
|
if alg == 'avg':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1, 2, 3, 4]))
|
|
else:
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('lrserver',
|
|
choices=[0.1, 0.5, 1.0]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('momentumsserver',
|
|
choices=[0.0, 0.9]))
|
|
elif mode in ['surrogate', 'raw']:
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('lr',
|
|
lower=1e-2,
|
|
upper=1.0,
|
|
log=True))
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('wd',
|
|
choices=[0.0, 0.001, 0.01, 0.1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('dropout', lower=.0, upper=.5))
|
|
if alg == 'avg':
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter(
|
|
'step', choices=[1, 2, 3, 4, 5, 6, 7, 8]))
|
|
else:
|
|
configuration_space.add_hyperparameter(
|
|
CS.CategoricalHyperparameter('step', choices=[1]))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('lrserver',
|
|
lower=1e-1,
|
|
upper=1.0,
|
|
log=True))
|
|
configuration_space.add_hyperparameter(
|
|
CS.UniformFloatHyperparameter('momentumsserver',
|
|
lower=0.0,
|
|
upper=1.0))
|
|
return configuration_space, fidelity_space
|
|
|
|
|
|
def initial_cfg(cfg):
|
|
# ---------------------------------------------------------------------- #
|
|
# benchmark related options
|
|
# ---------------------------------------------------------------------- #
|
|
cfg.benchmark = CN()
|
|
cfg.benchmark.cls = [{
|
|
'raw': RawBenchmark,
|
|
'tabular': TabularBenchmark,
|
|
'surrogate': SurrogateBenchmark
|
|
}]
|
|
|
|
# ********************************************************************** #
|
|
cfg.benchmark.type = 'raw'
|
|
cfg.benchmark.model = 'gcn'
|
|
cfg.benchmark.data = 'cora'
|
|
cfg.benchmark.device = 0
|
|
cfg.benchmark.sample_client = 1.0 # only for optimizer
|
|
cfg.benchmark.algo = 'avg' # ['avg', 'opt']
|
|
cfg.benchmark.out_dir = 'exp_results'
|
|
# ********************************************************************** #
|
|
|
|
# ---------------------------------------------------------------------- #
|
|
# cost related options
|
|
# ---------------------------------------------------------------------- #
|
|
cfg.cost = CN()
|
|
cfg.cost.type = 'estimated' # in ['raw', 'estimated']
|
|
cfg.cost.c = 1 # lambda for exponential distribution, time consumed in
|
|
# client
|
|
cfg.cost.time_server = 0 # time consumed in server, `0` for real time
|
|
cfg.cost.lag_const = 65535 # Max port number
|
|
|
|
# bandwidth for estimated cost
|
|
cfg.cost.bandwidth = CN()
|
|
cfg.cost.bandwidth.client_up = 0.25 * 1024 * 1024 * 8 / 32
|
|
cfg.cost.bandwidth.client_down = 0.75 * 1024 * 1024 * 8 / 32
|
|
cfg.cost.bandwidth.server_up = 0.25 * 1024 * 1024 * 8 / 32
|
|
cfg.cost.bandwidth.server_down = 0.75 * 1024 * 1024 * 8 / 32
|
|
|
|
# ---------------------------------------------------------------------- #
|
|
# optimizer related options
|
|
# ---------------------------------------------------------------------- #
|
|
cfg.optimizer = CN()
|
|
cfg.optimizer.type = 'de'
|
|
cfg.optimizer.min_budget = 1
|
|
cfg.optimizer.max_budget = 243
|
|
cfg.optimizer.n_iterations = 100000000 # No limits
|
|
cfg.optimizer.seed = 1
|
|
cfg.optimizer.limit_time = 86400 # one day
|
|
|
|
# ---------------------------------------------------------------------- #
|
|
# hpbandster related options (rs, bo_kde, hb, bohb)
|
|
# ---------------------------------------------------------------------- #
|
|
cfg.optimizer.hpbandster = CN()
|
|
cfg.optimizer.hpbandster.eta = 3
|
|
cfg.optimizer.hpbandster.max_stages = 5
|
|
|
|
# ---------------------------------------------------------------------- #
|
|
# smac related options (bo_gp, bo_rf)
|
|
# ---------------------------------------------------------------------- #
|
|
cfg.optimizer.smac = CN()
|
|
|
|
# ---------------------------------------------------------------------- #
|
|
# dehb related options (dehb, de)
|
|
# ---------------------------------------------------------------------- #
|
|
cfg.optimizer.dehb = CN()
|
|
cfg.optimizer.dehb.strategy = 'rand1_bin'
|
|
cfg.optimizer.dehb.mutation_factor = 0.5
|
|
cfg.optimizer.dehb.crossover_prob = 0.5
|
|
|
|
# dehb.dehb
|
|
cfg.optimizer.dehb.dehb = CN()
|
|
cfg.optimizer.dehb.dehb.gens = 1
|
|
cfg.optimizer.dehb.dehb.eta = 3
|
|
cfg.optimizer.dehb.dehb.async_strategy = 'immediate'
|
|
|
|
# dehb.de
|
|
cfg.optimizer.dehb.de = CN()
|
|
cfg.optimizer.dehb.de.pop_size = 20
|
|
|
|
# ---------------------------------------------------------------------- #
|
|
# optuna related options (tpe_md, tpe_hb)
|
|
# ---------------------------------------------------------------------- #
|
|
cfg.optimizer.optuna = CN()
|
|
cfg.optimizer.optuna.reduction_factor = 3
|
|
|
|
|
|
def add_configs(cfg):
|
|
# ---------------------------------------------------------------------- #
|
|
# HPO search space related options, which is fixed when mode is `raw`
|
|
# ---------------------------------------------------------------------- #
|
|
configuration_space, fidelity_space = get_cs(cfg.benchmark.data,
|
|
cfg.benchmark.model,
|
|
cfg.benchmark.type,
|
|
cfg.benchmark.algo)
|
|
|
|
cfg.benchmark.configuration_space = [configuration_space
|
|
] # avoid invalid type
|
|
cfg.benchmark.fidelity_space = [fidelity_space] # avoid invalid type
|
|
|
|
|
|
initial_cfg(fhb_cfg)
|