Skip to content
This repository has been archived by the owner on Mar 10, 2024. It is now read-only.

Commit

Permalink
Network Cleanup (automl#81)
Browse files Browse the repository at this point in the history 
* removed old supported_tasks dictionary from heads, added some docstrings and some small fixes

* removed old supported_tasks attribute and updated doc strings in base backbone and base head components

* removed old supported_tasks attribute from network backbones

* put time series backbones in separate files, add doc strings and refactored search space arguments

* split image networks into separate files, add doc strings and refactor search space

* fix typo

* add an intial simple backbone test similar to the network head test

* fix flake8

* fixed imports in backbones and heads

* added new network backbone and head tests

* enabled tests for adding custom backbones and heads, added required properties to base head and base backbone
  • Loading branch information
@bastiscode @nabenabe0928
bastiscode authored and nabenabe0928 committed Feb 19, 2021
1 parent 085c1d5 commit 8325ea9
Show file tree
Hide file tree
Showing 15 changed files with 832 additions and 555 deletions.
122 changes: 122 additions & 0 deletions autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
from typing import Any, Dict, List, Optional, Tuple

import ConfigSpace as CS
from ConfigSpace.configuration_space import ConfigurationSpace
from ConfigSpace.hyperparameters import (
CategoricalHyperparameter,
UniformIntegerHyperparameter
)

from torch import nn

from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent
from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations


class ConvNetImageBackbone(NetworkBackboneComponent):
"""
Standard Convolutional Neural Network backbone for images
"""

def __init__(self, **kwargs: Any):
super().__init__(**kwargs)
self.bn_args = {"eps": 1e-5, "momentum": 0.1}

def _get_layer_size(self, w: int, h: int) -> Tuple[int, int]:
cw = ((w - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
// self.config["conv_kernel_stride"]) + 1
ch = ((h - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
// self.config["conv_kernel_stride"]) + 1
cw, ch = cw // self.config["pool_size"], ch // self.config["pool_size"]
return cw, ch

def _add_layer(self, layers: List[nn.Module], in_filters: int, out_filters: int) -> None:
layers.append(nn.Conv2d(in_filters, out_filters,
kernel_size=self.config["conv_kernel_size"],
stride=self.config["conv_kernel_stride"],
padding=self.config["conv_kernel_padding"]))
layers.append(nn.BatchNorm2d(out_filters, **self.bn_args))
layers.append(_activations[self.config["activation"]]())
layers.append(nn.MaxPool2d(kernel_size=self.config["pool_size"], stride=self.config["pool_size"]))

def build_backbone(self, input_shape: Tuple[int, ...]) -> nn.Module:
channels, iw, ih = input_shape
layers: List[nn.Module] = []
init_filter = self.config["conv_init_filters"]
self._add_layer(layers, channels, init_filter)

cw, ch = self._get_layer_size(iw, ih)
for i in range(2, self.config["num_layers"] + 1):
cw, ch = self._get_layer_size(cw, ch)
if cw == 0 or ch == 0:
break
self._add_layer(layers, init_filter, init_filter * 2)
init_filter *= 2
backbone = nn.Sequential(*layers)
self.backbone = backbone
return backbone

@staticmethod
def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
return {
'shortname': 'ConvNetImageBackbone',
'name': 'ConvNetImageBackbone',
'handles_tabular': False,
'handles_image': True,
'handles_time_series': False,
}

@staticmethod
def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
num_layers: Tuple[Tuple, int] = ((2, 8), 4),
num_init_filters: Tuple[Tuple, int] = ((16, 64), 32),
activation: Tuple[Tuple, str] = (tuple(_activations.keys()),
list(_activations.keys())[0]),
kernel_size: Tuple[Tuple, int] = ((3, 5), 3),
stride: Tuple[Tuple, int] = ((1, 3), 1),
padding: Tuple[Tuple, int] = ((2, 3), 2),
pool_size: Tuple[Tuple, int] = ((2, 3), 2)
) -> ConfigurationSpace:
cs = CS.ConfigurationSpace()

min_num_layers, max_num_layers = num_layers[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('num_layers',
lower=min_num_layers,
upper=max_num_layers,
default_value=num_layers[1]))

cs.add_hyperparameter(CategoricalHyperparameter('activation',
choices=activation[0],
default_value=activation[1]))

min_init_filters, max_init_filters = num_init_filters[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_init_filters',
lower=min_init_filters,
upper=max_init_filters,
default_value=num_init_filters[1]))

min_kernel_size, max_kernel_size = kernel_size[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_size',
lower=min_kernel_size,
upper=max_kernel_size,
default_value=kernel_size[1]))

min_stride, max_stride = stride[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_stride',
lower=min_stride,
upper=max_stride,
default_value=stride[1]))

min_padding, max_padding = padding[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_padding',
lower=min_padding,
upper=max_padding,
default_value=padding[1]))

min_pool_size, max_pool_size = pool_size[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('pool_size',
lower=min_pool_size,
upper=max_pool_size,
default_value=pool_size[1]))

return cs
149 changes: 35 additions & 114 deletions ...omponents/setup/network_backbone/image.py → .../network_backbone/DenseNetImageBackone.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import logging
import math
from collections import OrderedDict
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, Optional, Tuple

import ConfigSpace as CS
from ConfigSpace.configuration_space import ConfigurationSpace
Expand All @@ -16,103 +15,7 @@
from torch.nn import functional as F

from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent

_activations: Dict[str, nn.Module] = {
"relu": nn.ReLU,
"tanh": nn.Tanh,
"sigmoid": nn.Sigmoid
}


class ConvNetImageBackbone(NetworkBackboneComponent):
supported_tasks = {"image_classification", "image_regression"}

def __init__(self, **kwargs: Any):
super().__init__(**kwargs)
self.bn_args = {"eps": 1e-5, "momentum": 0.1}

def _get_layer_size(self, w: int, h: int) -> Tuple[int, int]:
cw = ((w - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
// self.config["conv_kernel_stride"]) + 1
ch = ((h - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
// self.config["conv_kernel_stride"]) + 1
cw, ch = cw // self.config["pool_size"], ch // self.config["pool_size"]
return cw, ch

def _add_layer(self, layers: List[nn.Module], in_filters: int, out_filters: int) -> None:
layers.append(nn.Conv2d(in_filters, out_filters,
kernel_size=self.config["conv_kernel_size"],
stride=self.config["conv_kernel_stride"],
padding=self.config["conv_kernel_padding"]))
layers.append(nn.BatchNorm2d(out_filters, **self.bn_args))
layers.append(_activations[self.config["activation"]]())
layers.append(nn.MaxPool2d(kernel_size=self.config["pool_size"], stride=self.config["pool_size"]))

def build_backbone(self, input_shape: Tuple[int, ...]) -> nn.Module:
channels, iw, ih = input_shape
layers: List[nn.Module] = []
init_filter = self.config["conv_init_filters"]
self._add_layer(layers, channels, init_filter)

cw, ch = self._get_layer_size(iw, ih)
for i in range(2, self.config["num_layers"] + 1):
cw, ch = self._get_layer_size(cw, ch)
if cw == 0 or ch == 0:
logging.info("> reduce network size due to too small layers.")
break
self._add_layer(layers, init_filter, init_filter * 2)
init_filter *= 2
backbone = nn.Sequential(*layers)
self.backbone = backbone
return backbone

@staticmethod
def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
return {
'shortname': 'ConvNetImageBackbone',
'name': 'ConvNetImageBackbone',
'handles_tabular': False,
'handles_image': True,
'handles_time_series': False,
}

@staticmethod
def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, str]] = None,
min_num_layers: int = 2,
max_num_layers: int = 5,
min_init_filters: int = 16,
max_init_filters: int = 64,
min_kernel_size: int = 2,
max_kernel_size: int = 5,
min_stride: int = 1,
max_stride: int = 3,
min_padding: int = 2,
max_padding: int = 3,
min_pool_size: int = 2,
max_pool_size: int = 3) -> ConfigurationSpace:
cs = CS.ConfigurationSpace()

cs.add_hyperparameter(UniformIntegerHyperparameter('num_layers',
lower=min_num_layers,
upper=max_num_layers))
cs.add_hyperparameter(CategoricalHyperparameter('activation',
choices=list(_activations.keys())))
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_init_filters',
lower=min_init_filters,
upper=max_init_filters))
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_size',
lower=min_kernel_size,
upper=max_kernel_size))
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_stride',
lower=min_stride,
upper=max_stride))
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_padding',
lower=min_padding,
upper=max_padding))
cs.add_hyperparameter(UniformIntegerHyperparameter('pool_size',
lower=min_pool_size,
upper=max_pool_size))
return cs
from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations


class _DenseLayer(nn.Sequential):
Expand Down Expand Up @@ -177,7 +80,9 @@ def __init__(self,


class DenseNetBackbone(NetworkBackboneComponent):
supported_tasks = {"image_classification", "image_regression"}
"""
Dense Net Backbone for images (see https://arxiv.org/pdf/1608.06993.pdf)
"""

def __init__(self, **kwargs: Any):
super().__init__(**kwargs)
Expand Down Expand Up @@ -247,39 +152,55 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[
}

@staticmethod
def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, str]] = None,
min_growth_rate: int = 12,
max_growth_rate: int = 40,
min_num_blocks: int = 3,
max_num_blocks: int = 4,
min_num_layers: int = 4,
max_num_layers: int = 64) -> ConfigurationSpace:
def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
num_blocks: Tuple[Tuple, int] = ((3, 4), 3),
num_layers: Tuple[Tuple, int] = ((4, 64), 16),
growth_rate: Tuple[Tuple, int] = ((12, 40), 20),
activation: Tuple[Tuple, str] = (tuple(_activations.keys()),
list(_activations.keys())[0]),
use_dropout: Tuple[Tuple, bool] = ((True, False), False),
dropout: Tuple[Tuple, float] = ((0, 0.5), 0.2)
) -> ConfigurationSpace:
cs = CS.ConfigurationSpace()

min_growth_rate, max_growth_rate = growth_rate[0]
growth_rate_hp = UniformIntegerHyperparameter('growth_rate',
lower=min_growth_rate,
upper=max_growth_rate)
upper=max_growth_rate,
default_value=growth_rate[1])
cs.add_hyperparameter(growth_rate_hp)

min_num_blocks, max_num_blocks = num_blocks[0]
blocks_hp = UniformIntegerHyperparameter('blocks',
lower=min_num_blocks,
upper=max_num_blocks)
upper=max_num_blocks,
default_value=num_blocks[1])
cs.add_hyperparameter(blocks_hp)

activation_hp = CategoricalHyperparameter('activation',
choices=list(_activations.keys()))
choices=activation[0],
default_value=activation[1])
cs.add_hyperparameter(activation_hp)

use_dropout = CategoricalHyperparameter('use_dropout', choices=[True, False])
use_dropout = CategoricalHyperparameter('use_dropout',
choices=use_dropout[0],
default_value=use_dropout[1])

min_dropout, max_dropout = dropout[0]
dropout = UniformFloatHyperparameter('dropout',
lower=0.0,
upper=1.0)
lower=min_dropout,
upper=max_dropout,
default_value=dropout[1])

cs.add_hyperparameters([use_dropout, dropout])
cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True))

for i in range(1, max_num_blocks + 1):
min_num_layers, max_num_layers = num_layers[0]
layer_hp = UniformIntegerHyperparameter('layer_in_block_%d' % i,
lower=min_num_layers,
upper=max_num_layers)
upper=max_num_layers,
default_value=num_layers[1])
cs.add_hyperparameter(layer_hp)

if i > min_num_blocks:
Expand Down
Loading

0 comments on commit 8325ea9

Please sign in to comment.