diff --git a/.gitignore b/.gitignore index 57bd3c856..20255fcf3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ - # Visual Studio *.vs/* @@ -6,8 +5,9 @@ *.vscode/* # Python -*__pycache__/ +*__pycache__* *.pyc +.ipynb_checkpoints* # Zipped *.tar.gz @@ -24,19 +24,27 @@ results.json outputs/ jobs.txt .pylintrc +*worker_logs* # Build *build/ -*autonet.egg-info +*autoPyTorch.egg-info *.simg - - -# Datasets -/datasets/ +.DS_Store +dist/ # Meta GPU *meta_logs/ +runs.log +runs.log.lock +logs/ # ensemble data predictions_for_ensemble.npy test_predictions_for_ensemble.npy + +# testing +tests.ipynb + +# venv +env/ diff --git a/LICENSE b/LICENSE index 15652a459..58b3499b2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,14 +1,201 @@ -Copyright 2018 AutoML Freiburg -(contributions by Max Dippel, Michael Burkart, Matthias Urban and Marius Lindauer) + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + 1. Definitions. -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2019] The Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index fe5a17df7..27a28271f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # Auto-PyTorch -Copyright (C) 2018 [AutoML Group](http://www.automl.org/) +Copyright (C) 2019 [AutoML Group Freiburg](http://www.automl.org/) This a very early pre-alpha version of our upcoming Auto-PyTorch. -So far, Auto-PyTorch only supports featurized data. +So far, Auto-PyTorch supports featurized data (classification, regression) and image data (classification). ## Installation @@ -33,6 +33,8 @@ $ python setup.py install ## Examples +For a detailed tutorial, please refer to the jupyter notebook in https://github.com/automl/Auto-PyTorch/tree/master/examples/basics. + In a nutshell: ```py @@ -95,7 +97,8 @@ autoPyTorch = AutoNetClassification(networks=["resnet", "shapedresnet", "mlpnet" # Each hyperparameter belongs to a node in Auto-PyTorch's ML Pipeline. # The names of the hyperparameters are prefixed with the name of the node: NodeName:hyperparameter_name. # If a hyperparameter belongs to a component: NodeName:component_name:hyperparameter_name. -autoPyTorch.get_hyperparameter_search_space() +# Call with the same arguments as fit. +autoPyTorch.get_hyperparameter_search_space(X_train, y_train, validation_split=0.3) # You can configure the search space of every hyperparameter of every component: from autoPyTorch import HyperparameterSearchSpaceUpdates @@ -111,7 +114,7 @@ search_space_updates.append(node_name="NetworkSelector", autoPyTorch = AutoNetClassification(hyperparameter_search_space_updates=search_space_updates) ``` -Enable ensemble building: +Enable ensemble building (for featurized data): ```py from autoPyTorch import AutoNetEnsemble @@ -129,15 +132,14 @@ autoPyTorch = AutoNetClassification("tiny_cs", log_level='info', max_runtime=300 ## License This program is free software: you can redistribute it and/or modify -it under the terms of the 3-clause BSD license (please see the LICENSE file). +it under the terms of the Apache license 2.0 (please see the LICENSE file). This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -You should have received a copy of the 3-clause BSD license +You should have received a copy of the Apache license 2.0 along with this program (see LICENSE file). -If not, see . ## Reference @@ -156,6 +158,9 @@ If not, see . } ``` +**Note**: Previously, the name of the project was AutoNet. Since this was too generic, we changed the name to AutoPyTorch. AutoNet 2.0 in the reference mention above is indeed AutoPyTorch. + + ## Contact Auto-PyTorch is developed by the [AutoML Group of the University of Freiburg](http://www.automl.org/). diff --git a/autoPyTorch/__init__.py b/autoPyTorch/__init__.py index c20b924ec..3e5048fdc 100644 --- a/autoPyTorch/__init__.py +++ b/autoPyTorch/__init__.py @@ -2,6 +2,7 @@ hpbandster = os.path.abspath(os.path.join(__file__, '..', '..', 'submodules', 'HpBandSter')) sys.path.append(hpbandster) -from autoPyTorch.core.autonet_classes import AutoNetClassification, AutoNetMultilabel, AutoNetRegression +from autoPyTorch.core.autonet_classes import AutoNetClassification, AutoNetMultilabel, AutoNetRegression, AutoNetImageClassification, AutoNetImageClassificationMultipleDatasets +from autoPyTorch.data_management.data_manager import DataManager from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates from autoPyTorch.core.ensemble import AutoNetEnsemble diff --git a/autoPyTorch/components/ensembles/abstract_ensemble.py b/autoPyTorch/components/ensembles/abstract_ensemble.py index 875490083..bcfde8298 100644 --- a/autoPyTorch/components/ensembles/abstract_ensemble.py +++ b/autoPyTorch/components/ensembles/abstract_ensemble.py @@ -2,6 +2,8 @@ class AbstractEnsemble(object): + """Ensemble interface extracted from auto-sklearn""" + __metaclass__ = ABCMeta @abstractmethod diff --git a/autoPyTorch/components/ensembles/ensemble_selection.py b/autoPyTorch/components/ensembles/ensemble_selection.py index 4e8bcf5e0..bad2dcc9d 100644 --- a/autoPyTorch/components/ensembles/ensemble_selection.py +++ b/autoPyTorch/components/ensembles/ensemble_selection.py @@ -7,12 +7,13 @@ class EnsembleSelection(AbstractEnsemble): - def __init__(self, ensemble_size, metric, minimize, + """Ensemble Selection algorithm extracted from auto-sklearn""" + + def __init__(self, ensemble_size, metric, sorted_initialization_n_best=0, only_consider_n_best=0, bagging=False, mode='fast'): self.ensemble_size = ensemble_size - self.metric = metric - self.minimize = 1 if minimize else -1 + self.metric = metric.get_loss_value self.sorted_initialization_n_best = sorted_initialization_n_best self.only_consider_n_best = only_consider_n_best self.bagging = bagging @@ -56,7 +57,7 @@ def _fast(self, predictions, labels): ensemble.append(predictions[idx]) order.append(idx) ensemble_ = np.array(ensemble).mean(axis=0) - ensemble_performance = self.metric(ensemble_, labels) * self.minimize + ensemble_performance = self.metric(ensemble_, labels) trajectory.append(ensemble_performance) ensemble_size -= self.sorted_initialization_n_best @@ -82,7 +83,7 @@ def _fast(self, predictions, labels): continue fant_ensemble_prediction[:,:] = weighted_ensemble_prediction + \ (1. / float(s + 1)) * pred - scores[j] = self.metric(fant_ensemble_prediction, labels) * self.minimize + scores[j] = self.metric(fant_ensemble_prediction, labels) all_best = np.argwhere(scores == np.nanmin(scores)).flatten() best = np.random.choice(all_best) ensemble.append(predictions[best]) @@ -113,7 +114,7 @@ def _slow(self, predictions, labels): ensemble.append(predictions[idx]) order.append(idx) ensemble_ = np.array(ensemble).mean(axis=0) - ensemble_performance = self.metric(ensemble_, labels) * self.minimize + ensemble_performance = self.metric(ensemble_, labels) trajectory.append(ensemble_performance) ensemble_size -= self.sorted_initialization_n_best @@ -129,7 +130,7 @@ def _slow(self, predictions, labels): continue ensemble.append(pred) ensemble_prediction = np.mean(np.array(ensemble), axis=0) - scores[j] = self.metric(ensemble_prediction, labels) * self.minimize + scores[j] = self.metric(ensemble_prediction, labels) ensemble.pop() best = np.nanargmin(scores) ensemble.append(predictions[best]) @@ -160,7 +161,7 @@ def _sorted_initialization(self, predictions, labels, n_best): perf = np.zeros([predictions.shape[0]]) for idx, prediction in enumerate(predictions): - perf[idx] = self.metric(prediction, labels) * self.minimize + perf[idx] = self.metric(prediction, labels) indices = np.argsort(perf)[:n_best] return indices diff --git a/autoPyTorch/components/lr_scheduler/lr_schedulers.py b/autoPyTorch/components/lr_scheduler/lr_schedulers.py index a3498115c..434f28a51 100644 --- a/autoPyTorch/components/lr_scheduler/lr_schedulers.py +++ b/autoPyTorch/components/lr_scheduler/lr_schedulers.py @@ -6,8 +6,11 @@ from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter, get_hyperparameter +import numpy as np +import math import torch import torch.optim.lr_scheduler as lr_scheduler +from torch.optim import Optimizer import ConfigSpace as CS import ConfigSpace.hyperparameters as CSH @@ -16,9 +19,20 @@ __version__ = "0.0.1" __license__ = "BSD" + class AutoNetLearningRateSchedulerBase(object): - def __new__(cls, params, config): - scheduler = cls._get_scheduler(cls, params, config) + def __new__(cls, optimizer, config): + """Get a new instance of the scheduler + + Arguments: + cls {class} -- Type of scheduler + optimizer {Optmizer} -- A PyTorch Optimizer + config {dict} -- Sampled lr_scheduler config + + Returns: + AutoNetLearningRateSchedulerBase -- The learning rate scheduler object + """ + scheduler = cls._get_scheduler(cls, optimizer, config) if not hasattr(scheduler, "allows_early_stopping"): scheduler.allows_early_stopping = True if not hasattr(scheduler, "snapshot_before_restart"): @@ -32,12 +46,17 @@ def _get_scheduler(self, optimizer, config): def get_config_space(): return CS.ConfigurationSpace() + class SchedulerNone(AutoNetLearningRateSchedulerBase): def _get_scheduler(self, optimizer, config): return NoScheduling(optimizer=optimizer) + class SchedulerStepLR(AutoNetLearningRateSchedulerBase): + """ + Step learning rate scheduler + """ def _get_scheduler(self, optimizer, config): return lr_scheduler.StepLR(optimizer=optimizer, step_size=config['step_size'], gamma=config['gamma'], last_epoch=-1) @@ -52,8 +71,12 @@ def get_config_space( add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'gamma', gamma) return cs + class SchedulerExponentialLR(AutoNetLearningRateSchedulerBase): - + """ + Exponential learning rate scheduler + """ + def _get_scheduler(self, optimizer, config): return lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=config['gamma'], last_epoch=-1) @@ -65,11 +88,17 @@ def get_config_space( add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'gamma', gamma) return cs + class SchedulerReduceLROnPlateau(AutoNetLearningRateSchedulerBase): + """ + Reduce LR on plateau learning rate scheduler + """ def _get_scheduler(self, optimizer, config): - return lr_scheduler.ReduceLROnPlateau(optimizer=optimizer) - + return lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, + factor=config['factor'], + patience=config['patience']) + @staticmethod def get_config_space( factor=(0.05, 0.5), @@ -80,7 +109,112 @@ def get_config_space( add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'patience', patience) return cs + +class SchedulerAdaptiveLR(AutoNetLearningRateSchedulerBase): + """ + Adaptive cosine learning rate scheduler + """ + + def _get_scheduler(self, optimizer, config): + return AdaptiveLR(optimizer=optimizer, + T_max=config['T_max'], + T_mul=config['T_mult'], + patience=config['patience'], + threshold=config['threshold']) + + @staticmethod + def get_config_space( + T_max=(300,1000), + patience=(2,5), + T_mult=(1.0,2.0), + threshold=(0.001, 0.5) + ): + cs = CS.ConfigurationSpace() + add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'T_max', T_max) + add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'patience', patience) + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'T_mult', T_mult) + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'threshold', threshold) + return cs + + +class AdaptiveLR(object): + + def __init__(self, optimizer, mode='min', T_max=30, T_mul=2.0, eta_min=0, patience=3, threshold=0.1, min_lr=0, eps=1e-8, last_epoch=-1): + + if not isinstance(optimizer, Optimizer): + raise TypeError('{} is not an Optimizer'.format( + type(optimizer).__name__)) + + self.optimizer = optimizer + + if last_epoch == -1: + for group in optimizer.param_groups: + group.setdefault('initial_lr', group['lr']) + else: + for i, group in enumerate(optimizer.param_groups): + if 'initial_lr' not in group: + raise KeyError("param 'initial_lr' is not specified " + "in param_groups[{}] when resuming an optimizer".format(i)) + + self.base_lrs = list(map(lambda group: group['initial_lr'], optimizer.param_groups)) + self.last_epoch = last_epoch + + if isinstance(min_lr, list) or isinstance(min_lr, tuple): + if len(min_lr) != len(optimizer.param_groups): + raise ValueError("expected {} min_lrs, got {}".format( + len(optimizer.param_groups), len(min_lr))) + self.min_lrs = list(min_lr) + else: + self.min_lrs = [min_lr] * len(optimizer.param_groups) + + self.T_max = T_max + self.T_mul = T_mul + self.eta_min = eta_min + self.current_base_lrs = self.base_lrs + self.metric_values = [] + self.threshold = threshold + self.patience = patience + self.steps = 0 + + def step(self, metrics, epoch=None): + if epoch is None: + epoch = self.last_epoch + 1 + self.last_epoch = epoch + + self.metric_values.append(metrics) + if len(self.metric_values) > self.patience: + self.metric_values = self.metric_values[1:] + + if max(self.metric_values) - metrics > self.threshold: + self.current_base_lrs = self.get_lr() + self.steps = 0 + else: + self.steps += 1 + + self.last_metric_value = metrics + + for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): + param_group['lr'] = lr + + def get_lr(self): + ''' + Override this method to the existing get_lr() of the parent class + ''' + if self.steps >= self.T_max: + self.T_max = self.T_max * self.T_mul + self.current_base_lrs = self.base_lrs + self.metric_values = [] + self.steps = 0 + + return [self.eta_min + (base_lr - self.eta_min) * + (1 + math.cos(math.pi * self.steps / self.T_max)) / 2 + for base_lr in self.current_base_lrs] + + class SchedulerCyclicLR(AutoNetLearningRateSchedulerBase): + """ + Cyclic learning rate scheduler + """ def _get_scheduler(self, optimizer, config): maf = config['max_factor'] @@ -108,7 +242,11 @@ def get_config_space( add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'cycle_length', cycle_length) return cs + class SchedulerCosineAnnealingWithRestartsLR(AutoNetLearningRateSchedulerBase): + """ + Cosine annealing learning rate scheduler with warm restarts + """ def _get_scheduler(self, optimizer, config): scheduler = CosineAnnealingWithRestartsLR(optimizer, T_max=config['T_max'], T_mult=config['T_mult'],last_epoch=-1) @@ -141,7 +279,6 @@ def get_lr(self): return [None] -import math class CosineAnnealingWithRestartsLR(torch.optim.lr_scheduler._LRScheduler): r"""Copyright: pytorch @@ -195,3 +332,62 @@ def get_lr(self): if self.step_n >= self.restart_every: self.restart() return [self.cosine(base_lr) for base_lr in self.base_lrs] + + def needs_checkpoint(self): + return self.step_n + 1 >= self.restart_every + + +class SchedulerAlternatingCosineLR(AutoNetLearningRateSchedulerBase): + """ + Alternating cosine learning rate scheduler + """ + + def _get_scheduler(self, optimizer, config): + scheduler = AlternatingCosineLR(optimizer, T_max=config['T_max'], T_mul=config['T_mult'], amplitude_reduction=config['amp_reduction'], last_epoch=-1) + return scheduler + + @staticmethod + def get_config_space( + T_max=(1, 20), + T_mult=(1.0, 2.0), + amp_reduction=(0.1,1) + ): + cs = CS.ConfigurationSpace() + add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'T_max', T_max) + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'T_mult', T_mult) + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'amp_reduction', amp_reduction) + return cs + + +class AlternatingCosineLR(torch.optim.lr_scheduler._LRScheduler): + def __init__(self, optimizer, T_max, T_mul=1, amplitude_reduction=0.9, eta_min=0, last_epoch=-1): + ''' + Here last_epoch actually means last_step since the + learning rate is decayed after each batch step. + ''' + + self.T_max = T_max + self.T_mul = T_mul + self.eta_min = eta_min + self.cumulative_time = 0 + self.amplitude_mult = amplitude_reduction + self.base_lr_mult = 1 + self.frequency_mult = 1 + self.time_offset = 0 + self.last_step = 0 + super(AlternatingCosineLR, self).__init__(optimizer, last_epoch) + + def get_lr(self): + ''' + Override this method to the existing get_lr() of the parent class + ''' + if self.last_epoch >= self.T_max: + self.T_max = self.T_max * self.T_mul + self.time_offset = self.T_max / 2 + self.last_epoch = 0 + self.base_lr_mult *= self.amplitude_mult + self.frequency_mult = 2 + self.cumulative_time = 0 + return [self.eta_min + (base_lr * self.base_lr_mult - self.eta_min) * + (1 + math.cos(math.pi * (self.time_offset + self.cumulative_time) / self.T_max * self.frequency_mult)) / 2 + for base_lr in self.base_lrs] diff --git a/autoPyTorch/components/metrics/__init__.py b/autoPyTorch/components/metrics/__init__.py index aee6098dd..fc023c87a 100644 --- a/autoPyTorch/components/metrics/__init__.py +++ b/autoPyTorch/components/metrics/__init__.py @@ -1,3 +1,3 @@ from autoPyTorch.components.metrics.balanced_accuracy import balanced_accuracy from autoPyTorch.components.metrics.pac_score import pac_metric -from autoPyTorch.components.metrics.standard_metrics import accuracy, auc_metric, mean_distance, multilabel_accuracy \ No newline at end of file +from autoPyTorch.components.metrics.standard_metrics import accuracy, auc_metric, mean_distance, multilabel_accuracy, cross_entropy, top1, top3, top5 diff --git a/autoPyTorch/components/metrics/additional_logs.py b/autoPyTorch/components/metrics/additional_logs.py index f082c248e..3e3718dd0 100644 --- a/autoPyTorch/components/metrics/additional_logs.py +++ b/autoPyTorch/components/metrics/additional_logs.py @@ -1,5 +1,6 @@ class test_result(): + """Log the performance on the test set""" def __init__(self, autonet, X_test, Y_test): self.autonet = autonet self.X_test = X_test diff --git a/autoPyTorch/components/metrics/balanced_accuracy.py b/autoPyTorch/components/metrics/balanced_accuracy.py index e15698982..64969fb09 100644 --- a/autoPyTorch/components/metrics/balanced_accuracy.py +++ b/autoPyTorch/components/metrics/balanced_accuracy.py @@ -4,11 +4,9 @@ from sklearn.metrics.classification import _check_targets, type_of_target -def balanced_accuracy(y_pred, y_true): - return _balanced_accuracy(np.argmax(y_true, axis=1), np.argmax(y_pred, axis=1)) * 100 +def balanced_accuracy(solution, prediction): + """balanced accuracy implementation of auto-sklearn""" - -def _balanced_accuracy(solution, prediction): y_type, solution, prediction = _check_targets(solution, prediction) if y_type not in ["binary", "multiclass", 'multilabel-indicator']: @@ -65,4 +63,4 @@ def _balanced_accuracy(solution, prediction): else: raise ValueError(y_type) - return np.mean(bac) # average over all classes \ No newline at end of file + return np.mean(bac) # average over all classes diff --git a/autoPyTorch/components/metrics/pac_score.py b/autoPyTorch/components/metrics/pac_score.py index f3bdb5e99..198ff247c 100644 --- a/autoPyTorch/components/metrics/pac_score.py +++ b/autoPyTorch/components/metrics/pac_score.py @@ -4,11 +4,7 @@ from sklearn.metrics.classification import _check_targets, type_of_target -def pac_metric(y_pred, y_true): - return _pac_score(y_true, y_pred) * 100 - - -def _pac_score(solution, prediction): +def pac_metric(solution, prediction): """ Probabilistic Accuracy based on log_loss metric. We assume the solution is in {0, 1} and prediction in [0, 1]. diff --git a/autoPyTorch/components/metrics/standard_metrics.py b/autoPyTorch/components/metrics/standard_metrics.py index f78c20a68..b87541151 100644 --- a/autoPyTorch/components/metrics/standard_metrics.py +++ b/autoPyTorch/components/metrics/standard_metrics.py @@ -2,22 +2,47 @@ import numpy as np # classification metrics -def accuracy(y_pred, y_true): - return np.mean((undo_ohe(y_true) == undo_ohe(y_pred))) * 100 +def accuracy(y_true, y_pred): + return np.mean(y_true == y_pred) * 100 -def auc_metric(y_pred, y_true): - return (2 * metrics.roc_auc_score(y_true, y_pred) - 1) * 100 +def auc_metric(y_true, y_pred): + return (2 * metrics.roc_auc_score(y_true, y_pred) - 1) +def cross_entropy(y_true, y_pred): + if y_true==1: + return -np.log(y_pred) + else: + return -np.log(1-y_pred) -# multilabel metric -def multilabel_accuracy(y_pred, y_true): - return np.mean(y_true == (y_pred > 0.5)) * 100 +def top1(y_pred, y_true): + return topN(y_pred, y_true, 1) -# regression metric -def mean_distance(y_pred, y_true): - return np.mean(np.abs(y_true - y_pred)) +def top3(y_pred, y_true): + return topN(y_pred, y_true, 3) + +def top5(y_pred, y_true): + if y_pred.shape[1] < 5: + return -1 + return topN(y_pred, y_true, 5) + +def topN(output, target, topk): + """Computes the accuracy over the k top predictions for the specified values of k""" + with torch.no_grad(): + batch_size = target.size(0) + + _, pred = output.topk(topk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + correct_k = correct[:topk].view(-1).float().sum(0, keepdim=True) + return correct_k.mul_(100.0 / batch_size).item() -def undo_ohe(y): - if len(y.shape) == 1: - return(y) - return np.argmax(y, axis=1) \ No newline at end of file + +# multilabel metrics +def multilabel_accuracy(y_true, y_pred): + return np.mean(y_true == (y_pred > 0.5)) + + +# regression metrics +def mean_distance(y_true, y_pred): + return np.mean(np.abs(y_true - y_pred)) diff --git a/autoPyTorch/components/networks/base_net.py b/autoPyTorch/components/networks/base_net.py index 3fcfbc236..154dc9af9 100644 --- a/autoPyTorch/components/networks/base_net.py +++ b/autoPyTorch/components/networks/base_net.py @@ -64,4 +64,18 @@ def __init__(self, config, in_features, out_features, embedding, final_activatio def forward(self, x): x = self.embedding(x) - return super(BaseFeatureNet, self).forward(x) \ No newline at end of file + return super(BaseFeatureNet, self).forward(x) + + +class BaseImageNet(BaseNet): + def __init__(self, config, in_features, out_features, final_activation): + super(BaseImageNet, self).__init__(config, in_features, out_features, final_activation) + + if len(in_features) == 2: + self.channels = 1 + self.iw = in_features[0] + self.ih = in_features[1] + if len(in_features) == 3: + self.channels = in_features[0] + self.iw = in_features[1] + self.ih = in_features[2] diff --git a/autoPyTorch/components/networks/image/__init__.py b/autoPyTorch/components/networks/image/__init__.py new file mode 100644 index 000000000..9a0900524 --- /dev/null +++ b/autoPyTorch/components/networks/image/__init__.py @@ -0,0 +1,4 @@ +from autoPyTorch.components.networks.image.convnet import ConvNet +from autoPyTorch.components.networks.image.densenet import DenseNet +from autoPyTorch.components.networks.image.resnet import ResNet +from autoPyTorch.components.networks.image.mobilenet import MobileNet diff --git a/autoPyTorch/components/networks/image/convnet.py b/autoPyTorch/components/networks/image/convnet.py new file mode 100644 index 000000000..8b3344460 --- /dev/null +++ b/autoPyTorch/components/networks/image/convnet.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Basic Implementation of a convolutional network. +""" + +from __future__ import division, print_function + +import ConfigSpace as CS +import ConfigSpace.hyperparameters as CSH +import torch.nn as nn + +from autoPyTorch.components.networks.base_net import BaseImageNet + +__author__ = "Max Dippel, Michael Burkart and Matthias Urban" +__version__ = "0.0.1" +__license__ = "BSD" + + +class ConvNet(BaseImageNet): + def __init__(self, config, in_features, out_features, final_activation, *args, **kwargs): + super(ConvNet, self).__init__(config, in_features, out_features, final_activation) + self.layers = self._build_net(self.n_classes) + + + def forward(self, x): + x = self.layers(x) + x = x.reshape(x.size(0), -1) + x = self.last_layer(x) + if not self.training and self.final_activation is not None: + x = self.final_activation(x) + return x + + def _build_net(self, out_features): + layers = list() + init_filter = self.config["conv_init_filters"] + self._add_layer(layers, self.channels, init_filter, 1) + + cw, ch = self._get_layer_size(self.iw, self.ih) + self.dense_size = init_filter * cw * ch + print(cw, ch, self.dense_size) + for i in range(2, self.config["num_layers"]+1): + cw, ch = self._get_layer_size(cw, ch) + if cw == 0 or ch == 0: + print("> reduce network size due to too small layers.") + break + self._add_layer(layers, init_filter, init_filter * 2, i) + init_filter *= 2 + self.dense_size = init_filter * cw * ch + print(cw, ch, self.dense_size) + + self.last_layer = nn.Linear(self.dense_size, out_features) + nw = nn.Sequential(*layers) + #print(nw) + return nw + + def _get_layer_size(self, w, h): + cw = ((w - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"]) + //self.config["conv_kernel_stride"]) + 1 + ch = ((h - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"]) + //self.config["conv_kernel_stride"]) + 1 + cw, ch = cw // self.config["pool_size"], ch // self.config["pool_size"] + return cw, ch + + def _add_layer(self, layers, in_filters, out_filters, layer_id): + layers.append(nn.Conv2d(in_filters, out_filters, + kernel_size=self.config["conv_kernel_size"], + stride=self.config["conv_kernel_stride"], + padding=self.config["conv_kernel_padding"])) + layers.append(nn.BatchNorm2d(out_filters)) + layers.append(self.activation()) + layers.append(nn.MaxPool2d(kernel_size=self.config["pool_size"], stride=self.config["pool_size"])) + + @staticmethod + def get_config_space(user_updates=None): + cs = CS.ConfigurationSpace() + + cs.add_hyperparameter(CSH.CategoricalHyperparameter('activation', ['relu'])) #'sigmoid', 'tanh', + num_layers = CSH.UniformIntegerHyperparameter('num_layers', lower=2, upper=5) + cs.add_hyperparameter(num_layers) + cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('conv_init_filters', lower=16, upper=64)) + cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('conv_kernel_size', lower=2, upper=5)) + cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('conv_kernel_stride', lower=1, upper=3)) + cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('conv_kernel_padding', lower=2, upper=3)) + cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('pool_size', lower=2, upper=3)) + + return(cs) diff --git a/examples/basics/__init__.py b/autoPyTorch/components/networks/image/darts/__init__.py similarity index 100% rename from examples/basics/__init__.py rename to autoPyTorch/components/networks/image/darts/__init__.py diff --git a/autoPyTorch/components/networks/image/darts/darts_worker.py b/autoPyTorch/components/networks/image/darts/darts_worker.py new file mode 100644 index 000000000..c4c4b882c --- /dev/null +++ b/autoPyTorch/components/networks/image/darts/darts_worker.py @@ -0,0 +1,114 @@ +import os +import time +import argparse +#from copy import copy, deepcopy + +import ConfigSpace as CS +import ConfigSpace.hyperparameters as CSH +from hpbandster.core.worker import Worker +# from .helper import darts_cifar10 + + +PRIMITIVES = [ + #'none', + 'max_pool_3x3', + 'avg_pool_3x3', + 'skip_connect', + 'sep_conv_3x3', + 'sep_conv_5x5', + 'dil_conv_3x3', + 'dil_conv_5x5', +] + + +class DARTSWorker(Worker): + # def __init__(self, *args, **kwargs): + # super().__init__(*args, **kwargs) + # #self.darts_mainsourcepath = '/home/zelaa/Thesis/bohb-darts/workers/lib' + # self.darts_path = os.getcwd() + '/workers/lib/darts_space' + + # def compute(self, config, budget, config_id, working_directory): + # return darts_cifar10(config=config, + # budget=int(budget), + # config_id=config_id, + # directory=working_directory, + # darts_source=self.darts_path) + + @staticmethod + def get_config_space(): + config_space = CS.ConfigurationSpace() + + # here we instantiate one categorical hyperparameter for each edge in + # the DARTS cell + for i in range(14): + config_space.add_hyperparameter(CSH.CategoricalHyperparameter('edge_normal_{}'.format(i), + PRIMITIVES)) + config_space.add_hyperparameter(CSH.CategoricalHyperparameter('edge_reduce_{}'.format(i), + PRIMITIVES)) + # for the intermediate node 2 we add directly the two incoming edges to + # the config_space. All nodes are topologicaly sorted and the labels 0 + # and 1 correspond to the 2 input nodes of the cell. nodes 2, 3, 4, 5 + # are intermediate nodes. We define below a CategoricalHyperparameter + # for nodes 3, 4, 5 with each category representing two possible + # predecesor nodes indices (for node 2 there is only one possibility) + pred_nodes = {'3': ['0_1', '0_2', '1_2'], + '4': ['0_1', '0_2', '0_3', '1_2', '1_3', '2_3'], + '5': ['0_1', '0_2', '0_3', '0_4', '1_2', '1_3', '1_4', + '2_3', '2_4', '3_4'] + } + + for i in range(3, 6): + config_space.add_hyperparameter(CSH.CategoricalHyperparameter('inputs_node_normal_{}'.format(i), + pred_nodes[str(i)])) + config_space.add_hyperparameter(CSH.CategoricalHyperparameter('inputs_node_reduce_{}'.format(i), + pred_nodes[str(i)])) + + config_space.add_hyperparameter(CSH.Constant('layers', 20)) + config_space.add_hyperparameter(CSH.Constant('init_channels', 36)) + config_space.add_hyperparameter(CSH.Constant('drop_path_prob', 0.1)) + config_space.add_hyperparameter(CSH.CategoricalHyperparameter('auxiliary', [False])) + + # now we define the conditions constraining the inclusion of the edges + # on the optimization in order to be consistent with the DARTS original + # search space + for cell_type in ['normal', 'reduce']: + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_2'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_3'.format(cell_type)), + values=['0_1', '0_2'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_3'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_3'.format(cell_type)), + values=['0_1', '1_2'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_4'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_3'.format(cell_type)), + values=['0_2', '1_2'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_5'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_4'.format(cell_type)), + values=['0_1', '0_2', '0_3'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_6'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_4'.format(cell_type)), + values=['0_1', '1_2', '1_3'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_7'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_4'.format(cell_type)), + values=['0_2', '1_2', '2_3'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_8'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_4'.format(cell_type)), + values=['0_3', '1_3', '2_3'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_9'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_5'.format(cell_type)), + values=['0_1', '0_2', '0_3', '0_4'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_10'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_5'.format(cell_type)), + values=['0_1', '1_2', '1_3', '1_4'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_11'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_5'.format(cell_type)), + values=['0_2', '1_2', '2_3', '2_4'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_12'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_5'.format(cell_type)), + values=['0_3', '1_3', '2_3', '3_4'])) + config_space.add_condition(CS.InCondition(child=config_space.get_hyperparameter('edge_{}_13'.format(cell_type)), + parent=config_space.get_hyperparameter('inputs_node_{}_5'.format(cell_type)), + values=['0_4', '1_4', '2_4', '3_4'])) + + return config_space + + diff --git a/autoPyTorch/components/networks/image/darts/genotypes.py b/autoPyTorch/components/networks/image/darts/genotypes.py new file mode 100644 index 000000000..2a46099fc --- /dev/null +++ b/autoPyTorch/components/networks/image/darts/genotypes.py @@ -0,0 +1,98 @@ +from functools import wraps +from collections import namedtuple +import random +import sys + +Genotype = namedtuple('Genotype', 'normal normal_concat reduce reduce_concat') + +PRIMITIVES = [ + #'none', + 'max_pool_3x3', + 'avg_pool_3x3', + 'skip_connect', + 'sep_conv_3x3', + 'sep_conv_5x5', + 'dil_conv_3x3', + 'dil_conv_5x5' +] + +DARTS = Genotype(normal=[('sep_conv_3x3', 0), ('sep_conv_3x3', 1), + ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), + ('sep_conv_3x3', 1), ('skip_connect', 0), + ('skip_connect', 0), ('dil_conv_3x3', 2)], + normal_concat=[2, 3, 4, 5], reduce=[('max_pool_3x3', 0), + ('max_pool_3x3', 1), + ('skip_connect', 2), + ('max_pool_3x3', 1), + ('max_pool_3x3', 0), + ('skip_connect', 2), + ('skip_connect', 2), + ('max_pool_3x3', 1)], + reduce_concat=[2, 3, 4, 5]) + + +def generate_genotype(gene_function): + @wraps(gene_function) + def wrapper(config=None, steps=4): + concat = range(2, 6) + gene_normal, gene_reduce = gene_function(config, steps).values() + genotype = Genotype( + normal=gene_normal, normal_concat=concat, + reduce=gene_reduce, reduce_concat=concat + ) + return genotype + return wrapper + + +@generate_genotype +def get_gene_from_config(config, steps=4): + gene = {'normal': [], 'reduce': []} + + # node 2 + for cell_type in gene.keys(): + first_edge = (config['edge_{}_0'.format(cell_type)], 0) + second_edge = (config['edge_{}_1'.format(cell_type)], 1) + gene[cell_type].append(first_edge) + gene[cell_type].append(second_edge) + + # nodes 3, 4, 5 + for i, offset in zip(range(3, steps+2), [2, 5, 9]): + for cell_type in gene.keys(): + input_nodes = config['inputs_node_{}_{}'.format(cell_type, i)].split('_') + for node in input_nodes: + edge = (config['edge_{}_{}'.format(cell_type, int(node)+offset)], + int(node)) + gene[cell_type].append(edge) + return gene + + +@generate_genotype +def random_gene(config=None, steps=4): + gene = {'normal': [], 'reduce': []} + + n = 1 + for i in range(steps): + for cell_type in gene.keys(): + first_edge = (random.choice(PRIMITIVES), + random.randint(0, n)) + second_edge = (random.choice(PRIMITIVES), + random.randint(0, n)) + + gene[cell_type].append(first_edge) + gene[cell_type].append(second_edge) + n += 1 + return gene + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print("usage:\n python {} CONFIGS".format(sys.argv[0])) + sys.exit(1) + + with open('genotypes.py', 'a') as f: + _nr_random_genes = sys.argv[1] + for i in range(int(_nr_random_genes)): + gene = random_gene() + f.write('DARTS_%d = %s'%(i, gene)) + f.write('\n') + print(gene) diff --git a/autoPyTorch/components/networks/image/darts/model.py b/autoPyTorch/components/networks/image/darts/model.py new file mode 100644 index 000000000..2fa9d332f --- /dev/null +++ b/autoPyTorch/components/networks/image/darts/model.py @@ -0,0 +1,238 @@ +import torch +import torch.nn as nn +from .operations import * +from .utils import drop_path + + +class Cell(nn.Module): + + def __init__(self, genotype, C_prev_prev, C_prev, C, reduction, reduction_prev): + super(Cell, self).__init__() + # print(C_prev_prev, C_prev, C) + + if reduction_prev: + self.preprocess0 = FactorizedReduce(C_prev_prev, C) + else: + self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1, 1, 0) + self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0) + + if reduction: + op_names, indices = zip(*genotype.reduce) + concat = genotype.reduce_concat + else: + op_names, indices = zip(*genotype.normal) + concat = genotype.normal_concat + self._compile(C, op_names, indices, concat, reduction) + + def _compile(self, C, op_names, indices, concat, reduction): + assert len(op_names) == len(indices) + self._steps = len(op_names) // 2 + self._concat = concat + self.multiplier = len(concat) + + self._ops = nn.ModuleList() + for name, index in zip(op_names, indices): + stride = 2 if reduction and index < 2 else 1 + op = OPS[name](C, stride, True) + self._ops += [op] + self._indices = indices + + def forward(self, s0, s1, drop_prob): + s0 = self.preprocess0(s0) + s1 = self.preprocess1(s1) + + states = [s0, s1] + for i in range(self._steps): + h1 = states[self._indices[2*i]] + h2 = states[self._indices[2*i+1]] + op1 = self._ops[2*i] + op2 = self._ops[2*i+1] + h1 = op1(h1) + h2 = op2(h2) + if self.training and drop_prob > 0.: + if not isinstance(op1, Identity): + h1 = drop_path(h1, drop_prob) + if not isinstance(op2, Identity): + h2 = drop_path(h2, drop_prob) + s = h1 + h2 + states += [s] + return torch.cat([states[i] for i in self._concat], dim=1) + + +class AuxiliaryHeadCIFAR(nn.Module): + + def __init__(self, C, num_classes): + """assuming input size 8x8""" + super(AuxiliaryHeadCIFAR, self).__init__() + self.features = nn.Sequential( + nn.ReLU(inplace=True), + nn.AvgPool2d(5, stride=3, padding=0, count_include_pad=False), # image size = 2 x 2 + nn.Conv2d(C, 128, 1, bias=False), + nn.BatchNorm2d(128), + nn.ReLU(inplace=True), + nn.Conv2d(128, 768, 2, bias=False), + nn.BatchNorm2d(768), + nn.ReLU(inplace=True) + ) + self.classifier = nn.Linear(768, num_classes) + + def forward(self, x): + x = self.features(x) + x = self.classifier(x.view(x.size(0),-1)) + return x + + +class AuxiliaryHeadImageNet(nn.Module): + + def __init__(self, C, num_classes): + """assuming input size 14x14""" + super(AuxiliaryHeadImageNet, self).__init__() + self.features = nn.Sequential( + nn.ReLU(inplace=True), + nn.AvgPool2d(5, stride=2, padding=0, count_include_pad=False), + nn.Conv2d(C, 128, 1, bias=False), + nn.BatchNorm2d(128), + nn.ReLU(inplace=True), + nn.Conv2d(128, 768, 2, bias=False), + # NOTE: This batchnorm was omitted in my earlier implementation due to a typo. + # Commenting it out for consistency with the experiments in the paper. + # nn.BatchNorm2d(768), + nn.ReLU(inplace=True) + ) + self.classifier = nn.Linear(768, num_classes) + + def forward(self, x): + x = self.features(x) + x = self.classifier(x.view(x.size(0),-1)) + return x + + +from autoPyTorch.components.networks.base_net import BaseImageNet +class NetworkCIFAR(BaseImageNet): + + def __init__(self, C, num_classes, layers, auxiliary, genotype): + #super(NetworkCIFAR, self).__init__() + self._layers = layers + self._auxiliary = auxiliary + + stem_multiplier = 3 + C_curr = stem_multiplier*C + self.stem = nn.Sequential( + nn.Conv2d(3, C_curr, 3, padding=1, bias=False), + nn.BatchNorm2d(C_curr) + ) + + C_prev_prev, C_prev, C_curr = C_curr, C_curr, C + self.cells = nn.ModuleList() + reduction_prev = False + for i in range(layers): + if i in [layers//3, 2*layers//3]: + C_curr *= 2 + reduction = True + else: + reduction = False + cell = Cell(genotype, C_prev_prev, C_prev, C_curr, reduction, reduction_prev) + reduction_prev = reduction + self.cells += [cell] + C_prev_prev, C_prev = C_prev, cell.multiplier*C_curr + if i == 2*layers//3: + C_to_auxiliary = C_prev + + if auxiliary: + self.auxiliary_head = AuxiliaryHeadCIFAR(C_to_auxiliary, num_classes) + self.global_pooling = nn.AdaptiveAvgPool2d(1) + self.classifier = nn.Linear(C_prev, num_classes) + + def forward(self, input): + logits_aux = None + s0 = s1 = self.stem(input) + for i, cell in enumerate(self.cells): + s0, s1 = s1, cell(s0, s1, self.drop_path_prob) + if i == 2*self._layers//3: + if self._auxiliary and self.training: + logits_aux = self.auxiliary_head(s1) + out = self.global_pooling(s1) + logits = self.classifier(out.view(out.size(0),-1)) + return logits#, logits_aux + + + +class NetworkImageNet(BaseImageNet): + + def __init__(self, C, num_classes, layers, auxiliary, genotype): + # super(NetworkImageNet, self).__init__() + self._layers = layers + self._auxiliary = auxiliary + + self.stem0 = nn.Sequential( + nn.Conv2d(3, C // 2, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(C // 2), + nn.ReLU(inplace=True), + nn.Conv2d(C // 2, C, 3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(C), + ) + + self.stem1 = nn.Sequential( + nn.ReLU(inplace=True), + nn.Conv2d(C, C, 3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(C), + ) + + C_prev_prev, C_prev, C_curr = C, C, C + + self.cells = nn.ModuleList() + reduction_prev = True + for i in range(layers): + if i in [layers // 3, 2 * layers // 3]: + C_curr *= 2 + reduction = True + else: + reduction = False + cell = Cell(genotype, C_prev_prev, C_prev, C_curr, reduction, reduction_prev) + reduction_prev = reduction + self.cells += [cell] + C_prev_prev, C_prev = C_prev, cell.multiplier * C_curr + if i == 2 * layers // 3: + C_to_auxiliary = C_prev + + if auxiliary: + self.auxiliary_head = AuxiliaryHeadImageNet(C_to_auxiliary, num_classes) + self.global_pooling = nn.AdaptiveAvgPool2d((1, 1)) + self.classifier = nn.Linear(C_prev, num_classes) + + def forward(self, input): + logits_aux = None + s0 = self.stem0(input) + s1 = self.stem1(s0) + for i, cell in enumerate(self.cells): + s0, s1 = s1, cell(s0, s1, self.drop_path_prob) + if i == 2 * self._layers // 3: + if self._auxiliary and self.training: + logits_aux = self.auxiliary_head(s1) + out = self.global_pooling(s1) + logits = self.classifier(out.view(out.size(0), -1)) + return logits#, logits_aux + + +from .genotypes import get_gene_from_config +from .darts_worker import DARTSWorker +class DARTSImageNet(NetworkCIFAR): # use cifar10 base as we train ImageNet mostly with 64x64 images + def __init__(self, config, in_features, out_features, final_activation, **kwargs): + super(NetworkCIFAR, self).__init__(config, in_features, out_features, final_activation) + + self.drop_path_prob = config['drop_path_prob'] + topology = {key: config[key] for key in config if ('edge' in key) or ('inputs_node' in key)} + genotype = get_gene_from_config(topology) + super(DARTSImageNet, self).__init__(config['init_channels'], out_features, config['layers'], config['auxiliary'], genotype) + + def forward(self, x): + x = super(DARTSImageNet, self).forward(x) + + if not self.training and self.final_activation is not None: + x = self.final_activation(x) + return x + + @staticmethod + def get_config_space(**kwargs): + return DARTSWorker.get_config_space() + diff --git a/autoPyTorch/components/networks/image/darts/operations.py b/autoPyTorch/components/networks/image/darts/operations.py new file mode 100644 index 000000000..b0c62c575 --- /dev/null +++ b/autoPyTorch/components/networks/image/darts/operations.py @@ -0,0 +1,105 @@ +import torch +import torch.nn as nn + +OPS = { + 'none' : lambda C, stride, affine: Zero(stride), + 'avg_pool_3x3' : lambda C, stride, affine: nn.AvgPool2d(3, stride=stride, padding=1, count_include_pad=False), + 'max_pool_3x3' : lambda C, stride, affine: nn.MaxPool2d(3, stride=stride, padding=1), + 'skip_connect' : lambda C, stride, affine: Identity() if stride == 1 else FactorizedReduce(C, C, affine=affine), + 'sep_conv_3x3' : lambda C, stride, affine: SepConv(C, C, 3, stride, 1, affine=affine), + 'sep_conv_5x5' : lambda C, stride, affine: SepConv(C, C, 5, stride, 2, affine=affine), + 'sep_conv_7x7' : lambda C, stride, affine: SepConv(C, C, 7, stride, 3, affine=affine), + 'dil_conv_3x3' : lambda C, stride, affine: DilConv(C, C, 3, stride, 2, 2, affine=affine), + 'dil_conv_5x5' : lambda C, stride, affine: DilConv(C, C, 5, stride, 4, 2, affine=affine), + 'conv_7x1_1x7' : lambda C, stride, affine: nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C, C, (1,7), stride=(1, stride), padding=(0, 3), bias=False), + nn.Conv2d(C, C, (7,1), stride=(stride, 1), padding=(3, 0), bias=False), + nn.BatchNorm2d(C, affine=affine) + ), +} + +class ReLUConvBN(nn.Module): + + def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True): + super(ReLUConvBN, self).__init__() + self.op = nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C_in, C_out, kernel_size, stride=stride, padding=padding, bias=False), + nn.BatchNorm2d(C_out, affine=affine) + ) + + def forward(self, x): + return self.op(x) + +class DilConv(nn.Module): + + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine=True): + super(DilConv, self).__init__() + self.op = nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, groups=C_in, bias=False), + nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=False), + nn.BatchNorm2d(C_out, affine=affine), + ) + + def forward(self, x): + return self.op(x) + + +class SepConv(nn.Module): + + def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True): + super(SepConv, self).__init__() + self.op = nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=stride, padding=padding, groups=C_in, bias=False), + nn.Conv2d(C_in, C_in, kernel_size=1, padding=0, bias=False), + nn.BatchNorm2d(C_in, affine=affine), + nn.ReLU(inplace=False), + nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=1, padding=padding, groups=C_in, bias=False), + nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=False), + nn.BatchNorm2d(C_out, affine=affine), + ) + + def forward(self, x): + return self.op(x) + + +class Identity(nn.Module): + + def __init__(self): + super(Identity, self).__init__() + + def forward(self, x): + return x + + +class Zero(nn.Module): + + def __init__(self, stride): + super(Zero, self).__init__() + self.stride = stride + + def forward(self, x): + if self.stride == 1: + return x.mul(0.) + return x[:,:,::self.stride,::self.stride].mul(0.) + + +class FactorizedReduce(nn.Module): + + def __init__(self, C_in, C_out, affine=True): + super(FactorizedReduce, self).__init__() + assert C_out % 2 == 0 + self.relu = nn.ReLU(inplace=False) + self.conv_1 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) + self.conv_2 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) + self.bn = nn.BatchNorm2d(C_out, affine=affine) + + def forward(self, x): + x = self.relu(x) + out = torch.cat([self.conv_1(x), self.conv_2(x[:,:,1:,1:])], dim=1) + out = self.bn(out) + return out + diff --git a/autoPyTorch/components/networks/image/darts/utils.py b/autoPyTorch/components/networks/image/darts/utils.py new file mode 100644 index 000000000..fd5081dbd --- /dev/null +++ b/autoPyTorch/components/networks/image/darts/utils.py @@ -0,0 +1,166 @@ +import os +import numpy as np +import torch +import shutil +import torchvision.transforms as transforms +from torch.autograd import Variable + + +class AvgrageMeter(object): + + def __init__(self): + self.reset() + + def reset(self): + self.avg = 0 + self.sum = 0 + self.cnt = 0 + + def update(self, val, n=1): + self.sum += val * n + self.cnt += n + self.avg = self.sum / self.cnt + + +def accuracy(output, target, topk=(1,)): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0/batch_size)) + return res + + +class Cutout(object): + def __init__(self, length): + self.length = length + + def __call__(self, img): + h, w = img.size(1), img.size(2) + mask = np.ones((h, w), np.float32) + y = np.random.randint(h) + x = np.random.randint(w) + + y1 = np.clip(y - self.length // 2, 0, h) + y2 = np.clip(y + self.length // 2, 0, h) + x1 = np.clip(x - self.length // 2, 0, w) + x2 = np.clip(x + self.length // 2, 0, w) + + mask[y1: y2, x1: x2] = 0. + mask = torch.from_numpy(mask) + mask = mask.expand_as(img) + img *= mask + return img + + +def _data_transforms_cifar10(args): + CIFAR_MEAN = [0.49139968, 0.48215827, 0.44653124] + CIFAR_STD = [0.24703233, 0.24348505, 0.26158768] + + train_transform = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(CIFAR_MEAN, CIFAR_STD), + ]) + if args.cutout: + train_transform.transforms.append(Cutout(args.cutout_length)) + + valid_transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(CIFAR_MEAN, CIFAR_STD), + ]) + return train_transform, valid_transform + + +def count_parameters_in_MB(model): + return np.sum(np.prod(v.size()) for v in model.parameters())/1e6 + + +def save_checkpoint(state, is_best, save): + filename = os.path.join(save, 'checkpoint.pth.tar') + torch.save(state, filename) + if is_best: + best_filename = os.path.join(save, 'model_best.pth.tar') + shutil.copyfile(filename, best_filename) + + +def save(model, model_path): + torch.save(model.state_dict(), model_path) + + +def load(model, model_path, genotype): + pretrained_dict = torch.load(model_path) + model_dict = model.state_dict() + + # keep only the weights for the specified genotype, + # and prune all the other weights from the MixedOps + #pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict} + + edge_dict = {(0,2): 0, (0,3): 2, (0,4): 5, (0,5): 9, (1,2): 1, (1,3): 3, (1,4): 6, (1,5): 10, (2,3): 4, (2,4): 7, (3,4): 8, (2,5): 11, (3,5): 12, (4,5): 13} + + for layer in range(8): + first_number = layer + + for p in range(2): + if layer in [3, 6] and p == 0: + key = 'cells.{}.preprocess{}.conv_1.weight'.format(layer, p) + key = 'cells.{}.preprocess{}.conv_2.weight'.format(layer, p) + else: + key = 'cells.{}.preprocess{}.op.1.weight'.format(layer, p) + model_dict[key] = pretrained_dict[key] + + if layer in [2, 5]: + gene = genotype.reduce + else: + gene = genotype.normal + + for i in range(4): + for k in [2*i, 2*i + 1]: + op, j = gene[k] + second_number = edge_dict[(j, i + 2)] + if op == 'sep_conv_3x3': + third_number = 4 + for h in [1, 2, 5, 6]: + key_model = 'cells.{}._ops.{}.op.{}.weight'.format(layer, k, h) + key_pretrained = 'cells.{}._ops.{}._ops.{}.op.{}.weight'.format(first_number, second_number, third_number, h) + model_dict[key_model] = pretrained_dict[key_pretrained] + elif op == 'max_pool_3x3': + third_number = 1 + elif op == 'avg_pool_3x3': + third_number = 2 + + model.load_state_dict(model_dict) + + +def drop_path(x, drop_prob): + if drop_prob > 0.: + keep_prob = 1.-drop_prob + try: + mask = Variable(torch.cuda.FloatTensor(x.size(0), 1, 1, 1).bernoulli_(keep_prob)) + except: + mask = Variable(torch.FloatTensor(x.size(0), 1, 1, 1).bernoulli_(keep_prob)) + x.div_(keep_prob) + x.mul_(mask) + return x + + +def create_exp_dir(path, scripts_to_save=None): + import time, random + time.sleep(random.uniform(1, 2)) + if not os.path.exists(path): + os.mkdir(path) + print('Experiment dir : {}'.format(path)) + + if scripts_to_save is not None: + os.mkdir(os.path.join(path, 'scripts')) + for script in scripts_to_save: + dst_file = os.path.join(path, 'scripts', os.path.basename(script)) + shutil.copyfile(script, dst_file) + diff --git a/autoPyTorch/components/networks/image/densenet.py b/autoPyTorch/components/networks/image/densenet.py new file mode 100644 index 000000000..2931be7c0 --- /dev/null +++ b/autoPyTorch/components/networks/image/densenet.py @@ -0,0 +1,173 @@ +import re +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.model_zoo as model_zoo +from collections import OrderedDict + +import ConfigSpace +from autoPyTorch.components.networks.base_net import BaseImageNet +from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter, get_hyperparameter + +from autoPyTorch.components.networks.base_net import BaseImageNet + +class _DenseLayer(nn.Sequential): + def __init__(self, num_input_features, growth_rate, bn_size, drop_rate): + super(_DenseLayer, self).__init__() + self.add_module('norm1', nn.BatchNorm2d(num_input_features)), + self.add_module('relu1', nn.ReLU(inplace=True)), + self.add_module('conv1', nn.Conv2d(num_input_features, bn_size * + growth_rate, kernel_size=1, stride=1, bias=False)), + self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)), + self.add_module('relu2', nn.ReLU(inplace=True)), + self.add_module('conv2', nn.Conv2d(bn_size * growth_rate, growth_rate, + kernel_size=3, stride=1, padding=1, bias=False)), + self.drop_rate = drop_rate + + def forward(self, x): + new_features = super(_DenseLayer, self).forward(x) + if self.drop_rate > 0: + new_features = F.dropout(new_features, p=self.drop_rate, training=self.training) + return torch.cat([x, new_features], 1) + + +class _DenseBlock(nn.Sequential): + def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate): + super(_DenseBlock, self).__init__() + for i in range(num_layers): + layer = _DenseLayer(num_input_features + i * growth_rate, growth_rate, bn_size, drop_rate) + self.add_module('denselayer%d' % (i + 1), layer) + + +class _Transition(nn.Sequential): + def __init__(self, num_input_features, num_output_features, pool_size): + super(_Transition, self).__init__() + self.add_module('norm', nn.BatchNorm2d(num_input_features)) + self.add_module('relu', nn.ReLU(inplace=True)) + self.add_module('conv', nn.Conv2d(num_input_features, num_output_features, + kernel_size=1, stride=1, bias=False)) + self.add_module('pool', nn.AvgPool2d(kernel_size=pool_size, stride=pool_size)) + + +class DenseNet(BaseImageNet): + r"""Densenet-BC model class, based on + `"Densely Connected Convolutional Networks" `_ + Args: + growth_rate (int) - how many filters to add each layer (`k` in paper) + block_config (list of 4 ints) - how many layers in each pooling block + num_init_features (int) - the number of filters to learn in the first convolution layer + bn_size (int) - multiplicative factor for number of bottle neck layers + (i.e. bn_size * k features in the bottleneck layer) + drop_rate (float) - dropout rate after each dense layer + num_classes (int) - number of classification classes + """ + + def __init__(self, config, in_features, out_features, final_activation, *args, **kwargs): + + super(DenseNet, self).__init__(config, in_features, out_features, final_activation) + + growth_rate = config['growth_rate'] + block_config=[config['layer_in_block_%d' % (i+1)] for i in range(config['blocks'])] + num_init_features = 2 * growth_rate + bn_size = 4 + drop_rate = config['dropout'] if config['use_dropout'] else 0 + num_classes = self.n_classes + + image_size, min_image_size = min(self.iw, self.ih), 1 + + import math + division_steps = math.floor(math.log2(image_size) - math.log2(min_image_size) - 1e-5) + 1 + + if division_steps > len(block_config) + 1: + # First convolution + self.features = nn.Sequential(OrderedDict([ + ('conv0', nn.Conv2d(self.channels, num_init_features, kernel_size=7, stride=2, padding=3, bias=False)), + ('norm0', nn.BatchNorm2d(num_init_features)), + ('relu0', nn.ReLU(inplace=True)), + ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)), + ])) + division_steps -= 2 + else: + self.features = nn.Sequential(OrderedDict([ + ('conv0', nn.Conv2d(self.channels, num_init_features, kernel_size=3, stride=1, padding=1, bias=False)) + ])) + + + # Each denseblock + num_features = num_init_features + for i, num_layers in enumerate(block_config): + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, + bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) + self.features.add_module('denseblock%d' % (i + 1), block) + num_features = num_features + num_layers * growth_rate + if i != len(block_config) - 1: + trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2, pool_size=2 if i > len(block_config) - division_steps else 1) + self.features.add_module('transition%d' % (i + 1), trans) + num_features = num_features // 2 + + # Final batch norm + self.features.add_module('last_norm', nn.BatchNorm2d(num_features)) + + # Linear layer + self.classifier = nn.Linear(num_features, num_classes) + + # Official init from torch repo. + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + nn.init.constant_(m.bias, 0) + + self.layers = nn.Sequential(self.features) + + def forward(self, x): + features = self.features(x) + out = F.relu(features, inplace=True) + out = F.adaptive_avg_pool2d(out, (1, 1)).view(features.size(0), -1) + out = self.classifier(out) + + if not self.training and self.final_activation is not None: + out = self.final_activation(out) + return out + + @staticmethod + def get_config_space(growth_rate_range=(12, 40), nr_blocks=(3, 4), layer_range=([1, 12], [6, 24], [12, 64], [12, 64]), num_init_features=(32, 128), **kwargs): + + import ConfigSpace as CS + import ConfigSpace.hyperparameters as CSH + from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter + + cs = CS.ConfigurationSpace() + growth_rate_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'growth_rate', growth_rate_range) + cs.add_hyperparameter(growth_rate_hp) + # add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'bn_size', [2, 4]) + # add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'num_init_features', num_init_features, log=True) + # add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'bottleneck', [True, False]) + + blocks_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'blocks', nr_blocks) + cs.add_hyperparameter(blocks_hp) + use_dropout = add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'use_dropout', [True, False]) + dropout = add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'dropout', [0.0, 1.0]) + cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True)) + + if type(nr_blocks[0]) == int: + min_blocks = nr_blocks[0] + max_blocks = nr_blocks[1] + else: + min_blocks = nr_blocks[0][0] + max_blocks = nr_blocks[0][1] + + for i in range(1, max_blocks+1): + layer_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'layer_in_block_%d' % i, layer_range[i-1]) + cs.add_hyperparameter(layer_hp) + + if i > min_blocks: + cs.add_condition(CS.GreaterThanCondition(layer_hp, blocks_hp, i-1)) + + return cs + + + diff --git a/autoPyTorch/components/networks/image/densenet_flexible.py b/autoPyTorch/components/networks/image/densenet_flexible.py new file mode 100644 index 000000000..ab766aac5 --- /dev/null +++ b/autoPyTorch/components/networks/image/densenet_flexible.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Implementation of a Dense Net for image data. +""" + +import torch +import torch.nn as nn +import math + +import ConfigSpace +from autoPyTorch.components.networks.base_net import BaseImageNet +from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter, get_hyperparameter + +import inspect +from autoPyTorch.components.networks.base_net import BaseImageNet +from autoPyTorch.utils.modules import Reshape +from autoPyTorch.components.networks.activations import all_activations, get_activation +from .utils.utils import get_layer_params + +# https://github.com/liuzhuang13/DenseNet + +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + +import logging +logger = logging.getLogger('autonet') + +class PrintNode(nn.Module): + def __init__(self, msg): + super(PrintNode, self).__init__() + self.msg = msg + + def forward(self, x): + logger.debug(self.msg) + return x + + +class _DenseLayer(nn.Sequential): + def __init__(self, nChannels, growth_rate, drop_rate, bottleneck, kernel_size, activation): + super(_DenseLayer, self).__init__() + # self.add_module('p_layer1', PrintNode("layer1")) + self.add_module('norm1', nn.BatchNorm2d(nChannels)) + self.add_module('relu1', get_activation(activation, inplace=True)) + if bottleneck: + self.add_module('conv1', nn.Conv2d(nChannels, 4 * growth_rate, kernel_size=1, stride=1, bias=False)) + nChannels = 4 * growth_rate + if drop_rate > 0: + self.add_module('drop', nn.Dropout2d(p=drop_rate, inplace=True)) + # self.add_module('p_layer2', PrintNode("layer2")) + self.add_module('norm2', nn.BatchNorm2d(nChannels)) + self.add_module('relu2', get_activation(activation, inplace=True)) + self.add_module('conv2', nn.Conv2d(nChannels, growth_rate, kernel_size=kernel_size, stride=1, padding=int((kernel_size-1)/2), bias=False)) + if drop_rate > 0: + self.add_module('drop', nn.Dropout2d(p=drop_rate, inplace=True)) + + def forward(self, x): + new_features = super(_DenseLayer, self).forward(x) + # logger.debug('concat ' + str(x.shape) + ' and ' + str(new_features.shape)) + return torch.cat([x, new_features], 1) + + +class _DenseBlock(nn.Sequential): + def __init__(self, N, nChannels, growth_rate, drop_rate, bottleneck, kernel_size, activation): + super(_DenseBlock, self).__init__() + for i in range(N): + self.add_module('denselayer%d' % (i + 1), _DenseLayer(nChannels, growth_rate, drop_rate, bottleneck, kernel_size, activation)) + nChannels += growth_rate + + + +class _Transition(nn.Sequential): + def __init__(self, nChannels, nOutChannels, drop_rate, last, pool_size, kernel_size, stride, padding, activation): + super(_Transition, self).__init__() + # self.add_module('p_transition', PrintNode("transition")) + self.add_module('norm', nn.BatchNorm2d(nChannels)) + self.add_module('relu', get_activation(activation, inplace=True)) + # self.add_module('p_last', PrintNode("last transition " + str(last))) + if last: + self.add_module('pool', nn.AvgPool2d(kernel_size=pool_size, stride=pool_size)) + self.add_module('reshape', Reshape(nChannels)) + else: + self.add_module('conv', nn.Conv2d(nChannels, nOutChannels, kernel_size=1, stride=1, bias=False)) + if drop_rate > 0: + self.add_module('drop', nn.Dropout2d(p=drop_rate, inplace=True)) + self.add_module('pool', nn.AvgPool2d(kernel_size=kernel_size, stride=stride, padding=padding)) + + +class DenseNetFlexible(BaseImageNet): + + def __init__(self, config, in_features, out_features, final_activation, *args, **kwargs): + + super(DenseNetFlexible, self).__init__(config, in_features, out_features, final_activation) + + growth_rate=config['growth_rate'] + bottleneck=config['bottleneck'] + channel_reduction=config['channel_reduction'] + + in_size = min(self.iw, self.ih) + out_size = max(1, in_size * config['last_image_size']) + size_reduction = math.pow(in_size / out_size, 1 / (config['blocks'] + 1)) + + nChannels= 2 * growth_rate + + self.features = nn.Sequential() + + sizes = [max(1, round(in_size / math.pow(size_reduction, i+1))) for i in range(config['blocks'] + 2)] + + in_size, kernel_size, stride, padding = get_layer_params(in_size, sizes[0], config['first_conv_kernel']) + self.features.add_module('conv0', nn.Conv2d(self.channels, nChannels, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)) + self.features.add_module('norm0', nn.BatchNorm2d(nChannels)) + self.features.add_module('activ0', get_activation(config['first_activation'], inplace=True)) + + in_size, kernel_size, stride, padding = get_layer_params(in_size, sizes[1], config['first_pool_kernel']) + self.features.add_module('pool0', nn.MaxPool2d(kernel_size=kernel_size, stride=stride, padding=padding)) + # print(in_size) + + nOutChannels = nChannels + # Each denseblock + for i in range(1, config['blocks']+1): + nChannels = nOutChannels + + drop_rate = config['dropout_%d' % i] if config['use_dropout'] else 0 + + block = _DenseBlock(N=config['layer_in_block_%d' % i], nChannels=nChannels, bottleneck=bottleneck, + growth_rate=growth_rate, drop_rate=drop_rate, kernel_size=config['conv_kernel_%d' % i], + activation=config['activation_%d' % i]) + + self.features.add_module('denseblock%d' % i, block) + nChannels = nChannels + config['layer_in_block_%d' % i] * growth_rate + nOutChannels = max(1, math.floor(nChannels * channel_reduction)) + + out_size, kernel_size, stride, padding = get_layer_params(in_size, sizes[i+1], config['pool_kernel_%d' % i]) + transition = _Transition( nChannels=nChannels, nOutChannels=nOutChannels, + drop_rate=drop_rate, last=(i == config['blocks']), + pool_size=in_size, # only used in last transition -> reduce to '1x1 image' + kernel_size=kernel_size, stride=stride, padding=padding, + activation=config['activation_%d' % i]) + in_size = out_size + + self.features.add_module('trans%d' % i, transition) + + # Linear layer + self.classifier = nn.Linear(nChannels, out_features) + + # Official init from torch repo. + for m in self.modules(): + if isinstance(m, nn.Conv2d): + self.matrix_init(m.weight, config['conv_init']) + elif isinstance(m, nn.BatchNorm2d): + self.matrix_init(m.weight, config['batchnorm_weight_init']) + self.matrix_init(m.bias, config['batchnorm_bias_init']) + elif isinstance(m, nn.Linear): + self.matrix_init(m.bias, config['linear_bias_init']) + + # logger.debug(print(self)) + + self.layers = nn.Sequential(self.features) + + def matrix_init(self, matrix, init_type): + if init_type == 'kaiming_normal': + nn.init.kaiming_normal_(matrix) + elif init_type == 'constant_0': + nn.init.constant_(matrix, 0) + elif init_type == 'constant_1': + nn.init.constant_(matrix, 1) + elif init_type == 'constant_05': + nn.init.constant_(matrix, 0.5) + elif init_type == 'random': + return + else: + raise ValueError('Init type ' + init_type + ' is not supported') + + + def forward(self, x): + out = self.features(x) + out = self.classifier(out) + if not self.training and self.final_activation is not None: + out = self.final_activation(out) + return out + + @staticmethod + def get_config_space( growth_rate_range=(5, 128), nr_blocks=(1, 5), kernel_range=(2, 7), + layer_range=(5, 50), activations=all_activations.keys(), + conv_init=('random', 'kaiming_normal', 'constant_0', 'constant_1', 'constant_05'), + batchnorm_weight_init=('random', 'constant_0', 'constant_1', 'constant_05'), + batchnorm_bias_init=('random', 'constant_0', 'constant_1', 'constant_05'), + linear_bias_init=('random', 'constant_0', 'constant_1', 'constant_05'), **kwargs): + + import ConfigSpace as CS + import ConfigSpace.hyperparameters as CSH + from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter + + cs = CS.ConfigurationSpace() + growth_rate_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'growth_rate', growth_rate_range) + first_conv_kernel_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'first_conv_kernel', kernel_range) + first_pool_kernel_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'first_pool_kernel', kernel_range) + conv_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'conv_init', conv_init) + batchnorm_weight_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'batchnorm_weight_init', batchnorm_weight_init) + batchnorm_bias_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'batchnorm_bias_init', batchnorm_bias_init) + linear_bias_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'linear_bias_init', linear_bias_init) + first_activation_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'first_activation', set(activations).intersection(all_activations)) + blocks_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'blocks', nr_blocks) + + cs.add_hyperparameter(growth_rate_hp) + cs.add_hyperparameter(first_conv_kernel_hp) + cs.add_hyperparameter(first_pool_kernel_hp) + cs.add_hyperparameter(conv_init_hp) + cs.add_hyperparameter(batchnorm_weight_init_hp) + cs.add_hyperparameter(batchnorm_bias_init_hp) + cs.add_hyperparameter(linear_bias_init_hp) + cs.add_hyperparameter(first_activation_hp) + cs.add_hyperparameter(blocks_hp) + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'channel_reduction', [0.1, 0.9]) + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'last_image_size', [0, 1]) + add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'bottleneck', [True, False]) + use_dropout = add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'use_dropout', [True, False]) + + if type(nr_blocks[0]) == int: + min_blocks = nr_blocks[0] + max_blocks = nr_blocks[1] + else: + min_blocks = nr_blocks[0][0] + max_blocks = nr_blocks[0][1] + + for i in range(1, max_blocks+1): + layer_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'layer_in_block_%d' % i, layer_range) + pool_kernel_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'pool_kernel_%d' % i, kernel_range) + activation_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'activation_%d' % i, set(activations).intersection(all_activations)) + cs.add_hyperparameter(layer_hp) + cs.add_hyperparameter(pool_kernel_hp) + cs.add_hyperparameter(activation_hp) + dropout = add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'dropout_%d' % i, [0.0, 1.0]) + conv_kernel = add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'conv_kernel_%d' % i, [3, 5, 7]) + + + if i > min_blocks: + cs.add_condition(CS.GreaterThanCondition(layer_hp, blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(conv_kernel, blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(pool_kernel_hp, blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(activation_hp, blocks_hp, i-1)) + cs.add_condition(CS.AndConjunction(CS.EqualsCondition(dropout, use_dropout, True), CS.GreaterThanCondition(dropout, blocks_hp, i-1))) + else: + cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True)) + + return cs diff --git a/autoPyTorch/components/networks/image/mobilenet.py b/autoPyTorch/components/networks/image/mobilenet.py new file mode 100644 index 000000000..a2190b1a3 --- /dev/null +++ b/autoPyTorch/components/networks/image/mobilenet.py @@ -0,0 +1,258 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +import math + +import ConfigSpace +from autoPyTorch.components.networks.base_net import BaseImageNet +from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter, get_hyperparameter + +from torch.autograd import Variable +from autoPyTorch.components.networks.base_net import BaseImageNet + +from .utils.mobilenet_utils import GenEfficientNet, _decode_arch_def, _resolve_bn_args, _round_channels, swish, sigmoid, hard_swish, hard_sigmoid, SelectAdaptivePool2d + +# TODO +# EXPANSION RATIO (currenty hardcoded) +# ACTIVATION? (currently swish) + +class Arch_Encoder(): + """ Encode block definition string + Encodes a list of config space (dicts) through a string notation of arguments for further usage with _decode_architecure and timm. + E.g. ir_r2_k3_s2_e1_i32_o16_se0.25_noskip + + leading string - block type ( + ir = InvertedResidual, ds = DepthwiseSep, dsa = DeptwhiseSep with pw act, cn = ConvBnAct) + r - number of repeat blocks, + k - kernel size, + s - strides (1-9), + e - expansion ratio, + c - output channels, + se - squeeze/excitation ratio + n - activation fn ('re', 'r6', 'hs', or 'sw') + Args: + block hyperpar dict as coming from MobileNet class + Returns: + Architecture encoded as string for further usage with _decode_architecure and timm. + """ + + def __init__(self, block_types, nr_sub_blocks, kernel_sizes, strides, output_filters, se_ratios, skip_connections, expansion_rates=0): + self.block_types = block_types + self.nr_sub_blocks = nr_sub_blocks + self.kernel_sizes = kernel_sizes + self.strides = strides + self.expansion_rates = expansion_rates + self.output_filters = output_filters + self.se_ratios = se_ratios + self.skip_connections = skip_connections + + self.arch_encoded = [[""] for ind in range(len(self.block_types))] + self._encode_architecture() + + def _encode_architecture(self): + encoding_functions = [self._get_encoded_blocks, self._get_encoded_nr_sub_bocks, self._get_encoded_kernel_sizes, self._get_encoded_strides, + self._get_encoded_expansion_rates , self._get_encoded_output_filters, self._get_encoded_se_ratios, self._get_encoded_skip_connections] + + for func in encoding_functions: + return_val = func() + self._add_specifications(return_val) + + def _add_specifications(self, arguments): + for ind, arg in enumerate(arguments): + if len(self.arch_encoded[ind][0])!=0 and arg!="" and not self.arch_encoded[ind][0].endswith("_") : + self.arch_encoded[ind][0] = self.arch_encoded[ind][0] + "_" + self.arch_encoded[ind][0] = self.arch_encoded[ind][0] + arg + + def _get_encoded_blocks(self): + block_type_dict = {"inverted_residual":"ir", "dwise_sep_conv":"ds", "conv_bn_act":"cn"} + block_type_list = self._dict_to_list(self.block_types) + return [block_type_dict[item] for item in block_type_list] + + def _get_encoded_nr_sub_bocks(self): + nr_sub_blocks_dict = dict([(i, "r"+str(i)) for i in range(10)]) + nr_sub_blocks_list = self._dict_to_list(self.nr_sub_blocks) + return [nr_sub_blocks_dict[item] for item in nr_sub_blocks_list] + + def _get_encoded_kernel_sizes(self): + kernel_sizes_dict = dict([(i, "k"+str(i)) for i in range(10)]) + kernel_sizes_list = self._dict_to_list(self.kernel_sizes) + return [kernel_sizes_dict[item] for item in kernel_sizes_list] + + def _get_encoded_strides(self): + strides_dict = dict([(i, "s"+str(i)) for i in range(10)]) + strides_list = self._dict_to_list(self.strides) + return [strides_dict[item] for item in strides_list] + + def _get_encoded_expansion_rates(self): + if self.expansion_rates == 0: + exp_list = ["e1","e6","e6","e6","e6","e6","e6"] + return exp_list[0:len(self.block_types)] + else: + expansion_rates_dict = dict([(i, "e"+str(i)) for i in range(10)]) + expansion_rates_list = self._dict_to_list(self.expansion_rates) + return [expansion_rates_dict[item] for item in expansion_rates_list] + + def _get_encoded_output_filters(self): + output_filters_dict = dict([(i, "c"+str(i)) for i in range(5000)]) + output_filters_list = self._dict_to_list(self.output_filters) + return [output_filters_dict[item] for item in output_filters_list] + + def _get_encoded_se_ratios(self): + se_ratios_dict = {0:"", 0.25:"se0.25"} + se_ratios_list = self._dict_to_list(self.se_ratios) + return [se_ratios_dict[item] for item in se_ratios_list] + + def _get_encoded_skip_connections(self): + skip_connections_dict = {True : "", False: "no_skip"} + skip_connections_list = self._dict_to_list(self.skip_connections) + return [skip_connections_dict[item] for item in skip_connections_list] + + def _dict_to_list(self, input_dict): + output_list = [] + dict_len = len(input_dict) + for ind in range(dict_len): + output_list.append(input_dict["Group_" + str(ind+1)]) + return output_list + + def get_encoded_architecture(self): + return self.arch_encoded + + +class MobileNet(BaseImageNet): + """ + Implements a search space as in MnasNet (https://arxiv.org/abs/1807.11626) using inverted residuals. + """ + def __init__(self, config, in_features, out_features, final_activation, **kwargs): + super(MobileNet, self).__init__(config, in_features, out_features, final_activation) + + # Initialize hyperpars for architecture + nn.Module.config = config + self.final_activation = final_activation + self.nr_main_blocks = config['nr_main_blocks'] + self.initial_filters = config['initial_filters'] + + + self.nr_sub_blocks = dict([ + ('Group_%d' % (i+1), config['nr_sub_blocks_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + self.op_types = dict([ + ('Group_%d' % (i+1), config['op_type_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + self.kernel_sizes = dict([ + ('Group_%d' % (i+1), config['kernel_size_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + self.strides = dict([ + ('Group_%d' % (i+1), config['stride_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + self.output_filters = dict([ + ('Group_%d' % (i+1), config['out_filters_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + self.skip_cons = dict([ + ('Group_%d' % (i+1), config['skip_con_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + self.se_ratios = dict([ + ('Group_%d' % (i+1), config['se_ratio_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + + ########## Create model + encoder = Arch_Encoder(block_types=self.op_types, + nr_sub_blocks=self.nr_sub_blocks, + kernel_sizes=self.kernel_sizes, + strides=self.strides, + expansion_rates=0, + output_filters=self.output_filters, + se_ratios=self.se_ratios, + skip_connections=self.skip_cons) + arch_enc = encoder.get_encoded_architecture() + + kwargs["bn_momentum"] = 0.01 + + self.model = GenEfficientNet(_decode_arch_def(arch_enc, depth_multiplier=1.0), + num_classes=out_features, + stem_size=self.initial_filters, + channel_multiplier=1.0, + num_features=_round_channels(1280, 1.0, 8, None), + bn_args=_resolve_bn_args(kwargs), + act_fn=swish, + drop_connect_rate=0.2, + drop_rate=0.2, + **kwargs) + + def _cfg(url='', **kwargs): + return {'url': url, 'num_classes': 1000, 'input_size': (3, 224, 224), 'pool_size': (7, 7), + 'crop_pct': 0.875, 'interpolation': 'bicubic', + 'mean': (0.485, 0.456, 0.406), 'std': (0.229, 0.224, 0.225), + 'first_conv': 'conv_stem', 'classifier': 'classifier', **kwargs} + + self.model.default_cfg = _cfg(url='', input_size=in_features, pool_size=(10, 10), crop_pct=0.904, num_classes=out_features) + + def forward(self, x): + # make sure channels first + x = self.model(x) + if not self.training and self.final_activation is not None: + x = self.final_activation(x) + return x + + @staticmethod + def get_config_space( nr_main_blocks=[3, 7], initial_filters=([8, 32], True), nr_sub_blocks=([1, 4], False), + op_types = ["inverted_residual", "dwise_sep_conv"], kernel_sizes=[3, 5], strides=[1,2], + output_filters = [[12, 16, 20], + [18, 24, 30], + [24, 32, 40], + [48, 64, 80], + [72, 96, 120], + [120, 160, 200], + [240, 320, 400]], # the idea is to search for e.g. 0.75, 1, 1.25* output_filters(mainblock number) + skip_connection = [True, False], se_ratios = [0, 0.25], **kwargs): + + import ConfigSpace as CS + import ConfigSpace.hyperparameters as CSH + + cs = CS.ConfigurationSpace() + + + main_blocks_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, "nr_main_blocks", nr_main_blocks) + initial_filters_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, "initial_filters", initial_filters) + cs.add_hyperparameter(main_blocks_hp) + cs.add_hyperparameter(initial_filters_hp) + + if type(nr_main_blocks[0]) == int: + min_blocks = nr_main_blocks[0] + max_blocks = nr_main_blocks[1] + else: + min_blocks = nr_main_blocks[0][0] + max_blocks = nr_main_blocks[0][1] + + for i in range(1, max_blocks + 1): + sub_blocks_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'nr_sub_blocks_%d' % i, nr_sub_blocks) + op_type_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'op_type_%d' % i, op_types) + kernel_size_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'kernel_size_%d' % i, kernel_sizes) + stride_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'stride_%d' % i, strides) + out_filters_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'out_filters_%d' % i, output_filters[i-1]) # take output_filters list i-1 as options + se_ratio_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'se_ratio_%d' % i, se_ratios) + cs.add_hyperparameter(sub_blocks_hp) + cs.add_hyperparameter(op_type_hp) + cs.add_hyperparameter(kernel_size_hp) + cs.add_hyperparameter(stride_hp) + cs.add_hyperparameter(out_filters_hp) + cs.add_hyperparameter(se_ratio_hp) + skip_con = cs.add_hyperparameter(CSH.CategoricalHyperparameter('skip_con_%d' % i, [True, False])) + + if i > min_blocks: + cs.add_condition(CS.GreaterThanCondition(sub_blocks_hp, main_blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(op_type_hp, main_blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(kernel_size_hp, main_blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(stride_hp, main_blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(out_filters_hp, main_blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(skip_con, main_blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(se_ratio_hp, main_blocks_hp, i-1)) + + return cs diff --git a/autoPyTorch/components/networks/image/resnet.py b/autoPyTorch/components/networks/image/resnet.py new file mode 100644 index 000000000..c709f7a3d --- /dev/null +++ b/autoPyTorch/components/networks/image/resnet.py @@ -0,0 +1,294 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +import ConfigSpace +from autoPyTorch.components.networks.base_net import BaseImageNet +from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter, get_hyperparameter + +from autoPyTorch.components.networks.image.utils.utils import initialize_weights +from autoPyTorch.components.networks.image.utils.shakeshakeblock import shake_shake, generate_alpha_beta +from autoPyTorch.components.networks.image.utils.shakedrop import shake_drop, generate_alpha_beta_single + +class SkipConnection(nn.Module): + def __init__(self, in_channels, out_channels, stride): + super(SkipConnection, self).__init__() + + self.s1 = nn.Sequential() + self.s1.add_module('Skip_1_AvgPool', + nn.AvgPool2d(1, stride=stride)) + self.s1.add_module('Skip_1_Conv', + nn.Conv2d(in_channels, + int(out_channels / 2), + kernel_size=1, + stride=1, + padding=0, + bias=False)) + + self.s2 = nn.Sequential() + self.s2.add_module('Skip_2_AvgPool', + nn.AvgPool2d(1, stride=stride)) + self.s2.add_module('Skip_2_Conv', + nn.Conv2d(in_channels, + int(out_channels / 2) if out_channels % 2 == 0 else int(out_channels / 2) + 1, + kernel_size=1, + stride=1, + padding=0, + bias=False)) + + self.batch_norm = nn.BatchNorm2d(out_channels) + + def forward(self, x): + out1 = F.relu(x, inplace=False) + out1 = self.s1(out1) + + out2 = F.pad(x[:, :, 1:, 1:], (0, 1, 0, 1)) + out2 = self.s2(out2) + + out = torch.cat([out1, out2], dim=1) + out = self.batch_norm(out) + + return out + + +class ResidualBranch(nn.Module): + def __init__(self, in_channels, out_channels, filter_size, stride, branch_index): + super(ResidualBranch, self).__init__() + + self.residual_branch = nn.Sequential() + + self.residual_branch.add_module('Branch_{}:ReLU_1'.format(branch_index), + nn.ReLU(inplace=False)) + self.residual_branch.add_module('Branch_{}:Conv_1'.format(branch_index), + nn.Conv2d(in_channels, + out_channels, + kernel_size=filter_size, + stride=stride, + padding=round(filter_size / 3), + bias=False)) + self.residual_branch.add_module('Branch_{}:BN_1'.format(branch_index), + nn.BatchNorm2d(out_channels)) + self.residual_branch.add_module('Branch_{}:ReLU_2'.format(branch_index), + nn.ReLU(inplace=False)) + self.residual_branch.add_module('Branch_{}:Conv_2'.format(branch_index), + nn.Conv2d(out_channels, + out_channels, + kernel_size=filter_size, + stride=1, + padding=round(filter_size / 3), + bias=False)) + self.residual_branch.add_module('Branch_{}:BN_2'.format(branch_index), + nn.BatchNorm2d(out_channels)) + + def forward(self, x): + return self.residual_branch(x) + + +class BasicBlock(nn.Module): + def __init__(self, n_input_plane, n_output_plane, filter_size, res_branches, stride, shake_config): + super(BasicBlock, self).__init__() + + self.shake_config = shake_config + self.branches = nn.ModuleList([ResidualBranch(n_input_plane, n_output_plane, filter_size, stride, branch + 1) for branch in range(res_branches)]) + + # Skip connection + self.skip = nn.Sequential() + if n_input_plane != n_output_plane or stride != 1: + self.skip.add_module('Skip_connection', + SkipConnection(n_input_plane, n_output_plane, stride)) + + + def forward(self, x): + if len(self.branches) == 1: + out = self.branches[0](x) + if self.config.apply_shakeDrop: + alpha, beta = generate_alpha_beta_single(out.size(), self.shake_config if self.training else (False, False, False), x.is_cuda) + out = shake_drop(out, alpha, beta, self.config.death_rate, self.training) + else: + if self.config.apply_shakeShake: + alpha, beta = generate_alpha_beta(len(self.branches), x.size(0), self.shake_config if self.training else (False, False, False), x.is_cuda) + branches = [self.branches[i](x) for i in range(len(self.branches))] + out = shake_shake(alpha, beta, *branches) + else: + out = sum([self.branches[i](x) for i in range(len(self.branches))]) + + return out + self.skip(x) + + +class ResidualGroup(nn.Module): + def __init__(self, block, n_input_plane, n_output_plane, n_blocks, filter_size, res_branches, stride, shake_config): + super(ResidualGroup, self).__init__() + self.group = nn.Sequential() + self.n_blocks = n_blocks + + # The first residual block in each group is responsible for the input downsampling + self.group.add_module('Block_1', + block(n_input_plane, + n_output_plane, + filter_size, + res_branches, + stride=stride, + shake_config=shake_config)) + + # The following residual block do not perform any downsampling (stride=1) + for block_index in range(2, n_blocks + 1): + block_name = 'Block_{}'.format(block_index) + self.group.add_module(block_name, + block(n_output_plane, + n_output_plane, + filter_size, + res_branches, + stride=1, + shake_config=shake_config)) + + def forward(self, x): + return self.group(x) + + +class ResNet(BaseImageNet): + def __init__(self, config, in_features, out_features, final_activation, **kwargs): + super(ResNet, self).__init__(config, in_features, out_features, final_activation) + + nn.Module.config = config + self.final_activation = final_activation + self.nr_main_blocks = config['nr_main_blocks'] + config.initial_filters = config['initial_filters'] + config.death_rate = config['death_rate'] + + config.forward_shake = True + config.backward_shake = True + config.shake_image = True + config.apply_shakeDrop = True + config.apply_shakeShake = True + + self.nr_residual_blocks = dict([ + ('Group_%d' % (i+1), config['nr_residual_blocks_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + self.widen_factors = dict([ + ('Group_%d' % (i+1), config['widen_factor_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + self.res_branches = dict([ + ('Group_%d' % (i+1), config['res_branches_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + self.filters_size = dict([ + ('Group_%d' % (i+1), 3) #config['filters_size_%i' % (i+1)]) + for i in range(self.nr_main_blocks)]) + + shake_config = (config.forward_shake, config.backward_shake, + config.shake_image) + + ########## + self.model = nn.Sequential() + + # depth = sum([config.nr_convs * self.nr_residual_blocks['Group_{}'.format(i)] + 2 for i in range(1, self.nr_main_blocks + 1)]) + # print(' | Multi-branch ResNet-' + str(depth) + ' CIFAR-10') + + block = BasicBlock + + im_size = max(self.ih, self.iw) + + self.model.add_module('Conv_0', + nn.Conv2d(self.channels, + config.initial_filters, + kernel_size=7 if im_size > 200 else 3, + stride=2 if im_size > 200 else 1, + padding=3 if im_size > 200 else 1, + bias=False)) + self.model.add_module('BN_0', + nn.BatchNorm2d(config.initial_filters)) + + if im_size > 200: + self.model.add_module('ReLU_0', nn.ReLU(inplace=True)) + self.model.add_module('Pool_0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)) + + feature_maps_in = int(round(config.initial_filters * self.widen_factors['Group_1'])) + self.model.add_module('Group_1', + ResidualGroup(block, + config.initial_filters, + feature_maps_in, + self.nr_residual_blocks['Group_1'], + self.filters_size['Group_1'], + self.res_branches['Group_1'], + 1, #2 if im_size > 100 else 1, + shake_config)) + + # image_size, min_image_size = min(self.iw, self.ih), 5 + # division_steps = math.floor(math.log2(image_size) - math.log2(min_image_size) - 1e-5) + + for main_block_nr in range(2, self.nr_main_blocks + 1): + feature_maps_out = int(round(feature_maps_in * self.widen_factors['Group_{}'.format(main_block_nr)])) + self.model.add_module('Group_{}'.format(main_block_nr), + ResidualGroup(block, + feature_maps_in, + feature_maps_out, + self.nr_residual_blocks['Group_{}'.format(main_block_nr)], + self.filters_size['Group_{}'.format(main_block_nr)], + self.res_branches['Group_{}'.format(main_block_nr)], + 2, # if main_block_nr > self.nr_main_blocks - division_steps else 1, + shake_config)) + + #image_size = math.floor((image_size+1)/2.0) if main_block_nr > self.nr_main_blocks - division_steps else image_size + feature_maps_in = feature_maps_out + + self.feature_maps_out = feature_maps_in + self.model.add_module('ReLU_0', nn.ReLU(inplace=True)) + self.model.add_module('AveragePool', nn.AdaptiveAvgPool2d(1)) + self.fc = nn.Linear(self.feature_maps_out, out_features) + + self.apply(initialize_weights) + + self.layers = nn.Sequential(self.model) + + def forward(self, x): + x = self.model(x) + x = x.view(-1, self.feature_maps_out) + x = self.fc(x) + if not self.training and self.final_activation is not None: + x = self.final_activation(x) + return x + + @staticmethod + def get_config_space( nr_main_blocks=[1, 8], nr_residual_blocks=([1, 16], True), initial_filters=([8, 32], True), widen_factor=([0.5, 4], True), + res_branches=([1, 5], False), filters_size=[3, 3], **kwargs): + + import ConfigSpace as CS + import ConfigSpace.hyperparameters as CSH + + cs = CS.ConfigurationSpace() + + nr_main_blocks_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, "nr_main_blocks", nr_main_blocks) + cs.add_hyperparameter(nr_main_blocks_hp) + initial_filters_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, "initial_filters", initial_filters) + cs.add_hyperparameter(initial_filters_hp) + # add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'nr_convs', nr_convs, log=True) + death_rate_hp = get_hyperparameter(ConfigSpace.UniformFloatHyperparameter, "death_rate", ([0,1], False)) + cs.add_hyperparameter(death_rate_hp) + + if type(nr_main_blocks[0]) is int: + main_blocks_min = nr_main_blocks[0] + main_blocks_max = nr_main_blocks[1] + else: + main_blocks_min = nr_main_blocks[0][0] + main_blocks_max = nr_main_blocks[0][1] + + for i in range(1, main_blocks_max + 1): + blocks_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'nr_residual_blocks_%d' % i, nr_residual_blocks) + blocks = cs.add_hyperparameter(blocks_hp) + widen_hp = get_hyperparameter(ConfigSpace.UniformFloatHyperparameter, 'widen_factor_%d' % i, widen_factor) + widen = cs.add_hyperparameter(widen_hp) + branches_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'res_branches_%d' % i, res_branches) + branches = cs.add_hyperparameter(branches_hp) + # filters = add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'filters_size_%d' % i, filters_size, log=False) + + if i > main_blocks_min: + cs.add_condition(CS.GreaterThanCondition(blocks_hp, nr_main_blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(widen_hp, nr_main_blocks_hp, i-1)) + cs.add_condition(CS.GreaterThanCondition(branches_hp, nr_main_blocks_hp, i-1)) + # cs.add_condition(CS.GreaterThanCondition(filters, main_blocks, i-1)) + + return cs diff --git a/autoPyTorch/components/networks/image/resnet152.py b/autoPyTorch/components/networks/image/resnet152.py new file mode 100644 index 000000000..22dc6b476 --- /dev/null +++ b/autoPyTorch/components/networks/image/resnet152.py @@ -0,0 +1,192 @@ + +import torch.nn as nn +import torch.utils.model_zoo as model_zoo + + +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152'] + + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = conv1x1(inplanes, planes) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = conv3x3(planes, planes, stride) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = conv1x1(planes, planes * self.expansion) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +from autoPyTorch.components.networks.base_net import BaseImageNet +class ResNet(BaseImageNet): + + def __init__(self, block, layers, num_classes=1000, zero_init_residual=False): + self.inplanes = 64 + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + self.layers = nn.Sequential(self.conv1, self.bn1, self.relu, self.maxpool, self.layer1, self.layer2, self.layer3, self.layer4, self.avgpool) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + + return x + + +class ResNet152(ResNet): + def __init__(self, config, in_features, out_features, final_activation, **kwargs): + super(ResNet, self).__init__(config, in_features, out_features, final_activation) + super(ResNet152, self).__init__(Bottleneck, [3, 8, 36, 3], num_classes=out_features) + + + def forward(self, x): + x = super(ResNet152, self).forward(x) + + if not self.training and self.final_activation is not None: + x = self.final_activation(x) + return x + + +# def resnet152(pretrained=False, **kwargs): +# """Constructs a ResNet-152 model. +# Args: +# pretrained (bool): If True, returns a model pre-trained on ImageNet +# """ +# model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs) +# if pretrained: +# model.load_state_dict(model_zoo.load_url(model_urls['resnet152'])) +# return model diff --git a/autoPyTorch/components/networks/image/utils/__init__.py b/autoPyTorch/components/networks/image/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/autoPyTorch/components/networks/image/utils/conv2d_helpers.py b/autoPyTorch/components/networks/image/utils/conv2d_helpers.py new file mode 100644 index 000000000..75801c374 --- /dev/null +++ b/autoPyTorch/components/networks/image/utils/conv2d_helpers.py @@ -0,0 +1,135 @@ +# Copyright 2019 Ross Wightman +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + + +import torch +import torch.nn as nn +import torch.nn.functional as F +import math + + +def _is_static_pad(kernel_size, stride=1, dilation=1, **_): + return stride == 1 and (dilation * (kernel_size - 1)) % 2 == 0 + + +def _get_padding(kernel_size, stride=1, dilation=1, **_): + padding = ((stride - 1) + dilation * (kernel_size - 1)) // 2 + return padding + + +def _calc_same_pad(i, k, s, d): + return max((math.ceil(i / s) - 1) * s + (k - 1) * d + 1 - i, 0) + + +def _split_channels(num_chan, num_groups): + split = [num_chan // num_groups for _ in range(num_groups)] + split[0] += num_chan - sum(split) + return split + + +class Conv2dSame(nn.Conv2d): + """ Tensorflow like 'SAME' convolution wrapper for 2D convolutions + """ + def __init__(self, in_channels, out_channels, kernel_size, stride=1, + padding=0, dilation=1, groups=1, bias=True): + super(Conv2dSame, self).__init__( + in_channels, out_channels, kernel_size, stride, 0, dilation, + groups, bias) + + def forward(self, x): + ih, iw = x.size()[-2:] + kh, kw = self.weight.size()[-2:] + pad_h = _calc_same_pad(ih, kh, self.stride[0], self.dilation[0]) + pad_w = _calc_same_pad(iw, kw, self.stride[1], self.dilation[1]) + if pad_h > 0 or pad_w > 0: + x = F.pad(x, [pad_w//2, pad_w - pad_w//2, pad_h//2, pad_h - pad_h//2]) + return F.conv2d(x, self.weight, self.bias, self.stride, + self.padding, self.dilation, self.groups) + + +def conv2d_pad(in_chs, out_chs, kernel_size, **kwargs): + padding = kwargs.pop('padding', '') + kwargs.setdefault('bias', False) + if isinstance(padding, str): + # for any string padding, the padding will be calculated for you, one of three ways + padding = padding.lower() + if padding == 'same': + # TF compatible 'SAME' padding, has a performance and GPU memory allocation impact + if _is_static_pad(kernel_size, **kwargs): + # static case, no extra overhead + padding = _get_padding(kernel_size, **kwargs) + return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs) + else: + # dynamic padding + return Conv2dSame(in_chs, out_chs, kernel_size, **kwargs) + elif padding == 'valid': + # 'VALID' padding, same as padding=0 + return nn.Conv2d(in_chs, out_chs, kernel_size, padding=0, **kwargs) + else: + # Default to PyTorch style 'same'-ish symmetric padding + padding = _get_padding(kernel_size, **kwargs) + return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs) + else: + # padding was specified as a number or pair + return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs) + + +class MixedConv2d(nn.Module): + """ Mixed Grouped Convolution + Based on MDConv and GroupedConv in MixNet impl: + https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mixnet/custom_layers.py + """ + + def __init__(self, in_channels, out_channels, kernel_size=3, + stride=1, padding='', dilated=False, depthwise=False, **kwargs): + super(MixedConv2d, self).__init__() + + kernel_size = kernel_size if isinstance(kernel_size, list) else [kernel_size] + num_groups = len(kernel_size) + in_splits = _split_channels(in_channels, num_groups) + out_splits = _split_channels(out_channels, num_groups) + for idx, (k, in_ch, out_ch) in enumerate(zip(kernel_size, in_splits, out_splits)): + d = 1 + # FIXME make compat with non-square kernel/dilations/strides + if stride == 1 and dilated: + d, k = (k - 1) // 2, 3 + conv_groups = out_ch if depthwise else 1 + # use add_module to keep key space clean + self.add_module( + str(idx), + conv2d_pad( + in_ch, out_ch, k, stride=stride, + padding=padding, dilation=d, groups=conv_groups, **kwargs) + ) + self.splits = in_splits + + def forward(self, x): + x_split = torch.split(x, self.splits, 1) + x_out = [c(x) for x, c in zip(x_split, self._modules.values())] + x = torch.cat(x_out, 1) + return x + + +# helper method +def select_conv2d(in_chs, out_chs, kernel_size, **kwargs): + assert 'groups' not in kwargs # only use 'depthwise' bool arg + if isinstance(kernel_size, list): + # We're going to use only lists for defining the MixedConv2d kernel groups, + # ints, tuples, other iterables will continue to pass to normal conv and specify h, w. + return MixedConv2d(in_chs, out_chs, kernel_size, **kwargs) + else: + depthwise = kwargs.pop('depthwise', False) + groups = out_chs if depthwise else 1 + return conv2d_pad(in_chs, out_chs, kernel_size, groups=groups, **kwargs) diff --git a/autoPyTorch/components/networks/image/utils/mobilenet_utils.py b/autoPyTorch/components/networks/image/utils/mobilenet_utils.py new file mode 100644 index 000000000..4218554bb --- /dev/null +++ b/autoPyTorch/components/networks/image/utils/mobilenet_utils.py @@ -0,0 +1,753 @@ +# Copyright 2019 Ross Wightman +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +""" Generic EfficientNets +A generic class with building blocks to support a variety of models with efficient architectures: +* EfficientNet (B0-B7) +* MixNet (Small, Medium, and Large) +* MnasNet B1, A1 (SE), Small +* MobileNet V1, V2, and V3 +* FBNet-C (TODO A & B) +* ChamNet (TODO still guessing at architecture definition) +* Single-Path NAS Pixel1 +* And likely more... +TODO not all combinations and variations have been tested. Currently working on training hyper-params... +Hacked together by Ross Wightman +""" + +import math +import re +import logging +from copy import deepcopy + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .conv2d_helpers import select_conv2d + + +IMAGENET_DEFAULT_MEAN = (0.485, 0.456, 0.406) +IMAGENET_DEFAULT_STD = (0.229, 0.224, 0.225) + + +__all__ = ['GenEfficientNet'] + + +def _cfg(url='', **kwargs): + return { + 'url': url, 'num_classes': 1000, 'input_size': (3, 224, 224), 'pool_size': (7, 7), + 'crop_pct': 0.875, 'interpolation': 'bicubic', + 'mean': IMAGENET_DEFAULT_MEAN, 'std': IMAGENET_DEFAULT_STD, + 'first_conv': 'conv_stem', 'classifier': 'classifier', + **kwargs + } + + +_DEBUG = False + +# Default args for PyTorch BN impl +_BN_MOMENTUM_PT_DEFAULT = 0.01 +_BN_EPS_PT_DEFAULT = 1e-5 +_BN_ARGS_PT = dict(momentum=_BN_MOMENTUM_PT_DEFAULT, eps=_BN_EPS_PT_DEFAULT) + +# Defaults used for Google/Tensorflow training of mobile networks /w RMSprop as per +# papers and TF reference implementations. PT momentum equiv for TF decay is (1 - TF decay) +# NOTE: momentum varies btw .99 and .9997 depending on source +# .99 in official TF TPU impl +# .9997 (/w .999 in search space) for paper +# HERE CHANGED TO WORK WITH PYTORCH +_BN_MOMENTUM_TF_DEFAULT = 1 - 0.99 +_BN_EPS_TF_DEFAULT = 1e-3 +_BN_ARGS_TF = dict(momentum=_BN_MOMENTUM_TF_DEFAULT, eps=_BN_EPS_TF_DEFAULT) + + +def adaptive_pool_feat_mult(pool_type='avg'): + if pool_type == 'catavgmax': + return 2 + else: + return 1 + + +def adaptive_avgmax_pool2d(x, output_size=1): + x_avg = F.adaptive_avg_pool2d(x, output_size) + x_max = F.adaptive_max_pool2d(x, output_size) + return 0.5 * (x_avg + x_max) + + +def adaptive_catavgmax_pool2d(x, output_size=1): + x_avg = F.adaptive_avg_pool2d(x, output_size) + x_max = F.adaptive_max_pool2d(x, output_size) + return torch.cat((x_avg, x_max), 1) + + +def select_adaptive_pool2d(x, pool_type='avg', output_size=1): + """Selectable global pooling function with dynamic input kernel size + """ + if pool_type == 'avg': + x = F.adaptive_avg_pool2d(x, output_size) + elif pool_type == 'avgmax': + x = adaptive_avgmax_pool2d(x, output_size) + elif pool_type == 'catavgmax': + x = adaptive_catavgmax_pool2d(x, output_size) + elif pool_type == 'max': + x = F.adaptive_max_pool2d(x, output_size) + else: + assert False, 'Invalid pool type: %s' % pool_type + return x + + +class AdaptiveAvgMaxPool2d(nn.Module): + def __init__(self, output_size=1): + super(AdaptiveAvgMaxPool2d, self).__init__() + self.output_size = output_size + + def forward(self, x): + return adaptive_avgmax_pool2d(x, self.output_size) + + +class AdaptiveCatAvgMaxPool2d(nn.Module): + def __init__(self, output_size=1): + super(AdaptiveCatAvgMaxPool2d, self).__init__() + self.output_size = output_size + + def forward(self, x): + return adaptive_catavgmax_pool2d(x, self.output_size) + + +class SelectAdaptivePool2d(nn.Module): + """Selectable global pooling layer with dynamic input kernel size + """ + def __init__(self, output_size=1, pool_type='avg'): + super(SelectAdaptivePool2d, self).__init__() + self.output_size = output_size + self.pool_type = pool_type + if pool_type == 'avgmax': + self.pool = AdaptiveAvgMaxPool2d(output_size) + elif pool_type == 'catavgmax': + self.pool = AdaptiveCatAvgMaxPool2d(output_size) + elif pool_type == 'max': + self.pool = nn.AdaptiveMaxPool2d(output_size) + else: + if pool_type != 'avg': + assert False, 'Invalid pool type: %s' % pool_type + self.pool = nn.AdaptiveAvgPool2d(output_size) + + def forward(self, x): + return self.pool(x) + + def feat_mult(self): + return adaptive_pool_feat_mult(self.pool_type) + + def __repr__(self): + return self.__class__.__name__ + ' (' \ + + 'output_size=' + str(self.output_size) \ + + ', pool_type=' + self.pool_type + ')' + + +def _resolve_bn_args(kwargs): + bn_args = _BN_ARGS_TF.copy() if kwargs.pop('bn_tf', False) else _BN_ARGS_PT.copy() + bn_momentum = kwargs.pop('bn_momentum', None) + if bn_momentum is not None: + bn_args['momentum'] = bn_momentum + bn_eps = kwargs.pop('bn_eps', None) + if bn_eps is not None: + bn_args['eps'] = bn_eps + return bn_args + + +def _round_channels(channels, multiplier=1.0, divisor=8, channel_min=None): + """Round number of filters based on depth multiplier.""" + if not multiplier: + return channels + + channels *= multiplier + channel_min = channel_min or divisor + new_channels = max( + int(channels + divisor / 2) // divisor * divisor, + channel_min) + # Make sure that round down does not go down by more than 10%. + if new_channels < 0.9 * channels: + new_channels += divisor + return new_channels + + +def _parse_ksize(ss): + if ss.isdigit(): + return int(ss) + else: + return [int(k) for k in ss.split('.')] + + +def _decode_block_str(block_str, depth_multiplier=1.0): + """ Decode block definition string + Gets a list of block arg (dicts) through a string notation of arguments. + E.g. ir_r2_k3_s2_e1_i32_o16_se0.25_noskip + All args can exist in any order with the exception of the leading string which + is assumed to indicate the block type. + leading string - block type ( + ir = InvertedResidual, ds = DepthwiseSep, dsa = DeptwhiseSep with pw act, cn = ConvBnAct) + r - number of repeat blocks, + k - kernel size, + s - strides (1-9), + e - expansion ratio, + c - output channels, + se - squeeze/excitation ratio + n - activation fn ('re', 'r6', 'hs', or 'sw') + Args: + block_str: a string representation of block arguments. + Returns: + A list of block args (dicts) + Raises: + ValueError: if the string def not properly specified (TODO) + """ + assert isinstance(block_str, str) + ops = block_str.split('_') + block_type = ops[0] # take the block type off the front + ops = ops[1:] + options = {} + noskip = False + for op in ops: + # string options being checked on individual basis, combine if they grow + if op == 'noskip': + noskip = True + elif op.startswith('n'): + # activation fn + key = op[0] + v = op[1:] + if v == 're': + value = F.relu + elif v == 'r6': + value = F.relu6 + elif v == 'hs': + value = hard_swish + elif v == 'sw': + value = swish + else: + continue + options[key] = value + else: + # all numeric options + splits = re.split(r'(\d.*)', op) + if len(splits) >= 2: + key, value = splits[:2] + options[key] = value + + # if act_fn is None, the model default (passed to model init) will be used + act_fn = options['n'] if 'n' in options else None + exp_kernel_size = _parse_ksize(options['a']) if 'a' in options else 1 + pw_kernel_size = _parse_ksize(options['p']) if 'p' in options else 1 + + num_repeat = int(options['r']) + # each type of block has different valid arguments, fill accordingly + if block_type == 'ir': + block_args = dict( + block_type=block_type, + dw_kernel_size=_parse_ksize(options['k']), + exp_kernel_size=exp_kernel_size, + pw_kernel_size=pw_kernel_size, + out_chs=int(options['c']), + exp_ratio=float(options['e']), + se_ratio=float(options['se']) if 'se' in options else None, + stride=int(options['s']), + act_fn=act_fn, + noskip=noskip, + ) + elif block_type == 'ds' or block_type == 'dsa': + block_args = dict( + block_type=block_type, + dw_kernel_size=_parse_ksize(options['k']), + pw_kernel_size=pw_kernel_size, + out_chs=int(options['c']), + se_ratio=float(options['se']) if 'se' in options else None, + stride=int(options['s']), + act_fn=act_fn, + pw_act=block_type == 'dsa', + noskip=block_type == 'dsa' or noskip, + ) + elif block_type == 'cn': + block_args = dict( + block_type=block_type, + kernel_size=int(options['k']), + out_chs=int(options['c']), + stride=int(options['s']), + act_fn=act_fn, + ) + else: + assert False, 'Unknown block type (%s)' % block_type + + # return a list of block args expanded by num_repeat and + # scaled by depth_multiplier + num_repeat = int(math.ceil(num_repeat * depth_multiplier)) + return [deepcopy(block_args) for _ in range(num_repeat)] + + +def _decode_arch_def(arch_def, depth_multiplier=1.0): + arch_args = [] + for stack_idx, block_strings in enumerate(arch_def): + assert isinstance(block_strings, list) + stack_args = [] + for block_str in block_strings: + assert isinstance(block_str, str) + stack_args.extend(_decode_block_str(block_str, depth_multiplier)) + arch_args.append(stack_args) + return arch_args + + +def swish(x, inplace=False): + if inplace: + return x.mul_(x.sigmoid()) + else: + return x * x.sigmoid() + + +def sigmoid(x, inplace=False): + return x.sigmoid_() if inplace else x.sigmoid() + + +def hard_swish(x, inplace=False): + if inplace: + return x.mul_(F.relu6(x + 3.) / 6.) + else: + return x * F.relu6(x + 3.) / 6. + + +def hard_sigmoid(x, inplace=False): + if inplace: + return x.add_(3.).clamp_(0., 6.).div_(6.) + else: + return F.relu6(x + 3.) / 6. + + +class _BlockBuilder: + """ Build Trunk Blocks + This ended up being somewhat of a cross between + https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mnasnet_models.py + and + https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/modeling/backbone/fbnet_builder.py + """ + def __init__(self, channel_multiplier=1.0, channel_divisor=8, channel_min=None, + pad_type='', act_fn=None, se_gate_fn=sigmoid, se_reduce_mid=False, + bn_args=_BN_ARGS_PT, drop_connect_rate=0., verbose=False): + self.channel_multiplier = channel_multiplier + self.channel_divisor = channel_divisor + self.channel_min = channel_min + self.pad_type = pad_type + self.act_fn = act_fn + self.se_gate_fn = se_gate_fn + self.se_reduce_mid = se_reduce_mid + self.bn_args = bn_args + self.drop_connect_rate = drop_connect_rate + self.verbose = verbose + + # updated during build + self.in_chs = None + self.block_idx = 0 + self.block_count = 0 + + def _round_channels(self, chs): + return _round_channels(chs, self.channel_multiplier, self.channel_divisor, self.channel_min) + + def _make_block(self, ba): + bt = ba.pop('block_type') + ba['in_chs'] = self.in_chs + ba['out_chs'] = self._round_channels(ba['out_chs']) + ba['bn_args'] = self.bn_args + ba['pad_type'] = self.pad_type + # block act fn overrides the model default + ba['act_fn'] = ba['act_fn'] if ba['act_fn'] is not None else self.act_fn + assert ba['act_fn'] is not None + if bt == 'ir': + ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count + ba['se_gate_fn'] = self.se_gate_fn + ba['se_reduce_mid'] = self.se_reduce_mid + if self.verbose: + logging.info(' InvertedResidual {}, Args: {}'.format(self.block_idx, str(ba))) + block = InvertedResidual(**ba) + elif bt == 'ds' or bt == 'dsa': + ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count + if self.verbose: + logging.info(' DepthwiseSeparable {}, Args: {}'.format(self.block_idx, str(ba))) + block = DepthwiseSeparableConv(**ba) + elif bt == 'cn': + if self.verbose: + logging.info(' ConvBnAct {}, Args: {}'.format(self.block_idx, str(ba))) + block = ConvBnAct(**ba) + else: + assert False, 'Uknkown block type (%s) while building model.' % bt + self.in_chs = ba['out_chs'] # update in_chs for arg of next block + + return block + + def _make_stack(self, stack_args): + blocks = [] + # each stack (stage) contains a list of block arguments + for i, ba in enumerate(stack_args): + if self.verbose: + logging.info(' Block: {}'.format(i)) + if i >= 1: + # only the first block in any stack can have a stride > 1 + ba['stride'] = 1 + block = self._make_block(ba) + blocks.append(block) + self.block_idx += 1 # incr global idx (across all stacks) + return nn.Sequential(*blocks) + + def __call__(self, in_chs, block_args): + """ Build the blocks + Args: + in_chs: Number of input-channels passed to first block + block_args: A list of lists, outer list defines stages, inner + list contains strings defining block configuration(s) + Return: + List of block stacks (each stack wrapped in nn.Sequential) + """ + if self.verbose: + logging.info('Building model trunk with %d stages...' % len(block_args)) + self.in_chs = in_chs + self.block_count = sum([len(x) for x in block_args]) + self.block_idx = 0 + blocks = [] + # outer list of block_args defines the stacks ('stages' by some conventions) + for stack_idx, stack in enumerate(block_args): + if self.verbose: + logging.info('Stack: {}'.format(stack_idx)) + assert isinstance(stack, list) + stack = self._make_stack(stack) + blocks.append(stack) + return blocks + + +def _initialize_weight_goog(m): + # weight init as per Tensorflow Official impl + # https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mnasnet_model.py + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels # fan-out + m.weight.data.normal_(0, math.sqrt(2.0 / n)) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1.0) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + n = m.weight.size(0) # fan-out + init_range = 1.0 / math.sqrt(n) + m.weight.data.uniform_(-init_range, init_range) + m.bias.data.zero_() + + +def _initialize_weight_default(m): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1.0) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + nn.init.kaiming_uniform_(m.weight, mode='fan_in', nonlinearity='linear') + + +def drop_connect(inputs, training=False, drop_connect_rate=0.): + """Apply drop connect.""" + if not training: + return inputs + + keep_prob = 1 - drop_connect_rate + random_tensor = keep_prob + torch.rand( + (inputs.size()[0], 1, 1, 1), dtype=inputs.dtype, device=inputs.device) + random_tensor.floor_() # binarize + output = inputs.div(keep_prob) * random_tensor + return output + + +class ChannelShuffle(nn.Module): + # FIXME haven't used yet + def __init__(self, groups): + super(ChannelShuffle, self).__init__() + self.groups = groups + + def forward(self, x): + """Channel shuffle: [N,C,H,W] -> [N,g,C/g,H,W] -> [N,C/g,g,H,w] -> [N,C,H,W]""" + N, C, H, W = x.size() + g = self.groups + assert C % g == 0, "Incompatible group size {} for input channel {}".format( + g, C + ) + return ( + x.view(N, g, int(C / g), H, W) + .permute(0, 2, 1, 3, 4) + .contiguous() + .view(N, C, H, W) + ) + + +class SqueezeExcite(nn.Module): + def __init__(self, in_chs, reduce_chs=None, act_fn=F.relu, gate_fn=sigmoid): + super(SqueezeExcite, self).__init__() + self.act_fn = act_fn + self.gate_fn = gate_fn + reduced_chs = reduce_chs or in_chs + self.conv_reduce = nn.Conv2d(in_chs, reduced_chs, 1, bias=True) + self.conv_expand = nn.Conv2d(reduced_chs, in_chs, 1, bias=True) + + def forward(self, x): + # NOTE adaptiveavgpool can be used here, but seems to cause issues with NVIDIA AMP performance + x_se = x.view(x.size(0), x.size(1), -1).mean(-1).view(x.size(0), x.size(1), 1, 1) + x_se = self.conv_reduce(x_se) + x_se = self.act_fn(x_se, inplace=True) + x_se = self.conv_expand(x_se) + x = x * self.gate_fn(x_se) + return x + + +class ConvBnAct(nn.Module): + def __init__(self, in_chs, out_chs, kernel_size, + stride=1, pad_type='', act_fn=F.relu, bn_args=_BN_ARGS_PT): + super(ConvBnAct, self).__init__() + assert stride in [1, 2] + self.act_fn = act_fn + self.conv = select_conv2d(in_chs, out_chs, kernel_size, stride=stride, padding=pad_type) + self.bn1 = nn.BatchNorm2d(out_chs, **bn_args) + + def forward(self, x): + x = self.conv(x) + x = self.bn1(x) + x = self.act_fn(x, inplace=True) + return x + + +class DepthwiseSeparableConv(nn.Module): + """ DepthwiseSeparable block + Used for DS convs in MobileNet-V1 and in the place of IR blocks with an expansion + factor of 1.0. This is an alternative to having a IR with optional first pw conv. + """ + def __init__(self, in_chs, out_chs, dw_kernel_size=3, + stride=1, pad_type='', act_fn=F.relu, noskip=False, + pw_kernel_size=1, pw_act=False, + se_ratio=0., se_gate_fn=sigmoid, + bn_args=_BN_ARGS_PT, drop_connect_rate=0.): + super(DepthwiseSeparableConv, self).__init__() + assert stride in [1, 2] + self.has_se = se_ratio is not None and se_ratio > 0. + self.has_residual = (stride == 1 and in_chs == out_chs) and not noskip + self.has_pw_act = pw_act # activation after point-wise conv + self.act_fn = act_fn + self.drop_connect_rate = drop_connect_rate + + self.conv_dw = select_conv2d( + in_chs, in_chs, dw_kernel_size, stride=stride, padding=pad_type, depthwise=True) + self.bn1 = nn.BatchNorm2d(in_chs, **bn_args) + + # Squeeze-and-excitation + if self.has_se: + self.se = SqueezeExcite( + in_chs, reduce_chs=max(1, int(in_chs * se_ratio)), act_fn=act_fn, gate_fn=se_gate_fn) + + self.conv_pw = select_conv2d(in_chs, out_chs, pw_kernel_size, padding=pad_type) + self.bn2 = nn.BatchNorm2d(out_chs, **bn_args) + + def forward(self, x): + residual = x + + x = self.conv_dw(x) + x = self.bn1(x) + x = self.act_fn(x, inplace=True) + + if self.has_se: + x = self.se(x) + + x = self.conv_pw(x) + x = self.bn2(x) + if self.has_pw_act: + x = self.act_fn(x, inplace=True) + + if self.has_residual: + if self.drop_connect_rate > 0.: + x = drop_connect(x, self.training, self.drop_connect_rate) + x += residual + return x + + +class InvertedResidual(nn.Module): + """ Inverted residual block w/ optional SE""" + + def __init__(self, in_chs, out_chs, dw_kernel_size=3, + stride=1, pad_type='', act_fn=F.relu, noskip=False, + exp_ratio=1.0, exp_kernel_size=1, pw_kernel_size=1, + se_ratio=0., se_reduce_mid=False, se_gate_fn=sigmoid, + shuffle_type=None, bn_args=_BN_ARGS_PT, drop_connect_rate=0.): + super(InvertedResidual, self).__init__() + mid_chs = int(in_chs * exp_ratio) + self.has_se = se_ratio is not None and se_ratio > 0. + self.has_residual = (in_chs == out_chs and stride == 1) and not noskip + self.act_fn = act_fn + self.drop_connect_rate = drop_connect_rate + + # Point-wise expansion + self.conv_pw = select_conv2d(in_chs, mid_chs, exp_kernel_size, padding=pad_type) + self.bn1 = nn.BatchNorm2d(mid_chs, **bn_args) + + self.shuffle_type = shuffle_type + if shuffle_type is not None and isinstance(exp_kernel_size, list): + self.shuffle = ChannelShuffle(len(exp_kernel_size)) + + # Depth-wise convolution + self.conv_dw = select_conv2d( + mid_chs, mid_chs, dw_kernel_size, stride=stride, padding=pad_type, depthwise=True) + self.bn2 = nn.BatchNorm2d(mid_chs, **bn_args) + + # Squeeze-and-excitation + if self.has_se: + se_base_chs = mid_chs if se_reduce_mid else in_chs + self.se = SqueezeExcite( + mid_chs, reduce_chs=max(1, int(se_base_chs * se_ratio)), act_fn=act_fn, gate_fn=se_gate_fn) + + # Point-wise linear projection + self.conv_pwl = select_conv2d(mid_chs, out_chs, pw_kernel_size, padding=pad_type) + self.bn3 = nn.BatchNorm2d(out_chs, **bn_args) + + def forward(self, x): + residual = x + + # Point-wise expansion + x = self.conv_pw(x) + x = self.bn1(x) + x = self.act_fn(x, inplace=True) + + # FIXME haven't tried this yet + # for channel shuffle when using groups with pointwise convs as per FBNet variants + if self.shuffle_type == "mid": + x = self.shuffle(x) + + # Depth-wise convolution + x = self.conv_dw(x) + x = self.bn2(x) + x = self.act_fn(x, inplace=True) + + # Squeeze-and-excitation + if self.has_se: + x = self.se(x) + + # Point-wise linear projection + x = self.conv_pwl(x) + x = self.bn3(x) + + if self.has_residual: + if self.drop_connect_rate > 0.: + x = drop_connect(x, self.training, self.drop_connect_rate) + x += residual + + # NOTE maskrcnn_benchmark building blocks have an SE module defined here for some variants + + return x + + +class GenEfficientNet(nn.Module): + """ Generic EfficientNet + An implementation of efficient network architectures, in many cases mobile optimized networks: + * MobileNet-V1 + * MobileNet-V2 + * MobileNet-V3 + * MnasNet A1, B1, and small + * FBNet A, B, and C + * ChamNet (arch details are murky) + * Single-Path NAS Pixel1 + * EfficientNet B0-B7 + * MixNet S, M, L + """ + + def __init__(self, block_args, num_classes=1000, in_chans=3, stem_size=32, num_features=1280, + channel_multiplier=1.0, channel_divisor=8, channel_min=None, + pad_type='', act_fn=F.relu, drop_rate=0., drop_connect_rate=0., + se_gate_fn=sigmoid, se_reduce_mid=False, bn_args=_BN_ARGS_PT, + global_pool='avg', head_conv='default', weight_init='goog'): + super(GenEfficientNet, self).__init__() + self.num_classes = num_classes + self.drop_rate = drop_rate + self.act_fn = act_fn + self.num_features = num_features + + stem_size = _round_channels(stem_size, channel_multiplier, channel_divisor, channel_min) + self.conv_stem = select_conv2d(in_chans, stem_size, 3, stride=2, padding=pad_type) + self.bn1 = nn.BatchNorm2d(stem_size, **bn_args) + in_chs = stem_size + + builder = _BlockBuilder( + channel_multiplier, channel_divisor, channel_min, + pad_type, act_fn, se_gate_fn, se_reduce_mid, + bn_args, drop_connect_rate, verbose=_DEBUG) + self.blocks = nn.Sequential(*builder(in_chs, block_args)) + in_chs = builder.in_chs + + if not head_conv or head_conv == 'none': + self.efficient_head = False + self.conv_head = None + assert in_chs == self.num_features + else: + self.efficient_head = head_conv == 'efficient' + self.conv_head = select_conv2d(in_chs, self.num_features, 1, padding=pad_type) + self.bn2 = None if self.efficient_head else nn.BatchNorm2d(self.num_features, **bn_args) + + self.global_pool = SelectAdaptivePool2d(pool_type=global_pool) + self.classifier = nn.Linear(self.num_features * self.global_pool.feat_mult(), self.num_classes) + + for m in self.modules(): + if weight_init == 'goog': + _initialize_weight_goog(m) + else: + _initialize_weight_default(m) + + def get_classifier(self): + return self.classifier + + def reset_classifier(self, num_classes, global_pool='avg'): + self.global_pool = SelectAdaptivePool2d(pool_type=global_pool) + self.num_classes = num_classes + del self.classifier + if num_classes: + self.classifier = nn.Linear( + self.num_features * self.global_pool.feat_mult(), num_classes) + else: + self.classifier = None + + def forward_features(self, x, pool=True): + x = self.conv_stem(x) + x = self.bn1(x) + x = self.act_fn(x, inplace=True) + x = self.blocks(x) + if self.efficient_head: + # efficient head, currently only mobilenet-v3 performs pool before last 1x1 conv + x = self.global_pool(x) # always need to pool here regardless of flag + x = self.conv_head(x) + # no BN + x = self.act_fn(x, inplace=True) + if pool: + # expect flattened output if pool is true, otherwise keep dim + x = x.view(x.size(0), -1) + else: + if self.conv_head is not None: + x = self.conv_head(x) + x = self.bn2(x) + x = self.act_fn(x, inplace=True) + if pool: + x = self.global_pool(x) + x = x.view(x.size(0), -1) + return x + + def forward(self, x): + x = self.forward_features(x) + if self.drop_rate > 0.: + x = F.dropout(x, p=self.drop_rate, training=self.training) + return self.classifier(x) + diff --git a/autoPyTorch/components/networks/image/utils/shakedrop.py b/autoPyTorch/components/networks/image/utils/shakedrop.py new file mode 100644 index 000000000..3cfa6d3f4 --- /dev/null +++ b/autoPyTorch/components/networks/image/utils/shakedrop.py @@ -0,0 +1,60 @@ +import torch +from torch.autograd import Variable, Function + + +class ShakeDrop(Function): + @staticmethod + def forward(ctx, x, alpha, beta, death_rate, is_train): + gate = (torch.rand(1) > death_rate).numpy() + ctx.gate = gate + ctx.save_for_backward(x, alpha, beta) + + if is_train: + if not gate: + y = alpha * x + else: + y = x + else: + y = x.mul(1 - (death_rate * 1.0)) + + return y + + @staticmethod + def backward(ctx, grad_output): + x, alpha, beta = ctx.saved_variables + grad_x1 = grad_alpha = grad_beta = None + + if ctx.needs_input_grad[0]: + if not ctx.gate: + grad_x = grad_output * beta + else: + grad_x = grad_output + + return grad_x, grad_alpha, grad_beta, None, None + +shake_drop = ShakeDrop.apply + + +def generate_alpha_beta_single(tensor_size, shake_config, is_cuda): + forward_shake, backward_shake, shake_image = shake_config + + if forward_shake and not shake_image: + alpha = torch.rand(tensor_size).mul(2).add(-1) + elif forward_shake and shake_image: + alpha = torch.rand(tensor_size[0]).view(tensor_size[0], 1, 1, 1) + alpha.mul_(2).add_(-1) # alpha from -1 to 1 + else: + alpha = torch.FloatTensor([0.5]) + + if backward_shake and not shake_image: + beta = torch.rand(tensor_size) + elif backward_shake and shake_image: + beta = torch.rand(tensor_size[0]).view(tensor_size[0], 1, 1, 1) + else: + beta = torch.FloatTensor([0.5]) + + if is_cuda: + alpha = alpha.cuda() + beta = beta.cuda() + + return Variable(alpha), Variable(beta) \ No newline at end of file diff --git a/autoPyTorch/components/networks/image/utils/shakeshakeblock.py b/autoPyTorch/components/networks/image/utils/shakeshakeblock.py new file mode 100644 index 000000000..4ebc5085b --- /dev/null +++ b/autoPyTorch/components/networks/image/utils/shakeshakeblock.py @@ -0,0 +1,49 @@ +# coding: utf-8 + +import torch +from torch.autograd import Variable, Function + + +class ShakeShakeBlock(Function): + @staticmethod + def forward(ctx, alpha, beta, *args): + ctx.save_for_backward(beta) + + y = sum(alpha[i] * args[i] for i in range(len(args))) + return y + + @staticmethod + def backward(ctx, grad_output): + beta = ctx.saved_variables + grad_x = [beta[0][i] * grad_output for i in range(beta[0].shape[0])] + + return (None, None, *grad_x) + +shake_shake = ShakeShakeBlock.apply + + +def generate_alpha_beta(num_branches, batch_size, shake_config, is_cuda): + forward_shake, backward_shake, shake_image = shake_config + + if forward_shake and not shake_image: + alpha = torch.rand(num_branches) + elif forward_shake and shake_image: + alpha = torch.rand(num_branches, batch_size).view(num_branches, batch_size, 1, 1, 1) + else: + alpha = torch.ones(num_branches) + + if backward_shake and not shake_image: + beta = torch.rand(num_branches) + elif backward_shake and shake_image: + beta = torch.rand(num_branches, batch_size).view(num_branches, batch_size, 1, 1, 1) + else: + beta = torch.ones(num_branches) + + alpha = torch.nn.Softmax(0)(Variable(alpha)) + beta = torch.nn.Softmax(0)(Variable(beta)) + + if is_cuda: + alpha = alpha.cuda() + beta = beta.cuda() + + return alpha, beta \ No newline at end of file diff --git a/autoPyTorch/components/networks/image/utils/utils.py b/autoPyTorch/components/networks/image/utils/utils.py new file mode 100644 index 000000000..c743b4eb0 --- /dev/null +++ b/autoPyTorch/components/networks/image/utils/utils.py @@ -0,0 +1,44 @@ +import torch.nn as nn +import math + +def initialize_weights(module): + if isinstance(module, nn.Conv2d): + n = module.kernel_size[0] * module.kernel_size[1] * module.out_channels + module.weight.data.normal_(0, math.sqrt(2. / n)) + #nn.init.kaiming_normal(module.weight.data, mode='fan_out') + elif isinstance(module, nn.BatchNorm2d): + module.weight.data.fill_(1) + module.bias.data.zero_() + elif isinstance(module, nn.Linear): + module.bias.data.zero_() + +def get_layer_params(in_size, out_size, kernel_size): + kernel_size = int(kernel_size) + stride = int(max(1, math.ceil((in_size - kernel_size) / (out_size - 1)) if out_size > 1 else 1)) + cur_out_size = _get_out_size(in_size, kernel_size, stride, 0) + required_padding = (stride / 2) * (in_size - cur_out_size) + + cur_padding = int(math.ceil(required_padding)) + cur_out_size = _get_out_size(in_size, kernel_size, stride, cur_padding) + if cur_padding < kernel_size and cur_out_size <= in_size and cur_out_size >= 1: + return cur_out_size, kernel_size, stride, cur_padding + + cur_padding = int(math.floor(required_padding)) + cur_out_size = _get_out_size(in_size, kernel_size, stride, cur_padding) + if cur_padding < kernel_size and cur_out_size <= in_size and cur_out_size >= 1: + return cur_out_size, kernel_size, stride, cur_padding + + if stride > 1: + stride = int(stride - 1) + cur_padding = 0 + cur_out_size = int(_get_out_size(in_size, kernel_size, stride, cur_padding)) + if cur_padding < kernel_size and cur_out_size <= in_size and cur_out_size >= 1: + return cur_out_size, kernel_size, stride, cur_padding + + if (kernel_size % 2) == 0 and out_size == in_size: + return get_layer_params(in_size, out_size, kernel_size + 1) # an odd kernel can always keep the dimension (with stride 1) + + raise Exception('Could not find padding and stride to reduce ' + str(in_size) + ' to ' + str(out_size) + ' using kernel ' + str(kernel_size)) + +def _get_out_size(in_size, kernel_size, stride, padding): + return int(math.floor((in_size - kernel_size + 2 * padding) / stride + 1)) \ No newline at end of file diff --git a/autoPyTorch/components/optimizer/optimizer.py b/autoPyTorch/components/optimizer/optimizer.py index 7d5c0c86b..21197b9d0 100644 --- a/autoPyTorch/components/optimizer/optimizer.py +++ b/autoPyTorch/components/optimizer/optimizer.py @@ -11,10 +11,12 @@ import ConfigSpace as CS import ConfigSpace.hyperparameters as CSH + __author__ = "Max Dippel, Michael Burkart and Matthias Urban" __version__ = "0.0.1" __license__ = "BSD" + class AutoNetOptimizerBase(object): def __new__(cls, params, config): return cls._get_optimizer(cls, params, config) @@ -34,8 +36,24 @@ def _get_optimizer(self, params, config): @staticmethod def get_config_space( - learning_rate=((0.0001, 0.1), True), - weight_decay=(0.0001, 0.1) + learning_rate=((1e-4, 0.1), True), + weight_decay=(1e-5, 0.1) + ): + cs = CS.ConfigurationSpace() + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'learning_rate', learning_rate) + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'weight_decay', weight_decay) + return cs + + +class AdamWOptimizer(AutoNetOptimizerBase): + + def _get_optimizer(self, params, config): + return optim.AdamW(params=params, lr=config['learning_rate'], weight_decay=config['weight_decay']) + + @staticmethod + def get_config_space( + learning_rate=((1e-4, 0.1), True), + weight_decay=(1e-5, 0.1) ): cs = CS.ConfigurationSpace() add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'learning_rate', learning_rate) @@ -50,12 +68,32 @@ def _get_optimizer(self, params, config): @staticmethod def get_config_space( - learning_rate=((0.0001, 0.1), True), - momentum=((0.1, 0.9), True), - weight_decay=(0.0001, 0.1) + learning_rate=((1e-4, 0.1), True), + momentum=((0.1, 0.99), True), + weight_decay=(1e-5, 0.1) + ): + cs = CS.ConfigurationSpace() + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'learning_rate', learning_rate) + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'momentum', momentum) + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'weight_decay', weight_decay) + return cs + + +class RMSpropOptimizer(AutoNetOptimizerBase): + + def _get_optimizer(self, params, config): + return optim.RMSprop(params=params, lr=config['learning_rate'], momentum=config['momentum'], weight_decay=config['weight_decay'], centered=False) + + @staticmethod + def get_config_space( + learning_rate=((1e-4, 0.1), True), + momentum=((0.1, 0.99), True), + weight_decay=(1e-5, 0.1), + alpha=(0.1,0.99) ): cs = CS.ConfigurationSpace() add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'learning_rate', learning_rate) add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'momentum', momentum) add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'weight_decay', weight_decay) + add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'alpha', alpha) return cs diff --git a/autoPyTorch/components/preprocessing/image_preprocessing/__init__.py b/autoPyTorch/components/preprocessing/image_preprocessing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/autoPyTorch/components/preprocessing/image_preprocessing/archive.py b/autoPyTorch/components/preprocessing/image_preprocessing/archive.py new file mode 100644 index 000000000..6a12871a6 --- /dev/null +++ b/autoPyTorch/components/preprocessing/image_preprocessing/archive.py @@ -0,0 +1,28 @@ +# Copyright 2019 Kakao Brain +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +def fa_reduced_cifar10(): + p = [[["Contrast", 0.8320659688593578, 0.49884310562180767], ["TranslateX", 0.41849883971249136, 0.394023086494538]], [["Color", 0.3500483749890918, 0.43355143929883955], ["Color", 0.5120716140300229, 0.7508299643325016]], [["Rotate", 0.9447932604389472, 0.29723465088990375], ["Sharpness", 0.1564936149799504, 0.47169309978091745]], [["Rotate", 0.5430015349185097, 0.6518626678905443], ["Color", 0.5694844928020679, 0.3494533005430269]], [["AutoContrast", 0.5558922032451064, 0.783136004977799], ["TranslateY", 0.683914191471972, 0.7597025305860181]], [["TranslateX", 0.03489224481658926, 0.021025488042663354], ["Equalize", 0.4788637403857401, 0.3535481281496117]], [["Sharpness", 0.6428916269794158, 0.22791511918580576], ["Contrast", 0.016014045073950323, 0.26811312269487575]], [["Rotate", 0.2972727228410451, 0.7654251516829896], ["AutoContrast", 0.16005809254943348, 0.5380523650108116]], [["Contrast", 0.5823671057717301, 0.7521166301398389], ["TranslateY", 0.9949449214751978, 0.9612671341689751]], [["Equalize", 0.8372126687702321, 0.6944127225621206], ["Rotate", 0.25393282929784755, 0.3261658365286546]], [["Invert", 0.8222011603194572, 0.6597915864008403], ["Posterize", 0.31858707654447327, 0.9541013715579584]], [["Sharpness", 0.41314621282107045, 0.9437344470879956], ["Cutout", 0.6610495837889337, 0.674411664255093]], [["Contrast", 0.780121736705407, 0.40826152397463156], ["Color", 0.344019192125256, 0.1942922781355767]], [["Rotate", 0.17153139555621344, 0.798745732456474], ["Invert", 0.6010555860501262, 0.320742172554767]], [["Invert", 0.26816063450777416, 0.27152062163148327], ["Equalize", 0.6786829200236982, 0.7469412443514213]], [["Contrast", 0.3920564414367518, 0.7493644582838497], ["TranslateY", 0.8941657805606704, 0.6580846856375955]], [["Equalize", 0.875509207399372, 0.9061130537645283], ["Cutout", 0.4940280679087308, 0.7896229623628276]], [["Contrast", 0.3331423298065147, 0.7170041362529597], ["ShearX", 0.7425484291842793, 0.5285117152426109]], [["Equalize", 0.97344237365026, 0.4745759720473106], ["TranslateY", 0.055863458430295276, 0.9625142022954672]], [["TranslateX", 0.6810614083109192, 0.7509937355495521], ["TranslateY", 0.3866463019475701, 0.5185481505576112]], [["Sharpness", 0.4751529944753671, 0.550464012488733], ["Cutout", 0.9472914750534814, 0.5584925992985023]], [["Contrast", 0.054606784909375095, 0.17257080196712182], ["Cutout", 0.6077026782754803, 0.7996504165944938]], [["ShearX", 0.328798428243695, 0.2769563264079157], ["Cutout", 0.9037632437023772, 0.4915809476763595]], [["Cutout", 0.6891202672363478, 0.9951490996172914], ["Posterize", 0.06532762462628705, 0.4005246609075227]], [["TranslateY", 0.6908583592523334, 0.725612120376128], ["Rotate", 0.39907735501746666, 0.36505798032223147]], [["TranslateX", 0.10398364107399072, 0.5913918470536627], ["Rotate", 0.7169811539340365, 0.8283850670648724]], [["ShearY", 0.9526373530768361, 0.4482347365639251], ["Contrast", 0.4203947336351471, 0.41526799558953864]], [["Contrast", 0.24894431199700073, 0.09578870500994707], ["Solarize", 0.2273713345927395, 0.6214942914963707]], [["TranslateX", 0.06331228870032912, 0.8961907489444944], ["Cutout", 0.5110007859958743, 0.23704875994050723]], [["Cutout", 0.3769183548846172, 0.6560944580253987], ["TranslateY", 0.7201924599434143, 0.4132476526938319]], [["Invert", 0.6707431156338866, 0.11622795952464149], ["Posterize", 0.12075972752370845, 0.18024933294172307]], [["Color", 0.5010057264087142, 0.5277767327434318], ["Rotate", 0.9486115946366559, 0.31485546630220784]], [["ShearX", 0.31741302466630406, 0.1991215806270692], ["Invert", 0.3744727015523084, 0.6914113986757578]], [["Brightness", 0.40348479064392617, 0.8924182735724888], ["Brightness", 0.1973098763857779, 0.3939288933689655]], [["Color", 0.01208688664030888, 0.6055693000885217], ["Equalize", 0.433259451147881, 0.420711137966155]], [["Cutout", 0.2620018360076487, 0.11594468278143644], ["Rotate", 0.1310401567856766, 0.7244318146544101]], [["ShearX", 0.15249651845933576, 0.35277277071866986], ["Contrast", 0.28221794032094016, 0.42036586509397444]], [["Brightness", 0.8492912150468908, 0.26386920887886056], ["Solarize", 0.8764208056263386, 0.1258195122766067]], [["ShearX", 0.8537058239675831, 0.8415101816171269], ["AutoContrast", 0.23958568830416294, 0.9889049529564014]], [["Rotate", 0.6463207930684552, 0.8750192129056532], ["Contrast", 0.6865032211768652, 0.8564981333033417]], [["Equalize", 0.8877190311811044, 0.7370995897848609], ["TranslateX", 0.9979660314391368, 0.005683998913244781]], [["Color", 0.6420017551677819, 0.6225337265571229], ["Solarize", 0.8344504978566362, 0.8332856969941151]], [["ShearX", 0.7439332981992567, 0.9747608698582039], ["Equalize", 0.6259189804002959, 0.028017478098245174]], [["TranslateY", 0.39794770293366843, 0.8482966537902709], ["Rotate", 0.9312935630405351, 0.5300586925826072]], [["Cutout", 0.8904075572021911, 0.3522934742068766], ["Equalize", 0.6431186289473937, 0.9930577962126151]], [["Contrast", 0.9183553386089476, 0.44974266209396685], ["TranslateY", 0.8193684583123862, 0.9633741156526566]], [["ShearY", 0.616078299924283, 0.19219314358924766], ["Solarize", 0.1480945914138868, 0.05922109541654652]], [["Solarize", 0.25332455064128157, 0.18853037431947994], ["ShearY", 0.9518390093954243, 0.14603930044061142]], [["Color", 0.8094378664335412, 0.37029830225408433], ["Contrast", 0.29504113617467465, 0.065096365468442]], [["AutoContrast", 0.7075167558685455, 0.7084621693458267], ["Sharpness", 0.03555539453323875, 0.5651948313888351]], [["TranslateY", 0.5969982600930229, 0.9857264201029572], ["Rotate", 0.9898628564873607, 0.1985685534926911]], [["Invert", 0.14915939942810352, 0.6595839632446547], ["Posterize", 0.768535289994361, 0.5997358684618563]], [["Equalize", 0.9162691815967111, 0.3331035307653627], ["Color", 0.8169118187605557, 0.7653910258006366]], [["Rotate", 0.43489185299530897, 0.752215269135173], ["Brightness", 0.1569828560334806, 0.8002808712857853]], [["Invert", 0.931876215328345, 0.029428644395760872], ["Equalize", 0.6330036052674145, 0.7235531014288485]], [["ShearX", 0.5216138393704968, 0.849272958911589], ["AutoContrast", 0.19572688655120263, 0.9786551568639575]], [["ShearX", 0.9899586208275011, 0.22580547500610293], ["Brightness", 0.9831311903178727, 0.5055159610855606]], [["Brightness", 0.29179117009211486, 0.48003584672937294], ["Solarize", 0.7544252317330058, 0.05806581735063043]], [["AutoContrast", 0.8919800329537786, 0.8511261613698553], ["Contrast", 0.49199446084551035, 0.7302297140181429]], [["Cutout", 0.7079723710644835, 0.032565015538375874], ["AutoContrast", 0.8259782090388609, 0.7860708789468442]], [["Posterize", 0.9980262659801914, 0.6725084224935673], ["ShearY", 0.6195568269664682, 0.5444170291816751]], [["Posterize", 0.8687351834713217, 0.9978004914422602], ["Equalize", 0.4532646848325955, 0.6486748015710573]], [["Contrast", 0.2713928776950594, 0.15255249557027806], ["ShearY", 0.9276834387970199, 0.5266542862333478]], [["AutoContrast", 0.5240786618055582, 0.9325642258930253], ["Cutout", 0.38448627892037357, 0.21219415055662394]], [["TranslateX", 0.4299517937295352, 0.20133751201386152], ["TranslateX", 0.6753468310276597, 0.6985621035400441]], [["Rotate", 0.4006472499103597, 0.6704748473357586], ["Equalize", 0.674161668148079, 0.6528530101705237]], [["Equalize", 0.9139902833674455, 0.9015103149680278], ["Sharpness", 0.7289667720691948, 0.7623606352376232]], [["Cutout", 0.5911267429414259, 0.5953141187177585], ["Rotate", 0.5219064817468504, 0.11085141355857986]], [["TranslateX", 0.3620095133946267, 0.26194039409492476], ["Rotate", 0.3929841359545597, 0.4913406720338047]], [["Invert", 0.5175298901458896, 0.001661410821811482], ["Invert", 0.004656581318332242, 0.8157622192213624]], [["AutoContrast", 0.013609693335051465, 0.9318651749409604], ["Invert", 0.8980844358979592, 0.2268511862780368]], [["ShearY", 0.7717126261142194, 0.09975547983707711], ["Equalize", 0.7808494401429572, 0.4141412091009955]], [["TranslateX", 0.5878675721341552, 0.29813268038163376], ["Posterize", 0.21257276051591356, 0.2837285296666412]], [["Brightness", 0.4268335108566488, 0.4723784991635417], ["Cutout", 0.9386262901570471, 0.6597686851494288]], [["ShearX", 0.8259423807590159, 0.6215304795389204], ["Invert", 0.6663365779667443, 0.7729669184580387]], [["ShearY", 0.4801338723951297, 0.5220145420100984], ["Solarize", 0.9165803796596582, 0.04299335502862134]], [["Color", 0.17621114853558817, 0.7092601754635434], ["ShearX", 0.9014406936728542, 0.6028711944367818]], [["Rotate", 0.13073284972300658, 0.9088831512880851], ["ShearX", 0.4228105332316806, 0.7985249783662675]], [["Brightness", 0.9182753692730031, 0.0063635477774044436], ["Color", 0.4279825602663798, 0.28727149118585327]], [["Equalize", 0.578218285372267, 0.9611758542158054], ["Contrast", 0.5471552264150691, 0.8819635504027596]], [["Brightness", 0.3208589067274543, 0.45324733565167497], ["Solarize", 0.5218455808633233, 0.5946097503647126]], [["Equalize", 0.3790381278653, 0.8796082535775276], ["Solarize", 0.4875526773149246, 0.5186585878052613]], [["ShearY", 0.12026461479557571, 0.1336953429068397], ["Posterize", 0.34373988646025766, 0.8557727670803785]], [["Cutout", 0.2396745247507467, 0.8123036135209865], ["Equalize", 0.05022807681008945, 0.6648492261984383]], [["Brightness", 0.35226676470748264, 0.5950011514888855], ["Rotate", 0.27555076067000894, 0.9170063321486026]], [["ShearX", 0.320224630647278, 0.9683584649071976], ["Invert", 0.6905585196648905, 0.5929115667894518]], [["Color", 0.9941395717559652, 0.7474441679798101], ["Sharpness", 0.7559998478658021, 0.6656052889626682]], [["ShearY", 0.4004220568345669, 0.5737646992826074], ["Equalize", 0.9983495213746147, 0.8307907033362303]], [["Color", 0.13726809242038207, 0.9378850119950549], ["Equalize", 0.9853362454752445, 0.42670264496554156]], [["Invert", 0.13514636153298576, 0.13516363849081958], ["Sharpness", 0.2031189356693901, 0.6110226359872745]], [["TranslateX", 0.7360305209630797, 0.41849698571655614], ["Contrast", 0.8972161549144564, 0.7820296625565641]], [["Color", 0.02713118828682548, 0.717110684828096], ["TranslateY", 0.8118759006836348, 0.9120098002024992]], [["Sharpness", 0.2915428949403711, 0.7630303724396518], ["Solarize", 0.22030536162851078, 0.38654526772661757]], [["Equalize", 0.9949114839538582, 0.7193630656062793], ["AutoContrast", 0.00889496657931299, 0.2291400476524672]], [["Rotate", 0.7120948976490488, 0.7804359309791055], ["Cutout", 0.10445418104923654, 0.8022999156052766]], [["Equalize", 0.7941710117902707, 0.8648170634288153], ["Invert", 0.9235642581144047, 0.23810725859722381]], [["Cutout", 0.3669397998623156, 0.42612815083245004], ["Solarize", 0.5896322046441561, 0.40525016166956795]], [["Color", 0.8389858785714184, 0.4805764176488667], ["Rotate", 0.7483931487048825, 0.4731174601400677]], [["Sharpness", 0.19006538611394763, 0.9480745790240234], ["TranslateY", 0.13904429049439282, 0.04117685330615939]], [["TranslateY", 0.9958097661701637, 0.34853788612580905], ["Cutout", 0.2235829624082113, 0.3737887095480745]], [["ShearX", 0.635453761342424, 0.6063917273421382], ["Posterize", 0.8738297843709666, 0.4893042590265556]], [["Brightness", 0.7907245198402727, 0.7082189713070691], ["Color", 0.030313003541849737, 0.6927897798493439]], [["Cutout", 0.6965622481073525, 0.8103522907758203], ["ShearY", 0.6186794303078708, 0.28640671575703547]], [["ShearY", 0.43734910588450226, 0.32549342535621517], ["ShearX", 0.08154980987651872, 0.3286764923112455]], [["AutoContrast", 0.5262462005050853, 0.8175584582465848], ["Contrast", 0.8683217097363655, 0.548776281479276]], [["ShearY", 0.03957878500311707, 0.5102350637943197], ["Rotate", 0.13794708520303778, 0.38035687712954236]], [["Sharpness", 0.634288567312677, 0.6387948309075822], ["AutoContrast", 0.13437288694693272, 0.7150448869023095]], [["Contrast", 0.5198339640088544, 0.9409429390321714], ["Cutout", 0.09489154903321972, 0.6228488803821982]], [["Equalize", 0.8955909061806043, 0.7727336527163008], ["AutoContrast", 0.6459479564441762, 0.7065467781139214]], [["Invert", 0.07214420843537739, 0.15334721382249505], ["ShearX", 0.9242027778363903, 0.5809187849982554]], [["Equalize", 0.9144084379856188, 0.9457539278608998], ["Sharpness", 0.14337499858300173, 0.5978054365425495]], [["Posterize", 0.18894269796951202, 0.14676331276539045], ["Equalize", 0.846204299950047, 0.0720601838168885]], [["Contrast", 0.47354445405741163, 0.1793650330107468], ["Solarize", 0.9086106327264657, 0.7578807802091502]], [["AutoContrast", 0.11805466892967886, 0.6773620948318575], ["TranslateX", 0.584222568299264, 0.9475693349391936]], [["Brightness", 0.5833017701352768, 0.6892593824176294], ["AutoContrast", 0.9073141314561828, 0.5823085733964589]], [["TranslateY", 0.5711231614144834, 0.6436240447620021], ["Contrast", 0.21466964050052473, 0.8042843954486391]], [["Contrast", 0.22967904487976765, 0.2343103109298762], ["Invert", 0.5502897289159286, 0.386181060792375]], [["Invert", 0.7008423439928628, 0.4234003051405053], ["Rotate", 0.77270460187611, 0.6650852696828039]], [["Invert", 0.050618322309703534, 0.24277027926683614], ["TranslateX", 0.789703489736613, 0.5116446685339312]], [["Color", 0.363898083076868, 0.7870323584210503], ["ShearY", 0.009608425513626617, 0.6188625018465327]], [["TranslateY", 0.9447601615216088, 0.8605867115798349], ["Equalize", 0.24139180127003634, 0.9587337957930782]], [["Equalize", 0.3968589440144503, 0.626206375426996], ["Solarize", 0.3215967960673186, 0.826785464835443]], [["TranslateX", 0.06947339047121326, 0.016705969558222122], ["Contrast", 0.6203392406528407, 0.6433525559906872]], [["Solarize", 0.2479835265518212, 0.6335009955617831], ["Sharpness", 0.6260191862978083, 0.18998095149428562]], [["Invert", 0.9818841924943431, 0.03252098144087934], ["TranslateY", 0.9740718042586802, 0.32038951753031475]], [["Solarize", 0.8795784664090814, 0.7014953994354041], ["AutoContrast", 0.8508018319577783, 0.09321935255338443]], [["Color", 0.8067046326105318, 0.13732893832354054], ["Contrast", 0.7358549680271418, 0.7880588355974301]], [["Posterize", 0.5005885536838065, 0.7152229305267599], ["ShearX", 0.6714249591308944, 0.7732232697859908]], [["TranslateY", 0.5657943483353953, 0.04622399873706862], ["AutoContrast", 0.2787442688649845, 0.567024378767143]], [["ShearY", 0.7589839214283295, 0.041071003934029404], ["Equalize", 0.3719852873722692, 0.43285778682687326]], [["Posterize", 0.8841266183653291, 0.42441306955476366], ["Cutout", 0.06578801759412933, 0.5961125797961526]], [["Rotate", 0.4057875004314082, 0.20241115848366442], ["AutoContrast", 0.19331542807918067, 0.7175484678480565]], [["Contrast", 0.20331327116693088, 0.17135387852218742], ["Cutout", 0.6282459410351067, 0.6690015305529187]], [["ShearX", 0.4309850328306535, 0.99321178125828], ["AutoContrast", 0.01809604030453338, 0.693838277506365]], [["Rotate", 0.24343531125298268, 0.5326412444169899], ["Sharpness", 0.8663989992597494, 0.7643990609130789]], [["Rotate", 0.9785019204622459, 0.8941922576710696], ["ShearY", 0.3823185048761075, 0.9258854046017292]], [["ShearY", 0.5502613342963388, 0.6193478797355644], ["Sharpness", 0.2212116534610532, 0.6648232390110979]], [["TranslateY", 0.43222920981513757, 0.5657636397633089], ["ShearY", 0.9153733286073634, 0.4868521171273169]], [["Posterize", 0.12246560519738336, 0.9132288825898972], ["Cutout", 0.6058471327881816, 0.6426901876150983]], [["Color", 0.3693970222695844, 0.038929141432555436], ["Equalize", 0.6228052875653781, 0.05064436511347281]], [["Color", 0.7172600331356893, 0.2824542634766688], ["Color", 0.425293116261649, 0.1796441283313972]], [["Cutout", 0.7539608428122959, 0.9896141728228921], ["Solarize", 0.17811081117364758, 0.9064195503634402]], [["AutoContrast", 0.6761242607012717, 0.6484842446399923], ["AutoContrast", 0.1978135076901828, 0.42166879492601317]], [["ShearY", 0.25901666379802524, 0.4770778270322449], ["Solarize", 0.7640963173407052, 0.7548463227094349]], [["TranslateY", 0.9222487731783499, 0.33658389819616463], ["Equalize", 0.9159112511468139, 0.8877136302394797]], [["TranslateX", 0.8994836977137054, 0.11036053676846591], ["Sharpness", 0.9040333410652747, 0.007266095214664592]], [["Invert", 0.627758632524958, 0.8075245097227242], ["Color", 0.7525387912148516, 0.05950239294733184]], [["TranslateX", 0.43505193292761857, 0.38108822876120796], ["TranslateY", 0.7432578052364004, 0.685678116134759]], [["Contrast", 0.9293507582470425, 0.052266842951356196], ["Posterize", 0.45187123977747456, 0.8228290399726782]], [["ShearX", 0.07240786542746291, 0.8945667925365756], ["Brightness", 0.5305443506561034, 0.12025274552427578]], [["Invert", 0.40157564448143335, 0.5364745514006678], ["Posterize", 0.3316124671813876, 0.43002413237035997]], [["ShearY", 0.7152314630009072, 0.1938339083417453], ["Invert", 0.14102478508140615, 0.41047623580174253]], [["Equalize", 0.19862832613849246, 0.5058521685279254], ["Sharpness", 0.16481208629549782, 0.29126323102770557]], [["Equalize", 0.6951591703541872, 0.7294822018800076], ["ShearX", 0.8726656726111219, 0.3151484225786487]], [["Rotate", 0.17234370554263745, 0.9356543193000078], ["TranslateX", 0.4954374070084091, 0.05496727345849217]], [["Contrast", 0.347405480122842, 0.831553005022885], ["ShearX", 0.28946367213071134, 0.11905898704394013]], [["Rotate", 0.28096672507990683, 0.16181284050307398], ["Color", 0.6554918515385365, 0.8739728050797386]], [["Solarize", 0.05408073374114053, 0.5357087283758337], ["Posterize", 0.42457175211495335, 0.051807130609045515]], [["TranslateY", 0.6216669362331361, 0.9691341207381867], ["Rotate", 0.9833579358130944, 0.12227426932415297]], [["AutoContrast", 0.7572619475282892, 0.8062834082727393], ["Contrast", 0.1447865402875591, 0.40242646573228436]], [["Rotate", 0.7035658783466086, 0.9840285268256428], ["Contrast", 0.04613961510519471, 0.7666683217450163]], [["TranslateX", 0.4580462177951252, 0.6448678609474686], ["AutoContrast", 0.14845695613708987, 0.1581134188537895]], [["Color", 0.06795037145259564, 0.9115552821158709], ["TranslateY", 0.9972953449677655, 0.6791016521791214]], [["Cutout", 0.3586908443690823, 0.11578558293480945], ["Color", 0.49083981719164294, 0.6924851425917189]], [["Brightness", 0.7994717831637873, 0.7887316255321768], ["Posterize", 0.01280463502435425, 0.2799086732858721]], [["ShearY", 0.6733451536131859, 0.8122332639516706], ["AutoContrast", 0.20433889615637357, 0.29023346867819966]], [["TranslateY", 0.709913512385177, 0.6538196931503809], ["Invert", 0.06629795606579203, 0.40913219547548296]], [["Sharpness", 0.4704559834362948, 0.4235993305308414], ["Equalize", 0.7578132044306966, 0.9388824249397175]], [["AutoContrast", 0.5281702802395268, 0.8077253610116979], ["Equalize", 0.856446858814119, 0.0479755681647559]], [["Color", 0.8244145826797791, 0.038409264586238945], ["Equalize", 0.4933123249234237, 0.8251940933672189]], [["TranslateX", 0.23949314158035084, 0.13576027004706692], ["ShearX", 0.8547563771688399, 0.8309262160483606]], [["Cutout", 0.4655680937486001, 0.2819807000622825], ["Contrast", 0.8439552665937905, 0.4843617871587037]], [["TranslateX", 0.19142454476784831, 0.7516148119169537], ["AutoContrast", 0.8677128351329768, 0.34967990912346336]], [["Contrast", 0.2997868299880966, 0.919508054854469], ["AutoContrast", 0.3003418493384957, 0.812314984368542]], [["Invert", 0.1070424236198183, 0.614674386498809], ["TranslateX", 0.5010973510899923, 0.20828478805259465]], [["Contrast", 0.6775882415611454, 0.6938564815591685], ["Cutout", 0.4814634264207498, 0.3086844939744179]], [["TranslateY", 0.939427105020265, 0.02531043619423201], ["Contrast", 0.793754257944812, 0.6676072472565451]], [["Sharpness", 0.09833672397575444, 0.5937214638292085], ["Rotate", 0.32530675291753763, 0.08302275740932441]], [["Sharpness", 0.3096455511562728, 0.6726732004553959], ["TranslateY", 0.43268997648796537, 0.8755012330217743]], [["ShearY", 0.9290771880324833, 0.22114736271319912], ["Equalize", 0.5520199288501478, 0.34269650332060553]], [["AutoContrast", 0.39763980746649374, 0.4597414582725454], ["Contrast", 0.941507852412761, 0.24991270562477041]], [["Contrast", 0.19419400547588095, 0.9127524785329233], ["Invert", 0.40544905179551727, 0.770081532844878]], [["Invert", 0.30473757368608334, 0.23534811781828846], ["Cutout", 0.26090722356706686, 0.5478390909877727]], [["Posterize", 0.49434361308057373, 0.05018423270527428], ["Color", 0.3041910676883317, 0.2603810415446437]], [["Invert", 0.5149061746764011, 0.9507449210221298], ["TranslateY", 0.4458076521892904, 0.8235358255774426]], [["Cutout", 0.7900006753351625, 0.905578861382507], ["Cutout", 0.6707153655762056, 0.8236715672258502]], [["Solarize", 0.8750534386579575, 0.10337670467100568], ["Posterize", 0.6102379615481381, 0.9264503915416868]], [["ShearY", 0.08448689377082852, 0.13981233725811626], ["TranslateX", 0.13979689669329498, 0.768774869872818]], [["TranslateY", 0.35752572266759985, 0.22827299847812488], ["Solarize", 0.3906957174236011, 0.5663314388307709]], [["ShearY", 0.29155240367061563, 0.8427516352971683], ["ShearX", 0.988825367441916, 0.9371258864857649]], [["Posterize", 0.3470780859769458, 0.5467686612321239], ["Rotate", 0.5758606274160093, 0.8843838082656007]], [["Cutout", 0.07825368363221841, 0.3230799425855425], ["Equalize", 0.2319163865298529, 0.42133965674727325]], [["Invert", 0.41972172597448654, 0.34618622513582953], ["ShearX", 0.33638469398198834, 0.9098575535928108]], [["Invert", 0.7322652233340448, 0.7747502957687412], ["Cutout", 0.9643121397298106, 0.7983335094634907]], [["TranslateY", 0.30039942808098496, 0.229018798182827], ["TranslateY", 0.27009499739380194, 0.6435577237846236]], [["Color", 0.38245274994070644, 0.7030758568461645], ["ShearX", 0.4429321461666281, 0.6963787864044149]], [["AutoContrast", 0.8432798685515605, 0.5775214369578088], ["Brightness", 0.7140899735355927, 0.8545854720117658]], [["Rotate", 0.14418935535613786, 0.5637968282213426], ["Color", 0.7115231912479835, 0.32584796564566776]], [["Sharpness", 0.4023501062807533, 0.4162097130412771], ["Brightness", 0.5536372686153666, 0.03004023273348777]], [["TranslateX", 0.7526053265574295, 0.5365938133399961], ["Cutout", 0.07914142706557492, 0.7544953091603148]], [["TranslateY", 0.6932934644882822, 0.5302211727137424], ["Invert", 0.5040606028391255, 0.6074863635108957]], [["Sharpness", 0.5013938602431629, 0.9572417724333157], ["TranslateY", 0.9160516359783026, 0.41798927975391675]], [["ShearY", 0.5130018836722556, 0.30209438428424185], ["Color", 0.15017170588500262, 0.20653495360587826]], [["TranslateX", 0.5293300090022314, 0.6407011888285266], ["Rotate", 0.4809817860439001, 0.3537850070371702]], [["Equalize", 0.42243081336551014, 0.13472721311046565], ["Posterize", 0.4700309639484068, 0.5197704360874883]], [["AutoContrast", 0.40674959899687235, 0.7312824868168921], ["TranslateX", 0.7397527975920833, 0.7068339877944815]], [["TranslateY", 0.5880995184787206, 0.41294111378078946], ["ShearX", 0.3181387627799316, 0.4810010147143413]], [["Color", 0.9898680233928507, 0.13241525577655167], ["Contrast", 0.9824932511238534, 0.5081145010853807]], [["Invert", 0.1591854062582687, 0.9760371953250404], ["Color", 0.9913399302056851, 0.8388709501056177]], [["Rotate", 0.6427451962231163, 0.9486793975292853], ["AutoContrast", 0.8501937877930463, 0.021326757974406196]], [["Contrast", 0.13611684531087598, 0.3050858709483848], ["Posterize", 0.06618644756084646, 0.8776928511951034]], [["TranslateX", 0.41021065663839407, 0.4965319749091702], ["Rotate", 0.07088831484595115, 0.4435516708223345]], [["Sharpness", 0.3151707977154323, 0.28275482520179296], ["Invert", 0.36980384682133804, 0.20813616084536624]], [["Cutout", 0.9979060206661017, 0.39712948644725854], ["Brightness", 0.42451052896163466, 0.942623075649937]], [["Equalize", 0.5300853308425644, 0.010183500830128867], ["AutoContrast", 0.06930788523716991, 0.5403125318991522]], [["Contrast", 0.010385458959237814, 0.2588311035539086], ["ShearY", 0.9347048553928764, 0.10439028366854963]], [["ShearY", 0.9867649486508592, 0.8409258132716434], ["ShearX", 0.48031199530836444, 0.7703375364614137]], [["ShearY", 0.04835889473136512, 0.2671081675890492], ["Brightness", 0.7856432618509617, 0.8032169570159564]], [["Posterize", 0.11112884927351185, 0.7116956530752987], ["TranslateY", 0.7339151092128607, 0.3331241226029017]], [["Invert", 0.13527036207875454, 0.8425980515358883], ["Color", 0.7836395778298139, 0.5517059252678862]], [["Sharpness", 0.012541163521491816, 0.013197550692292892], ["Invert", 0.6295957932861318, 0.43276521236056054]], [["AutoContrast", 0.7681480991225756, 0.3634284648496289], ["Brightness", 0.09708289828517969, 0.45016725043529726]], [["Brightness", 0.5839450499487329, 0.47525965678316795], ["Posterize", 0.43096581990183735, 0.9332382960125196]], [["Contrast", 0.9725334964552795, 0.9142902966863341], ["Contrast", 0.12376116410622995, 0.4355916974126801]], [["TranslateX", 0.8572708473690132, 0.02544522678265526], ["Sharpness", 0.37902120723460364, 0.9606092969833118]], [["TranslateY", 0.8907359001296927, 0.8011363927236099], ["Color", 0.7693777154407178, 0.0936768686746503]], [["Equalize", 0.0002657688243309364, 0.08190798535970034], ["Rotate", 0.5215478065240905, 0.5773519995038368]], [["TranslateY", 0.3383007813932477, 0.5733428274739165], ["Sharpness", 0.2436110797174722, 0.4757790814590501]], [["Cutout", 0.0957402176213592, 0.8914395928996034], ["Cutout", 0.4959915628586883, 0.25890349461645246]], [["AutoContrast", 0.594787300189186, 0.9627455357634459], ["ShearY", 0.5136027621132064, 0.10419602450259002]], [["Solarize", 0.4684077211553732, 0.6592850629431414], ["Sharpness", 0.2382385935956325, 0.6589291408243176]], [["Cutout", 0.4478786947325877, 0.6893616643143388], ["TranslateX", 0.2761781720270474, 0.21750622627277727]], [["Sharpness", 0.39476077929016484, 0.930902796668923], ["Cutout", 0.9073012208742808, 0.9881122386614257]], [["TranslateY", 0.0933719180021565, 0.7206252503441172], ["ShearX", 0.5151400441789256, 0.6307540083648309]], [["AutoContrast", 0.7772689258806401, 0.8159317013156503], ["AutoContrast", 0.5932793713915097, 0.05262217353927168]], [["Equalize", 0.38017352056118914, 0.8084724050448412], ["ShearY", 0.7239725628380852, 0.4246314890359326]], [["Cutout", 0.741157483503503, 0.13244380646497977], ["Invert", 0.03395378056675935, 0.7140036618098844]], [["Rotate", 0.0662727247460636, 0.7099861732415447], ["Rotate", 0.3168532707508249, 0.3553167425022127]], [["AutoContrast", 0.7429303516734129, 0.07117444599776435], ["Posterize", 0.5379537435918104, 0.807221330263993]], [["TranslateY", 0.9788586874795164, 0.7967243851346594], ["Invert", 0.4479103376922362, 0.04260360776727545]], [["Cutout", 0.28318121763188997, 0.7748680701406292], ["AutoContrast", 0.9109258369403016, 0.17126397858002085]], [["Color", 0.30183727885272027, 0.46718354750112456], ["TranslateX", 0.9628952256033627, 0.10269543754135535]], [["AutoContrast", 0.6316709389784041, 0.84287698792044], ["Brightness", 0.5544761629904337, 0.025264772745200004]], [["Rotate", 0.08803313299532567, 0.306059720523696], ["Invert", 0.5222165872425064, 0.045935208620454304]], [["TranslateY", 0.21912346831923835, 0.48529224559004436], ["TranslateY", 0.15466734731903942, 0.8929485418495068]], [["ShearX", 0.17141022847016563, 0.8607600402165531], ["ShearX", 0.6890511341106859, 0.7540899265679949]], [["Invert", 0.9417455522972059, 0.9021733684991224], ["Solarize", 0.7693107057723746, 0.7268007946568782]], [["Posterize", 0.02376991543373752, 0.6768442864453844], ["Rotate", 0.7736875065112697, 0.6706331753139825]], [["Contrast", 0.3623841610390669, 0.15023657344457686], ["Equalize", 0.32975472189318666, 0.05629246869510651]], [["Sharpness", 0.7874882420165824, 0.49535778020457066], ["Posterize", 0.09485578893387558, 0.6170768580482466]], [["Brightness", 0.7099280202949585, 0.021523012961427335], ["Posterize", 0.2076371467666719, 0.17168118578815206]], [["Color", 0.8546367645761538, 0.832011891505731], ["Equalize", 0.6429734783051777, 0.2618995960561532]], [["Rotate", 0.8780793721476224, 0.5920897827664297], ["ShearX", 0.5338303685064825, 0.8605424531336439]], [["Sharpness", 0.7504493806631884, 0.9723552387375258], ["Sharpness", 0.3206385634203266, 0.45127845905824693]], [["ShearX", 0.23794709526711355, 0.06257530645720066], ["Solarize", 0.9132374030587093, 0.6240819934824045]], [["Sharpness", 0.790583587969259, 0.28551171786655405], ["Contrast", 0.39872982844590554, 0.09644706751019538]], [["Equalize", 0.30681999237432944, 0.5645045018157916], ["Posterize", 0.525966242669736, 0.7360106111256014]], [["TranslateX", 0.4881014179825114, 0.6317220208872226], ["ShearX", 0.2935158995550958, 0.23104608987381758]], [["Rotate", 0.49977116738568395, 0.6610761068306319], ["TranslateY", 0.7396566602715687, 0.09386747830045217]], [["ShearY", 0.5909773790018789, 0.16229529902832718], ["Equalize", 0.06461394468918358, 0.6661349001143908]], [["TranslateX", 0.7218443721851834, 0.04435720302810153], ["Cutout", 0.986686540951642, 0.734771197038724]], [["ShearX", 0.5353800096911666, 0.8120139502148365], ["Equalize", 0.4613239578449774, 0.5159528929124512]], [["Color", 0.0871713897628631, 0.7708895183198486], ["Solarize", 0.5811386808912219, 0.35260648120785887]], [["Posterize", 0.3910857927477053, 0.4329219555775561], ["Color", 0.9115983668789468, 0.6043069944145293]], [["Posterize", 0.07493067637060635, 0.4258000066006725], ["AutoContrast", 0.4740957581389772, 0.49069587151651295]], [["Rotate", 0.34086200894268937, 0.9812149332288828], ["Solarize", 0.6801012471371733, 0.17271491146753837]], [["Color", 0.20542270872895207, 0.5532087457727624], ["Contrast", 0.2718692536563381, 0.20313287569510108]], [["Equalize", 0.05199827210980934, 0.0832859890912212], ["AutoContrast", 0.8092395764794107, 0.7778945136511004]], [["Sharpness", 0.1907689513066838, 0.7705754572256907], ["Color", 0.3911178658498049, 0.41791326933095485]], [["Solarize", 0.19611855804748257, 0.2407807485604081], ["AutoContrast", 0.5343964972940493, 0.9034209455548394]], [["Color", 0.43586520148538865, 0.4711164626521439], ["ShearY", 0.28635408186820555, 0.8417816793020271]], [["Cutout", 0.09818482420382535, 0.1649767430954796], ["Cutout", 0.34582392911178494, 0.3927982995799828]], [["ShearX", 0.001253882705272269, 0.48661629027584596], ["Solarize", 0.9229221435457137, 0.44374894836659073]], [["Contrast", 0.6829734655718668, 0.8201750485099037], ["Cutout", 0.7886756837648936, 0.8423285219631946]], [["TranslateY", 0.857017093561528, 0.3038537151773969], ["Invert", 0.12809228606383538, 0.23637166191748027]], [["Solarize", 0.9829027723424164, 0.9723093910674763], ["Color", 0.6346495302126811, 0.5405494753107188]], [["AutoContrast", 0.06868643520377715, 0.23758659417688077], ["AutoContrast", 0.6648225411500879, 0.5618315648260103]], [["Invert", 0.44202305603311676, 0.9945938909685547], ["Equalize", 0.7991650497684454, 0.16014142656347097]], [["AutoContrast", 0.8778631604769588, 0.03951977631894088], ["ShearY", 0.8495160088963707, 0.35771447321250416]], [["Color", 0.5365078341001592, 0.21102444169782308], ["ShearX", 0.7168869678248874, 0.3904298719872734]], [["TranslateX", 0.6517203786101899, 0.6467598990650437], ["Invert", 0.26552491504364517, 0.1210812827294625]], [["Posterize", 0.35196021684368994, 0.8420648319941891], ["Invert", 0.7796829363930631, 0.9520895999240896]], [["Sharpness", 0.7391572148971984, 0.4853940393452846], ["TranslateX", 0.7641915295592839, 0.6351349057666782]], [["Posterize", 0.18485880221115913, 0.6117603277356728], ["Rotate", 0.6541660490605724, 0.5704041108375348]], [["TranslateY", 0.27517423188070533, 0.6610080904072458], ["Contrast", 0.6091250547289317, 0.7702443247557892]], [["Equalize", 0.3611798581067118, 0.6623615672642768], ["TranslateX", 0.9537265090885917, 0.06352772509358584]], [["ShearX", 0.09720029389103535, 0.7800423126320308], ["Invert", 0.30314352455858884, 0.8519925470889914]], [["Brightness", 0.06931529763458055, 0.57760829499712], ["Cutout", 0.637251974467394, 0.7184346129191052]], [["AutoContrast", 0.5026722100286064, 0.32025257156541886], ["Contrast", 0.9667478703047919, 0.14178519432669368]], [["Equalize", 0.5924463845816984, 0.7187610262181517], ["TranslateY", 0.7059479079159405, 0.06551471830655187]], [["Sharpness", 0.18161164512332928, 0.7576138481173385], ["Brightness", 0.19191138767695282, 0.7865880269424701]], [["Brightness", 0.36780861866078696, 0.0677855546737901], ["AutoContrast", 0.8491446654142264, 0.09217782099938121]], [["TranslateY", 0.06011399855120858, 0.8374487034710264], ["TranslateY", 0.8373922962070498, 0.1991295720254297]], [["Posterize", 0.702559916122481, 0.30257509683007755], ["Rotate", 0.249899495398891, 0.9370437251176267]], [["ShearX", 0.9237874098232075, 0.26241907483351146], ["Brightness", 0.7221766836146657, 0.6880749752986671]], [["Cutout", 0.37994098189193104, 0.7836874473657957], ["ShearX", 0.9212861960976824, 0.8140948561570449]], [["Posterize", 0.2584098274786417, 0.7990847652004848], ["Invert", 0.6357731737590063, 0.1066304859116326]], [["Sharpness", 0.4412790857539922, 0.9692465283229825], ["Color", 0.9857401617339051, 0.26755393929808713]], [["Equalize", 0.22348671644912665, 0.7370019910830038], ["Posterize", 0.5396106339575417, 0.5559536849843303]], [["Equalize", 0.8742967663495852, 0.2797122599926307], ["Rotate", 0.4697322053105951, 0.8769872942579476]], [["Sharpness", 0.44279911640509206, 0.07729581896071613], ["Cutout", 0.3589177366154631, 0.2704031551235969]], [["TranslateX", 0.614216412574085, 0.47929659784170453], ["Brightness", 0.6686234118438007, 0.05700784068205689]], [["ShearY", 0.17920614630857634, 0.4699685075827862], ["Color", 0.38251870810870003, 0.7262706923005887]], [["Solarize", 0.4951799001144561, 0.212775278026479], ["TranslateX", 0.8666105646463097, 0.6750496637519537]], [["Color", 0.8110864170849051, 0.5154263861958484], ["Sharpness", 0.2489044083898776, 0.3763372541462343]], [["Cutout", 0.04888193613483871, 0.06041664638981603], ["Color", 0.06438587718683708, 0.5797881428892969]], [["Rotate", 0.032427448352152166, 0.4445797818376559], ["Posterize", 0.4459357828482998, 0.5879865187630777]], [["ShearX", 0.1617179557693058, 0.050796802246318884], ["Cutout", 0.8142465452060423, 0.3836391305618707]], [["TranslateY", 0.1806857249209416, 0.36697730355422675], ["Rotate", 0.9897576550818276, 0.7483432452225264]], [["Brightness", 0.18278016458098223, 0.952352527690299], ["Cutout", 0.3269735224453044, 0.3924869905012752]], [["ShearX", 0.870832707718742, 0.3214743207190739], ["Cutout", 0.6805560681792573, 0.6984188155282459]], [["TranslateX", 0.4157118388833776, 0.3964216288135384], ["TranslateX", 0.3253012682285006, 0.624835513104391]], [["Contrast", 0.7678168037628158, 0.31033802162621793], ["ShearX", 0.27022424855977134, 0.3773245605126201]], [["TranslateX", 0.37812621869017593, 0.7657993810740699], ["Rotate", 0.18081890120092914, 0.8893511219618171]], [["Posterize", 0.8735859716088367, 0.18243793043074286], ["TranslateX", 0.90435994250313, 0.24116383818819453]], [["Invert", 0.06666709253664793, 0.3881076083593933], ["TranslateX", 0.3783333964963522, 0.14411014979589543]], [["Equalize", 0.8741147867162096, 0.14203839235846816], ["TranslateX", 0.7801536758037405, 0.6952401607812743]], [["Cutout", 0.6095335117944475, 0.5679026063718094], ["Posterize", 0.06433868172233115, 0.07139559616012303]], [["TranslateY", 0.3020364047315408, 0.21459810361176246], ["Cutout", 0.7097677414888889, 0.2942144632587549]], [["Brightness", 0.8223662419048653, 0.195700694016108], ["Invert", 0.09345407040803999, 0.779843655582099]], [["TranslateY", 0.7353462929356228, 0.0468520680237382], ["Cutout", 0.36530918247940425, 0.3897292909049672]], [["Invert", 0.9676896451721213, 0.24473302189463453], ["Invert", 0.7369271521408992, 0.8193267003356975]], [["Sharpness", 0.8691871972054326, 0.4441713912682772], ["ShearY", 0.47385584832119887, 0.23521684584675429]], [["ShearY", 0.9266946026184021, 0.7611986713358834], ["TranslateX", 0.6195820760253926, 0.14661428669483678]], [["Sharpness", 0.08470870576026868, 0.3380219099907229], ["TranslateX", 0.3062343307496658, 0.7135777338095889]], [["Sharpness", 0.5246448204194909, 0.3193061215236702], ["ShearX", 0.8160637208508432, 0.9720697396582731]], [["Posterize", 0.5249259956549405, 0.3492042382504774], ["Invert", 0.8183138799547441, 0.11107271762524618]], [["TranslateY", 0.210869733350744, 0.7138905840721885], ["Sharpness", 0.7773226404450125, 0.8005353621959782]], [["Posterize", 0.33067522385556025, 0.32046239220630124], ["AutoContrast", 0.18918147708798405, 0.4646281070474484]], [["TranslateX", 0.929502026131094, 0.8029128121556285], ["Invert", 0.7319794306118105, 0.5421878712623392]], [["ShearX", 0.25645940834182723, 0.42754710760160963], ["ShearX", 0.44640695310173306, 0.8132185532296811]], [["Color", 0.018436846416536312, 0.8439313862001113], ["Sharpness", 0.3722867661453415, 0.5103570873163251]], [["TranslateX", 0.7285989086776543, 0.4809027697099264], ["TranslateY", 0.9740807004893643, 0.8241085438636939]], [["Posterize", 0.8721868989693397, 0.5700907310383815], ["Posterize", 0.4219074410577852, 0.8032643572845402]], [["Contrast", 0.9811380092558266, 0.8498397471632105], ["Sharpness", 0.8380884329421594, 0.18351306571903125]], [["TranslateY", 0.3878939366762001, 0.4699103438753077], ["Invert", 0.6055556353233807, 0.8774727658400134]], [["TranslateY", 0.052317005261018346, 0.39471450378745787], ["ShearX", 0.8612486845942395, 0.28834103278807466]], [["Color", 0.511993351208063, 0.07251427040525904], ["Solarize", 0.9898097047354855, 0.299761565689576]], [["Equalize", 0.2721248231619904, 0.6870975927455507], ["Cutout", 0.8787327242363994, 0.06228061428917098]], [["Invert", 0.8931880335225408, 0.49720931867378193], ["Posterize", 0.9619698792159256, 0.17859639696940088]], [["Posterize", 0.0061688075074411985, 0.08082938731035938], ["Brightness", 0.27745128028826993, 0.8638528796903816]], [["ShearY", 0.9140200609222026, 0.8240421430867707], ["Invert", 0.651734417415332, 0.08871906369930926]], [["Color", 0.45585010413511196, 0.44705070078574316], ["Color", 0.26394624901633146, 0.11242877788650807]], [["ShearY", 0.9200278466372522, 0.2995901331149652], ["Cutout", 0.8445407215116278, 0.7410524214287446]], [["ShearY", 0.9950483746990132, 0.112964468262847], ["ShearY", 0.4118332303218585, 0.44839613407553636]], [["Contrast", 0.7905821952255192, 0.23360046159385106], ["Posterize", 0.8611787233956044, 0.8984260048943528]], [["TranslateY", 0.21448061359312853, 0.8228112806838331], ["Contrast", 0.8992297266152983, 0.9179231590570998]], [["Invert", 0.3924194798946006, 0.31830516468371495], ["Rotate", 0.8399556845248508, 0.3764892022932781]], [["Cutout", 0.7037916990046816, 0.9214620769502728], ["AutoContrast", 0.02913794613018239, 0.07808607528954048]], [["ShearY", 0.6041490474263381, 0.6094184590800105], ["Equalize", 0.2932954517354919, 0.5840888946081727]], [["ShearX", 0.6056801676269449, 0.6948580442549543], ["Cutout", 0.3028001021044615, 0.15117101733894078]], [["Brightness", 0.8011486803860253, 0.18864079729374195], ["Solarize", 0.014965327213230961, 0.8842620292527029]], [["Invert", 0.902244007904273, 0.5634673798052033], ["Equalize", 0.13422913507398349, 0.4110956745883727]], [["TranslateY", 0.9981773319103838, 0.09568550987216096], ["Color", 0.7627662124105109, 0.8494409737419493]], [["Cutout", 0.3013527640416782, 0.03377226729898486], ["ShearX", 0.5727964831614619, 0.8784196638222834]], [["TranslateX", 0.6050722426803684, 0.3650103962378708], ["TranslateX", 0.8392084589130886, 0.6479816470292911]], [["Rotate", 0.5032806606500023, 0.09276980118866307], ["TranslateY", 0.7800234515261191, 0.18896454379343308]], [["Invert", 0.9266027256244017, 0.8246111062199752], ["Contrast", 0.12112023357797697, 0.33870762271759436]], [["Brightness", 0.8688784756993134, 0.17263759696106606], ["ShearX", 0.5133700431071326, 0.6686811994542494]], [["Invert", 0.8347840440941976, 0.03774897445901726], ["Brightness", 0.24925057499276548, 0.04293631677355758]], [["Color", 0.5998145279485104, 0.4820093200092529], ["TranslateY", 0.6709586184077769, 0.07377334081382858]], [["AutoContrast", 0.7898846202957984, 0.325293526672498], ["Contrast", 0.5156435596826767, 0.2889223168660645]], [["ShearX", 0.08147389674998307, 0.7978924681113669], ["Contrast", 0.7270003309106291, 0.009571215234092656]], [["Sharpness", 0.417607614440786, 0.9532566433338661], ["Posterize", 0.7186586546796782, 0.6936509907073302]], [["ShearX", 0.9555300215926675, 0.1399385550263872], ["Color", 0.9981041061848231, 0.5037462398323248]], [["Equalize", 0.8003487831375474, 0.5413759363796945], ["ShearY", 0.0026607045117773565, 0.019262273030984933]], [["TranslateY", 0.04845391502469176, 0.10063445212118283], ["Cutout", 0.8273170186786745, 0.5045257728554577]], [["TranslateX", 0.9690985344978033, 0.505202991815533], ["TranslateY", 0.7255326592928096, 0.02103609500701631]], [["Solarize", 0.4030771176836736, 0.8424237871457034], ["Cutout", 0.28705805963928965, 0.9601617893682582]], [["Sharpness", 0.16865290353070606, 0.6899673563468826], ["Posterize", 0.3985430034869616, 0.6540651997730774]], [["ShearY", 0.21395578485362032, 0.09519358818949009], ["Solarize", 0.6692821708524135, 0.6462523623552485]], [["AutoContrast", 0.912360598054091, 0.029800239085051583], ["Invert", 0.04319256403746308, 0.7712501517098587]], [["ShearY", 0.9081969961839055, 0.4581560239984739], ["AutoContrast", 0.5313894814729159, 0.5508393335751848]], [["ShearY", 0.860528568424097, 0.8196987216301588], ["Posterize", 0.41134650331494205, 0.3686632018978778]], [["AutoContrast", 0.8753670810078598, 0.3679438326304749], ["Invert", 0.010444228965415858, 0.9581244779208277]], [["Equalize", 0.07071836206680682, 0.7173594756186462], ["Brightness", 0.06111434312497388, 0.16175064669049277]], [["AutoContrast", 0.10522219073562122, 0.9768776621069855], ["TranslateY", 0.2744795945215529, 0.8577967957127298]], [["AutoContrast", 0.7628146493166175, 0.996157376418147], ["Contrast", 0.9255565598518469, 0.6826126662976868]], [["TranslateX", 0.017225816199011312, 0.2470332491402908], ["Solarize", 0.44048494909493807, 0.4492422515972162]], [["ShearY", 0.38885252627795064, 0.10272256704901939], ["Equalize", 0.686154959829183, 0.8973517148655337]], [["Rotate", 0.29628991573592967, 0.16639926575004715], ["ShearX", 0.9013782324726413, 0.0838318162771563]], [["Color", 0.04968391374688563, 0.6138600739645352], ["Invert", 0.11177127838716283, 0.10650198522261578]], [["Invert", 0.49655016367624016, 0.8603374164829688], ["ShearY", 0.40625439617553727, 0.4516437918820778]], [["TranslateX", 0.15015718916062992, 0.13867777502116208], ["Brightness", 0.3374464418810188, 0.7613355669536931]], [["Invert", 0.644644393321966, 0.19005804481199562], ["AutoContrast", 0.2293259789431853, 0.30335723256340186]], [["Solarize", 0.004968793254801596, 0.5370892072646645], ["Contrast", 0.9136902637865596, 0.9510587477779084]], [["Rotate", 0.38991518440867123, 0.24796987467455756], ["Sharpness", 0.9911180315669776, 0.5265657122981591]], [["Solarize", 0.3919646484436238, 0.6814994037194909], ["Sharpness", 0.4920838987787103, 0.023425724294012018]], [["TranslateX", 0.25107587874378867, 0.5414936560189212], ["Cutout", 0.7932919623814599, 0.9891303444820169]], [["Brightness", 0.07863012174272999, 0.045175652208389594], ["Solarize", 0.889609658064552, 0.8228793315963948]], [["Cutout", 0.20477096178169596, 0.6535063675027364], ["ShearX", 0.9216318577173639, 0.2908690977359947]], [["Contrast", 0.7035118947423187, 0.45982709058312454], ["Contrast", 0.7130268070749464, 0.8635123354235471]], [["Sharpness", 0.26319477541228997, 0.7451278726847078], ["Rotate", 0.8170499362173754, 0.13998593411788207]], [["Rotate", 0.8699365715164192, 0.8878057721750832], ["Equalize", 0.06682350555715044, 0.7164702080630689]], [["ShearY", 0.3137466057521987, 0.6747433496011368], ["Rotate", 0.42118828936218133, 0.980121180104441]], [["Solarize", 0.8470375049950615, 0.15287589264139223], ["Cutout", 0.14438435054693055, 0.24296463267973512]], [["TranslateY", 0.08822241792224905, 0.36163911974799356], ["TranslateY", 0.11729726813270003, 0.6230889726445291]], [["ShearX", 0.7720112337718541, 0.2773292905760122], ["Sharpness", 0.756290929398613, 0.27830353710507705]], [["Color", 0.33825031007968287, 0.4657590047522816], ["ShearY", 0.3566628994713067, 0.859750504071925]], [["TranslateY", 0.06830147433378053, 0.9348778582086664], ["TranslateX", 0.15509346516378553, 0.26320778885339435]], [["Posterize", 0.20266751150740858, 0.008351463842578233], ["Sharpness", 0.06506971109417259, 0.7294471760284555]], [["TranslateY", 0.6278911394418829, 0.8702181892620695], ["Invert", 0.9367073860264247, 0.9219230428944211]], [["Sharpness", 0.1553425337673321, 0.17601557714491345], ["Solarize", 0.7040449681338888, 0.08764313147327729]], [["Equalize", 0.6082233904624664, 0.4177428549911376], ["AutoContrast", 0.04987405274618151, 0.34516208204700916]], [["Brightness", 0.9616085936167699, 0.14561237331885468], ["Solarize", 0.8927707736296572, 0.31176907850205704]], [["Brightness", 0.6707778304730988, 0.9046457117525516], ["Brightness", 0.6801448953060988, 0.20015313057149042]], [["Color", 0.8292680845499386, 0.5181603879593888], ["Brightness", 0.08549161770369762, 0.6567870536463203]], [["ShearY", 0.267802208078051, 0.8388133819588173], ["Sharpness", 0.13453409120796123, 0.10028351311149486]], [["Posterize", 0.775796593610272, 0.05359034561289766], ["Cutout", 0.5067360625733027, 0.054451986840317934]], [["TranslateX", 0.5845238647690084, 0.7507147553486293], ["Brightness", 0.2642051786121197, 0.2578358927056452]], [["Cutout", 0.10787517610922692, 0.8147986902794228], ["Contrast", 0.2190149206329539, 0.902210615462459]], [["TranslateX", 0.5663614214181296, 0.05309965916414028], ["ShearX", 0.9682797885154938, 0.41791929533938466]], [["ShearX", 0.2345325577621098, 0.383780128037189], ["TranslateX", 0.7298083748149163, 0.644325797667087]], [["Posterize", 0.5138725709682734, 0.7901809917259563], ["AutoContrast", 0.7966018627776853, 0.14529337543427345]], [["Invert", 0.5973031989249785, 0.417399314592829], ["Solarize", 0.9147539948653116, 0.8221272315548086]], [["Posterize", 0.601596043336383, 0.18969646160963938], ["Color", 0.7527275484079655, 0.431793831326888]], [["Equalize", 0.6731483454430538, 0.7866786558207602], ["TranslateX", 0.97574396899191, 0.5970255778044692]], [["Cutout", 0.15919495850169718, 0.8916094305850562], ["Invert", 0.8351348834751027, 0.4029937360314928]], [["Invert", 0.5894085405226027, 0.7283806854157764], ["Brightness", 0.3973976860470554, 0.949681121498567]], [["AutoContrast", 0.3707914135327408, 0.21192068592079616], ["ShearX", 0.28040127351140676, 0.6754553511344856]], [["Solarize", 0.07955132378694896, 0.15073572961927306], ["ShearY", 0.5735850168851625, 0.27147326850217746]], [["Equalize", 0.678653949549764, 0.8097796067861455], ["Contrast", 0.2283048527510083, 0.15507804874474185]], [["Equalize", 0.286013868374536, 0.186785848694501], ["Posterize", 0.16319021740810458, 0.1201304443285659]], [["Sharpness", 0.9601590830563757, 0.06267915026513238], ["AutoContrast", 0.3813920685124327, 0.294224403296912]], [["Brightness", 0.2703246632402241, 0.9168405377492277], ["ShearX", 0.6156009855831097, 0.4955986055846403]], [["Color", 0.9065504424987322, 0.03393612216080133], ["ShearY", 0.6768595880405884, 0.9981068127818191]], [["Equalize", 0.28812842368483904, 0.300387487349145], ["ShearY", 0.28812248704858345, 0.27105076231533964]], [["Brightness", 0.6864882730513477, 0.8205553299102412], ["Cutout", 0.45995236371265424, 0.5422030370297759]], [["Color", 0.34941404877084326, 0.25857961830158516], ["AutoContrast", 0.3451390878441899, 0.5000938249040454]], [["Invert", 0.8268247541815854, 0.6691380821226468], ["Cutout", 0.46489193601530476, 0.22620873109485895]], [["Rotate", 0.17879730528062376, 0.22670425330593935], ["Sharpness", 0.8692795688221834, 0.36586055020855723]], [["Brightness", 0.31203975139659634, 0.6934046293010939], ["Cutout", 0.31649437872271236, 0.08078625004157935]], [["Cutout", 0.3119482836150119, 0.6397160035509996], ["Contrast", 0.8311248624784223, 0.22897510169718616]], [["TranslateX", 0.7631157841429582, 0.6482890521284557], ["Brightness", 0.12681196272427664, 0.3669813784257344]], [["TranslateX", 0.06027722649179801, 0.3101104512201861], ["Sharpness", 0.5652076706249394, 0.05210008400968136]], [["AutoContrast", 0.39213552101583127, 0.5047021194355596], ["ShearY", 0.7164003055682187, 0.8063370761002899]], [["Solarize", 0.9574307011238342, 0.21472064809226854], ["AutoContrast", 0.8102612285047174, 0.716870148067014]], [["Rotate", 0.3592634277567387, 0.6452602893051465], ["AutoContrast", 0.27188430331411506, 0.06003099168464854]], [["Cutout", 0.9529536554825503, 0.5285505311027461], ["Solarize", 0.08478231903311029, 0.15986449762728216]], [["TranslateY", 0.31176130458018936, 0.5642853506158253], ["Equalize", 0.008890883901317648, 0.5146121040955942]], [["Color", 0.40773645085566157, 0.7110398926612682], ["Color", 0.18233100156439364, 0.7830036002758337]], [["Posterize", 0.5793809197821732, 0.043748553135581236], ["Invert", 0.4479962016131668, 0.7349663010359488]], [["TranslateX", 0.1994882312299382, 0.05216859488899439], ["Rotate", 0.48288726352035416, 0.44713829026777585]], [["Posterize", 0.22122838185154603, 0.5034546841241283], ["TranslateX", 0.2538745835410222, 0.6129055170893385]], [["Color", 0.6786559960640814, 0.4529749369803212], ["Equalize", 0.30215879674415336, 0.8733394611096772]], [["Contrast", 0.47316062430673456, 0.46669538897311447], ["Invert", 0.6514906551984854, 0.3053339444067804]], [["Equalize", 0.6443202625334524, 0.8689731394616441], ["Color", 0.7549183794057628, 0.8889001426329578]], [["Solarize", 0.616709740662654, 0.7792180816399313], ["ShearX", 0.9659155537406062, 0.39436937531179495]], [["Equalize", 0.23694011299406226, 0.027711152164392128], ["TranslateY", 0.1677339686527083, 0.3482126536808231]], [["Solarize", 0.15234175951790285, 0.7893840414281341], ["TranslateX", 0.2396395768284183, 0.27727219214979715]], [["Contrast", 0.3792017455380605, 0.32323660409845334], ["Contrast", 0.1356037413846466, 0.9127772969992305]], [["ShearX", 0.02642732222284716, 0.9184662576502115], ["Equalize", 0.11504884472142995, 0.8957638893097964]], [["TranslateY", 0.3193812913345325, 0.8828100030493128], ["ShearY", 0.9374975727563528, 0.09909415611083694]], [["AutoContrast", 0.025840721736048122, 0.7941037581373024], ["TranslateY", 0.498518003323313, 0.5777122846572548]], [["ShearY", 0.6042199307830248, 0.44809668754508836], ["Cutout", 0.3243978207701482, 0.9379740926294765]], [["ShearY", 0.6858549297583574, 0.9993252035788924], ["Sharpness", 0.04682428732773203, 0.21698099707915652]], [["ShearY", 0.7737469436637263, 0.8810127181224531], ["ShearY", 0.8995655445246451, 0.4312416220354539]], [["TranslateY", 0.4953094136709374, 0.8144161580138571], ["Solarize", 0.26301211718928097, 0.518345311180405]], [["Brightness", 0.8820246486031275, 0.571075863786249], ["ShearX", 0.8586669146703955, 0.0060476383595142735]], [["Sharpness", 0.20519233710982254, 0.6144574759149729], ["Posterize", 0.07976625267460813, 0.7480145046726968]], [["ShearY", 0.374075419680195, 0.3386105402023202], ["ShearX", 0.8228083637082115, 0.5885174783155361]], [["Brightness", 0.3528780713814561, 0.6999884884306623], ["Sharpness", 0.3680348120526238, 0.16953358258959617]], [["Brightness", 0.24891223104442084, 0.7973853494920095], ["TranslateX", 0.004256803835524736, 0.0470216343108546]], [["Posterize", 0.1947344282646012, 0.7694802711054367], ["Cutout", 0.9594385534844785, 0.5469744140592429]], [["Invert", 0.19012504762806026, 0.7816140211434693], ["TranslateY", 0.17479746932338402, 0.024249345245078602]], [["Rotate", 0.9669262055946796, 0.510166180775991], ["TranslateX", 0.8990602034610352, 0.6657802719304693]], [["ShearY", 0.5453049050407278, 0.8476872739603525], ["Cutout", 0.14226529093962592, 0.15756960661106634]], [["Equalize", 0.5895291156113004, 0.6797218994447763], ["TranslateY", 0.3541442192192753, 0.05166001155849864]], [["Equalize", 0.39530681662726097, 0.8448335365081087], ["Brightness", 0.6785483272734143, 0.8805568647038574]], [["Cutout", 0.28633258271917905, 0.7750870268336066], ["Equalize", 0.7221097824537182, 0.5865506280531162]], [["Posterize", 0.9044429629421187, 0.4620266401793388], ["Invert", 0.1803008045494473, 0.8073190766288534]], [["Sharpness", 0.7054649148075851, 0.3877207948962055], ["TranslateX", 0.49260224225927285, 0.8987462620731029]], [["Sharpness", 0.11196934729294483, 0.5953704422694938], ["Contrast", 0.13969334315069737, 0.19310569898434204]], [["Posterize", 0.5484346101051778, 0.7914140118600685], ["Brightness", 0.6428044691630473, 0.18811316670808076]], [["Invert", 0.22294834094984717, 0.05173157689962704], ["Cutout", 0.6091129168510456, 0.6280845506243643]], [["AutoContrast", 0.5726444076195267, 0.2799840903601295], ["Cutout", 0.3055752727786235, 0.591639807512993]], [["Brightness", 0.3707116723204462, 0.4049175910826627], ["Rotate", 0.4811601625588309, 0.2710760253723644]], [["ShearY", 0.627791719653608, 0.6877498291550205], ["TranslateX", 0.8751753308366824, 0.011164650018719358]], [["Posterize", 0.33832547954522263, 0.7087039872581657], ["Posterize", 0.6247474435007484, 0.7707784192114796]], [["Contrast", 0.17620186308493468, 0.9946224854942095], ["Solarize", 0.5431896088395964, 0.5867904203742308]], [["ShearX", 0.4667959516719652, 0.8938082224109446], ["TranslateY", 0.7311343008292865, 0.6829842246020277]], [["ShearX", 0.6130281467237769, 0.9924010909612302], ["Brightness", 0.41039241699696916, 0.9753218875311392]], [["TranslateY", 0.0747250386427123, 0.34602725521067534], ["Rotate", 0.5902597465515901, 0.361094672021087]], [["Invert", 0.05234890878959486, 0.36914978664919407], ["Sharpness", 0.42140532878231374, 0.19204058551048275]], [["ShearY", 0.11590485361909497, 0.6518540857972316], ["Invert", 0.6482444740361704, 0.48256237896163945]], [["Rotate", 0.4931329446923608, 0.037076242417301675], ["Contrast", 0.9097939772412852, 0.5619594905306389]], [["Posterize", 0.7311032479626216, 0.4796364593912915], ["Color", 0.13912123993932402, 0.03997286439663705]], [["AutoContrast", 0.6196602944085344, 0.2531430457527588], ["Rotate", 0.5583937060431972, 0.9893379795224023]], [["AutoContrast", 0.8847753125072959, 0.19123028952580057], ["TranslateY", 0.494361716097206, 0.14232297727461696]], [["Invert", 0.6212360716340707, 0.033898871473033165], ["AutoContrast", 0.30839896957008295, 0.23603569542166247]], [["Equalize", 0.8255583546605049, 0.613736933157845], ["AutoContrast", 0.6357166629525485, 0.7894617347709095]], [["Brightness", 0.33840706322846814, 0.07917167871493658], ["ShearY", 0.15693175752528676, 0.6282773652129153]], [["Cutout", 0.7550520024859294, 0.08982367300605598], ["ShearX", 0.5844942417320858, 0.36051195083380105]]] + return p + + +def fa_reduced_imagenet(): + p = [[["ShearY", 0.14143816458479197, 0.513124791615952], ["Sharpness", 0.9290316227291179, 0.9788406212603302]], [["Color", 0.21502874228385338, 0.3698477943880306], ["TranslateY", 0.49865058747734736, 0.4352676987103321]], [["Brightness", 0.6603452126485386, 0.6990174510500261], ["Cutout", 0.7742953773992511, 0.8362550883640804]], [["Posterize", 0.5188375788270497, 0.9863648925446865], ["TranslateY", 0.8365230108655313, 0.6000972236440252]], [["ShearY", 0.9714994964711299, 0.2563663552809896], ["Equalize", 0.8987567223581153, 0.1181761775609772]], [["Sharpness", 0.14346409304565366, 0.5342189791746006], ["Sharpness", 0.1219714162835897, 0.44746801278319975]], [["TranslateX", 0.08089260772173967, 0.028011721602479833], ["TranslateX", 0.34767877352421406, 0.45131294688688794]], [["Brightness", 0.9191164585327378, 0.5143232242627864], ["Color", 0.9235247849934283, 0.30604586249462173]], [["Contrast", 0.4584173187505879, 0.40314219914942756], ["Rotate", 0.550289356406774, 0.38419022293237126]], [["Posterize", 0.37046156420799325, 0.052693291117634544], ["Cutout", 0.7597581409366909, 0.7535799791937421]], [["Color", 0.42583964114658746, 0.6776641859552079], ["ShearY", 0.2864805671096011, 0.07580175477739545]], [["Brightness", 0.5065952125552232, 0.5508640233704984], ["Brightness", 0.4760021616081475, 0.3544313318097987]], [["Posterize", 0.5169630851995185, 0.9466018906715961], ["Posterize", 0.5390336503396841, 0.1171015788193209]], [["Posterize", 0.41153170909576176, 0.7213063942615204], ["Rotate", 0.6232230424824348, 0.7291984098675746]], [["Color", 0.06704687234714028, 0.5278429246040438], ["Sharpness", 0.9146652195810183, 0.4581415618941407]], [["ShearX", 0.22404644446773492, 0.6508620171913467], ["Brightness", 0.06421961538672451, 0.06859528721039095]], [["Rotate", 0.29864103693134797, 0.5244313199644495], ["Sharpness", 0.4006161706584276, 0.5203708477368657]], [["AutoContrast", 0.5748186910788027, 0.8185482599354216], ["Posterize", 0.9571441684265188, 0.1921474117448481]], [["ShearY", 0.5214786760436251, 0.8375629059785009], ["Invert", 0.6872393349333636, 0.9307694335024579]], [["Contrast", 0.47219838080793364, 0.8228524484275648], ["TranslateY", 0.7435518856840543, 0.5888865560614439]], [["Posterize", 0.10773482839638836, 0.6597021018893648], ["Contrast", 0.5218466423129691, 0.562985661685268]], [["Rotate", 0.4401753067886466, 0.055198255925702475], ["Rotate", 0.3702153509335602, 0.5821574425474759]], [["TranslateY", 0.6714729117832363, 0.7145542887432927], ["Equalize", 0.0023263758097700205, 0.25837341854887885]], [["Cutout", 0.3159707561240235, 0.19539664199170742], ["TranslateY", 0.8702824829864558, 0.5832348977243467]], [["AutoContrast", 0.24800812729140026, 0.08017301277245716], ["Brightness", 0.5775505849482201, 0.4905904775616114]], [["Color", 0.4143517886294533, 0.8445937742921498], ["ShearY", 0.28688910858536587, 0.17539366839474402]], [["Brightness", 0.6341134194059947, 0.43683815933640435], ["Brightness", 0.3362277685899835, 0.4612826163288225]], [["Sharpness", 0.4504035748829761, 0.6698294470467474], ["Posterize", 0.9610055612671645, 0.21070714173174876]], [["Posterize", 0.19490421920029832, 0.7235798208354267], ["Rotate", 0.8675551331308305, 0.46335565746433094]], [["Color", 0.35097958351003306, 0.42199181561523186], ["Invert", 0.914112788087429, 0.44775583211984815]], [["Cutout", 0.223575616055454, 0.6328591417299063], ["TranslateY", 0.09269465212259387, 0.5101073959070608]], [["Rotate", 0.3315734525975911, 0.9983593458299167], ["Sharpness", 0.12245416662856974, 0.6258689139914664]], [["ShearY", 0.696116760180471, 0.6317805202283014], ["Color", 0.847501151593963, 0.4440116609830195]], [["Solarize", 0.24945891607225948, 0.7651150206105561], ["Cutout", 0.7229677092930331, 0.12674657348602494]], [["TranslateX", 0.43461945065713675, 0.06476571036747841], ["Color", 0.6139316940180952, 0.7376264330632316]], [["Invert", 0.1933003530637138, 0.4497819016184308], ["Invert", 0.18391634069983653, 0.3199769100951113]], [["Color", 0.20418296626476137, 0.36785101882029814], ["Posterize", 0.624658293920083, 0.8390081535735991]], [["Sharpness", 0.5864963540530814, 0.586672446690273], ["Posterize", 0.1980280647652339, 0.222114611452575]], [["Invert", 0.3543654961628104, 0.5146369635250309], ["Equalize", 0.40751271919434434, 0.4325310837291978]], [["ShearY", 0.22602859359451877, 0.13137880879778158], ["Posterize", 0.7475029061591305, 0.803900538461099]], [["Sharpness", 0.12426276165599924, 0.5965912716602046], ["Invert", 0.22603903038966913, 0.4346802001255868]], [["TranslateY", 0.010307035630661765, 0.16577665156754046], ["Posterize", 0.4114319141395257, 0.829872913683949]], [["TranslateY", 0.9353069865746215, 0.5327821671247214], ["Color", 0.16990443486261103, 0.38794866007484197]], [["Cutout", 0.1028174322829021, 0.3955952903458266], ["ShearY", 0.4311995281335693, 0.48024695395374734]], [["Posterize", 0.1800334334284686, 0.0548749478418862], ["Brightness", 0.7545808536793187, 0.7699080551646432]], [["Color", 0.48695305373084197, 0.6674269768464615], ["ShearY", 0.4306032279086781, 0.06057690550239343]], [["Brightness", 0.4919399683825053, 0.677338905806407], ["Brightness", 0.24112708387760828, 0.42761103121157656]], [["Posterize", 0.4434818644882532, 0.9489450593207714], ["Posterize", 0.40957675116385955, 0.015664946759584186]], [["Posterize", 0.41307949855153797, 0.6843276552020272], ["Rotate", 0.8003545094091291, 0.7002300783416026]], [["Color", 0.7038570031770905, 0.4697612983649519], ["Sharpness", 0.9700016496081002, 0.25185103545948884]], [["AutoContrast", 0.714641656154856, 0.7962423001719023], ["Sharpness", 0.2410097684093468, 0.5919171048019731]], [["TranslateX", 0.8101567644494714, 0.7156447005337443], ["Solarize", 0.5634727831229329, 0.8875158446846]], [["Sharpness", 0.5335258857303261, 0.364743126378182], ["Color", 0.453280875871377, 0.5621962714743068]], [["Cutout", 0.7423678127672542, 0.7726370777867049], ["Invert", 0.2806161382641934, 0.6021111986900146]], [["TranslateY", 0.15190341320343761, 0.3860373175487939], ["Cutout", 0.9980805818665679, 0.05332384819400854]], [["Posterize", 0.36518675678786605, 0.2935819027397963], ["TranslateX", 0.26586180351840005, 0.303641300745208]], [["Brightness", 0.19994509744377761, 0.90813953707639], ["Equalize", 0.8447217761297836, 0.3449396603478335]], [["Sharpness", 0.9294773669936768, 0.999713346583839], ["Brightness", 0.1359744825665662, 0.1658489221872924]], [["TranslateX", 0.11456529257659381, 0.9063795878367734], ["Equalize", 0.017438134319894553, 0.15776887259743755]], [["ShearX", 0.9833726383270114, 0.5688194948373335], ["Equalize", 0.04975615490994345, 0.8078130016227757]], [["Brightness", 0.2654654830488695, 0.8989789725280538], ["TranslateX", 0.3681535065952329, 0.36433345713161036]], [["Rotate", 0.04956524209892327, 0.5371942433238247], ["ShearY", 0.0005527499145153714, 0.56082571605602]], [["Rotate", 0.7918337108932019, 0.5906896260060501], ["Posterize", 0.8223967034091191, 0.450216998388943]], [["Color", 0.43595106766978337, 0.5253013785221605], ["Sharpness", 0.9169421073531799, 0.8439997639348893]], [["TranslateY", 0.20052300197155504, 0.8202662448307549], ["Sharpness", 0.2875792108435686, 0.6997181624527842]], [["Color", 0.10568089980973616, 0.3349467065132249], ["Brightness", 0.13070947282207768, 0.5757725013960775]], [["AutoContrast", 0.3749999712869779, 0.6665578760607657], ["Brightness", 0.8101178402610292, 0.23271946112218125]], [["Color", 0.6473605933679651, 0.7903409763232029], ["ShearX", 0.588080941572581, 0.27223524148254086]], [["Cutout", 0.46293361616697304, 0.7107761001833921], ["AutoContrast", 0.3063766931658412, 0.8026114219854579]], [["Brightness", 0.7884854981520251, 0.5503669863113797], ["Brightness", 0.5832456158675261, 0.5840349298921661]], [["Solarize", 0.4157539625058916, 0.9161905834309929], ["Sharpness", 0.30628197221802017, 0.5386291658995193]], [["Sharpness", 0.03329610069672856, 0.17066672983670506], ["Invert", 0.9900547302690527, 0.6276238841220477]], [["Solarize", 0.551015648982762, 0.6937104775938737], ["Color", 0.8838491591064375, 0.31596634380795385]], [["AutoContrast", 0.16224182418148447, 0.6068227969351896], ["Sharpness", 0.9599468096118623, 0.4885289719905087]], [["TranslateY", 0.06576432526133724, 0.6899544605400214], ["Posterize", 0.2177096480169678, 0.9949164789616582]], [["Solarize", 0.529820544480292, 0.7576047224165541], ["Sharpness", 0.027047878909321643, 0.45425231553970685]], [["Sharpness", 0.9102526010473146, 0.8311987141993857], ["Invert", 0.5191838751826638, 0.6906136644742229]], [["Solarize", 0.4762773516008588, 0.7703654263842423], ["Color", 0.8048437792602289, 0.4741523094238038]], [["Sharpness", 0.7095055508594206, 0.7047344238075169], ["Sharpness", 0.5059623654132546, 0.6127255499234886]], [["TranslateY", 0.02150725921966186, 0.3515764519224378], ["Posterize", 0.12482170119714735, 0.7829851754051393]], [["Color", 0.7983830079184816, 0.6964694521670339], ["Brightness", 0.3666527856286296, 0.16093151636495978]], [["AutoContrast", 0.6724982375829505, 0.536777706678488], ["Sharpness", 0.43091754837597646, 0.7363240924241439]], [["Brightness", 0.2889770401966227, 0.4556557902380539], ["Sharpness", 0.8805303296690755, 0.6262218017754902]], [["Sharpness", 0.5341939854581068, 0.6697109101429343], ["Rotate", 0.6806606655137529, 0.4896914517968317]], [["Sharpness", 0.5690509737059344, 0.32790632371915096], ["Posterize", 0.7951894258661069, 0.08377850335209162]], [["Color", 0.6124132978216081, 0.5756485920709012], ["Brightness", 0.33053544654445344, 0.23321841707002083]], [["TranslateX", 0.0654795026615917, 0.5227246924310244], ["ShearX", 0.2932320531132063, 0.6732066478183716]], [["Cutout", 0.6226071187083615, 0.01009274433736012], ["ShearX", 0.7176799968189801, 0.3758780240463811]], [["Rotate", 0.18172339508029314, 0.18099184896819184], ["ShearY", 0.7862658331645667, 0.295658135767252]], [["Contrast", 0.4156099177015862, 0.7015784500878446], ["Sharpness", 0.6454135310009, 0.32335858947955287]], [["Color", 0.6215885089922037, 0.6882673235388836], ["Brightness", 0.3539881732605379, 0.39486736455795496]], [["Invert", 0.8164816716866418, 0.7238192000817796], ["Sharpness", 0.3876355847343607, 0.9870077619731956]], [["Brightness", 0.1875628712629315, 0.5068115936257], ["Sharpness", 0.8732419122060423, 0.5028019258530066]], [["Sharpness", 0.6140734993408259, 0.6458239834366959], ["Rotate", 0.5250107862824867, 0.533419456933602]], [["Sharpness", 0.5710893143725344, 0.15551651073007305], ["ShearY", 0.6548487860151722, 0.021365083044319146]], [["Color", 0.7610250354649954, 0.9084452893074055], ["Brightness", 0.6934611792619156, 0.4108071412071374]], [["ShearY", 0.07512550098923898, 0.32923768385754293], ["ShearY", 0.2559588911696498, 0.7082337365398496]], [["Cutout", 0.5401319018926146, 0.004750568603408445], ["ShearX", 0.7473354415031975, 0.34472481968368773]], [["Rotate", 0.02284154583679092, 0.1353450082435801], ["ShearY", 0.8192458031684238, 0.2811653613473772]], [["Contrast", 0.21142896718139154, 0.7230739568811746], ["Sharpness", 0.6902690582665707, 0.13488436112901683]], [["Posterize", 0.21701219600958138, 0.5900695769640687], ["Rotate", 0.7541095031505971, 0.5341162375286219]], [["Posterize", 0.5772853064792737, 0.45808311743269936], ["Brightness", 0.14366050177823675, 0.4644871239446629]], [["Cutout", 0.8951718842805059, 0.4970074074310499], ["Equalize", 0.3863835903119882, 0.9986531042150006]], [["Equalize", 0.039411354473938925, 0.7475477254908457], ["Sharpness", 0.8741966378291861, 0.7304822679596362]], [["Solarize", 0.4908704265218634, 0.5160677350249471], ["Color", 0.24961813832742435, 0.09362352627360726]], [["Rotate", 7.870457075154214e-05, 0.8086950025500952], ["Solarize", 0.10200484521793163, 0.12312889222989265]], [["Contrast", 0.8052564975559727, 0.3403813036543645], ["Solarize", 0.7690158533600184, 0.8234626822018851]], [["AutoContrast", 0.680362728854513, 0.9415320040873628], ["TranslateY", 0.5305871824686941, 0.8030609611614028]], [["Cutout", 0.1748050257378294, 0.06565343731910589], ["TranslateX", 0.1812738872339903, 0.6254461448344308]], [["Brightness", 0.4230502644722749, 0.3346463682905031], ["ShearX", 0.19107198973659312, 0.6715789128604919]], [["ShearX", 0.1706528684548394, 0.7816570201200446], ["TranslateX", 0.494545185948171, 0.4710810058360291]], [["TranslateX", 0.42356251508933324, 0.23865307292867322], ["TranslateX", 0.24407503619326745, 0.6013778508137331]], [["AutoContrast", 0.7719512185744232, 0.3107905373009763], ["ShearY", 0.49448082925617176, 0.5777951230577671]], [["Cutout", 0.13026983827940525, 0.30120438757485657], ["Brightness", 0.8857896834516185, 0.7731541459513939]], [["AutoContrast", 0.6422800349197934, 0.38637401090264556], ["TranslateX", 0.25085431400995084, 0.3170642592664873]], [["Sharpness", 0.22336654455367122, 0.4137774852324138], ["ShearY", 0.22446851054920894, 0.518341735882535]], [["Color", 0.2597579403253848, 0.7289643913060193], ["Sharpness", 0.5227416670468619, 0.9239943674030637]], [["Cutout", 0.6835337711563527, 0.24777620448593812], ["AutoContrast", 0.37260245353051846, 0.4840361183247263]], [["Posterize", 0.32756602788628375, 0.21185124493743707], ["ShearX", 0.25431504951763967, 0.19585996561416225]], [["AutoContrast", 0.07930627591849979, 0.5719381348340309], ["AutoContrast", 0.335512380071304, 0.4208050118308541]], [["Rotate", 0.2924360268257798, 0.5317629242879337], ["Sharpness", 0.4531050021499891, 0.4102650087199528]], [["Equalize", 0.5908862210984079, 0.468742362277498], ["Brightness", 0.08571766548550425, 0.5629320703375056]], [["Cutout", 0.52751122383816, 0.7287774744737556], ["Equalize", 0.28721628275296274, 0.8075179887475786]], [["AutoContrast", 0.24208377391366226, 0.34616549409607644], ["TranslateX", 0.17454707403766834, 0.5278055700078459]], [["Brightness", 0.5511881924749478, 0.999638675514418], ["Equalize", 0.14076197797220913, 0.2573030693317552]], [["ShearX", 0.668731433926434, 0.7564253049646743], ["Color", 0.63235486543845, 0.43954436063340785]], [["ShearX", 0.40511960873276237, 0.5710419512142979], ["Contrast", 0.9256769948746423, 0.7461350716211649]], [["Cutout", 0.9995917204023061, 0.22908419326246265], ["TranslateX", 0.5440902956629469, 0.9965570051216295]], [["Color", 0.22552987172228894, 0.4514558960849747], ["Sharpness", 0.638058150559443, 0.9987829481002615]], [["Contrast", 0.5362775837534763, 0.7052133185951871], ["ShearY", 0.220369845547023, 0.7593922994775721]], [["ShearX", 0.0317785822935219, 0.775536785253455], ["TranslateX", 0.7939510227015061, 0.5355620618496535]], [["Cutout", 0.46027969917602196, 0.31561199122527517], ["Color", 0.06154066467629451, 0.5384660000729091]], [["Sharpness", 0.7205483743301113, 0.552222392539886], ["Posterize", 0.5146496404711752, 0.9224333144307473]], [["ShearX", 0.00014547730356910538, 0.3553954298642108], ["TranslateY", 0.9625736029090676, 0.57403418640424]], [["Posterize", 0.9199917903297341, 0.6690259107633706], ["Posterize", 0.0932558110217602, 0.22279303372106138]], [["Invert", 0.25401453476874863, 0.3354329544078385], ["Posterize", 0.1832673201325652, 0.4304718799821412]], [["TranslateY", 0.02084122674367607, 0.12826181437197323], ["ShearY", 0.655862534043703, 0.3838330909470975]], [["Contrast", 0.35231797644104523, 0.3379356652070079], ["Cutout", 0.19685599014304822, 0.1254328595280942]], [["Sharpness", 0.18795594984191433, 0.09488678946484895], ["ShearX", 0.33332876790679306, 0.633523782574133]], [["Cutout", 0.28267175940290246, 0.7901991550267817], ["Contrast", 0.021200195312951198, 0.4733128702798515]], [["ShearX", 0.966231043411256, 0.7700673327786812], ["TranslateX", 0.7102390777763321, 0.12161245817120675]], [["Cutout", 0.5183324259533826, 0.30766086003013055], ["Color", 0.48399078150128927, 0.4967477809069189]], [["Sharpness", 0.8160855187385873, 0.47937658961644], ["Posterize", 0.46360395447862535, 0.7685454058155061]], [["ShearX", 0.10173571421694395, 0.3987290690178754], ["TranslateY", 0.8939980277379345, 0.5669994143735713]], [["Posterize", 0.6768089584801844, 0.7113149244621721], ["Posterize", 0.054896856043358935, 0.3660837250743921]], [["AutoContrast", 0.5915576211896306, 0.33607718177676493], ["Contrast", 0.3809408206617828, 0.5712201773913784]], [["AutoContrast", 0.012321347472748323, 0.06379072432796573], ["Rotate", 0.0017964439160045656, 0.7598026295973337]], [["Contrast", 0.6007100085192627, 0.36171972473370206], ["Invert", 0.09553573684975913, 0.12218510774295901]], [["AutoContrast", 0.32848604643836266, 0.2619457656206414], ["Invert", 0.27082113532501784, 0.9967965642293485]], [["AutoContrast", 0.6156282120903395, 0.9422706516080884], ["Sharpness", 0.4215509247379262, 0.4063347716503587]], [["Solarize", 0.25059210436331264, 0.7215305521159305], ["Invert", 0.1654465185253614, 0.9605851884186778]], [["AutoContrast", 0.4464438610980994, 0.685334175815482], ["Cutout", 0.24358625461158645, 0.4699066834058694]], [["Rotate", 0.5931657741857909, 0.6813978655574067], ["AutoContrast", 0.9259100547738681, 0.4903201223870492]], [["Color", 0.8203976071280751, 0.9777824466585101], ["Posterize", 0.4620669369254169, 0.2738895968716055]], [["Contrast", 0.13754352055786848, 0.3369433962088463], ["Posterize", 0.48371187792441916, 0.025718004361451302]], [["Rotate", 0.5208233630704999, 0.1760188899913535], ["TranslateX", 0.49753461392937226, 0.4142935276250922]], [["Cutout", 0.5967418240931212, 0.8028675552639539], ["Cutout", 0.20021854152659121, 0.19426330549590076]], [["ShearY", 0.549583567386676, 0.6601326640171705], ["Cutout", 0.6111813470383047, 0.4141935587984994]], [["Brightness", 0.6354891977535064, 0.31591459747846745], ["AutoContrast", 0.7853952208711621, 0.6555861906702081]], [["AutoContrast", 0.7333725370546154, 0.9919410576081586], ["Cutout", 0.9984177877923588, 0.2938253683694291]], [["Color", 0.33219296307742263, 0.6378995578424113], ["AutoContrast", 0.15432820754183288, 0.7897899838932103]], [["Contrast", 0.5905289460222578, 0.8158577207653422], ["Cutout", 0.3980284381203051, 0.43030531250317217]], [["TranslateX", 0.452093693346745, 0.5251475931559115], ["Rotate", 0.991422504871258, 0.4556503729269001]], [["Color", 0.04560406292983776, 0.061574671308480766], ["Brightness", 0.05161079440128734, 0.6718398142425688]], [["Contrast", 0.02913302416506853, 0.14402056093217708], ["Rotate", 0.7306930378774588, 0.47088249057922094]], [["Solarize", 0.3283072384190169, 0.82680847744367], ["Invert", 0.21632614168418854, 0.8792241691482687]], [["Equalize", 0.4860808352478527, 0.9440534949023064], ["Cutout", 0.31395897639184694, 0.41805859306017523]], [["Rotate", 0.2816043232522335, 0.5451282807926706], ["Color", 0.7388520447173302, 0.7706503658143311]], [["Color", 0.9342776719536201, 0.9039981381514299], ["Rotate", 0.6646389177840164, 0.5147917008383647]], [["Cutout", 0.08929430082050335, 0.22416445996932374], ["Posterize", 0.454485751267457, 0.500958345348237]], [["TranslateX", 0.14674201106374488, 0.7018633472428202], ["Sharpness", 0.6128796723832848, 0.743535235614809]], [["TranslateX", 0.5189900164469432, 0.6491132403587601], ["Contrast", 0.26309555778227806, 0.5976857969656114]], [["Solarize", 0.23569808291972655, 0.3315781686591778], ["ShearY", 0.07292078937544964, 0.7460326987587573]], [["ShearY", 0.7090542757477153, 0.5246437008439621], ["Sharpness", 0.9666919148538443, 0.4841687888767071]], [["Solarize", 0.3486952615189488, 0.7012877201721799], ["Invert", 0.1933387967311534, 0.9535472742828175]], [["AutoContrast", 0.5393460721514914, 0.6924005011697713], ["Cutout", 0.16988156769247176, 0.3667207571712882]], [["Rotate", 0.5815329514554719, 0.5390406879316949], ["AutoContrast", 0.7370538341589625, 0.7708822194197815]], [["Color", 0.8463701017918459, 0.9893491045831084], ["Invert", 0.06537367901579016, 0.5238468509941635]], [["Contrast", 0.8099771812443645, 0.39371603893945184], ["Posterize", 0.38273629875646487, 0.46493786058573966]], [["Color", 0.11164686537114032, 0.6771450570033168], ["Posterize", 0.27921361289661406, 0.7214300893597819]], [["Contrast", 0.5958265906571906, 0.5963959447666958], ["Sharpness", 0.2640889223630885, 0.3365870842641453]], [["Color", 0.255634146724125, 0.5610029792926452], ["ShearY", 0.7476893976084721, 0.36613194760395557]], [["ShearX", 0.2167581882130063, 0.022978065071245002], ["TranslateX", 0.1686864409720319, 0.4919575435512007]], [["Solarize", 0.10702753776284957, 0.3954707963684698], ["Contrast", 0.7256100635368403, 0.48845259655719686]], [["Sharpness", 0.6165615058519549, 0.2624079463213861], ["ShearX", 0.3804820351860919, 0.4738994677544202]], [["TranslateX", 0.18066394808448177, 0.8174509422318228], ["Solarize", 0.07964569396290502, 0.45495935736800974]], [["Sharpness", 0.2741884021129658, 0.9311045302358317], ["Cutout", 0.0009101326429323388, 0.5932102256756948]], [["Rotate", 0.8501796375826188, 0.5092564038282137], ["Brightness", 0.6520146983999912, 0.724091283316938]], [["Brightness", 0.10079744898900078, 0.7644088017429471], ["AutoContrast", 0.33540215138213575, 0.1487538541758792]], [["ShearY", 0.10632545944757177, 0.9565164562996977], ["Rotate", 0.275833816849538, 0.6200731548023757]], [["Color", 0.6749819274397422, 0.41042188598168844], ["AutoContrast", 0.22396590966461932, 0.5048018491863738]], [["Equalize", 0.5044277111650255, 0.2649182381110667], ["Brightness", 0.35715133289571355, 0.8653260893016869]], [["Cutout", 0.49083594426355326, 0.5602781291093129], ["Posterize", 0.721795488514384, 0.5525847430754974]], [["Sharpness", 0.5081835448947317, 0.7453323423804428], ["TranslateX", 0.11511932212234266, 0.4337766796030984]], [["Solarize", 0.3817050641766593, 0.6879004573473403], ["Invert", 0.0015041436267447528, 0.9793134066888262]], [["AutoContrast", 0.5107410439697935, 0.8276720355454423], ["Cutout", 0.2786270701864015, 0.43993387208414564]], [["Rotate", 0.6711202569428987, 0.6342930903972932], ["Posterize", 0.802820231163559, 0.42770002619222053]], [["Color", 0.9426854321337312, 0.9055431782458764], ["AutoContrast", 0.3556422423506799, 0.2773922428787449]], [["Contrast", 0.10318991257659992, 0.30841372533347416], ["Posterize", 0.4202264962677853, 0.05060395018085634]], [["Invert", 0.549305630337048, 0.886056156681853], ["Cutout", 0.9314157033373055, 0.3485836940307909]], [["ShearX", 0.5642891775895684, 0.16427372934801418], ["Invert", 0.228741164726475, 0.5066345406806475]], [["ShearY", 0.5813123201003086, 0.33474363490586106], ["Equalize", 0.11803439432255824, 0.8583936440614798]], [["Sharpness", 0.1642809706111211, 0.6958675237301609], ["ShearY", 0.5989560762277414, 0.6194018060415276]], [["Rotate", 0.05092104774529638, 0.9358045394527796], ["Cutout", 0.6443254331615441, 0.28548414658857657]], [["Brightness", 0.6986036769232594, 0.9618046340942727], ["Sharpness", 0.5564490243465492, 0.6295231286085622]], [["Brightness", 0.42725649792574105, 0.17628028916784244], ["Equalize", 0.4425109360966546, 0.6392872650036018]], [["ShearY", 0.5758622795525444, 0.8773349286588288], ["ShearX", 0.038525646435423666, 0.8755366512394268]], [["Sharpness", 0.3704459924265827, 0.9236361456197351], ["Color", 0.6379842432311235, 0.4548767717224531]], [["Contrast", 0.1619523824549347, 0.4506528800882731], ["AutoContrast", 0.34513874426188385, 0.3580290330996726]], [["Contrast", 0.728699731513527, 0.6932238009822878], ["Brightness", 0.8602917375630352, 0.5341445123280423]], [["Equalize", 0.3574552353044203, 0.16814745124536548], ["Rotate", 0.24191717169379262, 0.3279497108179034]], [["ShearY", 0.8567478695576244, 0.37746117240238164], ["ShearX", 0.9654125389830487, 0.9283047610798827]], [["ShearY", 0.4339052480582405, 0.5394548246617406], ["Cutout", 0.5070570647967001, 0.7846286976687882]], [["AutoContrast", 0.021620100406875065, 0.44425839772845227], ["AutoContrast", 0.33978157614075183, 0.47716564815092244]], [["Contrast", 0.9727600659025666, 0.6651758819229426], ["Brightness", 0.9893133904996626, 0.39176397622636105]], [["Equalize", 0.283428620586305, 0.18727922861893637], ["Rotate", 0.3556063466797136, 0.3722839913107821]], [["ShearY", 0.7276172841941864, 0.4834188516302227], ["ShearX", 0.010783217950465884, 0.9756458772142235]], [["ShearY", 0.2901753295101581, 0.5684700238749064], ["Cutout", 0.655585564610337, 0.9490071307790201]], [["AutoContrast", 0.008507193981450278, 0.4881150103902877], ["AutoContrast", 0.6561989723231185, 0.3715071329838596]], [["Contrast", 0.7702505530948414, 0.6961371266519999], ["Brightness", 0.9953051630261895, 0.3861962467326121]], [["Equalize", 0.2805270012472756, 0.17715406116880994], ["Rotate", 0.3111256593947474, 0.15824352183820073]], [["Brightness", 0.9888680802094193, 0.4856236485253163], ["ShearX", 0.022370252047332284, 0.9284975906226682]], [["ShearY", 0.4065719044318099, 0.7468528006921563], ["AutoContrast", 0.19494427109708126, 0.8613186475174786]], [["AutoContrast", 0.023296727279367765, 0.9170949567425306], ["AutoContrast", 0.11663051100921168, 0.7908646792175343]], [["AutoContrast", 0.7335191671571732, 0.4958357308292425], ["Color", 0.7964964008349845, 0.4977687544324929]], [["ShearX", 0.19905221600021472, 0.3033081933150046], ["Equalize", 0.9383410219319321, 0.3224669877230161]], [["ShearX", 0.8265450331466404, 0.6509091423603757], ["Sharpness", 0.7134181178748723, 0.6472835976443643]], [["ShearY", 0.46962439525486044, 0.223433110541722], ["Rotate", 0.7749806946212373, 0.5337060376916906]], [["Posterize", 0.1652499695106796, 0.04860659068586126], ["Brightness", 0.6644577712782511, 0.4144528269429337]], [["TranslateY", 0.6220449565731829, 0.4917495676722932], ["Posterize", 0.6255000355409635, 0.8374266890984867]], [["AutoContrast", 0.4887160797052227, 0.7106426020530529], ["Sharpness", 0.7684218571497236, 0.43678474722954763]], [["Invert", 0.13178101535845366, 0.8301141976359813], ["Color", 0.002820877424219378, 0.49444413062487075]], [["TranslateX", 0.9920683666478188, 0.5862245842588877], ["Posterize", 0.5536357075855376, 0.5454300367281468]], [["Brightness", 0.8150181219663427, 0.1411060258870707], ["Sharpness", 0.8548823004164599, 0.77008691072314]], [["Brightness", 0.9580478020413399, 0.7198667636628974], ["ShearY", 0.8431585033377366, 0.38750016565010803]], [["Solarize", 0.2331505347152334, 0.25754361489084787], ["TranslateY", 0.447431373734262, 0.5782399531772253]], [["TranslateY", 0.8904927998691309, 0.25872872455072315], ["AutoContrast", 0.7129888139716263, 0.7161603231650524]], [["ShearY", 0.6336216800247362, 0.5247508616674911], ["Cutout", 0.9167315119726633, 0.2060557387978919]], [["ShearX", 0.001661782345968199, 0.3682225725445044], ["Solarize", 0.12303352043754572, 0.5014989548584458]], [["Brightness", 0.9723625105116246, 0.6555444729681099], ["Contrast", 0.5539208721135375, 0.7819973409318487]], [["Equalize", 0.3262607499912611, 0.0006745572802121513], ["Contrast", 0.35341551623767103, 0.36814689398886347]], [["ShearY", 0.7478539900243613, 0.37322078030129185], ["TranslateX", 0.41558847793529247, 0.7394615158544118]], [["Invert", 0.13735541232529067, 0.5536403864332143], ["Cutout", 0.5109718190377135, 0.0447509485253679]], [["AutoContrast", 0.09403602327274725, 0.5909250807862687], ["ShearY", 0.53234060616395, 0.5316981359469398]], [["ShearX", 0.5651922367876323, 0.6794110241313183], ["Posterize", 0.7431624856363638, 0.7896861463783287]], [["Brightness", 0.30949179379286806, 0.7650569096019195], ["Sharpness", 0.5461629122105034, 0.6814369444005866]], [["Sharpness", 0.28459340191768434, 0.7802208350806028], ["Rotate", 0.15097973114238117, 0.5259683294104645]], [["ShearX", 0.6430803693700531, 0.9333735880102375], ["Contrast", 0.7522209520030653, 0.18831747966185058]], [["Contrast", 0.4219455937915647, 0.29949769435499646], ["Color", 0.6925322933509542, 0.8095523885795443]], [["ShearX", 0.23553236193043048, 0.17966207900468323], ["AutoContrast", 0.9039700567886262, 0.21983629944639108]], [["ShearX", 0.19256223146671514, 0.31200739880443584], ["Sharpness", 0.31962196883294713, 0.6828107668550425]], [["Cutout", 0.5947690279080912, 0.21728220253899178], ["Rotate", 0.6757188879871141, 0.489460599679474]], [["ShearY", 0.18365897125470526, 0.3988571115918058], ["Brightness", 0.7727489489504, 0.4790369956329955]], [["Contrast", 0.7090301084131432, 0.5178303607560537], ["ShearX", 0.16749258277688506, 0.33061773301592356]], [["ShearX", 0.3706690885419934, 0.38510677124319415], ["AutoContrast", 0.8288356276501032, 0.16556487668770264]], [["TranslateY", 0.16758043046445614, 0.30127092823893986], ["Brightness", 0.5194636577132354, 0.6225165310621702]], [["Cutout", 0.6087289363049726, 0.10439287037803044], ["Rotate", 0.7503452083033819, 0.7425316019981433]], [["ShearY", 0.24347189588329932, 0.5554979486672325], ["Brightness", 0.9468115239174161, 0.6132449358023568]], [["Brightness", 0.7144508395807994, 0.4610594769966929], ["ShearX", 0.16466683833092968, 0.3382903812375781]], [["Sharpness", 0.27743648684265465, 0.17200038071656915], ["Color", 0.47404262107546236, 0.7868991675614725]], [["Sharpness", 0.8603993513633618, 0.324604728411791], ["TranslateX", 0.3331597130403763, 0.9369586812977804]], [["Color", 0.1535813630595832, 0.4700116846558207], ["Color", 0.5435647971896318, 0.7639291483525243]], [["Brightness", 0.21486188101947656, 0.039347277341450576], ["Cutout", 0.7069526940684954, 0.39273934115015696]], [["ShearY", 0.7267130888840517, 0.6310800726389485], ["AutoContrast", 0.662163190824139, 0.31948540372237766]], [["ShearX", 0.5123132117185981, 0.1981015909438834], ["AutoContrast", 0.9009347363863067, 0.26790399126924036]], [["Brightness", 0.24245061453231648, 0.2673478678291436], ["ShearX", 0.31707976089283946, 0.6800582845544948]], [["Cutout", 0.9257780138367764, 0.03972673526848819], ["Rotate", 0.6807858944518548, 0.46974332280612097]], [["ShearY", 0.1543443071262312, 0.6051682587030671], ["Brightness", 0.9758203119828304, 0.4941406868162414]], [["Contrast", 0.07578049236491124, 0.38953819133407647], ["ShearX", 0.20194918288164293, 0.4141510791947318]], [["Color", 0.27826402243792286, 0.43517491081531157], ["AutoContrast", 0.6159269026143263, 0.2021846783488046]], [["AutoContrast", 0.5039377966534692, 0.19241507605941105], ["Invert", 0.5563931144385394, 0.7069728937319112]], [["Sharpness", 0.19031632433810566, 0.26310171056096743], ["Color", 0.4724537593175573, 0.6715201448387876]], [["ShearY", 0.2280910467786642, 0.33340559088059313], ["ShearY", 0.8858560034869303, 0.2598627441471076]], [["ShearY", 0.07291814128021593, 0.5819462692986321], ["Cutout", 0.27605696060512147, 0.9693427371868695]], [["Posterize", 0.4249871586563321, 0.8256952014328607], ["Posterize", 0.005907466926447169, 0.8081353382152597]], [["Brightness", 0.9071305290601128, 0.4781196213717954], ["Posterize", 0.8996214311439275, 0.5540717376630279]], [["Brightness", 0.06560728936236392, 0.9920627849065685], ["TranslateX", 0.04530789794044952, 0.5318568944702607]], [["TranslateX", 0.6800263601084814, 0.4611536772507228], ["Rotate", 0.7245888375283157, 0.0914772551375381]], [["Sharpness", 0.879556061897963, 0.42272481462067535], ["TranslateX", 0.4600350422524085, 0.5742175429334919]], [["AutoContrast", 0.5005776243176145, 0.22597121331684505], ["Invert", 0.10763286370369299, 0.6841782704962373]], [["Sharpness", 0.7422908472000116, 0.6850324203882405], ["TranslateX", 0.3832914614128403, 0.34798646673324896]], [["ShearY", 0.31939465302679326, 0.8792088167639516], ["Brightness", 0.4093604352811235, 0.21055483197261338]], [["AutoContrast", 0.7447595860998638, 0.19280222555998586], ["TranslateY", 0.317754779431227, 0.9983454520593591]], [["Equalize", 0.27706973689750847, 0.6447455020660622], ["Contrast", 0.5626579126863761, 0.7920049962776781]], [["Rotate", 0.13064369451773816, 0.1495367590684905], ["Sharpness", 0.24893941981801215, 0.6295943894521504]], [["ShearX", 0.6856269993063254, 0.5167938584189854], ["Sharpness", 0.24835352574609537, 0.9990550493102627]], [["AutoContrast", 0.461654115871693, 0.43097388896245004], ["Cutout", 0.366359682416437, 0.08011826474215511]], [["AutoContrast", 0.993892672935951, 0.2403608711236933], ["ShearX", 0.6620817870694181, 0.1744814077869482]], [["ShearY", 0.6396747719986443, 0.15031017143644265], ["Brightness", 0.9451954879495629, 0.26490678840264714]], [["Color", 0.19311480787397262, 0.15712300697448575], ["Posterize", 0.05391448762015258, 0.6943963643155474]], [["Sharpness", 0.6199669674684085, 0.5412492335319072], ["Invert", 0.14086213450149815, 0.2611850277919339]], [["Posterize", 0.5533129268803405, 0.5332478159319912], ["ShearX", 0.48956244029096635, 0.09223930853562916]], [["ShearY", 0.05871590849449765, 0.19549715278943228], ["TranslateY", 0.7208521362741379, 0.36414003004659434]], [["ShearY", 0.7316263417917531, 0.0629747985768501], ["Contrast", 0.036359793501448245, 0.48658745414898386]], [["Rotate", 0.3301497610942963, 0.5686622043085637], ["ShearX", 0.40581487555676843, 0.5866127743850192]], [["ShearX", 0.6679039628249283, 0.5292270693200821], ["Sharpness", 0.25901391739310703, 0.9778360586541461]], [["AutoContrast", 0.27373222012596854, 0.14456771405730712], ["Contrast", 0.3877220783523938, 0.7965158941894336]], [["Solarize", 0.29440905483979096, 0.06071633809388455], ["Equalize", 0.5246736285116214, 0.37575084834661976]], [["TranslateY", 0.2191269464520395, 0.7444942293988484], ["Posterize", 0.3840878524812771, 0.31812671711741247]], [["Solarize", 0.25159267140731356, 0.5833264622559661], ["Brightness", 0.07552262572348738, 0.33210648549288435]], [["AutoContrast", 0.9770099298399954, 0.46421915310428197], ["AutoContrast", 0.04707358934642503, 0.24922048012183493]], [["Cutout", 0.5379685806621965, 0.02038212605928355], ["Brightness", 0.5900728303717965, 0.28807872931416956]], [["Sharpness", 0.11596624872886108, 0.6086947716949325], ["AutoContrast", 0.34876470059667525, 0.22707897759730578]], [["Contrast", 0.276545513135698, 0.8822580384226156], ["Rotate", 0.04874027684061846, 0.6722214281612163]], [["ShearY", 0.595839851757025, 0.4389866852785822], ["Equalize", 0.5225492356128832, 0.2735290854063459]], [["Sharpness", 0.9918029636732927, 0.9919926583216121], ["Sharpness", 0.03672376137997366, 0.5563865980047012]], [["AutoContrast", 0.34169589759999847, 0.16419911552645738], ["Invert", 0.32995953043129234, 0.15073174739720568]], [["Posterize", 0.04600255098477292, 0.2632612790075844], ["TranslateY", 0.7852153329831825, 0.6990722310191976]], [["AutoContrast", 0.4414653815356372, 0.2657468780017082], ["Posterize", 0.30647061536763337, 0.3688222724948656]], [["Contrast", 0.4239361091421837, 0.6076562806342001], ["Cutout", 0.5780707784165284, 0.05361325256745192]], [["Sharpness", 0.7657895907855394, 0.9842407321667671], ["Sharpness", 0.5416352696151596, 0.6773681575200902]], [["AutoContrast", 0.13967381098331305, 0.10787258006315015], ["Posterize", 0.5019536507897069, 0.9881978222469807]], [["Brightness", 0.030528346448984903, 0.31562058762552847], ["TranslateY", 0.0843808140595676, 0.21019213305350526]], [["AutoContrast", 0.6934579165006736, 0.2530484168209199], ["Rotate", 0.0005751408130693636, 0.43790043943210005]], [["TranslateX", 0.611258547664328, 0.25465240215894935], ["Sharpness", 0.5001446909868196, 0.36102204109889413]], [["Contrast", 0.8995127327150193, 0.5493190695343996], ["Brightness", 0.242708780669213, 0.5461116653329015]], [["AutoContrast", 0.3751825351022747, 0.16845985803896962], ["Cutout", 0.25201103287363663, 0.0005893331783358435]], [["ShearX", 0.1518985779435941, 0.14768180777304504], ["Color", 0.85133530274324, 0.4006641163378305]], [["TranslateX", 0.5489668255504668, 0.4694591826554948], ["Rotate", 0.1917354490155893, 0.39993269385802177]], [["ShearY", 0.6689267479532809, 0.34304285013663577], ["Equalize", 0.24133154048883143, 0.279324043138247]], [["Contrast", 0.3412544002099494, 0.20217358823930232], ["Color", 0.8606984790510235, 0.14305503544676373]], [["Cutout", 0.21656155695311988, 0.5240101349572595], ["Brightness", 0.14109877717636352, 0.2016827341210295]], [["Sharpness", 0.24764371218833872, 0.19655480259925423], ["Posterize", 0.19460398862039913, 0.4975414350200679]], [["Brightness", 0.6071850094982323, 0.7270716448607151], ["Solarize", 0.111786402398499, 0.6325641684614275]], [["Contrast", 0.44772949532200856, 0.44267502710695955], ["AutoContrast", 0.360117506402693, 0.2623958228760273]], [["Sharpness", 0.8888131688583053, 0.936897400764746], ["Sharpness", 0.16080674198274894, 0.5681119841445879]], [["AutoContrast", 0.8004456226590612, 0.1788600469525269], ["Brightness", 0.24832285390647374, 0.02755350284841604]], [["ShearY", 0.06910320102646594, 0.26076407321544054], ["Contrast", 0.8633703022354964, 0.38968514704043056]], [["AutoContrast", 0.42306251382780613, 0.6883260271268138], ["Rotate", 0.3938724346852023, 0.16740881249086037]], [["Contrast", 0.2725343884286728, 0.6468194318074759], ["Sharpness", 0.32238942646494745, 0.6721149242783824]], [["AutoContrast", 0.942093919956842, 0.14675331481712853], ["Posterize", 0.5406276708262192, 0.683901182218153]], [["Cutout", 0.5386811894643584, 0.04498833938429728], ["Posterize", 0.17007257321724775, 0.45761177118620633]], [["Contrast", 0.13599408935104654, 0.53282738083886], ["Solarize", 0.26941667995081114, 0.20958261079465895]], [["Color", 0.6600788518606634, 0.9522228302165842], ["Invert", 0.0542722262516899, 0.5152431169321683]], [["Contrast", 0.5328934819727553, 0.2376220512388278], ["Posterize", 0.04890422575781711, 0.3182233123739474]], [["AutoContrast", 0.9289628064340965, 0.2976678437448435], ["Color", 0.20936893798507963, 0.9649612821434217]], [["Cutout", 0.9019423698575457, 0.24002036989728096], ["Brightness", 0.48734445615892974, 0.047660899809176316]], [["Sharpness", 0.09347824275711591, 0.01358686275590612], ["Posterize", 0.9248539660538934, 0.4064232632650468]], [["Brightness", 0.46575675383704634, 0.6280194775484345], ["Invert", 0.17276207634499413, 0.21263495428839635]], [["Brightness", 0.7238014711679732, 0.6178946027258592], ["Equalize", 0.3815496086340364, 0.07301281068847276]], [["Contrast", 0.754557393588416, 0.895332753570098], ["Color", 0.32709957750707447, 0.8425486003491515]], [["Rotate", 0.43406698081696576, 0.28628263254953723], ["TranslateY", 0.43949548709125374, 0.15927082198238685]], [["Brightness", 0.0015838339831640708, 0.09341692553352654], ["AutoContrast", 0.9113966907329718, 0.8345900469751112]], [["ShearY", 0.46698796308585017, 0.6150701348176804], ["Invert", 0.14894062704815722, 0.2778388046184728]], [["Color", 0.30360499169455957, 0.995713092016834], ["Contrast", 0.2597016288524961, 0.8654420870658932]], [["Brightness", 0.9661642031891435, 0.7322006407169436], ["TranslateY", 0.4393502786333408, 0.33934762664274265]], [["Color", 0.9323638351992302, 0.912776309755293], ["Brightness", 0.1618274755371618, 0.23485741708056307]], [["Color", 0.2216470771158821, 0.3359240197334976], ["Sharpness", 0.6328691811471494, 0.6298393874452548]], [["Solarize", 0.4772769142265505, 0.7073470698713035], ["ShearY", 0.2656114148206966, 0.31343097010487253]], [["Solarize", 0.3839017339304234, 0.5985505779429036], ["Equalize", 0.002412059429196589, 0.06637506181196245]], [["Contrast", 0.12751196553017863, 0.46980311434237976], ["Sharpness", 0.3467487455865491, 0.4054907610444406]], [["AutoContrast", 0.9321813669127206, 0.31328471589533274], ["Rotate", 0.05801738717432747, 0.36035756254444273]], [["TranslateX", 0.52092390458353, 0.5261722561643886], ["Contrast", 0.17836804476171306, 0.39354333443158535]], [["Posterize", 0.5458100909925713, 0.49447244994482603], ["Brightness", 0.7372536822363605, 0.5303409097463796]], [["Solarize", 0.1913974941725724, 0.5582966653986761], ["Equalize", 0.020733669175727026, 0.9377467166472878]], [["Equalize", 0.16265732137763889, 0.5206282340874929], ["Sharpness", 0.2421533133595281, 0.506389065871883]], [["AutoContrast", 0.9787324801448523, 0.24815051941486466], ["Rotate", 0.2423487151245957, 0.6456493129745148]], [["TranslateX", 0.6809867726670327, 0.6949687002397612], ["Contrast", 0.16125673359747458, 0.7582679978218987]], [["Posterize", 0.8212000950994955, 0.5225012157831872], ["Brightness", 0.8824891856626245, 0.4499216779709508]], [["Solarize", 0.12061313332505218, 0.5319371283368052], ["Equalize", 0.04120865969945108, 0.8179402157299602]], [["Rotate", 0.11278256686005855, 0.4022686554165438], ["ShearX", 0.2983451019112792, 0.42782525461812604]], [["ShearY", 0.8847385513289983, 0.5429227024179573], ["Rotate", 0.21316428726607445, 0.6712120087528564]], [["TranslateX", 0.46448081241068717, 0.4746090648963252], ["Brightness", 0.19973580961271142, 0.49252862676553605]], [["Posterize", 0.49664100539481526, 0.4460713166484651], ["Brightness", 0.6629559985581529, 0.35192346529003693]], [["Color", 0.22710733249173676, 0.37943185764616194], ["ShearX", 0.015809774971472595, 0.8472080190835669]], [["Contrast", 0.4187366322381491, 0.21621979869256666], ["AutoContrast", 0.7631045030367304, 0.44965231251615134]], [["Sharpness", 0.47240637876720515, 0.8080091811749525], ["Cutout", 0.2853425420104144, 0.6669811510150936]], [["Posterize", 0.7830320527127324, 0.2727062685529881], ["Solarize", 0.527834000867504, 0.20098218845222998]], [["Contrast", 0.366380535288225, 0.39766001659663075], ["Cutout", 0.8708808878088891, 0.20669525734273086]], [["ShearX", 0.6815427281122932, 0.6146858582671569], ["AutoContrast", 0.28330622372053493, 0.931352024154997]], [["AutoContrast", 0.8668174463154519, 0.39961453880632863], ["AutoContrast", 0.5718557712359253, 0.6337062930797239]], [["ShearY", 0.8923152519411871, 0.02480062504737446], ["Cutout", 0.14954159341231515, 0.1422219808492364]], [["Rotate", 0.3733718175355636, 0.3861928572224287], ["Sharpness", 0.5651126520194574, 0.6091103847442831]], [["Posterize", 0.8891714191922857, 0.29600154265251016], ["TranslateY", 0.7865351723963945, 0.5664998548985523]], [["TranslateX", 0.9298214806998273, 0.729856565052017], ["AutoContrast", 0.26349082482341846, 0.9638882609038888]], [["Sharpness", 0.8387378377527128, 0.42146721129032494], ["AutoContrast", 0.9860522000876452, 0.4200699464169384]], [["ShearY", 0.019609159303115145, 0.37197835936879514], ["Cutout", 0.22199340461754258, 0.015932573201085848]], [["Rotate", 0.43871085583928443, 0.3283504258860078], ["Sharpness", 0.6077702068037776, 0.6830305349618742]], [["Contrast", 0.6160211756538094, 0.32029451083389626], ["Cutout", 0.8037631428427006, 0.4025688837399259]], [["TranslateY", 0.051637820936985435, 0.6908417834391846], ["Sharpness", 0.7602756948473368, 0.4927111506643095]], [["Rotate", 0.4973618638052235, 0.45931479729281227], ["TranslateY", 0.04701789716427618, 0.9408779705948676]], [["Rotate", 0.5214194592768602, 0.8371249272013652], ["Solarize", 0.17734812472813338, 0.045020798970228315]], [["ShearX", 0.7457999920079351, 0.19025612553075893], ["Sharpness", 0.5994846101703786, 0.5665094068864229]], [["Contrast", 0.6172655452900769, 0.7811432139704904], ["Cutout", 0.09915620454670282, 0.3963692287596121]], [["TranslateX", 0.2650112299235817, 0.7377261946165307], ["AutoContrast", 0.5019539734059677, 0.26905046992024506]], [["Contrast", 0.6646299821370135, 0.41667784809592945], ["Cutout", 0.9698457154992128, 0.15429001887703997]], [["Sharpness", 0.9467079029475773, 0.44906457469098204], ["Cutout", 0.30036908747917396, 0.4766149689663106]], [["Equalize", 0.6667517691051055, 0.5014839828447363], ["Solarize", 0.4127890336820831, 0.9578274770236529]], [["Cutout", 0.6447384874120834, 0.2868806107728985], ["Cutout", 0.4800990488106021, 0.4757538246206956]], [["Solarize", 0.12560195032363236, 0.5557473475801568], ["Equalize", 0.019957161871490228, 0.5556797187823773]], [["Contrast", 0.12607637375759484, 0.4300633627435161], ["Sharpness", 0.3437273670109087, 0.40493203127714417]], [["AutoContrast", 0.884353334807183, 0.5880138314357569], ["Rotate", 0.9846032404597116, 0.3591877296622974]], [["TranslateX", 0.6862295865975581, 0.5307482119690076], ["Contrast", 0.19439251187251982, 0.3999195825722808]], [["Posterize", 0.4187641835025246, 0.5008988942651585], ["Brightness", 0.6665805605402482, 0.3853288204214253]], [["Posterize", 0.4507470690013903, 0.4232437206624681], ["TranslateX", 0.6054107416317659, 0.38123828040922203]], [["AutoContrast", 0.29562338573283276, 0.35608605102687474], ["TranslateX", 0.909954785390274, 0.20098894888066549]], [["Contrast", 0.6015278411777212, 0.6049140992035096], ["Cutout", 0.47178713636517855, 0.5333747244651914]], [["TranslateX", 0.490851976691112, 0.3829593925141144], ["Sharpness", 0.2716675173824095, 0.5131696240367152]], [["Posterize", 0.4190558294646337, 0.39316689077269873], ["Rotate", 0.5018526072725914, 0.295712490156129]], [["AutoContrast", 0.29624715560691617, 0.10937329832409388], ["Posterize", 0.8770505275992637, 0.43117765012206943]], [["Rotate", 0.6649970092751698, 0.47767131373391974], ["ShearX", 0.6257923540490786, 0.6643337040198358]], [["Sharpness", 0.5553620705849509, 0.8467799429696928], ["Cutout", 0.9006185811918932, 0.3537270716262]], [["ShearY", 0.0007619678283789788, 0.9494591850536303], ["Invert", 0.24267733654007673, 0.7851608409575828]], [["Contrast", 0.9730916198112872, 0.404670123321921], ["Sharpness", 0.5923587793251186, 0.7405792404430281]], [["Cutout", 0.07393909593373034, 0.44569630026328344], ["TranslateX", 0.2460593252211425, 0.4817527814541055]], [["Brightness", 0.31058654119340867, 0.7043749950260936], ["ShearX", 0.7632161538947713, 0.8043681264908555]], [["AutoContrast", 0.4352334371415373, 0.6377550087204297], ["Rotate", 0.2892714673415678, 0.49521052050510556]], [["Equalize", 0.509071051375276, 0.7352913414974414], ["ShearX", 0.5099959429711828, 0.7071566714593619]], [["Posterize", 0.9540506532512889, 0.8498853304461906], ["ShearY", 0.28199061357155397, 0.3161715627214629]], [["Posterize", 0.6740855359097433, 0.684004694936616], ["Posterize", 0.6816720350737863, 0.9654766942980918]], [["Solarize", 0.7149344531717328, 0.42212789795181643], ["Brightness", 0.686601460864528, 0.4263050070610551]], [["Cutout", 0.49577164991501, 0.08394890892056037], ["Rotate", 0.5810369852730606, 0.3320732965776973]], [["TranslateY", 0.1793755480490623, 0.6006520265468684], ["Brightness", 0.3769016576438939, 0.7190746300828186]], [["TranslateX", 0.7226363597757153, 0.3847027238123509], ["Brightness", 0.7641713191794035, 0.36234003077512544]], [["TranslateY", 0.1211227055347106, 0.6693523474608023], ["Brightness", 0.13011180247738063, 0.5126647617294864]], [["Equalize", 0.1501070550869129, 0.0038548909451806557], ["Posterize", 0.8266535939653881, 0.5502199643499207]], [["Sharpness", 0.550624117428359, 0.2023044586648523], ["Brightness", 0.06291556314780017, 0.7832635398703937]], [["Color", 0.3701578205508141, 0.9051537973590863], ["Contrast", 0.5763972727739397, 0.4905511239739898]], [["Rotate", 0.7678527224046323, 0.6723066265307555], ["Solarize", 0.31458533097383207, 0.38329324335154524]], [["Brightness", 0.292050127929522, 0.7047582807953063], ["ShearX", 0.040541891910333805, 0.06639328601282746]], [["TranslateY", 0.4293891393238555, 0.6608516902234284], ["Sharpness", 0.7794685477624004, 0.5168044063408147]], [["Color", 0.3682450402286552, 0.17274523597220048], ["ShearY", 0.3936056470397763, 0.5702597289866161]], [["Equalize", 0.43436990310624657, 0.9207072627823626], ["Contrast", 0.7608688260846083, 0.4759023148841439]], [["Brightness", 0.7926088966143935, 0.8270093925674497], ["ShearY", 0.4924174064969461, 0.47424347505831244]], [["Contrast", 0.043917555279430476, 0.15861903591675125], ["ShearX", 0.30439480405505853, 0.1682659341098064]], [["TranslateY", 0.5598255583454538, 0.721352536005039], ["Posterize", 0.9700921973303752, 0.6882015184440126]], [["AutoContrast", 0.3620887415037668, 0.5958176322317132], ["TranslateX", 0.14213781552733287, 0.6230799786459947]], [["Color", 0.490366889723972, 0.9863152892045195], ["Color", 0.817792262022319, 0.6755656429452775]], [["Brightness", 0.7030707021937771, 0.254633187122679], ["Color", 0.13977318232688843, 0.16378180123959793]], [["AutoContrast", 0.2933247831326118, 0.6283663376211102], ["Sharpness", 0.85430478154147, 0.9753613184208796]], [["Rotate", 0.6674299955457268, 0.48571208708018976], ["Contrast", 0.47491370175907016, 0.6401079552479657]], [["Sharpness", 0.37589579644127863, 0.8475131989077025], ["TranslateY", 0.9985149867598191, 0.057815729375099975]], [["Equalize", 0.0017194373841596389, 0.7888361311461602], ["Contrast", 0.6779293670669408, 0.796851411454113]], [["TranslateY", 0.3296782119072306, 0.39765117357271834], ["Sharpness", 0.5890554357001884, 0.6318339473765834]], [["Posterize", 0.25423810893163856, 0.5400430289894207], ["Sharpness", 0.9273643918988342, 0.6480913470982622]], [["Cutout", 0.850219975768305, 0.4169812455601289], ["Solarize", 0.5418755745870089, 0.5679666650495466]], [["Brightness", 0.008881361977310959, 0.9282562314720516], ["TranslateY", 0.7736066471553994, 0.20041167606029642]], [["Brightness", 0.05382537581401925, 0.6405265501035952], ["Contrast", 0.30484329473639593, 0.5449338155734242]], [["Color", 0.613257119787967, 0.4541503912724138], ["Brightness", 0.9061572524724674, 0.4030159294447347]], [["Brightness", 0.02739111568942537, 0.006028056532326534], ["ShearX", 0.17276751958646486, 0.05967365780621859]], [["TranslateY", 0.4376298213047888, 0.7691816164456199], ["Sharpness", 0.8162292718857824, 0.6054926462265117]], [["Color", 0.37963069679121214, 0.5946919433483344], ["Posterize", 0.08485417284005387, 0.5663580913231766]], [["Equalize", 0.49785780226818316, 0.9999137109183761], ["Sharpness", 0.7685879484682496, 0.6260846154212211]], [["AutoContrast", 0.4190931409670763, 0.2374852525139795], ["Posterize", 0.8797422264608563, 0.3184738541692057]], [["Rotate", 0.7307269024632872, 0.41523609600701106], ["ShearX", 0.6166685870692289, 0.647133807748274]], [["Sharpness", 0.5633713231039904, 0.8276694754755876], ["Cutout", 0.8329340776895764, 0.42656043027424073]], [["ShearY", 0.14934828370884312, 0.8622510773680372], ["Invert", 0.25925989086863277, 0.8813283584888576]], [["Contrast", 0.9457071292265932, 0.43228655518614034], ["Sharpness", 0.8485316947644338, 0.7590298998732413]], [["AutoContrast", 0.8386103589399184, 0.5859583131318076], ["Solarize", 0.466758711343543, 0.9956215363818983]], [["Rotate", 0.9387133710926467, 0.19180564509396503], ["Rotate", 0.5558247609706255, 0.04321698692007105]], [["ShearX", 0.3608716600695567, 0.15206159451532864], ["TranslateX", 0.47295292905710146, 0.5290760596129888]], [["TranslateX", 0.8357685981547495, 0.5991305115727084], ["Posterize", 0.5362929404188211, 0.34398525441943373]], [["ShearY", 0.6751984031632811, 0.6066293622133011], ["Contrast", 0.4122723990263818, 0.4062467515095566]], [["Color", 0.7515349936021702, 0.5122124665429213], ["Contrast", 0.03190514292904123, 0.22903520154660545]], [["Contrast", 0.5448962625054385, 0.38655673938910545], ["AutoContrast", 0.4867400684894492, 0.3433111101096984]], [["Rotate", 0.0008372434310827959, 0.28599951781141714], ["Equalize", 0.37113686925530087, 0.5243929348114981]], [["Color", 0.720054993488857, 0.2010177651701808], ["TranslateX", 0.23036196506059398, 0.11152764304368781]], [["Cutout", 0.859134208332423, 0.6727345740185254], ["ShearY", 0.02159833505865088, 0.46390076266538544]], [["Sharpness", 0.3428232157391428, 0.4067874527486514], ["Brightness", 0.5409415136577347, 0.3698432231874003]], [["Solarize", 0.27303978936454776, 0.9832186173589548], ["ShearY", 0.08831127213044043, 0.4681870331149774]], [["TranslateY", 0.2909309268736869, 0.4059460811623174], ["Sharpness", 0.6425125139803729, 0.20275737203293587]], [["Contrast", 0.32167626214661627, 0.28636162794046977], ["Invert", 0.4712405253509603, 0.7934644799163176]], [["Color", 0.867993060896951, 0.96574321666213], ["Color", 0.02233897320328512, 0.44478933557303063]], [["AutoContrast", 0.1841254751814967, 0.2779992148017741], ["Color", 0.3586283093530607, 0.3696246850445087]], [["Posterize", 0.2052935984046965, 0.16796913860308244], ["ShearX", 0.4807226832843722, 0.11296747254563266]], [["Cutout", 0.2016411266364791, 0.2765295444084803], ["Brightness", 0.3054112810424313, 0.695924264931216]], [["Rotate", 0.8405872184910479, 0.5434142541450815], ["Cutout", 0.4493615138203356, 0.893453735250007]], [["Contrast", 0.8433310507685494, 0.4915423577963278], ["ShearX", 0.22567799557913246, 0.20129892537008834]], [["Contrast", 0.045954277103674224, 0.5043900167190442], ["Cutout", 0.5552992473054611, 0.14436447810888237]], [["AutoContrast", 0.7719296115130478, 0.4440417544621306], ["Sharpness", 0.13992809206158283, 0.7988278670709781]], [["Color", 0.7838574233513952, 0.5971351401625151], ["TranslateY", 0.13562290583925385, 0.2253039635819158]], [["Cutout", 0.24870301109385806, 0.6937886690381568], ["TranslateY", 0.4033400068952813, 0.06253378991880915]], [["TranslateX", 0.0036059390486775644, 0.5234723884081843], ["Solarize", 0.42724862530733526, 0.8697702564187633]], [["Equalize", 0.5446026737834311, 0.9367992979112202], ["ShearY", 0.5943478903735789, 0.42345889214100046]], [["ShearX", 0.18611885697957506, 0.7320849092947314], ["ShearX", 0.3796416430900566, 0.03817761920009881]], [["Posterize", 0.37636778506979124, 0.26807924785236537], ["Brightness", 0.4317372554383255, 0.5473346211870932]], [["Brightness", 0.8100436240916665, 0.3817612088285007], ["Brightness", 0.4193974619003253, 0.9685902764026623]], [["Contrast", 0.701776402197012, 0.6612786008858009], ["Color", 0.19882787177960912, 0.17275597188875483]], [["Color", 0.9538303302832989, 0.48362384535228686], ["ShearY", 0.2179980837345602, 0.37027290936457313]], [["TranslateY", 0.6068028691503798, 0.3919346523454841], ["Cutout", 0.8228303342563138, 0.18372280287814613]], [["Equalize", 0.016416758802906828, 0.642838949194916], ["Cutout", 0.5761717838655257, 0.7600661153497648]], [["Color", 0.9417761826818639, 0.9916074035986558], ["Equalize", 0.2524209308597042, 0.6373703468715077]], [["Brightness", 0.75512589439513, 0.6155072321007569], ["Contrast", 0.32413476940254515, 0.4194739830159837]], [["Sharpness", 0.3339450765586968, 0.9973297539194967], ["AutoContrast", 0.6523930242124429, 0.1053482471037186]], [["ShearX", 0.2961391955838801, 0.9870036064904368], ["ShearY", 0.18705025965909403, 0.4550895821154484]], [["TranslateY", 0.36956447983807883, 0.36371471767143543], ["Sharpness", 0.6860051967688487, 0.2850190720087796]], [["Cutout", 0.13017742151902967, 0.47316674150067195], ["Invert", 0.28923829959551883, 0.9295585654924601]], [["Contrast", 0.7302368472279086, 0.7178974949876642], ["TranslateY", 0.12589674152030433, 0.7485392909494947]], [["Color", 0.6474693117772619, 0.5518269515590674], ["Contrast", 0.24643004970708016, 0.3435581358079418]], [["Contrast", 0.5650327855750835, 0.4843031798040887], ["Brightness", 0.3526684005761239, 0.3005305004600969]], [["Rotate", 0.09822284968122225, 0.13172798244520356], ["Equalize", 0.38135066977857157, 0.5135129123554154]], [["Contrast", 0.5902590645585712, 0.2196062383730596], ["ShearY", 0.14188379126120954, 0.1582612142182743]], [["Cutout", 0.8529913814417812, 0.89734031211874], ["Color", 0.07293767043078672, 0.32577659205278897]], [["Equalize", 0.21401668971453247, 0.040015259500028266], ["ShearY", 0.5126400895338797, 0.4726484828276388]], [["Brightness", 0.8269430025954498, 0.9678362841865166], ["ShearY", 0.17142069814830432, 0.4726727848289514]], [["Brightness", 0.699707089334018, 0.2795501395789335], ["ShearX", 0.5308818178242845, 0.10581814221896294]], [["Equalize", 0.32519644258946145, 0.15763390340309183], ["TranslateX", 0.6149090364414208, 0.7454832565718259]], [["AutoContrast", 0.5404508567155423, 0.7472387762067986], ["Equalize", 0.05649876539221024, 0.5628180219887216]]] + return p diff --git a/autoPyTorch/components/preprocessing/image_preprocessing/augmentation_transforms.py b/autoPyTorch/components/preprocessing/image_preprocessing/augmentation_transforms.py new file mode 100644 index 000000000..fb716c905 --- /dev/null +++ b/autoPyTorch/components/preprocessing/image_preprocessing/augmentation_transforms.py @@ -0,0 +1,439 @@ +# Copyright 2018 The TensorFlow Authors All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Transforms used in the Augmentation Policies.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import random +import numpy as np +# pylint:disable=g-multiple-import +from PIL import ImageOps, ImageEnhance, ImageFilter, Image +# pylint:enable=g-multiple-import + + +IMAGE_SIZE = 32 +# What is the dataset mean and std of the images on the training set +MEANS = [0.49139968, 0.48215841, 0.44653091] +STDS = [0.24703223, 0.24348513, 0.26158784] +PARAMETER_MAX = 10 # What is the max 'level' a transform could be predicted + + +def random_flip(x): + """Flip the input x horizontally with 50% probability.""" + if np.random.rand(1)[0] > 0.5: + return np.fliplr(x) + return x + + +def zero_pad_and_crop(img, amount=4): + """Zero pad by `amount` zero pixels on each side then take a random crop. + Args: + img: numpy image that will be zero padded and cropped. + amount: amount of zeros to pad `img` with horizontally and verically. + Returns: + The cropped zero padded img. The returned numpy array will be of the same + shape as `img`. + """ + padded_img = np.zeros((img.shape[0] + amount * 2, img.shape[1] + amount * 2, + img.shape[2])) + padded_img[amount:img.shape[0] + amount, amount: + img.shape[1] + amount, :] = img + top = np.random.randint(low=0, high=2 * amount) + left = np.random.randint(low=0, high=2 * amount) + new_img = padded_img[top:top + img.shape[0], left:left + img.shape[1], :] + return new_img + + +def create_cutout_mask(img_height, img_width, num_channels, size): + """Creates a zero mask used for cutout of shape `img_height` x `img_width`. + Args: + img_height: Height of image cutout mask will be applied to. + img_width: Width of image cutout mask will be applied to. + num_channels: Number of channels in the image. + size: Size of the zeros mask. + Returns: + A mask of shape `img_height` x `img_width` with all ones except for a + square of zeros of shape `size` x `size`. This mask is meant to be + elementwise multiplied with the original image. Additionally returns + the `upper_coord` and `lower_coord` which specify where the cutout mask + will be applied. + """ + if size>1: + print("SIZE AND CHANNELS", size, num_channels) + print("IMAGE HEIGTH AND WIDTH", img_height, img_width) + assert img_height == img_width + + # Sample center where cutout mask will be applied + height_loc = np.random.randint(low=0, high=img_height) + width_loc = np.random.randint(low=0, high=img_width) + print("HEIGHT LOC AND WIDTH LOC HEIGTH AND WIDTH", height_loc, width_loc) + + # Determine upper right and lower left corners of patch + upper_coord = (max(0, height_loc - size // 2), max(0, width_loc - size // 2)) + lower_coord = (min(img_height, height_loc + size // 2), + min(img_width, width_loc + size // 2)) + print("UPPER AND LOWER COORD", upper_coord, lower_coord) + mask_height = lower_coord[0] - upper_coord[0] + mask_width = lower_coord[1] - upper_coord[1] + print("MASK HEIGTH AND WIDTH", mask_height, mask_width) + assert mask_height > 0 + assert mask_width > 0 + + mask = np.ones((img_height, img_width, num_channels)) + zeros = np.zeros((mask_height, mask_width, num_channels)) + mask[upper_coord[0]:lower_coord[0], upper_coord[1]:lower_coord[1], :] = ( + zeros) + + else: + height_loc = np.random.randint(low=0, high=img_height) + width_loc = np.random.randint(low=0, high=img_width) + upper_coord = (height_loc,width_loc) + lower_coord = upper_coord + mask = np.ones((img_height, img_width, num_channels)) + mask[height_loc, width_loc] = 0 + + return mask, upper_coord, lower_coord + + +def cutout_numpy(img, size=16): + """Apply cutout with mask of shape `size` x `size` to `img`. + The cutout operation is from the paper https://arxiv.org/abs/1708.04552. + This operation applies a `size`x`size` mask of zeros to a random location + within `img`. + Args: + img: Numpy image that cutout will be applied to. + size: Height/width of the cutout mask that will be + Returns: + A numpy tensor that is the result of applying the cutout mask to `img`. + """ + img_height, img_width, num_channels = (img.shape[0], img.shape[1], + img.shape[2]) + assert len(img.shape) == 3 + mask, _, _ = create_cutout_mask(img_height, img_width, num_channels, size) + return img * mask + + +def float_parameter(level, maxval): + """Helper function to scale `val` between 0 and maxval . + Args: + level: Level of the operation that will be between [0, `PARAMETER_MAX`]. + maxval: Maximum value that the operation can have. This will be scaled + to level/PARAMETER_MAX. + Returns: + A float that results from scaling `maxval` according to `level`. + """ + return float(level) * maxval / PARAMETER_MAX + + +def int_parameter(level, maxval): + """Helper function to scale `val` between 0 and maxval . + Args: + level: Level of the operation that will be between [0, `PARAMETER_MAX`]. + maxval: Maximum value that the operation can have. This will be scaled + to level/PARAMETER_MAX. + Returns: + An int that results from scaling `maxval` according to `level`. + """ + return int(level * maxval / PARAMETER_MAX) + + +def pil_wrap(img): + """Convert the `img` numpy tensor to a PIL Image.""" + return Image.fromarray( + np.uint8((img * STDS + MEANS) * 255.0)).convert('RGBA') + + +def pil_unwrap(pil_img): + """Converts the PIL img to a numpy array.""" + pic_array = (np.array(pil_img.getdata()).reshape((32, 32, 4)) / 255.0) + i1, i2 = np.where(pic_array[:, :, 3] == 0) + pic_array = (pic_array[:, :, :3] - MEANS) / STDS + pic_array[i1, i2] = [0, 0, 0] + return pic_array + + +def apply_policy(policy, img): + """Apply the `policy` to the numpy `img`. + Args: + policy: A list of tuples with the form (name, probability, level) where + `name` is the name of the augmentation operation to apply, `probability` + is the probability of applying the operation and `level` is what strength + the operation to apply. + img: Numpy image that will have `policy` applied to it. + Returns: + The result of applying `policy` to `img`. + """ + pil_img = img # pil_wrap(img) + + for xform in policy: + assert len(xform) == 3 + name, probability, level = xform + xform_fn = NAME_TO_TRANSFORM[name].pil_transformer(probability, level) + pil_img = xform_fn(pil_img) + return pil_img #pil_unwrap(pil_img) + + +class TransformFunction(object): + """Wraps the Transform function for pretty printing options.""" + + def __init__(self, func, name): + self.f = func + self.name = name + + def __repr__(self): + return '<' + self.name + '>' + + def __call__(self, pil_img): + return self.f(pil_img) + + +class TransformT(object): + """Each instance of this class represents a specific transform.""" + + def __init__(self, name, xform_fn): + self.name = name + self.xform = xform_fn + + def pil_transformer(self, probability, level): + + def return_function(im): + if random.random() < probability: + im = self.xform(im, level) + return im + + name = self.name + '({:.1f},{})'.format(probability, level) + return TransformFunction(return_function, name) + + def do_transform(self, image, level): + f = self.pil_transformer(PARAMETER_MAX, level) + return pil_unwrap(f(pil_wrap(image))) + + +################## Transform Functions ################## +identity = TransformT('identity', lambda pil_img, level: pil_img) +flip_lr = TransformT( + 'FlipLR', + lambda pil_img, level: pil_img.transpose(Image.FLIP_LEFT_RIGHT)) +flip_ud = TransformT( + 'FlipUD', + lambda pil_img, level: pil_img.transpose(Image.FLIP_TOP_BOTTOM)) +# pylint:disable=g-long-lambda +auto_contrast = TransformT( + 'AutoContrast', + lambda pil_img, level: ImageOps.autocontrast( + pil_img.convert('RGB')).convert('RGBA')) +equalize = TransformT( + 'Equalize', + lambda pil_img, level: ImageOps.equalize( + pil_img.convert('RGB')).convert('RGBA')) +invert = TransformT( + 'Invert', + lambda pil_img, level: ImageOps.invert( + pil_img.convert('RGB')).convert('RGBA')) +# pylint:enable=g-long-lambda +blur = TransformT( + 'Blur', lambda pil_img, level: pil_img.filter(ImageFilter.BLUR)) +smooth = TransformT( + 'Smooth', + lambda pil_img, level: pil_img.filter(ImageFilter.SMOOTH)) + + +def _rotate_impl(pil_img, level): + """Rotates `pil_img` from -30 to 30 degrees depending on `level`.""" + degrees = int_parameter(level, 30) + if random.random() > 0.5: + degrees = -degrees + return pil_img.rotate(degrees) + + +rotate = TransformT('Rotate', _rotate_impl) + + +def _posterize_impl(pil_img, level): + """Applies PIL Posterize to `pil_img`.""" + level = int_parameter(level, 4) + return ImageOps.posterize(pil_img.convert('RGB'), 4 - level).convert('RGBA') + + +posterize = TransformT('Posterize', _posterize_impl) + + +def _shear_x_impl(pil_img, level): + """Applies PIL ShearX to `pil_img`. + The ShearX operation shears the image along the horizontal axis with `level` + magnitude. + Args: + pil_img: Image in PIL object. + level: Strength of the operation specified as an Integer from + [0, `PARAMETER_MAX`]. + Returns: + A PIL Image that has had ShearX applied to it. + """ + level = float_parameter(level, 0.3) + if random.random() > 0.5: + level = -level + return pil_img.transform(pil_img.size, Image.AFFINE, (1, level, 0, 0, 1, 0)) + + +shear_x = TransformT('ShearX', _shear_x_impl) + + +def _shear_y_impl(pil_img, level): + """Applies PIL ShearY to `pil_img`. + The ShearY operation shears the image along the vertical axis with `level` + magnitude. + Args: + pil_img: Image in PIL object. + level: Strength of the operation specified as an Integer from + [0, `PARAMETER_MAX`]. + Returns: + A PIL Image that has had ShearX applied to it. + """ + level = float_parameter(level, 0.3) + if random.random() > 0.5: + level = -level + return pil_img.transform(pil_img.size, Image.AFFINE, (1, 0, 0, level, 1, 0)) + + +shear_y = TransformT('ShearY', _shear_y_impl) + + +def _translate_x_impl(pil_img, level): + """Applies PIL TranslateX to `pil_img`. + Translate the image in the horizontal direction by `level` + number of pixels. + Args: + pil_img: Image in PIL object. + level: Strength of the operation specified as an Integer from + [0, `PARAMETER_MAX`]. + Returns: + A PIL Image that has had TranslateX applied to it. + """ + level = int_parameter(level, 10) + if random.random() > 0.5: + level = -level + return pil_img.transform(pil_img.size, Image.AFFINE, (1, 0, level, 0, 1, 0)) + + +translate_x = TransformT('TranslateX', _translate_x_impl) + + +def _translate_y_impl(pil_img, level): + """Applies PIL TranslateY to `pil_img`. + Translate the image in the vertical direction by `level` + number of pixels. + Args: + pil_img: Image in PIL object. + level: Strength of the operation specified as an Integer from + [0, `PARAMETER_MAX`]. + Returns: + A PIL Image that has had TranslateY applied to it. + """ + level = int_parameter(level, 10) + if random.random() > 0.5: + level = -level + return pil_img.transform(pil_img.size, Image.AFFINE, (1, 0, 0, 0, 1, level)) + + +translate_y = TransformT('TranslateY', _translate_y_impl) + + +def _crop_impl(pil_img, level, interpolation=Image.BILINEAR): + """Applies a crop to `pil_img` with the size depending on the `level`.""" + cropped = pil_img.crop((level, level, IMAGE_SIZE - level, IMAGE_SIZE - level)) + resized = cropped.resize((IMAGE_SIZE, IMAGE_SIZE), interpolation) + return resized + + +crop_bilinear = TransformT('CropBilinear', _crop_impl) + + +def _solarize_impl(pil_img, level): + """Applies PIL Solarize to `pil_img`. + Translate the image in the vertical direction by `level` + number of pixels. + Args: + pil_img: Image in PIL object. + level: Strength of the operation specified as an Integer from + [0, `PARAMETER_MAX`]. + Returns: + A PIL Image that has had Solarize applied to it. + """ + level = int_parameter(level, 256) + return ImageOps.solarize(pil_img.convert('RGB'), 256 - level).convert('RGBA') + + +solarize = TransformT('Solarize', _solarize_impl) + + +def _cutout_pil_impl(pil_img, level): + """Apply cutout to pil_img at the specified level.""" + size = int_parameter(level, 20) + if size <= 0: + return pil_img + img_height, img_width, num_channels = (32, 32, 3) + _, upper_coord, lower_coord = ( + create_cutout_mask(img_height, img_width, num_channels, size)) + pixels = pil_img.load() # create the pixel map + for i in range(upper_coord[0], lower_coord[0]): # for every col: + for j in range(upper_coord[1], lower_coord[1]): # For every row + pixels[i, j] = (125, 122, 113, 0) # set the colour accordingly + return pil_img + +cutout = TransformT('Cutout', _cutout_pil_impl) + + +def _enhancer_impl(enhancer): + """Sets level to be between 0.1 and 1.8 for ImageEnhance transforms of PIL.""" + def impl(pil_img, level): + v = float_parameter(level, 1.8) + .1 # going to 0 just destroys it + return enhancer(pil_img).enhance(v) + return impl + + +color = TransformT('Color', _enhancer_impl(ImageEnhance.Color)) +contrast = TransformT('Contrast', _enhancer_impl(ImageEnhance.Contrast)) +brightness = TransformT('Brightness', _enhancer_impl( + ImageEnhance.Brightness)) +sharpness = TransformT('Sharpness', _enhancer_impl(ImageEnhance.Sharpness)) + +ALL_TRANSFORMS = [ + flip_lr, + flip_ud, + auto_contrast, + equalize, + invert, + rotate, + posterize, + crop_bilinear, + solarize, + color, + contrast, + brightness, + sharpness, + shear_x, + shear_y, + translate_x, + translate_y, + cutout, + blur, + smooth +] + +NAME_TO_TRANSFORM = {t.name: t for t in ALL_TRANSFORMS} +TRANSFORM_NAMES = NAME_TO_TRANSFORM.keys() diff --git a/autoPyTorch/components/preprocessing/image_preprocessing/operations.py b/autoPyTorch/components/preprocessing/image_preprocessing/operations.py new file mode 100644 index 000000000..0a1d72add --- /dev/null +++ b/autoPyTorch/components/preprocessing/image_preprocessing/operations.py @@ -0,0 +1,283 @@ +import numpy as np +import math +import random +import os + +from PIL import Image, ImageOps, ImageEnhance + +class Operation(object): + """ + Base class of all operations. + """ + def __init__(self, prob, magnitude): + self.prob = prob + self.magnitude = magnitude + + def __str__(self): + return self.__class__.__name__ + + def __call__(self, image): + raise NotImplementedError("Need to instantiate a subclass of this class!") + +class Equalize(Operation): + """ + Equalize the image histogram. + """ + def __init__(self, prob, magnitude): + super(Equalize, self).__init__(prob, None) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + return ImageOps.equalize(image) + +class Invert(Operation): + """ + Invert the pixels of the image. + """ + def __init__(self, prob, magnitude): + super(Invert, self).__init__(prob, None) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + return ImageOps.invert(image) + +class AutoContrast(Operation): + """ + Maximize the image contrast, by making the darkest pixel black and + the lightest pixel white. + """ + def __init__(self, prob, magnitude): + super(AutoContrast, self).__init__(prob, None) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + return ImageOps.autocontrast(image) + +class Posterize(Operation): + """ + Reduce the number of bits for each pixel magnitude bits. + """ + def __init__(self, prob, magnitude): + super(Posterize, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(4, 8, 10) + bits = int(round(magnitude_range[self.magnitude])) + return ImageOps.posterize(image, bits) + +class Solarize(Operation): + """ + Invert all pixels above a threshold value of magnitude. + """ + def __init__(self, prob, magnitude): + super(Solarize, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(0, 256, 10) + threshold = magnitude_range[self.magnitude] + return ImageOps.solarize(image, threshold) + +class Contrast(Operation): + """ + Control the contrast of the image. + A magnitude=0 gives a gray image, + whereas magnitude=1 gives the original image. + """ + def __init__(self, prob, magnitude): + super(Contrast, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(0.1, 1.9, 10) + factor = magnitude_range[self.magnitude] + enhancer = ImageEnhance.Contrast(image) + return enhancer.enhance(factor) + +class Color(Operation): + """ + Adjust the color balance of the image, + in a manner similar to the controls on a colour TV set. + A magnitude=0 gives a black & white image, + whereas magnitude=1 gives the original image. + """ + def __init__(self, prob, magnitude): + super(Color, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(0.1, 1.9, 10) + factor = magnitude_range[self.magnitude] + enhancer = ImageEnhance.Color(image) + return enhancer.enhance(factor) + +class Brightness(Operation): + """ + Adjust the brightness of the image. + A magnitude=0 gives a black image, + whereas magnitude=1 gives the original image. + """ + def __init__(self, prob, magnitude): + super(Brightness, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(0.1, 1.9, 10) + factor = magnitude_range[self.magnitude] + enhancer = ImageEnhance.Brightness(image) + return enhancer.enhance(factor) + +class Sharpness(Operation): + """ + Adjust the sharpness of the image. + A magnitude=0 gives a blurred image, + whereas magnitude=1 gives the original image. + """ + def __init__(self, prob, magnitude): + super(Sharpness, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(0.1, 1.9, 10) + factor = magnitude_range[self.magnitude] + enhancer = ImageEnhance.Sharpness(image) + return enhancer.enhance(factor) + +class Rotate(Operation): + """ + Rotate the image magnitude degrees. + """ + def __init(self, prob, magnitude): + super(Rotate, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(-30, 30, 10) + degrees = magnitude_range[self.magnitude] + return image.rotate(degrees, expand=False, resample=Image.BICUBIC) + +class TranslateX(Operation): + """ + Translate the image in the horizontal axis + direction by magnitude number of pixels. + """ + def __init__(self, prob, magnitude): + super(TranslateX, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(-15, 15, 10) + pixels = magnitude_range[self.magnitude] + return image.transform(image.size, Image.AFFINE, (1, 0, pixels, 0, 1, 0)) + +class TranslateY(Operation): + """ + Translate the image in the vertical axis + direction by magnitude number of pixels. + """ + def __init__(self, prob, magnitude): + super(TranslateY, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(-15, 15, 10) + pixels = magnitude_range[self.magnitude] + return image.transform(image.size, Image.AFFINE, (1, 0, 0, 0, 1, pixels)) + + +class ShearX(Operation): + """ + Shear image along horizontal axis with rate magnitude. + """ + def __init__(self, prob, magnitude): + super(ShearX, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(-0.3, 0.3, 10) + rate = magnitude_range[self.magnitude] + + w, h = image.size + + phi = math.tan(abs(rate)) + shift_in_pixels = phi * h + matrix_offset = shift_in_pixels + if rate <= 0: + matrix_offset = 0 + phi = -1 * phi + + transform_matrix = (1, phi, -matrix_offset, 0, 1, 0) + + image = image.transform((int(round(w + shift_in_pixels)), h), + Image.AFFINE, + transform_matrix) + + if rate <= 0: + image = image.crop((0, 0, w, h)) + else: + image = image.crop((abs(shift_in_pixels), 0, w + abs(shift_in_pixels), h)) + + return image + +class ShearY(Operation): + """ + Shear image along vertical axis with rate magnitude. + """ + def __init__(self, prob, magnitude): + super(ShearY, self).__init__(prob, magnitude) + + def __call__(self, image): + if random.uniform(0, 1) > self.prob: + return image + else: + magnitude_range = np.linspace(-0.3, 0.3, 10) + rate = magnitude_range[self.magnitude] + + w, h = image.size + + phi = math.tan(abs(rate)) + shift_in_pixels = phi * h + matrix_offset = shift_in_pixels + if rate <= 0: + matrix_offset = 0 + phi = -1 * phi + + transform_matrix = (1, 0, 0, phi, 1, -matrix_offset) + + image = image.transform((w, int(round(h + shift_in_pixels))), + Image.AFFINE, + transform_matrix) + + if rate <= 0: + image = image.crop((0, 0, w, h)) + else: + image = image.crop((0, abs(shift_in_pixels), w, h + abs(shift_in_pixels))) + + return image diff --git a/autoPyTorch/components/preprocessing/image_preprocessing/transforms.py b/autoPyTorch/components/preprocessing/image_preprocessing/transforms.py new file mode 100644 index 000000000..11effac21 --- /dev/null +++ b/autoPyTorch/components/preprocessing/image_preprocessing/transforms.py @@ -0,0 +1,177 @@ +from __future__ import absolute_import + +from torchvision.transforms import * +from .augmentation_transforms import * + +import random +import math +import torch +import numpy as np + +from .operations import * + + +class RandomErasing(object): + """ + Class that performs Random Erasing in Random Erasing Data Augmentation by Zhong et al. + + Args: + probability: The probability that the operation will be performed. + sl: min erasing area + sh: max erasing area + r1: min aspect ratio + mean: erasing value + """ + + def __init__(self, probability=0.5, sl=0.02, sh=0.4, r1=0.3, mean=[0.4914, 0.4822, 0.4465]): + self.probability = probability + self.sl = sl + self.sh = sh + self.r1 = r1 + self.mean = mean + + def __call__(self, img): + if random.uniform(0, 1) > self.probability: + return img + + for attempt in range(100): + area = img.size()[1] * img.size()[2] + + target_area = random.uniform(self.sl, self.sh) * area + aspect_ratio = random.uniform(self.r1, 1/self.r1) + + h = int(round(math.sqrt(target_area * aspect_ratio))) + w = int(round(math.sqrt(target_area / aspect_ratio))) + + if w < img.size()[2] and h < img.size()[1]: + x1 = random.randint(0, img.size()[1] - h) + y1 = random.randint(0, img.size()[2] - w) + if img.size()[0] == 3: + img[0, x1:x1+h, y1:y1+w] = self.mean[0] + img[1, x1:x1+h, y1:y1+w] = self.mean[1] + img[2, x1:x1+h, y1:y1+w] = self.mean[2] + else: + img[0, x1:x1+h, y1:y1+w] = self.mean[1] + return img + + return img + + +class Cutout(object): + """ + Randomly mask out one or more patches from an image. + Args: + n_holes (int): Number of patches to cut out of each image. + length (int): The length (in pixels) of each square patch. + """ + def __init__(self, n_holes, length, probability): + self.n_holes = n_holes + self.length = length + self.probability = probability + + def __call__(self, img): + """ + Args: + img (Tensor): Tensor image of size (C, H, W). + Returns: + Tensor: Image with n_holes of dimension length x length cut out of it. + """ + if random.uniform(0, 1) > self.probability: + return img + + h = img.size(1) + w = img.size(2) + + mask = np.ones((h, w), np.float32) + + for n in range(self.n_holes): + y = np.random.randint(h) + x = np.random.randint(w) + + y1 = int(np.clip(y - self.length / 2, 0, h)) + y2 = int(np.clip(y + self.length / 2, 0, h)) + x1 = int(np.clip(x - self.length / 2, 0, w)) + x2 = int(np.clip(x + self.length / 2, 0, w)) + + mask[y1: y2, x1: x2] = 0. + + mask = torch.from_numpy(mask) + mask = mask.expand_as(img) + img = img * mask + + return img + + +class AutoAugment(object): + + def __init__(self): + pass + + def __call__(self, img): + """ + Args: + img (Tensor): Tensor image of size (C, H, W). + """ + + # + # ImageNet policies proposed in https://arxiv.org/abs/1805.09501 + # + policies = [ + [('Posterize', 0.4, 8), ('Rotate', 0.6,9)], + [('Solarize', 0.6, 5), ('AutoContrast', 0.6, 5)], + [('Equalize', 0.8, 8), ('Equalize', 0.6, 3)], + [('Posterize', 0.6, 7), ('Posterize', 0.6, 3)], + [('Equalize', 0.4, 7), ('Solarize', 0.2, 4)], + [('Equalize', 0.4, 4), ('Rotate', 0.8, 8)], + [('Solarize', 0.6, 3), ('Equalize', 0.6, 7)], + [('Posterize', 0.8, 5), ('Equalize', 1.0, 2)], + [('Rotate', 0.2, 3), ('Solarize', 0.6, 8)], + [('Equalize', 0.6, 8), ('Posterize', 0.4, 6)], + [('Rotate', 0.8, 8), ('Color', 0.4, 0)], + [('Rotate', 0.4, 9), ('Equalize', 0.6, 2)], + [('Equalize', 0.0, 7), ('Equalize', 0.8, 8)], + [('Invert', 0.6, 4), ('Equalize', 1.0, 8)], + [('Color', 0.6, 4), ('Contrast', 1.0, 8)], + [('Rotate', 0.8, 8), ('Color', 1.0, 2)], + [('Color', 0.8, 8), ('Solarize', 0.8, 7)], + [('Sharpness', 0.4, 7), ('Invert', 0.6, 8)], + [('ShearX', 0.6, 5), ('Equalize', 1.0, 9)], + [('Color', 0.4, 0), ('Equalize', 0.6, 3)], + [('Equalize', 0.4, 7), ('Solarize', 0.2, 4)], + [('Solarize', 0.6, 5), ('AutoContrast', 0.6, 5)], + [('Invert', 0.6, 4), ('Equalize', 1.0, 8)], + [('Color', 0.6, 4), ('Contrast', 1.0, 8)], + [('Equalize', 0.8, 8), ('Equalize', 0.6, 3)], + ] + + policy = random.choice(policies) + + img = apply_policy(policy, img) + + return img.convert('RGB') + + +class FastAutoAugment(object): + + # + # ImageNet policies proposed in https://arxiv.org/abs/1905.00397 + # + + + def __init__(self): + + from .archive import fa_reduced_cifar10 + + self.policies = fa_reduced_cifar10() + + def __call__(self, img): + """ + Args: + img (Tensor): Tensor image of size (C, H, W). + """ + + policy = random.choice(self.policies) + + img = apply_policy(policy, img) + + return img.convert('RGB') diff --git a/autoPyTorch/components/preprocessing/loss_weight_strategies.py b/autoPyTorch/components/preprocessing/loss_weight_strategies.py index 39e9ec4a6..cddce005a 100644 --- a/autoPyTorch/components/preprocessing/loss_weight_strategies.py +++ b/autoPyTorch/components/preprocessing/loss_weight_strategies.py @@ -8,19 +8,27 @@ class LossWeightStrategyWeighted(): def __call__(self, pipeline_config, X, Y): - + counts = np.sum(Y, axis=0) total_weight = Y.shape[0] - weight_per_class = total_weight / Y.shape[1] - weights = (np.ones(Y.shape[1]) * weight_per_class) / np.maximum(counts, 1) + if len(Y.shape) > 1: + weight_per_class = total_weight / Y.shape[1] + weights = (np.ones(Y.shape[1]) * weight_per_class) / np.maximum(counts, 1) + else: + classes, counts = np.unique(Y, axis=0, return_counts=True) + classes, counts = classes[::-1], counts[::-1] + weight_per_class = total_weight / classes.shape[0] + weights = (np.ones(classes.shape[0]) * weight_per_class) / counts + return weights class LossWeightStrategyWeightedBinary(): def __call__(self, pipeline_config, X, Y): - + counts_one = np.sum(Y, axis=0) counts_zero = counts_one + (-Y.shape[0]) weights = counts_zero / np.maximum(counts_one, 1) return weights + diff --git a/autoPyTorch/components/training/budget_types.py b/autoPyTorch/components/training/budget_types.py index ded6b2a3f..5e21c680e 100644 --- a/autoPyTorch/components/training/budget_types.py +++ b/autoPyTorch/components/training/budget_types.py @@ -21,9 +21,9 @@ def on_batch_end(self, **kwargs): # OVERRIDE def on_epoch_end(self, trainer, **kwargs): - elapsed = time.time() - self.start_time + elapsed = time.time() - trainer.fit_start_time trainer.model.budget_trained = elapsed - trainer.logger.debug("Budget used: " + str(elapsed) + "/" + str(self.end_time - self.start_time)) + trainer.logger.debug("Budget used: " + str(elapsed) + "/" + str(trainer.budget - self.compensate)) if time.time() >= self.end_time: trainer.logger.debug("Budget exhausted!") @@ -47,4 +47,33 @@ def on_epoch_end(self, trainer, epoch, **kwargs): if epoch >= self.target: trainer.logger.debug("Budget exhausted!") return True - return False \ No newline at end of file + return False + +class BudgetTypeTrainingTime(BaseTrainingTechnique): + default_min_budget = 120 + default_max_budget = 6000 + + # OVERRIDE + def set_up(self, trainer, pipeline_config, **kwargs): + super(BudgetTypeTrainingTime, self).set_up(trainer, pipeline_config) + self.end_time = trainer.budget + time.time() + self.start_time = time.time() + + if self.start_time >= self.end_time: + raise Exception("Budget exhausted before training started") + + # OVERRIDE + def on_batch_end(self, **kwargs): + return time.time() >= self.end_time + + # OVERRIDE + def on_epoch_end(self, trainer, **kwargs): + elapsed = time.time() - self.start_time + trainer.model.budget_trained = elapsed + trainer.logger.debug("Budget used: " + str(elapsed) + + "/" + str(self.end_time - self.start_time)) + + if time.time() >= self.end_time: + trainer.logger.debug("Budget exhausted!") + return True + return False diff --git a/autoPyTorch/components/training/early_stopping.py b/autoPyTorch/components/training/early_stopping.py index 019941768..aa3e40759 100644 --- a/autoPyTorch/components/training/early_stopping.py +++ b/autoPyTorch/components/training/early_stopping.py @@ -11,8 +11,8 @@ class EarlyStopping(BaseTrainingTechnique): def set_up(self, trainer, pipeline_config, **kwargs): super(EarlyStopping, self).set_up(trainer, pipeline_config) self.reset_parameters = pipeline_config["early_stopping_reset_parameters"] - self.minimize = pipeline_config["minimize"] self.patience = pipeline_config["early_stopping_patience"] + self.loss_transform = trainer.metrics[0].loss_transform # does not work with e.g. cosine anealing with warm restarts if hasattr(trainer, "lr_scheduler") and not trainer.lr_scheduler.allows_early_stopping: @@ -21,8 +21,6 @@ def set_up(self, trainer, pipeline_config, **kwargs): # initialize current best performance to +/- infinity if trainer.model.current_best_epoch_performance is None: trainer.model.current_best_epoch_performance = float("inf") - if not self.minimize: - trainer.model.current_best_epoch_performance = -float("inf") trainer.logger.debug("Using Early stopping with patience: " + str(self.patience)) trainer.logger.debug("Reset Parameters to parameters with best validation performance: " + str(self.reset_parameters)) @@ -35,11 +33,10 @@ def on_epoch_end(self, trainer, log, **kwargs): return False if self.reset_parameters and (not hasattr(trainer, "lr_scheduler") or not trainer.lr_scheduler.snapshot_before_restart): log["best_parameters"] = False - current_performance = log["val_" + trainer.metrics[0]] + current_performance = self.loss_transform(log["val_" + trainer.metrics[0]]) # new best performance - if ((self.minimize and current_performance < trainer.model.current_best_epoch_performance) or - (not self.minimize and current_performance > trainer.model.current_best_epoch_performance)): + if current_performance < trainer.model.current_best_epoch_performance: trainer.model.num_epochs_no_progress = 0 trainer.model.current_best_epoch_performance = current_performance trainer.logger.debug("New best performance!") diff --git a/autoPyTorch/components/training/image/__init__.py b/autoPyTorch/components/training/image/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/autoPyTorch/components/training/image/base_training.py b/autoPyTorch/components/training/image/base_training.py new file mode 100644 index 000000000..637a6396f --- /dev/null +++ b/autoPyTorch/components/training/image/base_training.py @@ -0,0 +1,172 @@ +import ConfigSpace +from torch.autograd import Variable + +class BaseTrainingTechnique(): + def __init__(self, training_components=None): + """Initialize the training technique. Should be called in a fit Method of a Pipeline node. + + Keyword Arguments: + training_components {dict} -- Maps a names to a training components necessary for this training technique (default: {None}) + """ + + self.training_components = training_components or dict() + + # VIRTUAL + def set_up(self, training_components, pipeline_config, logger): + """Set up the training component + + Arguments: + training_components {dict} -- All training components of training. + pipeline_config {dict} -- Configuration of the Pipeline. + logger {Logger} -- Logger. + """ + + self.logger = logger + + # VIRTUAL + def before_train_batches(self, training_components, log, epoch): + """Function that gets called before the train_batches method of each epoch in training. + + Arguments: + training_components {dict} -- All training components used in training. + log {dict} -- The log of the current epoch. + epoch {int} -- The current epoch of training. + """ + + pass + + # VIRTUAL + def after_train_batches(self, training_components, log, epoch): + """Function that gets called after the train_batches method of each epoch in training. + Is able to stop training by returning True. + + Arguments: + training_components {dict} -- All training components used in training. + log {dict} -- The log of the current epoch. + epoch {int} -- The current epoch of training. + + Returns: + bool -- If training should be stopped. + """ + + return False + + # VIRTUAL + def during_train_batches(self, batch_loss, training_components): + """Function that gets called in the train_batches method of training. + Is able to cancel the current epoch by returning True. + + Arguments: + batch_loss {tensor} -- The batch loss of the current batch. + training_components {dict} -- All training components used in training. + + Returns: + bool -- If the current epoch should be canceled. + """ + + return False + + # VIRTUAL + def select_log(self, logs, training_components): + """Select one log from the list of all epoch logs. + + Arguments: + logs {list} -- A list of log. For each epoch of training there is one entry. + training_components {dict} -- All training components used in training. + + Returns: + log -- The selected log. Return None if undecided. + """ + + return False + + # VIRTUAL + def needs_eval_on_valid_each_epoch(self): + """Specify if the training technique needs the network to be evaluated on the validation set. + + Returns: + bool -- If the network should be evaluated on the validation set. + """ + + return False + + # VIRTUAL + def needs_eval_on_train_each_epoch(self): + """Specify if the training technique needs the network to be evaluated on the training set. + + Returns: + bool -- If the network should be evaluated on the training set. + """ + + + return False + + # VIRTUAL + @staticmethod + def get_pipeline_config_options(): + """Return a list of ConfigOption used for this training technique. + + Returns: + list -- A list of ConfigOptions. + """ + + return [] + + +class BaseBatchLossComputationTechnique(): + + # VIRTUAL + def set_up(self, pipeline_config, hyperparameter_config, logger): + """Initialize the batch loss computation technique. + + Arguments: + pipeline_config {dict} -- The configuration of the pipeline. + hyperparameter_config {dict} -- The hyperparameter config sampled by BOHB. + logger {Logger} -- Logger. + """ + self.logger = logger + + # VIRTUAL + def prepare_data(self, X_batch, y_batch): + """Method that gets called, before batch is but into network. + + Arguments: + X_batch {tensor} -- The features of the batch. + y_batch {tensor} -- The targets of the batch. + """ + + return X_batch, {'y_batch' : y_batch} + + # VIRTUAL + def criterion(self, y_batch): + return lambda criterion, pred: criterion(pred, y_batch) + + # VIRTUAL + def evaluate(self, metric, y_pred, y_batch): + return metric(y_pred, y_batch) + + + # VIRTUAL + @staticmethod + def get_pipeline_config_options(): + """A list of ConfigOptions used for this technique. + + Returns: + list -- A list of ConfigOptions for this technique. + """ + + return [] + + # VIRTUAL + @staticmethod + def get_hyperparameter_search_space(**pipeline_config): + """Get the hyperparameter config space for this technique. + + Returns: + ConfigurationSpace -- The hyperparameter config space for this technique + """ + + return ConfigSpace.ConfigurationSpace() + + + \ No newline at end of file diff --git a/autoPyTorch/components/training/image/budget_types.py b/autoPyTorch/components/training/image/budget_types.py new file mode 100644 index 000000000..ee7becba6 --- /dev/null +++ b/autoPyTorch/components/training/image/budget_types.py @@ -0,0 +1,50 @@ +from autoPyTorch.components.training.image.base_training import BaseTrainingTechnique +import time + +class BudgetTypeTime(BaseTrainingTechnique): + default_min_budget = 120 + default_max_budget = 6000 + compensate = 10 # will be modified by cv + + # OVERRIDE + def set_up(self, training_components, pipeline_config, logger): + super(BudgetTypeTime, self).set_up(training_components, pipeline_config, logger) + self.end_time = training_components["budget"] - self.compensate + training_components["fit_start_time"] + self.start_time = time.time() + + if self.start_time >= self.end_time: + raise Exception("Budget exhausted before training started") + + # OVERRIDE + def during_train_batches(self, batch_loss, training_components): + return time.time() >= self.end_time + + # OVERRIDE + def after_train_batches(self, training_components, log, epoch): + elapsed = time.time() - self.start_time + training_components["network"].budget_trained = elapsed + self.logger.debug("Budget used: " + str(elapsed) + "/" + str(self.end_time - self.start_time)) + + if time.time() >= self.end_time: + self.logger.debug("Budget exhausted!") + return True + return False + +class BudgetTypeEpochs(BaseTrainingTechnique): + default_min_budget = 5 + default_max_budget = 150 + + # OVERRIDE + def set_up(self, training_components, pipeline_config, logger): + super(BudgetTypeEpochs, self).set_up(training_components, pipeline_config, logger) + self.target = training_components["budget"] + + # OVERRIDE + def after_train_batches(self, training_components, log, epoch): + training_components["network"].budget_trained = epoch + self.logger.debug("Budget used: " + str(epoch) + "/" + str(self.target)) + + if epoch >= self.target: + self.logger.debug("Budget exhausted!") + return True + return False diff --git a/autoPyTorch/components/training/image/checkpoints/__init__.py b/autoPyTorch/components/training/image/checkpoints/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/autoPyTorch/components/training/image/checkpoints/load_specific.py b/autoPyTorch/components/training/image/checkpoints/load_specific.py new file mode 100644 index 000000000..493d71119 --- /dev/null +++ b/autoPyTorch/components/training/image/checkpoints/load_specific.py @@ -0,0 +1,58 @@ + +import os +import math +import torch +import torch.nn as nn + + +import logging + + +def load_model(model, checkpoint): + + if checkpoint is None: + return model + + pretrained_state = checkpoint['state'] + model_state = model.state_dict() + + pretrained_state = { k:v for k,v in pretrained_state.items() if k in model_state and v.size() == model_state[k].size() } + logging.getLogger('autonet').debug('=> Resuming model using ' + str(len(pretrained_state.keys())) + '/' + str(len(model_state.keys())) + ' parameters') + model_state.update(pretrained_state) + model.load_state_dict(model_state) + + return model + +# def load_optimizer(optimizer, checkpoint, device): + +# if checkpoint is None: +# return optimizer + +# opti_state = optimizer.state_dict() +# pretrained_state = checkpoint['optimizer'] + +# logging.getLogger('autonet').debug(str(len(pretrained_state['state']))) +# logging.getLogger('autonet').debug(str(len(opti_state['param_groups'][0]['params']))) +# logging.getLogger('autonet').debug(str(len(pretrained_state['param_groups'][0]['params']))) +# logging.getLogger('autonet').debug(str(set(pretrained_state['param_groups'][0]['params']).intersection(set(opti_state['param_groups'][0]['params'])))) + + +# pretrained_state = {k: pretrained_state[k] for state in opti_state.items() for k, v in enumerate(state) if state in pretrained_state and k in pretrained_state[state] and v.size() == opti_state[state][k].size()} +# logging.getLogger('autonet').debug('=> Resuming optimizer using ' + str(len(pretrained_state.keys())) + '/' + str(len(opti_state.keys()))) +# opti_state.update(pretrained_state) +# optimizer.load_state_dict(opti_state) + +# for state in optimizer.state.values(): +# for k, v in state.items(): +# if isinstance(v, torch.Tensor): +# state[k] = v.to(device) +# return optimizer + +# def load_scheduler(scheduler, checkpoint): + +# if checkpoint is None: +# return scheduler + +# loaded_scheduler = checkpoint['scheduler'] +# loaded_scheduler.optimizer = scheduler.optimizer +# return loaded_scheduler \ No newline at end of file diff --git a/autoPyTorch/components/training/image/checkpoints/save_load.py b/autoPyTorch/components/training/image/checkpoints/save_load.py new file mode 100644 index 000000000..014662f1c --- /dev/null +++ b/autoPyTorch/components/training/image/checkpoints/save_load.py @@ -0,0 +1,39 @@ +import torch +import os + +import logging + + +def get_checkpoint_name(config_id, budget): + return 'checkpoint_' + str(config_id) + '_Budget_' + str(int(budget)) + '.pt' + +def get_checkpoint_dir(working_directory): + return os.path.join(working_directory, 'checkpoints') + +def save_checkpoint(path, config_id, budget, model, optimizer, scheduler): + + name = get_checkpoint_name(config_id, budget) + os.makedirs(path, exist_ok=True) + + path = os.path.join(path, name) + + torch.save({ + 'state': model.state_dict(), + }, open(path, 'wb')) + + logging.getLogger('autonet').debug('=> Model {} saved to {}'.format(str(type(model)), path)) + return path + + +def load_checkpoint(path, config_id, budget): + name = get_checkpoint_name(config_id, budget) + + path = os.path.join(path, name) + if not os.path.exists(path): + return None + + logging.getLogger('autonet').debug('=> Loading checkpoint ' + path) + checkpoint = torch.load(path) + return checkpoint + + diff --git a/autoPyTorch/components/training/image/early_stopping.py b/autoPyTorch/components/training/image/early_stopping.py new file mode 100644 index 000000000..51f5f7678 --- /dev/null +++ b/autoPyTorch/components/training/image/early_stopping.py @@ -0,0 +1,84 @@ +from autoPyTorch.components.training.image.base_training import BaseTrainingTechnique +from autoPyTorch.utils.config.config_option import ConfigOption, to_bool + +class EarlyStopping(BaseTrainingTechnique): + """ Stop training when there is no improvement on the validation set for a specified number of epochs. + Is able to take a snapshot of the parameters, where the performance of the validation set is best. + There is no further split of the data. Therefore the validation performance reported to BOHB will become an optimistic estimator. + """ + + # OVERRIDE + def set_up(self, training_components, pipeline_config, logger): + super(EarlyStopping, self).set_up(training_components, pipeline_config, logger) + self.reset_parameters = pipeline_config["early_stopping_reset_parameters"] + self.minimize = pipeline_config["minimize"] + self.patience = pipeline_config["early_stopping_patience"] + + # does not work with e.g. cosine anealing with warm restarts + if "lr_scheduler" in training_components and not training_components["lr_scheduler"].allows_early_stopping: + self.patience = float("inf") + + # initialize current best performance to +/- infinity + if training_components["network"].current_best_epoch_performance is None: + training_components["network"].current_best_epoch_performance = float("inf") + if not self.minimize: + training_components["network"].current_best_epoch_performance = -float("inf") + + self.logger.debug("Using Early stopping with patience: " + str(self.patience)) + self.logger.debug("Reset Parameters to parameters with best validation performance: " + str(self.reset_parameters)) + + # OVERRIDE + def after_train_batches(self, training_components, log, epoch): + if "val_" + training_components["train_metric_name"] not in log: + if self.patience < float("inf"): + self.logger.debug("No Early stopping because no validation set performance available") + return False + if self.reset_parameters and ("lr_scheduler" not in training_components or not training_components["lr_scheduler"].snapshot_before_restart): + log["best_parameters"] = False + current_performance = log["val_" + training_components["train_metric_name"]] + + # new best performance + if ((self.minimize and current_performance < training_components["network"].current_best_epoch_performance) or + (not self.minimize and current_performance > training_components["network"].current_best_epoch_performance)): + training_components["network"].num_epochs_no_progress = 0 + training_components["network"].current_best_epoch_performance = current_performance + self.logger.debug("New best performance!") + + if self.reset_parameters and ("lr_scheduler" not in training_components or not training_components["lr_scheduler"].snapshot_before_restart): + self.logger.debug("Early stopping takes snapshot of current parameters") + log["best_parameters"] = True + training_components["network"].snapshot() + + # do early stopping + elif training_components["network"].num_epochs_no_progress > self.patience: + self.logger.debug("Early stopping patience exhausted. Stopping Early!") + training_components["network"].stopped_early = True + return True + + # no improvement + else: + self.logger.debug("No improvement") + training_components["network"].num_epochs_no_progress += 1 + return False + + # OVERRIDE + def select_log(self, logs, training_components): + # select the log where a snapshot has been taken + if self.reset_parameters and ("lr_scheduler" not in training_components or not training_components["lr_scheduler"].snapshot_before_restart): + self.logger.debug("Using logs of parameters with best validation performance") + logs = [log for log in logs if log["best_parameters"]] or logs + logs = logs[-1] + return logs + return False + + def needs_eval_on_valid_each_epoch(self): + return self.reset_parameters or self.patience < float("inf") + + # OVERRIDE + @staticmethod + def get_pipeline_config_options(): + options = [ + ConfigOption("early_stopping_patience", default=float("inf"), type=float), + ConfigOption("early_stopping_reset_parameters", default=False, type=to_bool) + ] + return options diff --git a/autoPyTorch/components/training/image/lr_scheduling.py b/autoPyTorch/components/training/image/lr_scheduling.py new file mode 100644 index 000000000..e207a2665 --- /dev/null +++ b/autoPyTorch/components/training/image/lr_scheduling.py @@ -0,0 +1,39 @@ +from autoPyTorch.components.training.image.base_training import BaseTrainingTechnique + +class LrScheduling(BaseTrainingTechnique): + """Schedule the learning rate with given learning rate scheduler. + The learning rate scheduler is usually set in a LrSchedulerSelector pipeline node. + """ + + # OVERRIDE + def after_train_batches(self, training_components, log, epoch): + + # do one step of lr scheduling + if callable(getattr(training_components["lr_scheduler"], "get_lr", None)): + log['lr'] = training_components["lr_scheduler"].get_lr()[0] + try: + training_components["lr_scheduler"].step(epoch=(epoch + 1), metrics=log['loss']) + except: + training_components["lr_scheduler"].step(epoch=(epoch + 1)) + self.logger.debug("Perform learning rate scheduling") + + # check if lr scheduler has converged, if possible + if not training_components["lr_scheduler"].snapshot_before_restart: + return False + training_components["lr_scheduler"].get_lr() + log["lr_scheduler_converged"] = False + if training_components["lr_scheduler"].restarted_at == (epoch + 1): + self.logger.debug("Learning rate scheduler converged. Taking Snapshot of models parameters.") + training_components["network"].snapshot() + log["lr_scheduler_converged"] = True + return False + + def select_log(self, logs, training_components): + + # select the log where the lr scheduler has converged, if possible. + if training_components["lr_scheduler"].snapshot_before_restart: + self.logger.debug("Using logs where lr scheduler converged") + logs = [log for log in logs if log["lr_scheduler_converged"]] or logs + logs = logs[-1] + return logs + return False diff --git a/autoPyTorch/components/training/image/mixup.py b/autoPyTorch/components/training/image/mixup.py new file mode 100644 index 000000000..7fb6d3309 --- /dev/null +++ b/autoPyTorch/components/training/image/mixup.py @@ -0,0 +1,32 @@ +from autoPyTorch.components.training.image.base_training import BaseBatchLossComputationTechnique +import numpy as np +from torch.autograd import Variable +import ConfigSpace +import torch + +class Mixup(BaseBatchLossComputationTechnique): + def set_up(self, pipeline_config, hyperparameter_config, logger): + super(Mixup, self).set_up(pipeline_config, hyperparameter_config, logger) + self.alpha = hyperparameter_config["alpha"] + + def prepare_data(self, x, y): + + lam = np.random.beta(self.alpha, self.alpha) if self.alpha > 0. else 1. + batch_size = x.size()[0] + index = torch.randperm(batch_size).cuda() if x.is_cuda else torch.randperm(batch_size) + + mixed_x = lam * x + (1 - lam) * x[index, :] + y_a, y_b = y, y[index] + return mixed_x, { 'y_a': y_a, 'y_b': y_b, 'lam' : lam } + + def criterion(self, y_a, y_b, lam): + return lambda criterion, pred: lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b) + + def evaluate(self, metric, y_pred, y_a, y_b, lam): + return lam * metric(y_pred, y_a) + (1 - lam) * metric(y_pred, y_b) + + @staticmethod + def get_hyperparameter_search_space(**pipeline_config): + cs = ConfigSpace.ConfigurationSpace() + cs.add_hyperparameter(ConfigSpace.hyperparameters.UniformFloatHyperparameter("alpha", lower=0, upper=1, default_value=1)) + return cs diff --git a/autoPyTorch/components/training/image/trainer.py b/autoPyTorch/components/training/image/trainer.py new file mode 100644 index 000000000..49a2a466f --- /dev/null +++ b/autoPyTorch/components/training/image/trainer.py @@ -0,0 +1,252 @@ +import time +import os +import torch +import torch.nn as nn + +import random +from torch.autograd import Variable +from .checkpoints.save_load import save_checkpoint + +# from util.transforms import mixup_data, mixup_criterion +# from checkpoints import save_checkpoint + +class Trainer(object): + def __init__(self, loss_computation, model, criterion, budget, optimizer, scheduler, budget_type, device, images_to_plot=0, checkpoint_path=None, config_id=None): + self.checkpoint_path = checkpoint_path + self.config_id = config_id + + self.scheduler = scheduler + # if self.scheduler and not hasattr(self.scheduler, 'cumulative_time'): + # self.scheduler.cumulative_time = 0 + self.optimizer = optimizer + self.device = device + + self.budget = budget + self.loss_computation = loss_computation + + self.images_plot_count = images_to_plot + + self.budget_type = budget_type + self.cumulative_time = 0 + + self.train_loss_sum = 0 + self.train_iterations = 0 + + self.latest_checkpoint = None + + try: + if torch.cuda.device_count() > 1: + model = nn.DataParallel(model) + self.model = model.to(self.device) + except: + print("CUDA unavailable, continue using CPU.") + self.model = model.to("cpu") + + try: + self.criterion = criterion.to(self.device) + except: + print("No criterion specified.") + self.criterion = None + + def train(self, epoch, train_loader, metrics): + ''' + Trains the model for a single epoch + ''' + + # train_size = int(0.9 * len(train_loader.dataset.train_data) / self.config.batch_size) + loss_sum = 0.0 + N = 0 + + # print('\33[1m==> Training epoch # {}\033[0m'.format(str(epoch))) + + + classified = [] + misclassified = [] + + self.model.train() + + budget_exceeded = False + metric_results = [0] * len(metrics) + start_time = time.time() + for step, (data, targets) in enumerate(train_loader): + # import matplotlib.pyplot as plt + # img = plt.imshow(data.numpy()[0,1,:]) + # plt.show() + + # images += list(data.numpy()) + # print('Data:', data.size(), ' - Label:', targets.size()) + + data = data.to(self.device) + targets = targets.to(self.device) + + data, criterion_kwargs = self.loss_computation.prepare_data(data, targets) + batch_size = data.size(0) + + outputs = self.model(data) + loss_func = self.loss_computation.criterion(**criterion_kwargs) + loss = loss_func(self.criterion, outputs) + + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + # print('Train:', ' '.join(str(outputs).split('\n')[0:2])) + + if self.images_plot_count > 0: + with torch.no_grad(): + _, pred = outputs.topk(1, 1, True, True) + pred = pred.t() + correct = pred.eq(targets.view(1, -1).expand_as(pred)).cpu().numpy()[0] + data = data.cpu().numpy() + classified += list(data[correct.astype(bool)]) + misclassified += list(data[(1-correct).astype(bool)]) + if len(classified) > self.images_plot_count: + classified = random.sample(classified, self.images_plot_count) + if len(misclassified) > self.images_plot_count: + misclassified = random.sample(misclassified, self.images_plot_count) + + # self.scheduler.cumulative_time += delta_time + # self.scheduler.last_step = self.scheduler.cumulative_time - delta_time - 1e-10 + + tmp = time.time() + + with torch.no_grad(): + for i, metric in enumerate(metrics): + metric_results[i] += self.loss_computation.evaluate(metric, outputs, **criterion_kwargs) * batch_size + + loss_sum += loss.item() * batch_size + N += batch_size + + #print('Update', (metric_results[0] / N), 'loss', (loss_sum / N), 'lr', self.optimizer.param_groups[0]['lr']) + + if self.budget_type == 'time' and self.cumulative_time + (time.time() - start_time) >= self.budget: + # print(' * Stopping at Epoch: [%d][%d/%d] for a budget of %.3f s' % (epoch, step + 1, train_size, self.config.budget)) + budget_exceeded = True + break + + if N==0: # Fixes a bug during initialization + N=1 + + if self.images_plot_count > 0: + import tensorboard_logger as tl + tl.log_images('Train_Classified/Image', classified, step=epoch) + tl.log_images('Train_Misclassified/Image', misclassified, step=epoch) + + if self.checkpoint_path and self.scheduler.snapshot_before_restart and self.scheduler.needs_checkpoint(): + self.latest_checkpoint = save_checkpoint(self.checkpoint_path, self.config_id, self.budget, self.model, self.optimizer, self.scheduler) + + try: + self.scheduler.step(epoch=epoch) + except: + self.scheduler.step(metrics=loss_sum / N, epoch=epoch) + + self.cumulative_time += (time.time() - start_time) + #print('LR', self.optimizer.param_groups[0]['lr'], 'Update', (metric_results[0] / N), 'loss', (loss_sum / N)) + + return [res / N for res in metric_results], loss_sum / N, budget_exceeded + + + def evaluate(self, test_loader, metrics, epoch=0): + + N = 0 + metric_results = [0] * len(metrics) + + classified = [] + misclassified = [] + + self.model.eval() + + with torch.no_grad(): + for step, (data, targets) in enumerate(test_loader): + + # import matplotlib.pyplot as plt + # img = plt.imshow(data.numpy()[0,1,:]) + # plt.show() + + try: + data = data.to(self.device) + targets = targets.to(self.device) + except: + data = data.to("cpu") + targets = targets.to("cpu") + + batch_size = data.size(0) + + outputs = self.model(data) + + if self.images_plot_count > 0: + _, pred = outputs.topk(1, 1, True, True) + pred = pred.t() + correct = pred.eq(targets.view(1, -1).expand_as(pred)).cpu().numpy()[0] + data = data.cpu().numpy() + classified += list(data[correct.astype(bool)]) + misclassified += list(data[(1-correct).astype(bool)]) + if len(classified) > self.images_plot_count: + classified = random.sample(classified, self.images_plot_count) + if len(misclassified) > self.images_plot_count: + misclassified = random.sample(misclassified, self.images_plot_count) + + # print('Valid:', ' '.join(str(outputs).split('\n')[0:2])) + # print('Shape:', outputs.shape, 'Sums', str(outputs.cpu().numpy().sum(1)).replace('\n', '')) + + for i, metric in enumerate(metrics): + metric_results[i] += metric(outputs.data, targets.data) * batch_size + + N += batch_size + + if self.images_plot_count > 0: + import tensorboard_logger as tl + tl.log_images('Valid_Classified/Image', classified, step=epoch) + tl.log_images('Valid_Misclassified/Image', misclassified, step=epoch) + + self.model.train() + + return [res / N for res in metric_results] + + + def class_to_probability_mapping(self, test_loader): + + N = 0 + + import numpy as np + import torch.nn as nn + + probs = None; + class_to_index = dict() + target_count = [] + + self.model.eval() + + with torch.no_grad(): + for i, (data, targets) in enumerate(test_loader): + + data = data.to(self.device) + targets = targets.to(self.device) + + batch_size = data.size(0) + + outputs = self.model(data) + + for i, output in enumerate(outputs): + target = targets[i].cpu().item() + np_output = output.cpu().numpy() + if target not in class_to_index: + if probs is None: + probs = np.array([np_output]) + else: + probs = np.vstack((probs, np_output)) + class_to_index[target] = probs.shape[0] - 1 + target_count.append(0) + else: + probs[class_to_index[target]] = probs[class_to_index[target]] + np_output + + target_count[class_to_index[target]] += 1 + + N += batch_size + + probs = probs / np.array(target_count)[:, None] #np.max(probs, axis=1)[:, None] + probs = torch.from_numpy(probs) + # probs = nn.Softmax(1)(probs) + + self.model.train() + return probs, class_to_index diff --git a/autoPyTorch/components/training/trainer.py b/autoPyTorch/components/training/trainer.py index 1de6f1234..1f8cbf413 100644 --- a/autoPyTorch/components/training/trainer.py +++ b/autoPyTorch/components/training/trainer.py @@ -66,10 +66,10 @@ def on_epoch_start(self, log, epoch): def on_epoch_end(self, log, epoch): return any([t.on_epoch_end(trainer=self, log=log, epoch=epoch) for t in self.training_techniques]) - def final_eval(self, opt_metric_name, logs, train_loader, valid_loader, minimize, best_over_epochs, refit): + def final_eval(self, opt_metric_name, logs, train_loader, valid_loader, best_over_epochs, refit): # select log if best_over_epochs: - final_log = (min if minimize else max)(logs, key=lambda log: log[opt_metric_name]) + final_log = min(logs, key=lambda log: self.metrics[0].loss_transform(log[opt_metric_name])) else: final_log = None for t in self.training_techniques: @@ -87,10 +87,10 @@ def final_eval(self, opt_metric_name, logs, train_loader, valid_loader, minimize for i, metric in enumerate(self.metrics): if valid_metric_results: - final_log['val_' + metric.__name__] = valid_metric_results[i] - if self.eval_additional_logs_on_snapshot: + final_log['val_' + metric.name] = valid_metric_results[i] + if self.eval_additional_logs_on_snapshot and not refit: for additional_log in self.log_functions: - final_log[additional_log.__name__] = additional_log(self.model, None) + final_log[additional_log.name] = additional_log(self.model, None) return final_log def train(self, epoch, train_loader): @@ -163,4 +163,3 @@ def compute_metrics(self, outputs_data, targets_data): outputs_data = np.vstack(outputs_data) targets_data = np.vstack(targets_data) return [metric(outputs_data, targets_data) for metric in self.metrics] - \ No newline at end of file diff --git a/autoPyTorch/core/api.py b/autoPyTorch/core/api.py index 80fb16d6d..fe680a37d 100644 --- a/autoPyTorch/core/api.py +++ b/autoPyTorch/core/api.py @@ -4,10 +4,12 @@ import numpy as np +import scipy.sparse import torch import torch.nn as nn import copy import os +import json from autoPyTorch.pipeline.base.pipeline import Pipeline @@ -15,16 +17,20 @@ from autoPyTorch.pipeline.nodes.cross_validation import CrossValidation from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector from autoPyTorch.pipeline.nodes.optimization_algorithm import OptimizationAlgorithm +from autoPyTorch.pipeline.nodes.create_dataset_info import CreateDatasetInfo +from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector +from autoPyTorch.pipeline.nodes.image.network_selector_datasetinfo import NetworkSelectorDatasetInfo from autoPyTorch.utils.config.config_file_parser import ConfigFileParser class AutoNet(): + """Find an optimal neural network given a ML-task using BOHB""" preset_folder_name = None def __init__(self, config_preset="medium_cs", pipeline=None, **autonet_config): """Superclass for all AutoNet variations, that specifies the API of AutoNet. - + Keyword Arguments: pipeline {Pipeline} -- Define your own Autonet Pipeline (default: {None}) **autonet_config -- Configure AutoNet for your needs. You can also configure AutoNet in fit(). Call print_help() for more info. @@ -33,6 +39,7 @@ def __init__(self, config_preset="medium_cs", pipeline=None, **autonet_config): self.base_config = autonet_config self.autonet_config = None self.fit_result = None + self.dataset_info = None if config_preset is not None: parser = self.get_autonet_config_file_parser() @@ -56,7 +63,6 @@ def print_help(self): print() config_file_parser.print_help(self.base_config) - def get_current_autonet_config(self): """Return the current AutoNet configuration @@ -68,14 +74,33 @@ def get_current_autonet_config(self): return self.autonet_config return self.pipeline.get_pipeline_config(**self.base_config) - def get_hyperparameter_search_space(self, dataset_info=None): - """Return the hyperparameter search space of AutoNet + def get_hyperparameter_search_space(self, X_train=None, Y_train=None, X_valid=None, Y_valid=None, **autonet_config): + """Return hyperparameter search space of Auto-PyTorch. Does depend on the dataset and the configuration! + You can either pass the dataset and the configuration or use dataset and configuration of last fit call. + + Keyword Arguments: + X_train {array} -- Training data. ConfigSpace depends on Training data. + Y_train {array} -- Targets of training data. + X_valid {array} -- Validation data. Will be ignored if cv_splits > 1. (default: {None}) + Y_valid {array} -- Validation data. Will be ignored if cv_splits > 1. (default: {None}) + autonet_config{dict} -- if not given and fit already called, config of last fit will be used Returns: - ConfigurationSpace -- The ConfigurationSpace that should be optimized + ConfigurationSpace -- The configuration space that should be optimized. """ + X_train, Y_train, X_valid, Y_valid = self.check_data_array_types(X_train, Y_train, X_valid, Y_valid) + dataset_info = self.dataset_info + pipeline_config = dict(self.base_config, **autonet_config) if autonet_config else \ + self.get_current_autonet_config() + if X_train is not None and Y_train is not None: + dataset_info_node = self.pipeline[CreateDatasetInfo.get_name()] + dataset_info = dataset_info_node.fit(pipeline_config=pipeline_config, + X_train=X_train, + Y_train=Y_train, + X_valid=X_valid, + Y_valid=Y_valid)["dataset_info"] - return self.pipeline.get_hyperparameter_search_space(dataset_info=dataset_info, **self.get_current_autonet_config()) + return self.pipeline.get_hyperparameter_search_space(dataset_info=dataset_info, **pipeline_config) @classmethod def get_default_pipeline(cls): @@ -110,21 +135,25 @@ def fit(self, X_train, Y_train, X_valid=None, Y_valid=None, refit=True, **autone Returns: optimized_hyperparameter_config -- The best found hyperparameter config. - final_metric_score -- The final score of the specified train metric. **autonet_config -- Configure AutoNet for your needs. You can also configure AutoNet in the constructor(). Call print_help() for more info. """ + X_train, Y_train, X_valid, Y_valid = self.check_data_array_types(X_train, Y_train, X_valid, Y_valid) self.autonet_config = self.pipeline.get_pipeline_config(**dict(self.base_config, **autonet_config)) self.fit_result = self.pipeline.fit_pipeline(pipeline_config=self.autonet_config, X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid) + try: + self.dataset_info = self.pipeline[CreateDatasetInfo.get_name()].fit_output["dataset_info"] + except: + self.dataset_info = None self.pipeline.clean() - if not self.fit_result["optimized_hyperparameter_config"]: + if "optimized_hyperparameter_config" not in self.fit_result.keys() or not self.fit_result["optimized_hyperparameter_config"]: # MODIFY raise RuntimeError("No models fit during training, please retry with a larger max_runtime.") if (refit): self.refit(X_train, Y_train, X_valid, Y_valid) - return self.fit_result["optimized_hyperparameter_config"], self.fit_result['final_metric_score'] + return self.fit_result def refit(self, X_train, Y_train, X_valid=None, Y_valid=None, hyperparameter_config=None, autonet_config=None, budget=None, rescore=False): """Refit AutoNet to given hyperparameters. This will skip hyperparameter search. @@ -144,6 +173,7 @@ def refit(self, X_train, Y_train, X_valid=None, Y_valid=None, hyperparameter_con Raises: ValueError -- No hyperparameter config available """ + X_train, Y_train, X_valid, Y_valid = self.check_data_array_types(X_train, Y_train, X_valid, Y_valid) if (autonet_config is None): autonet_config = self.autonet_config if (autonet_config is None): @@ -162,10 +192,13 @@ def refit(self, X_train, Y_train, X_valid=None, Y_valid=None, hyperparameter_con refit_data = {'hyperparameter_config': hyperparameter_config, 'budget': budget, 'rescore': rescore} - - result = self.pipeline.fit_pipeline(pipeline_config=autonet_config, refit=refit_data, - X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid) - return result["final_metric_score"] + + autonet_config = copy.deepcopy(autonet_config) + autonet_config['cv_splits'] = 1 + autonet_config['increase_number_of_trained_datasets'] = False #if training multiple datasets else ignored + + return self.pipeline.fit_pipeline(pipeline_config=autonet_config, refit=refit_data, + X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid) def predict(self, X, return_probabilities=False): """Predict the targets for a data matrix X. @@ -181,16 +214,23 @@ def predict(self, X, return_probabilities=False): """ # run predict pipeline - autonet_config = self.autonet_config or self.base_config + X, = self.check_data_array_types(X) + autonet_config = self.get_current_autonet_config() + Y_pred = self.pipeline.predict_pipeline(pipeline_config=autonet_config, X=X)['Y'] - # reverse one hot encoding - OHE = self.pipeline[OneHotEncoding.get_name()] - result = OHE.reverse_transform_y(Y_pred, OHE.fit_output['y_one_hot_encoder']) - return result if not return_probabilities else (result, Y_pred) + # reverse one hot encoding + if OneHotEncoding.get_name() in self.pipeline: + OHE = self.pipeline[OneHotEncoding.get_name()] + result = OHE.reverse_transform_y(Y_pred, OHE.fit_output['y_one_hot_encoder']) + return result if not return_probabilities else (result, Y_pred) + else: + result = dict() + result['Y'] = Y_pred + return result if not return_probabilities else (result, Y_pred) - def score(self, X_test, Y_test): - """Calculate the sore on test data using the specified train_metric + def score(self, X_test, Y_test, return_loss_value=False): + """Calculate the sore on test data using the specified optimize_metric Arguments: X_test {array} -- The test data matrix. @@ -200,14 +240,106 @@ def score(self, X_test, Y_test): score -- The score for the test data. """ + # Update config if needed + X_test, Y_test = self.check_data_array_types(X_test, Y_test) + autonet_config = self.get_current_autonet_config() + + res = self.pipeline.predict_pipeline(pipeline_config=autonet_config, X=X_test) + if 'score' in res: + # in case of default dataset like CIFAR10 - the pipeline will compute the score of the according pytorch test set + return res['score'] + Y_pred = res['Y'] # run predict pipeline - autonet_config = self.autonet_config or self.base_config - self.pipeline.predict_pipeline(pipeline_config=autonet_config, X=X_test) - Y_pred = self.pipeline[OptimizationAlgorithm.get_name()].predict_output['Y'] + #self.pipeline.predict_pipeline(pipeline_config=autonet_config, X=X_test) + #Y_pred = self.pipeline[OptimizationAlgorithm.get_name()].predict_output['Y'] + # one hot encode Y - OHE = self.pipeline[OneHotEncoding.get_name()] - Y_test = OHE.transform_y(Y_test, OHE.fit_output['y_one_hot_encoder']) + try: + OHE = self.pipeline[OneHotEncoding.get_name()] + Y_test = OHE.transform_y(Y_test, OHE.fit_output['y_one_hot_encoder']) + except: + print("No one-hot encodig possible. Continuing without.") + pass + + metric = self.pipeline[MetricSelector.get_name()].fit_output['optimize_metric'] + + if return_loss_value: + return metric.get_loss_value(Y_pred, Y_test) + return metric(torch.from_numpy(Y_pred.astype(np.float32)), torch.from_numpy(Y_test.astype(np.float32))) + + def get_pytorch_model(self): + """Returns a pytorch sequential model of the current incumbent configuration. Not possible for all models. + + Arguments: + + Returns: + model -- PyTorch sequential model of the current incumbent configuration + """ + try: + if NetworkSelector.get_name() in self.pipeline: + return self.pipeline[NetworkSelector.get_name()].fit_output["network"].layers + else: + return self.pipeline[NetworkSelectorDatasetInfo.get_name()].fit_output["network"].layers + except: + print("Can not get PyTorch Sequential model for incumbent config. Returning Auto-PyTorch model") + if NetworkSelector.get_name() in self.pipeline: + return self.pipeline[NetworkSelector.get_name()].fit_output["network"] + else: + return self.pipeline[NetworkSelectorDatasetInfo.get_name()].fit_output["network"] + + def initialize_from_checkpoint(self, hyperparameter_config, checkpoint, in_features, out_features, final_activation=None): + """ + + Arguments: + config_file: json with output as from .fit method + in_features: array-like object, channels first + out_features: int, number of classes + final_activation: + + Returns: + PyTorch Sequential model + + """ + # load state dict + state_dict = torch.load(checkpoint, map_location=torch.device('cpu'))["state"] + + # read config file + if type(hyperparameter_config)==dict: + config = hyperparameter_config + else: + with open(hyperparameter_config, 'r') as file: + config = json.load(file)[1] + + # get model + network_type = config['NetworkSelectorDatasetInfo:network'] + network_type = self.pipeline[NetworkSelectorDatasetInfo.get_name()].networks[network_type] + model = network_type(config=config, + in_features=in_features, + out_features=out_features, + final_activation=final_activation) - metric = self.pipeline[MetricSelector.get_name()].fit_output['train_metric'] - return metric(Y_pred, Y_test) + # Apply state dict + pretrained_state = state_dict + model_state = model.state_dict() + + pretrained_state = { k:v for k,v in pretrained_state.items() if k in model_state and v.size() == model_state[k].size() } + model_state.update(pretrained_state) + model.load_state_dict(model_state) + + # Add to pipeline + self.pipeline[NetworkSelectorDatasetInfo.get_name()].fit_output["network"] = model + + return model + + def check_data_array_types(self, *arrays): + result = [] + for array in arrays: + if array is None or scipy.sparse.issparse(array): + result.append(array) + continue + + result.append(np.asanyarray(array)) + if not result[-1].shape: + raise RuntimeError("Given data-array is of unexpected type %s. Please pass numpy arrays instead." % type(array)) + return result diff --git a/autoPyTorch/core/autonet_classes/__init__.py b/autoPyTorch/core/autonet_classes/__init__.py index 1979c846d..96e55e5e5 100644 --- a/autoPyTorch/core/autonet_classes/__init__.py +++ b/autoPyTorch/core/autonet_classes/__init__.py @@ -1,3 +1,5 @@ from autoPyTorch.core.autonet_classes.autonet_feature_classification import AutoNetClassification from autoPyTorch.core.autonet_classes.autonet_feature_regression import AutoNetRegression -from autoPyTorch.core.autonet_classes.autonet_feature_multilabel import AutoNetMultilabel \ No newline at end of file +from autoPyTorch.core.autonet_classes.autonet_feature_multilabel import AutoNetMultilabel +from autoPyTorch.core.autonet_classes.autonet_image_classification import AutoNetImageClassification +from autoPyTorch.core.autonet_classes.autonet_image_classification_multiple_datasets import AutoNetImageClassificationMultipleDatasets diff --git a/autoPyTorch/core/autonet_classes/autonet_feature_classification.py b/autoPyTorch/core/autonet_classes/autonet_feature_classification.py index 951bd7a4f..a6183e1f9 100644 --- a/autoPyTorch/core/autonet_classes/autonet_feature_classification.py +++ b/autoPyTorch/core/autonet_classes/autonet_feature_classification.py @@ -3,6 +3,7 @@ class AutoNetClassification(AutoNetFeatureData): preset_folder_name = "feature_classification" + # OVERRIDE @staticmethod def _apply_default_pipeline_settings(pipeline): from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector @@ -18,7 +19,7 @@ def _apply_default_pipeline_settings(pipeline): import torch.nn as nn from sklearn.model_selection import StratifiedKFold - from autoPyTorch.components.metrics import accuracy, auc_metric, pac_metric, balanced_accuracy + from autoPyTorch.components.metrics import accuracy, auc_metric, pac_metric, balanced_accuracy, cross_entropy from autoPyTorch.components.preprocessing.loss_weight_strategies import LossWeightStrategyWeighted AutoNetFeatureData._apply_default_pipeline_settings(pipeline) @@ -32,10 +33,16 @@ def _apply_default_pipeline_settings(pipeline): loss_selector.add_loss_module('cross_entropy_weighted', nn.CrossEntropyLoss, LossWeightStrategyWeighted(), True) metric_selector = pipeline[MetricSelector.get_name()] - metric_selector.add_metric('accuracy', accuracy) - metric_selector.add_metric('auc_metric', auc_metric) - metric_selector.add_metric('pac_metric', pac_metric) - metric_selector.add_metric('balanced_accuracy', balanced_accuracy) + metric_selector.add_metric('accuracy', accuracy, loss_transform=True, + requires_target_class_labels=True) + metric_selector.add_metric('auc_metric', auc_metric, loss_transform=True, + requires_target_class_labels=False) + metric_selector.add_metric('pac_metric', pac_metric, loss_transform=True, + requires_target_class_labels=False) + metric_selector.add_metric('balanced_accuracy', balanced_accuracy, loss_transform=True, + requires_target_class_labels=True) + metric_selector.add_metric('cross_entropy', cross_entropy, loss_transform=True, + requires_target_class_labels=False) resample_selector = pipeline[ResamplingStrategySelector.get_name()] resample_selector.add_over_sampling_method('random', RandomOverSamplingWithReplacement) @@ -58,4 +65,4 @@ def _apply_default_pipeline_settings(pipeline): return pipeline def flatten(x): - return x.reshape((-1, )) \ No newline at end of file + return x.reshape((-1, )) diff --git a/autoPyTorch/core/autonet_classes/autonet_feature_data.py b/autoPyTorch/core/autonet_classes/autonet_feature_data.py index 0514a5862..c844d4807 100644 --- a/autoPyTorch/core/autonet_classes/autonet_feature_data.py +++ b/autoPyTorch/core/autonet_classes/autonet_feature_data.py @@ -9,6 +9,11 @@ class AutoNetFeatureData(AutoNet): @classmethod def get_default_ensemble_pipeline(cls): + """Construct a default pipeline, include nodes for Ensemble. + + Returns: + Pipeline -- The constructed default pipeline + """ from autoPyTorch.pipeline.base.pipeline import Pipeline from autoPyTorch.pipeline.nodes import AutoNetSettings, OptimizationAlgorithm, \ CrossValidation, Imputation, NormalizationStrategySelector, OneHotEncoding, PreprocessorSelector, ResamplingStrategySelector, \ @@ -50,6 +55,11 @@ def get_default_ensemble_pipeline(cls): @classmethod def get_default_pipeline(cls): + """Construct a default pipeline, do not include nodes for Ensemble. + + Returns: + Pipeline -- The constructed default pipeline + """ from autoPyTorch.pipeline.base.pipeline import Pipeline from autoPyTorch.pipeline.nodes import AutoNetSettings, OptimizationAlgorithm, \ CrossValidation, Imputation, NormalizationStrategySelector, OneHotEncoding, PreprocessorSelector, ResamplingStrategySelector, \ @@ -87,15 +97,20 @@ def get_default_pipeline(cls): @staticmethod def _apply_default_pipeline_settings(pipeline): + """Add the components to the pipeline + + Arguments: + pipeline {pipeline} -- The pipelines to add the components to + """ from autoPyTorch.pipeline.nodes import NormalizationStrategySelector, PreprocessorSelector, EmbeddingSelector, NetworkSelector, \ OptimizerSelector, LearningrateSchedulerSelector, TrainNode, CrossValidation, InitializationSelector from autoPyTorch.components.networks.feature import MlpNet, ResNet, ShapedMlpNet, ShapedResNet from autoPyTorch.components.networks.initialization import SimpleInitializer, SparseInitialization - from autoPyTorch.components.optimizer.optimizer import AdamOptimizer, SgdOptimizer + from autoPyTorch.components.optimizer.optimizer import AdamOptimizer, AdamWOptimizer, SgdOptimizer, RMSpropOptimizer from autoPyTorch.components.lr_scheduler.lr_schedulers import SchedulerCosineAnnealingWithRestartsLR, SchedulerNone, \ - SchedulerCyclicLR, SchedulerExponentialLR, SchedulerReduceLROnPlateau, SchedulerReduceLROnPlateau, SchedulerStepLR + SchedulerCyclicLR, SchedulerExponentialLR, SchedulerReduceLROnPlateau, SchedulerReduceLROnPlateau, SchedulerStepLR, SchedulerAdaptiveLR, SchedulerAlternatingCosineLR from autoPyTorch.components.networks.feature import LearnedEntityEmbedding from sklearn.preprocessing import MinMaxScaler, StandardScaler, MaxAbsScaler @@ -135,19 +150,23 @@ def _apply_default_pipeline_settings(pipeline): opt_selector = pipeline[OptimizerSelector.get_name()] opt_selector.add_optimizer('adam', AdamOptimizer) + opt_selector.add_optimizer('adamw', AdamWOptimizer) opt_selector.add_optimizer('sgd', SgdOptimizer) + opt_selector.add_optimizer('rmsprop', RMSpropOptimizer) lr_selector = pipeline[LearningrateSchedulerSelector.get_name()] - lr_selector.add_lr_scheduler('cosine_annealing', SchedulerCosineAnnealingWithRestartsLR) - lr_selector.add_lr_scheduler('cyclic', SchedulerCyclicLR) - lr_selector.add_lr_scheduler('exponential', SchedulerExponentialLR) - lr_selector.add_lr_scheduler('step', SchedulerStepLR) - lr_selector.add_lr_scheduler('plateau', SchedulerReduceLROnPlateau) - lr_selector.add_lr_scheduler('none', SchedulerNone) + lr_selector.add_lr_scheduler('cosine_annealing', SchedulerCosineAnnealingWithRestartsLR) + lr_selector.add_lr_scheduler('cyclic', SchedulerCyclicLR) + lr_selector.add_lr_scheduler('exponential', SchedulerExponentialLR) + lr_selector.add_lr_scheduler('step', SchedulerStepLR) + lr_selector.add_lr_scheduler('adapt', SchedulerAdaptiveLR) + lr_selector.add_lr_scheduler('plateau', SchedulerReduceLROnPlateau) + lr_selector.add_lr_scheduler('alternating_cosine', SchedulerAlternatingCosineLR) + lr_selector.add_lr_scheduler('none', SchedulerNone) train_node = pipeline[TrainNode.get_name()] train_node.add_training_technique("early_stopping", EarlyStopping) train_node.add_batch_loss_computation_technique("mixup", Mixup) cv = pipeline[CrossValidation.get_name()] - cv.add_cross_validator("k_fold", KFold) \ No newline at end of file + cv.add_cross_validator("k_fold", KFold) diff --git a/autoPyTorch/core/autonet_classes/autonet_feature_multilabel.py b/autoPyTorch/core/autonet_classes/autonet_feature_multilabel.py index 6b6bd9db7..5b70e6915 100644 --- a/autoPyTorch/core/autonet_classes/autonet_feature_multilabel.py +++ b/autoPyTorch/core/autonet_classes/autonet_feature_multilabel.py @@ -3,6 +3,7 @@ class AutoNetMultilabel(AutoNetFeatureData): preset_folder_name = "feature_multilabel" + # OVERRIDE @staticmethod def _apply_default_pipeline_settings(pipeline): from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector @@ -25,12 +26,15 @@ def _apply_default_pipeline_settings(pipeline): loss_selector.add_loss_module('bce_with_logits_weighted', nn.BCEWithLogitsLoss, LossWeightStrategyWeightedBinary(), False) metric_selector = pipeline[MetricSelector.get_name()] - metric_selector.add_metric('multilabel_accuracy', multilabel_accuracy) - metric_selector.add_metric('auc_metric', auc_metric) - metric_selector.add_metric('pac_metric', pac_metric) + metric_selector.add_metric('multilabel_accuracy', multilabel_accuracy, + loss_transform=True, requires_target_class_labels=True) + metric_selector.add_metric('auc_metric', auc_metric, loss_transform=True, + requires_target_class_labels=False) + metric_selector.add_metric('pac_metric', pac_metric, loss_transform=True, + requires_target_class_labels=False) train_node = pipeline[TrainNode.get_name()] train_node.default_minimize_value = False cv = pipeline[CrossValidation.get_name()] - cv.use_stratified_cv_split_default = False \ No newline at end of file + cv.use_stratified_cv_split_default = False diff --git a/autoPyTorch/core/autonet_classes/autonet_feature_regression.py b/autoPyTorch/core/autonet_classes/autonet_feature_regression.py index 6dc1bb2f8..65d8b6490 100644 --- a/autoPyTorch/core/autonet_classes/autonet_feature_regression.py +++ b/autoPyTorch/core/autonet_classes/autonet_feature_regression.py @@ -8,6 +8,7 @@ class AutoNetRegression(AutoNetFeatureData): preset_folder_name = "feature_regression" + # OVERRIDE @staticmethod def _apply_default_pipeline_settings(pipeline): from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector @@ -28,10 +29,10 @@ def _apply_default_pipeline_settings(pipeline): loss_selector.add_loss_module('l1_loss', nn.L1Loss) metric_selector = pipeline[MetricSelector.get_name()] - metric_selector.add_metric('mean_distance', mean_distance) + metric_selector.add_metric('mean_distance', mean_distance, loss_transform=False, requires_target_class_labels=False) train_node = pipeline[TrainNode.get_name()] train_node.default_minimize_value = True cv = pipeline[CrossValidation.get_name()] - cv.use_stratified_cv_split_default = False \ No newline at end of file + cv.use_stratified_cv_split_default = False diff --git a/autoPyTorch/core/autonet_classes/autonet_image_classification.py b/autoPyTorch/core/autonet_classes/autonet_image_classification.py new file mode 100644 index 000000000..d9173aba2 --- /dev/null +++ b/autoPyTorch/core/autonet_classes/autonet_image_classification.py @@ -0,0 +1,43 @@ +from autoPyTorch.core.autonet_classes.autonet_image_data import AutoNetImageData + + +class AutoNetImageClassification(AutoNetImageData): + preset_folder_name = "image_classification" + + @staticmethod + def _apply_default_pipeline_settings(pipeline): + import torch.nn as nn + from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector + from autoPyTorch.pipeline.nodes.image.simple_train_node import SimpleTrainNode + from autoPyTorch.pipeline.nodes.image.cross_validation_indices import CrossValidationIndices + from autoPyTorch.pipeline.nodes.image.loss_module_selector_indices import LossModuleSelectorIndices + from autoPyTorch.pipeline.nodes.image.network_selector_datasetinfo import NetworkSelectorDatasetInfo + from autoPyTorch.components.metrics import accuracy, auc_metric, pac_metric, balanced_accuracy, cross_entropy + from autoPyTorch.components.preprocessing.loss_weight_strategies import LossWeightStrategyWeighted + + AutoNetImageData._apply_default_pipeline_settings(pipeline) + + net_selector = pipeline[NetworkSelectorDatasetInfo.get_name()] + net_selector.add_final_activation('softmax', nn.Softmax(1)) + + loss_selector = pipeline[LossModuleSelectorIndices.get_name()] + loss_selector.add_loss_module('cross_entropy', nn.CrossEntropyLoss, None, True) + loss_selector.add_loss_module('cross_entropy_weighted', nn.CrossEntropyLoss, LossWeightStrategyWeighted(), True) + + metric_selector = pipeline[MetricSelector.get_name()] + metric_selector.add_metric('accuracy', accuracy, loss_transform=True, + requires_target_class_labels=False) + metric_selector.add_metric('auc_metric', auc_metric, loss_transform=True, + requires_target_class_labels=False) + metric_selector.add_metric('pac_metric', pac_metric, loss_transform=True, + requires_target_class_labels=False) + metric_selector.add_metric('balanced_accuracy', balanced_accuracy, loss_transform=True, + requires_target_class_labels=True) + metric_selector.add_metric('cross_entropy', cross_entropy, loss_transform=True, + requires_target_class_labels=False) + + train_node = pipeline[SimpleTrainNode.get_name()] + train_node.default_minimize_value = False + + cv = pipeline[CrossValidationIndices.get_name()] + cv.use_stratified_cv_split_default = True diff --git a/autoPyTorch/core/autonet_classes/autonet_image_classification_multiple_datasets.py b/autoPyTorch/core/autonet_classes/autonet_image_classification_multiple_datasets.py new file mode 100644 index 000000000..95f6a9859 --- /dev/null +++ b/autoPyTorch/core/autonet_classes/autonet_image_classification_multiple_datasets.py @@ -0,0 +1,56 @@ +from autoPyTorch.core.autonet_classes.autonet_image_classification import AutoNetImageClassification + + +class AutoNetImageClassificationMultipleDatasets(AutoNetImageClassification): + preset_folder_name = "image_classification_multiple_datasets" + + @classmethod + def get_default_pipeline(cls): + from autoPyTorch.pipeline.base.pipeline import Pipeline + from autoPyTorch.pipeline.nodes.image.optimization_algorithm_no_timelimit import OptimizationAlgorithmNoTimeLimit + from autoPyTorch.pipeline.nodes.optimizer_selector import OptimizerSelector + from autoPyTorch.pipeline.nodes.log_functions_selector import LogFunctionsSelector + from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector + + from autoPyTorch.pipeline.nodes.image.simple_scheduler_selector import SimpleLearningrateSchedulerSelector + from autoPyTorch.pipeline.nodes.image.cross_validation_indices import CrossValidationIndices + from autoPyTorch.pipeline.nodes.image.autonet_settings_no_shuffle import AutoNetSettingsNoShuffle + from autoPyTorch.pipeline.nodes.image.network_selector_datasetinfo import NetworkSelectorDatasetInfo + from autoPyTorch.pipeline.nodes.image.loss_module_selector_indices import LossModuleSelectorIndices + from autoPyTorch.pipeline.nodes.image.image_augmentation import ImageAugmentation + from autoPyTorch.pipeline.nodes.image.create_image_dataloader import CreateImageDataLoader + from autoPyTorch.pipeline.nodes.image.create_dataset_info import CreateDatasetInfo + from autoPyTorch.pipeline.nodes.image.simple_train_node import SimpleTrainNode + from autoPyTorch.pipeline.nodes.image.multiple_datasets import MultipleDatasets + from autoPyTorch.pipeline.nodes.image.image_dataset_reader import ImageDatasetReader + + # build the pipeline + pipeline = Pipeline([ + AutoNetSettingsNoShuffle(), + OptimizationAlgorithmNoTimeLimit([ + + MultipleDatasets([ + + ImageDatasetReader(), + CreateDatasetInfo(), + CrossValidationIndices([ + + NetworkSelectorDatasetInfo(), + OptimizerSelector(), + SimpleLearningrateSchedulerSelector(), + + LogFunctionsSelector(), + MetricSelector(), + + LossModuleSelectorIndices(), + + ImageAugmentation(), + CreateImageDataLoader(), + SimpleTrainNode() + ]) + ]) + ]) + ]) + + cls._apply_default_pipeline_settings(pipeline) + return pipeline diff --git a/autoPyTorch/core/autonet_classes/autonet_image_data.py b/autoPyTorch/core/autonet_classes/autonet_image_data.py new file mode 100644 index 000000000..1437182ae --- /dev/null +++ b/autoPyTorch/core/autonet_classes/autonet_image_data.py @@ -0,0 +1,133 @@ +import numpy as np +import torch +from autoPyTorch.core.api import AutoNet + + +__author__ = "Max Dippel, Michael Burkart and Matthias Urban" +__version__ = "0.0.1" +__license__ = "BSD" + + +class AutoNetImageData(AutoNet): + + @classmethod + def get_default_pipeline(cls): + from autoPyTorch.pipeline.base.pipeline import Pipeline + from autoPyTorch.pipeline.nodes.image.optimization_algorithm_no_timelimit import OptimizationAlgorithmNoTimeLimit + from autoPyTorch.pipeline.nodes.one_hot_encoding import OneHotEncoding + from autoPyTorch.pipeline.nodes.optimizer_selector import OptimizerSelector + from autoPyTorch.pipeline.nodes.log_functions_selector import LogFunctionsSelector + from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector + + from autoPyTorch.pipeline.nodes.image.simple_scheduler_selector import SimpleLearningrateSchedulerSelector + from autoPyTorch.pipeline.nodes.image.cross_validation_indices import CrossValidationIndices + from autoPyTorch.pipeline.nodes.image.autonet_settings_no_shuffle import AutoNetSettingsNoShuffle + from autoPyTorch.pipeline.nodes.image.network_selector_datasetinfo import NetworkSelectorDatasetInfo + from autoPyTorch.pipeline.nodes.image.loss_module_selector_indices import LossModuleSelectorIndices + from autoPyTorch.pipeline.nodes.image.image_augmentation import ImageAugmentation + from autoPyTorch.pipeline.nodes.image.create_image_dataloader import CreateImageDataLoader + from autoPyTorch.pipeline.nodes.image.create_dataset_info import CreateDatasetInfo + from autoPyTorch.pipeline.nodes.image.simple_train_node import SimpleTrainNode + from autoPyTorch.pipeline.nodes.image.image_dataset_reader import ImageDatasetReader + from autoPyTorch.pipeline.nodes.image.single_dataset import SingleDataset + + # build the pipeline + pipeline = Pipeline([ + AutoNetSettingsNoShuffle(), + OptimizationAlgorithmNoTimeLimit([ + + SingleDataset([ + + ImageDatasetReader(), + CreateDatasetInfo(), + CrossValidationIndices([ + + NetworkSelectorDatasetInfo(), + OptimizerSelector(), + SimpleLearningrateSchedulerSelector(), + + LogFunctionsSelector(), + MetricSelector(), + + LossModuleSelectorIndices(), + + ImageAugmentation(), + CreateImageDataLoader(), + SimpleTrainNode() + ]) + ]) + ]) + ]) + + + cls._apply_default_pipeline_settings(pipeline) + return pipeline + + @staticmethod + def _apply_default_pipeline_settings(pipeline): + from autoPyTorch.pipeline.nodes.optimizer_selector import OptimizerSelector + from autoPyTorch.pipeline.nodes.image.simple_scheduler_selector import SimpleLearningrateSchedulerSelector + + from autoPyTorch.pipeline.nodes.image.network_selector_datasetinfo import NetworkSelectorDatasetInfo + from autoPyTorch.pipeline.nodes.image.simple_train_node import SimpleTrainNode + from autoPyTorch.pipeline.nodes.image.create_image_dataloader import CreateImageDataLoader + from autoPyTorch.pipeline.nodes.image.image_augmentation import ImageAugmentation + + from autoPyTorch.components.networks.image import DenseNet, ResNet, MobileNet + from autoPyTorch.components.networks.image.densenet_flexible import DenseNetFlexible + from autoPyTorch.components.networks.image.resnet152 import ResNet152 + from autoPyTorch.components.networks.image.darts.model import DARTSImageNet + + from autoPyTorch.components.optimizer.optimizer import AdamOptimizer, AdamWOptimizer, SgdOptimizer, RMSpropOptimizer + from autoPyTorch.components.lr_scheduler.lr_schedulers import SchedulerCosineAnnealingWithRestartsLR, SchedulerNone, \ + SchedulerCyclicLR, SchedulerExponentialLR, SchedulerReduceLROnPlateau, SchedulerReduceLROnPlateau, SchedulerStepLR, SchedulerAlternatingCosineLR, SchedulerAdaptiveLR, SchedulerExponentialLR + + from autoPyTorch.components.training.image.early_stopping import EarlyStopping + from autoPyTorch.components.training.image.mixup import Mixup + + net_selector = pipeline[NetworkSelectorDatasetInfo.get_name()] + net_selector.add_network('densenet', DenseNet) + net_selector.add_network('densenet_flexible', DenseNetFlexible) + net_selector.add_network('resnet', ResNet) + net_selector.add_network('resnet152', ResNet152) + net_selector.add_network('darts', DARTSImageNet) + net_selector.add_network('mobilenet', MobileNet) + net_selector._apply_search_space_update('resnet:nr_main_blocks', [2, 4], log=False) + net_selector._apply_search_space_update('resnet:widen_factor_1', [0.5, 8], log=True) + + opt_selector = pipeline[OptimizerSelector.get_name()] + opt_selector.add_optimizer('adam', AdamOptimizer) + opt_selector.add_optimizer('adamw', AdamWOptimizer) + opt_selector.add_optimizer('sgd', SgdOptimizer) + opt_selector.add_optimizer('rmsprop', RMSpropOptimizer) + + lr_selector = pipeline[SimpleLearningrateSchedulerSelector.get_name()] + lr_selector.add_lr_scheduler('cosine_annealing', SchedulerCosineAnnealingWithRestartsLR) + lr_selector.add_lr_scheduler('cyclic', SchedulerCyclicLR) + lr_selector.add_lr_scheduler('step', SchedulerStepLR) + lr_selector.add_lr_scheduler('adapt', SchedulerAdaptiveLR) + lr_selector.add_lr_scheduler('plateau', SchedulerReduceLROnPlateau) + lr_selector.add_lr_scheduler('alternating_cosine',SchedulerAlternatingCosineLR) + lr_selector.add_lr_scheduler('exponential', SchedulerExponentialLR) + lr_selector.add_lr_scheduler('none', SchedulerNone) + + lr_selector._apply_search_space_update('step:step_size', [1, 100], log=True) + lr_selector._apply_search_space_update('step:gamma', [0.001, 0.99], log=True) + lr_selector._apply_search_space_update('cosine_annealing:T_max', [1, 100], log=True) + lr_selector._apply_search_space_update('cosine_annealing:T_mult', [1., 2.], log=False) + + train_node = pipeline[SimpleTrainNode.get_name()] + #train_node.add_training_technique("early_stopping", EarlyStopping) + train_node.add_batch_loss_computation_technique("mixup", Mixup) + + data_node = pipeline[CreateImageDataLoader.get_name()] + + data_node._apply_search_space_update('batch_size', [32, 160], log=True) + + augment_node = pipeline[ImageAugmentation.get_name()] + augment_node._apply_search_space_update('augment', [False, True]) + augment_node._apply_search_space_update('autoaugment', [False, True]) + augment_node._apply_search_space_update('fastautoaugment', [False, True]) + augment_node._apply_search_space_update('length', [2,6]) + augment_node._apply_search_space_update('cutout', [False, True]) + augment_node._apply_search_space_update('cutout_holes', [1, 50]) diff --git a/autoPyTorch/core/ensemble.py b/autoPyTorch/core/ensemble.py index fbd2f0ce0..907f27206 100644 --- a/autoPyTorch/core/ensemble.py +++ b/autoPyTorch/core/ensemble.py @@ -4,8 +4,12 @@ from autoPyTorch.pipeline.nodes.one_hot_encoding import OneHotEncoding from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector from autoPyTorch.pipeline.nodes.ensemble import EnableComputePredictionsForEnsemble, SavePredictionsForEnsemble, BuildEnsemble, EnsembleServer +from autoPyTorch.pipeline.nodes.create_dataset_info import CreateDatasetInfo class AutoNetEnsemble(AutoNet): + """Build an ensemble of several neural networks that were evaluated during the architecure search""" + + # OVERRIDE def __init__(self, autonet, config_preset="medium_cs", **autonet_config): if isinstance(autonet, AutoNet): self.pipeline = autonet.pipeline @@ -29,6 +33,7 @@ def __init__(self, autonet, config_preset="medium_cs", **autonet_config): self.base_config.update(autonet_config) self.trained_autonets = None + self.dataset_info = None if config_preset is not None: parser = self.get_autonet_config_file_parser() @@ -37,16 +42,21 @@ def __init__(self, autonet, config_preset="medium_cs", **autonet_config): c.update(self.base_config) self.base_config = c + # OVERRIDE def fit(self, X_train, Y_train, X_valid=None, Y_valid=None, refit=True, **autonet_config): + X_train, Y_train, X_valid, Y_valid = self.check_data_array_types(X_train, Y_train, X_valid, Y_valid) self.autonet_config = self.pipeline.get_pipeline_config(**dict(self.base_config, **autonet_config)) self.fit_result = self.pipeline.fit_pipeline(pipeline_config=self.autonet_config, X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid) + self.dataset_info = self.pipeline[CreateDatasetInfo.get_name()].fit_output["dataset_info"] self.pipeline.clean() if refit: self.refit(X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid) - return self.fit_result["ensemble_configs"], self.fit_result["ensemble_final_metric_score"], self.fit_result["ensemble"] + return self.fit_result + # OVERRIDE def refit(self, X_train, Y_train, X_valid=None, Y_valid=None, ensemble_configs=None, ensemble=None, autonet_config=None): + X_train, Y_train, X_valid, Y_valid = self.check_data_array_types(X_train, Y_train, X_valid, Y_valid) if (autonet_config is None): autonet_config = self.autonet_config if (autonet_config is None): @@ -69,8 +79,10 @@ def refit(self, X_train, Y_train, X_valid=None, Y_valid=None, ensemble_configs=N hyperparameter_config=hyperparameter_config, autonet_config=autonet_config, budget=budget) self.trained_autonets[tuple(identifier)] = autonet + # OVERRIDE def predict(self, X, return_probabilities=False, return_metric=False): # run predict pipeline + X, = self.check_data_array_types(X) prediction = None models_with_weights = self.fit_result["ensemble"].get_models_with_weights(self.trained_autonets) autonet_config = self.autonet_config or self.base_config @@ -78,7 +90,7 @@ def predict(self, X, return_probabilities=False, return_metric=False): current_prediction = autonet.pipeline.predict_pipeline(pipeline_config=autonet_config, X=X)["Y"] prediction = current_prediction if prediction is None else prediction + weight * current_prediction OHE = autonet.pipeline[OneHotEncoding.get_name()] - metric = autonet.pipeline[MetricSelector.get_name()].fit_output['train_metric'] + metric = autonet.pipeline[MetricSelector.get_name()].fit_output['optimize_metric'] # reverse one hot encoding result = OHE.reverse_transform_y(prediction, OHE.fit_output['y_one_hot_encoder']) @@ -91,8 +103,10 @@ def predict(self, X, return_probabilities=False, return_metric=False): result.append(metric) return tuple(result) + # OVERRIDE def score(self, X_test, Y_test): # run predict pipeline + X_test, Y_test = self.check_data_array_types(X_test, Y_test) _, Y_pred, metric = self.predict(X_test, return_probabilities=True, return_metric=True) Y_test, _ = self.pipeline[OneHotEncoding.get_name()].complete_y_tranformation(Y_test) return metric(Y_pred, Y_test) diff --git a/autoPyTorch/core/hpbandster_extensions/run_with_time.py b/autoPyTorch/core/hpbandster_extensions/run_with_time.py index 30bc48fc8..68e8a1e38 100644 --- a/autoPyTorch/core/hpbandster_extensions/run_with_time.py +++ b/autoPyTorch/core/hpbandster_extensions/run_with_time.py @@ -18,7 +18,7 @@ def run_with_time(self, runtime=1, n_iterations=float("inf"), min_n_workers=1, i """ self.wait_for_workers(min_n_workers) - + iteration_kwargs.update({'result_logger': self.result_logger}) if self.time_ref is None: @@ -30,15 +30,15 @@ def run_with_time(self, runtime=1, n_iterations=float("inf"), min_n_workers=1, i self.thread_cond.acquire() start_time = time.time() - kill = False + while True: - if (not kill and runtime < time.time() - start_time): - # wait for running jobs and finish - kill = True - self.logger.info('HBMASTER: Timelimit reached: wait for remaining %i jobs'%self.num_running_jobs) - self._queue_wait() + + # Check if timelimit is reached + if (runtime < time.time() - start_time): + self.logger.info('HBMASTER: Timelimit reached: wait for remaining %i jobs'%self.num_running_jobs) + break next_run = None # find a new run to schedule @@ -47,16 +47,10 @@ def run_with_time(self, runtime=1, n_iterations=float("inf"), min_n_workers=1, i if not next_run is None: break if next_run is not None: - if kill: - # register new run as finished - this will be interpreted as a crashed job - config_id, config, budget = next_run - job = Job(config_id, config=config, budget=budget, working_directory=self.working_directory) - self.iterations[job.id[0] - self.iterations[0].HPB_iter].register_result(job) - else: - self.logger.debug('HBMASTER: schedule new run for iteration %i'%i) - self._submit_job(*next_run) + self.logger.debug('HBMASTER: schedule new run for iteration %i'%i) + self._submit_job(*next_run) continue - elif not kill and n_iterations > 0: + elif n_iterations > 0: next_HPB_iter = len(self.iterations) + (self.iterations[0].HPB_iter if len(self.iterations) > 0 else 0) self.iterations.append(self.get_next_iteration(next_HPB_iter, iteration_kwargs)) n_iterations -= 1 @@ -69,6 +63,27 @@ def run_with_time(self, runtime=1, n_iterations=float("inf"), min_n_workers=1, i else: break + # clean up / cancel remaining iteration runs + next_run = True + n_canceled = 0 + while next_run is not None: + next_run = None + for i in self.active_iterations(): + next_run = self.iterations[i].get_next_run() + if not next_run is None: + config_id, config, budget = next_run + job = Job(config_id, config=config, budget=budget, working_directory=self.working_directory) + self.iterations[job.id[0]].register_result(job) # register dummy job - will be interpreted as canceled job + n_canceled += 1 + break + + self.logger.debug('HBMASTER: Canceled %i remaining runs'%n_canceled) + + # wait for remaining jobs + while self.num_running_jobs > 0: + self.thread_cond.wait(60) + self.logger.debug('HBMASTER: Job finished: wait for remaining %i jobs'%self.num_running_jobs) + self.thread_cond.release() for i in self.warmstart_iteration: diff --git a/autoPyTorch/core/presets/image_classification/__init__.py b/autoPyTorch/core/presets/image_classification/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/autoPyTorch/core/presets/image_classification/full_cs.txt b/autoPyTorch/core/presets/image_classification/full_cs.txt new file mode 100644 index 000000000..e69de29bb diff --git a/autoPyTorch/core/presets/image_classification/medium_cs.txt b/autoPyTorch/core/presets/image_classification/medium_cs.txt new file mode 100644 index 000000000..f9fb6096f --- /dev/null +++ b/autoPyTorch/core/presets/image_classification/medium_cs.txt @@ -0,0 +1,4 @@ +lr_scheduler=[cosine_annealing, step] +networks=[resnet, mobilenet] +batch_loss_computation_techniques=[standard, mixup] +optimizer=[adamw, sgd] diff --git a/autoPyTorch/core/presets/image_classification/tiny_cs.txt b/autoPyTorch/core/presets/image_classification/tiny_cs.txt new file mode 100644 index 000000000..ebefdde80 --- /dev/null +++ b/autoPyTorch/core/presets/image_classification/tiny_cs.txt @@ -0,0 +1,4 @@ +lr_scheduler=[cosine_annealing] +networks=[resnet] +batch_loss_computation_techniques=[standard] +optimizer=[adamw] diff --git a/autoPyTorch/core/presets/image_classification_multiple_datasets/__init__.py b/autoPyTorch/core/presets/image_classification_multiple_datasets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/autoPyTorch/core/presets/image_classification_multiple_datasets/full_cs.txt b/autoPyTorch/core/presets/image_classification_multiple_datasets/full_cs.txt new file mode 100644 index 000000000..e69de29bb diff --git a/autoPyTorch/core/presets/image_classification_multiple_datasets/medium_cs.txt b/autoPyTorch/core/presets/image_classification_multiple_datasets/medium_cs.txt new file mode 100644 index 000000000..f9fb6096f --- /dev/null +++ b/autoPyTorch/core/presets/image_classification_multiple_datasets/medium_cs.txt @@ -0,0 +1,4 @@ +lr_scheduler=[cosine_annealing, step] +networks=[resnet, mobilenet] +batch_loss_computation_techniques=[standard, mixup] +optimizer=[adamw, sgd] diff --git a/autoPyTorch/core/presets/image_classification_multiple_datasets/tiny_cs.txt b/autoPyTorch/core/presets/image_classification_multiple_datasets/tiny_cs.txt new file mode 100644 index 000000000..ebefdde80 --- /dev/null +++ b/autoPyTorch/core/presets/image_classification_multiple_datasets/tiny_cs.txt @@ -0,0 +1,4 @@ +lr_scheduler=[cosine_annealing] +networks=[resnet] +batch_loss_computation_techniques=[standard] +optimizer=[adamw] diff --git a/autoPyTorch/core/worker.py b/autoPyTorch/core/worker.py index 3487cd732..b12128244 100644 --- a/autoPyTorch/core/worker.py +++ b/autoPyTorch/core/worker.py @@ -11,10 +11,27 @@ __version__ = "0.0.1" __license__ = "BSD" -class ModuleWorker(Worker): +class AutoNetWorker(Worker): + """Worker that evaluates the hyperparameter configurations of the ML-pipelines""" + def __init__(self, pipeline, pipeline_config, X_train, Y_train, X_valid, Y_valid, dataset_info, budget_type, max_budget, shutdownables, use_pynisher, *args, **kwargs): + """Initialize the worker. + + Arguments: + pipeline {Pipeline} -- The ML pipeline to evaluate + pipeline_config {dict} -- The configuration of the pipeline + X_train {array} -- The data used for training the neural networks + Y_train {array} -- The labels used for evaluating the neural networks + X_valid {array} -- The data used for evaluating the neural networks + Y_valid {array} -- The data used for evaluating the neural networks + dataset_info {DatasetInfo} -- Object containing basic information about the dataset + budget_type {BaseTrainingTechnique} -- The type of budget used for the optimization + max_budget {float} -- The maximum budget + shutdownables {list} -- For each element of the object, the shutdown() method is called when the worker is shutting down. + use_pynisher {bool} -- Whether to use pynisher to guarantee resource limits + """ self.X_train = X_train #torch.from_numpy(X_train).float() self.Y_train = Y_train #torch.from_numpy(Y_train).long() self.X_valid = X_valid @@ -38,6 +55,7 @@ def __init__(self, pipeline, pipeline_config, super().__init__(*args, **kwargs) + # OVERRIDE def compute(self, config, budget, working_directory, config_id, **kwargs): self.autonet_logger.debug("Budget " + str(budget) + " config: " + str(config)) @@ -45,6 +63,7 @@ def compute(self, config, budget, working_directory, config_id, **kwargs): start_time = time.time() self.autonet_logger.debug("Starting optimization!") + # guarantee time and memory limits using pynisher if self.guarantee_limits: import pynisher time_limit=None @@ -53,9 +72,11 @@ def compute(self, config, budget, working_directory, config_id, **kwargs): grace_time = 10 time_limit = int(budget + 240) + # start optimization limit_train = pynisher.enforce_limits(mem_in_mb=self.pipeline_config['memory_limit_mb'], wall_time_in_s=time_limit)(self.optimize_pipeline) result = limit_train(config, config_id, budget, start_time) + # check for exceptions if (limit_train.exit_status == pynisher.TimeoutException): raise Exception("Time limit reached. Took " + str((time.time()-start_time)) + " seconds with budget " + str(budget)) elif (limit_train.exit_status == pynisher.MemorylimitException): @@ -73,11 +94,22 @@ def compute(self, config, budget, working_directory, config_id, **kwargs): # that is not really elegant but we can want to achieve some kind of feedback network_name = [v for k, v in config.items() if k.endswith('network')] or "None" - self.autonet_logger.info("Training " + str(network_name) + " with budget " + str(budget) + " resulted in score: " + str(loss) + " took " + str((time.time()-start_time)) + " seconds") + self.autonet_logger.info("Training " + str(network_name) + " with budget " + str(budget) + " resulted in optimize-metric-loss: " + str(loss) + " took " + str((time.time()-start_time)) + " seconds") return result def optimize_pipeline(self, config, config_id, budget, optimize_start_time): + """Fit the pipeline using the sampled hyperparameter configuration. + + Arguments: + config {dict} -- The sampled hyperparameter configuration. + config_id {tuple} -- An ID for the configuration. Assigned by BOHB. + budget {float} -- The budget to evaluate the hyperparameter configuration. + optimize_start_time {float} -- The time when optimization started. + + Returns: + dict -- The result of fitting the pipeline. + """ try: self.autonet_logger.info("Fit optimization pipeline") return self.pipeline.fit_pipeline(hyperparameter_config=config, pipeline_config=self.pipeline_config, diff --git a/autoPyTorch/core/worker_no_timelimit.py b/autoPyTorch/core/worker_no_timelimit.py new file mode 100644 index 000000000..d99e3459e --- /dev/null +++ b/autoPyTorch/core/worker_no_timelimit.py @@ -0,0 +1,130 @@ +import logging +import torch +import time +import random +from hpbandster.core.worker import Worker + +from autoPyTorch.components.training.image.budget_types import BudgetTypeTime + +__author__ = "Max Dippel, Michael Burkart and Matthias Urban" +__version__ = "0.0.1" +__license__ = "BSD" + + +class ModuleWorkerNoTimeLimit(Worker): + def __init__(self, pipeline, pipeline_config, constant_hyperparameter, + X_train, Y_train, X_valid, Y_valid, budget_type, max_budget, working_directory, permutations=None, *args, **kwargs): + self.X_train = X_train #torch.from_numpy(X_train).float() + self.Y_train = Y_train #torch.from_numpy(Y_train).long() + self.X_valid = X_valid + self.Y_valid = Y_valid + + if permutations is None: + self.permutations = [idx for idx in range(len(X_train))] + else: + self.permutations = permutations + + self.max_budget = max_budget + self.budget_type = budget_type + + self.pipeline = pipeline + self.pipeline_config = pipeline_config + self.constant_hyperparameter = constant_hyperparameter + + self.working_directory = working_directory + + self.autonet_logger = logging.getLogger('autonet') + # self.end_time = None + + # We can only use user defined limits (memory) if we have the required module 'resource' - not available on windows! + self.guarantee_limits = module_exists("resource") and module_exists("pynisher") + if (not self.guarantee_limits): + self.autonet_logger.info("Can not guarantee memory and time limit because module 'resource' is not available") + + super().__init__(*args, **kwargs) + + def compute(self, config, budget, working_directory, config_id, **kwargs): + + start_time = time.time() + + self.autonet_logger.debug("Starting optimization!") + + config.update(self.constant_hyperparameter) + + self.autonet_logger.debug("Budget " + str(budget) + " config: " + str(config)) + + if self.guarantee_limits and self.budget_type == 'time': + import pynisher + + limit_train = pynisher.enforce_limits(wall_time_in_s=int(budget * 4))(self.optimize_pipeline) + result, randomstate = limit_train(config, budget, config_id, random.getstate()) + + if (limit_train.exit_status == pynisher.MemorylimitException): + raise Exception("Memory limit reached. Took " + str((time.time()-start_time)) + " seconds with budget " + str(budget)) + elif (limit_train.exit_status != 0): + self.autonet_logger.info('Exception occurred using config:\n' + str(config)) + raise Exception("Exception in train pipeline. Took " + str((time.time()-start_time)) + " seconds with budget " + str(budget)) + + else: + result, randomstate = self.optimize_pipeline(config, budget, config_id, random.getstate()) + + random.setstate(randomstate) + + loss = result['loss'] + if 'losses' in result.keys(): + losses = result['losses'] + else: + losses = loss + info = result['info'] + + self.autonet_logger.debug("Result: " + str(loss) + " info: " + str(info)) + + # that is not really elegant but we can want to achieve some kind of feedback + network_name = [v for k, v in config.items() if k.endswith('network')] or "None" + + self.autonet_logger.info("Training " + str(network_name) + " with budget " + str(budget) + " resulted in score: " + str(loss) + " took " + str((time.time()-start_time)) + " seconds") + + if 'use_tensorboard_logger' in self.pipeline_config and self.pipeline_config['use_tensorboard_logger']: + import os + log_file = os.path.join(self.working_directory, "worker_logs_" + str(self.pipeline_config['task_id']), 'results.log') + sep = '\t' + with open(log_file, 'a+') as f: + f.write('Result: ' + str(round(loss, 2)) + sep + \ + 'Budget: ' + str(round(budget)) + '/' + str(round(self.pipeline_config['max_budget'])) + sep + \ + 'Used time: ' + str(round((time.time()-start_time))) + 'sec (' + str(round((time.time()-start_time)/budget, 2)) + 'x)' + sep + \ + 'ID: ' + str(config_id) + '\n') + + return { + 'loss': loss, + 'info': info, + 'losses': losses + } + + def optimize_pipeline(self, config, budget, config_id, random_state): + + random.setstate(random_state) + + if self.permutations is not None: + current_sh_run = config_id[0] + self.pipeline_config["dataset_order"] = self.permutations[current_sh_run%len(self.permutations)].tolist() + + try: + self.autonet_logger.info("Fit optimization pipeline") + return self.pipeline.fit_pipeline(hyperparameter_config=config, pipeline_config=self.pipeline_config, + X_train=self.X_train, Y_train=self.Y_train, X_valid=self.X_valid, Y_valid=self.Y_valid, + budget=budget, budget_type=self.budget_type, max_budget=self.max_budget, + config_id=config_id, working_directory=self.working_directory), random.getstate() + except Exception as e: + if 'use_tensorboard_logger' in self.pipeline_config and self.pipeline_config['use_tensorboard_logger']: + import tensorboard_logger as tl + tl.log_value('Exceptions/' + str(e), budget, int(time.time())) + #self.autonet_logger.exception('Exception occurred') + raise e + +def module_exists(module_name): + try: + __import__(module_name) + except ImportError: + return False + else: + return True diff --git a/autoPyTorch/data_management/data_loader.py b/autoPyTorch/data_management/data_loader.py new file mode 100644 index 000000000..f7cac7380 --- /dev/null +++ b/autoPyTorch/data_management/data_loader.py @@ -0,0 +1,47 @@ +import os +import math +from PIL import Image +import requests +from io import BytesIO +from torchvision import transforms, utils + + + +class DataLoader(): + def __init__(self): + pass + + def load(self, url, size): + try: + response = requests.get(url) + img = Image.open(BytesIO(response.content)).convert('RGB') + except: + return None + t = transforms.Compose([transforms.Resize(size), + transforms.CenterCrop(size), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + # transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + ]) + res = (t(img) * 255).int() + return res.reshape((size*size*3)) + + def save_imagenet_subset(self, root, name, class_wnids, image_size, max_images=None): + with open(os.path.join(root, name) + '.data', 'w+') as data: + with open(os.path.join(root, name) + '.label', 'w+') as label: + for i, wnid in enumerate(class_wnids): + urls = requests.get('http://www.image-net.org/api/text/imagenet.synset.geturls?wnid=' + wnid).content + urls = urls.split(b"\n") + images = 0 + for u in range(len(urls)): + if max_images is not None and images+1 > max_images / len(class_wnids): + break + img = self.load(urls[u].decode('utf-8'), image_size) + if img is None: + continue + images += 1 + data.write(' '.join([str(rgb) for rgb in img.numpy()]) + '\n') + label.write(str(i) + '\n') + missing = math.floor(max_images/len(class_wnids)) - images + if missing > 0: + print('Wnid', wnid, 'needs', missing, 'more images.') \ No newline at end of file diff --git a/autoPyTorch/data_management/data_manager.py b/autoPyTorch/data_management/data_manager.py index cc0afe55a..b0e0d7777 100644 --- a/autoPyTorch/data_management/data_manager.py +++ b/autoPyTorch/data_management/data_manager.py @@ -17,11 +17,20 @@ from enum import Enum class ProblemType(Enum): FeatureClassification = 1 + ImageClassification = 2 FeatureRegression = 3 FeatureMultilabel = 4 + ImageClassificationMultipleDatasets = 5 class DataManager(object): + """ Load data from multiple sources and formants""" + def __init__(self, verbose=0): + """Construct the DataManager + + Keyword Arguments: + verbose {bool} -- Whether to print stuff. (default: {0}) + """ self.verbose = verbose self.X_train, self.Y_train = None, None self.X_test, self.Y_test = None, None @@ -33,6 +42,16 @@ def __init__(self, verbose=0): self.categorical_features = None def read_data(self, file_name, test_split=0.0, is_classification=None, random_seed=0, **kwargs): + """Read the data. + + Arguments: + file_name {str} -- The name of the file to load. Different Readers are associated with different filenames. + + Keyword Arguments: + test_split {float} -- Amount of data to use as test split (default: {0.0}) + is_classification {bool} -- Whether the data is a classification task (default: {None}) + random_seed {int} -- a random seed (default: {0}) + """ print("Read:" + file_name) reader = self._get_reader(file_name, is_classification) reader.read() @@ -53,6 +72,18 @@ def read_data(self, file_name, test_split=0.0, is_classification=None, random_se self._split_data(test_split, random_seed) def _get_reader(self, file_name, is_classification): + """Get the reader associated with the filename. + + Arguments: + file_name {str} -- The file to load + is_classification {bool} -- Whether the data is a classification task or not + + Raises: + ValueError: The given file type is not supported + + Returns: + DataReader -- A reader that is able to read the data type + """ if file_name.endswith(".csv"): reader = CSVReader(file_name, is_classification=is_classification) elif file_name.startswith("openml:"): @@ -65,6 +96,17 @@ def _get_reader(self, file_name, is_classification): return reader def generate_classification(self, num_classes, num_features, num_samples, test_split=0.1, seed=0): + """Generate a classification task + + Arguments: + num_classes {int} -- Number of classes + num_features {int} -- Number of features + num_samples {int} -- Number of samples + + Keyword Arguments: + test_split {float} -- Size of test split (default: {0.1}) + seed {int} -- A random seed (default: {0}) + """ #X, Y = make_classification(n_samples=800, n_features=num_feats, n_classes=num_classes, n_informative=4) X, y = make_multilabel_classification( n_samples=num_samples, n_features=num_features, n_classes=num_classes, n_labels=0.01, @@ -78,6 +120,16 @@ def generate_classification(self, num_classes, num_features, num_samples, test_s self._split_data(test_split, seed) def generate_regression(self, num_features, num_samples, test_split=0.1, seed=0): + """Generate a regression task + + Arguments: + num_features {int} -- Number of features + num_samples {int} -- Number of samples + + Keyword Arguments: + test_split {float} -- Size of test split (default: {0.1}) + seed {int} -- a random seed (default: {0}) + """ X, Y = make_regression(n_samples=num_samples, n_features=num_features, random_state=seed) self.categorical_features = [False] * num_features self.problem_type = ProblemType.FeatureRegression @@ -85,6 +137,12 @@ def generate_regression(self, num_features, num_samples, test_split=0.1, seed=0) self._split_data(test_split, seed) def _split_data(self, test_split, seed): + """Split the data in test (, valid) and training set. + + Arguments: + test_split {[type]} -- [description] + seed {[type]} -- [description] + """ valid_specified = self.X_valid is not None and self.Y_valid is not None test_specified = self.X_test is not None and self.Y_test is not None @@ -100,7 +158,114 @@ def _split_data(self, test_split, seed): self.X_train = self.X self.Y_train = self.Y + +class ImageManager(DataManager): + + def read_data(self, file_name, test_split=0.0, is_classification=None, **kwargs): + self.is_classification = True + self.is_multilabel = False + + if isinstance(file_name, list): + import numpy as np + arr = np.array(file_name) + self.X_train = arr + self.Y_train = np.array([0] * len(file_name)) + self.X_valid = self.Y_valid = self.X_test = self.Y_test = None + self.problem_type = ProblemType.ImageClassificationMultipleDatasets + elif file_name.endswith(".csv"): + import pandas as pd + import math + import numpy as np + self.data = np.array(pd.read_csv(file_name, header=None)) + + self.X_train = np.array(self.data[:,0]) + self.Y_train = np.array(self.data[:,1]) + + self.X_valid = self.Y_valid = self.X_test = self.Y_test = None + + if test_split > 0: + samples = self.X_train.shape[0] + indices = list(range(samples)) + np.random.shuffle(indices) + split = samples * test_split + test_indices, train_indices = indices[:math.ceil(split)], indices[math.floor(split):] + self.X_test, self.Y_test = self.X_train[test_indices], self.Y_train[test_indices] + self.X_train, self.Y_train = self.X_train[train_indices], self.Y_train[train_indices] + + self.problem_type = ProblemType.ImageClassification + + def generate_classification(self, problem="MNIST", test_split=0.1, force_download=False, train_size=-1, test_size=-1): + self.is_classification = True + data = None + conversion = False + if problem == "MNIST": + data = torchvision.datasets.MNIST + elif problem == "Fashion-MNIST": + data = torchvision.datasets.FashionMNIST + elif problem == "CIFAR": + conversion = True + data = torchvision.datasets.CIFAR10 + else: + raise ValueError("Dataset not supported: " + problem) + + + train_dataset = data(root='datasets/torchvision/' + problem + '/', + train=True, + transform=transforms.ToTensor(), + download=True) + + test_dataset = data(root='datasets/torchvision/' + problem + '/', + train=False, + transform=transforms.ToTensor()) + images_train = [] + labels_train = [] + + train_size = train_dataset.__len__() if train_size == -1 else min(train_size, train_dataset.__len__()) + test_size = test_dataset.__len__() if test_size == -1 else min(test_size, test_dataset.__len__()) + + for i in range(train_size): + sys.stdout.write("Reading " + problem + " train data ["+ str(train_size)+"] - progress: %d%% \r" % (int(100 * (i + 1)/ train_size) )) + sys.stdout.flush() + image, label = train_dataset.__getitem__(i) + if conversion: + label = torch.tensor(label) + images_train.append(image.numpy()) + labels_train.append(label.numpy()) + + self.X_train = np.array(images_train) + self.Y_train = np.array(labels_train) + + images_test = [] + labels_test = [] + print() + for i in range(test_size): + sys.stdout.write("Reading " + problem + " test data ["+ str(test_size)+"] - progress: %d%% \r" % (int(100 * (i + 1) / test_size) )) + sys.stdout.flush() + image, label = test_dataset.__getitem__(i) + if conversion: + label = torch.tensor(label) + images_test.append(image.numpy()) + labels_test.append(label.numpy()) + + self.problem_type = ProblemType.ImageClassification + self.X_test = np.array(images_test) + self.Y_test = np.array(labels_test) + + self.categorical_features = None + print() + def deterministic_shuffle_and_split(X, Y, split, seed): + """Split the data deterministically given the seed + + Arguments: + X {array} -- The feature data + Y {array} -- The targets + split {float} -- The size of the split + seed {int} -- A random seed + + Returns: + tuple -- Tuple of full data and the two splits + """ rng = np.random.RandomState(seed) p = rng.permutation(X.shape[0]) @@ -110,4 +275,4 @@ def deterministic_shuffle_and_split(X, Y, split, seed): split = int(split * X.shape[0]) return X, Y, X[0:-split], Y[0:-split], X[-split:], Y[-split:] else: - return X, Y, X, Y, None, None \ No newline at end of file + return X, Y, X, Y, None, None diff --git a/autoPyTorch/data_management/data_reader.py b/autoPyTorch/data_management/data_reader.py index 5d7d7d85d..36ed25352 100644 --- a/autoPyTorch/data_management/data_reader.py +++ b/autoPyTorch/data_management/data_reader.py @@ -231,4 +231,37 @@ def read_binary_sparse_datafile(self, filepath, shape): col_indizes.append(int(value) - 1) row_indizes.append(row) print("Done") - return csr_matrix(([1] * len(row_indizes), (row_indizes, col_indizes)), shape=shape) \ No newline at end of file + return csr_matrix(([1] * len(row_indizes), (row_indizes, col_indizes)), shape=shape) + + +class OpenMLImageReader(OpenMlReader): + def __init__(self, dataset_id, is_classification = None, api_key=None, nChannels=1): + self.channels = nChannels + super(OpenMLImageReader, self).__init__(dataset_id, is_classification, api_key) + + def read(self, auto_convert=True, **kwargs): + """ + Read the data from given openml datset file. + + Arguments: + auto_convert: Automatically convert data after reading. + *args, **kwargs: arguments for converting. + """ + + dataset = self.openml.datasets.get_dataset(self.dataset_id) + self.data = dataset.get_data() + + + self.num_entries = len(self.data) + self.num_features = len(self.data[0]) - 1 + + + self.X = self.data[0:self.num_entries, 0:self.num_features] / 255 + + image_size = int(math.sqrt(self.num_features / self.channels)) + self.X = np.reshape(self.X, (self.X.shape[0], self.channels, image_size, image_size)) + + self.Y = self.data[0:self.num_entries, -1] + self.num_classes = len(np.unique(self.Y)) + if self.is_classification is None: + self.is_classification = dataset.get_features_by_type("nominal")[-1] == self.num_features diff --git a/autoPyTorch/data_management/image_loader.py b/autoPyTorch/data_management/image_loader.py new file mode 100644 index 000000000..00dad12c1 --- /dev/null +++ b/autoPyTorch/data_management/image_loader.py @@ -0,0 +1,119 @@ +import torch.utils.data as data + +import os +import os.path + +import logging +logging.getLogger('PIL').setLevel(logging.CRITICAL) +from PIL import Image + +def default_loader(path): + return Image.open(path).convert('RGB') + +from multiprocessing import Process, RawValue, Lock +import time + +class ThreadCounter(object): + def __init__(self): + # RawValue because we don't need it to create a Lock: + self.val = RawValue('d', 0) + self.num = RawValue('i', 0) + self.lock = Lock() + + def add(self, value): + with self.lock: + self.val.value += value + self.num.value += 1 + + def value(self): + with self.lock: + return self.val.value + + def avg(self): + with self.lock: + return self.val.value / self.num.value + + def reset(self): + with self.lock: + self.val.value = 0 + self.num.value = 0 + +class ImageFilelist(data.Dataset): + def __init__(self, image_file_list, label_list, transform=None, target_transform=None, loader=default_loader, cache_size=0, image_size=None): + self.image_file_list = image_file_list + self.label_list = label_list + self.transform = transform + self.target_transform = target_transform + self.loader = loader + # self.readTime = ThreadCounter() + # self.augmentTime = ThreadCounter() + # self.loadTime = ThreadCounter() + self.fill_cache(cache_size, image_size) + + def get_times(self, prefix): + times = dict() + # times.update({prefix + k: v for k, v in self.transform.get_times().items()}) + # times[prefix + 'read_time'] = self.readTime.value() + # times[prefix + 'read_time_avg'] = self.readTime.avg() + # times[prefix + 'augment_time'] = self.augmentTime.value() + # times[prefix + 'augment_time_avg'] = self.augmentTime.avg() + # times[prefix + 'load_time'] = self.loadTime.value() + return times + + def fill_cache(self, cache_size, image_size_pixels): + self.cache = dict() + if cache_size == 0: + return + import sys + max_image_size = 0 + cur_size = 0 + for i, impath in enumerate(self.image_file_list): + img = self.loader(impath) + image_size = sys.getsizeof(img) + max_image_size = max(max_image_size, image_size) + cur_size += image_size + if image_size_pixels is not None: + img = img.resize(image_size_pixels) + self.cache[impath] = img + # logging.getLogger('autonet').info('Load image: ' + str(sys.getsizeof(self.cache[impath])) + ' bytes - Cache: ' + str(cur_size)) + if cur_size + max_image_size > cache_size: + break + logging.getLogger('autonet').info('Could load ' + str(i+1) + '/' + str(len(self.image_file_list)) + ' images into cache, used ' + str(cur_size) + '/' + str(cache_size) + ' bytes') + + def __getitem__(self, index): + impath = self.image_file_list[index] + target = self.label_list[index] + # start_time = time.time() + img = self.cache[impath] if impath in self.cache else self.loader(impath) + # self.readTime.add(time.time() - start_time) + # start_time = time.time() + if self.transform is not None: + img = self.transform(img) + if self.target_transform is not None: + target = self.target_transform(target) + # self.augmentTime.add(time.time() - start_time) + # self.loadTime.add(time.time() - start_time) + return img, target + + def __len__(self): + return len(self.image_file_list) + +class XYDataset(data.Dataset): + def __init__(self, X, Y, transform=None, target_transform=None): + self.X = X + self.Y = Y + self.transform = transform + self.target_transform = target_transform + + def __getitem__(self, index): + img = self.X[index] + target = self.Y[index] + + if self.transform is not None: + img = self.transform(img) + if self.target_transform is not None: + target = self.target_transform(target) + return img, target + + def __len__(self): + return len(self.image_file_list) \ No newline at end of file diff --git a/autoPyTorch/pipeline/base/node.py b/autoPyTorch/pipeline/base/node.py index 98a9201d7..45d43448e 100644 --- a/autoPyTorch/pipeline/base/node.py +++ b/autoPyTorch/pipeline/base/node.py @@ -5,6 +5,7 @@ import gc import inspect +import logging class Node(): @@ -12,19 +13,44 @@ def __init__(self): self.child_node = None self.fit_output = None self.predict_output = None + self.logger = logging.getLogger('autonet') def fit(self, **kwargs): + """Fit pipeline node. + Each node computes its fit function in linear order by fit_traverse().. + All args have to be specified in a parent node fit output. + + Returns: + dict -- output values that will be passed to child nodes, if required + """ return dict() def predict(self, **kwargs): + """Predict pipeline node. + Each node computes its predict function in linear order by predict_traverse(). + All args have to be specified in a parent node predict output or in the fit output of this node + + Returns: + dict -- output values that will be passed to child nodes, if required + """ return dict() def get_fit_argspec(self): + """Get the necessary keywords of the fit method for this node + + Returns: + tuple -- The keywords and their defaults + """ possible_keywords, _, _, defaults, _, _, _ = inspect.getfullargspec(self.fit) possible_keywords = [k for k in possible_keywords if k != 'self'] return possible_keywords, defaults def get_predict_argspec(self): + """Get the necessary keywords of the predict method for this node + + Returns: + tuple -- The keywords and their defaults + """ possible_keywords, _, _, defaults, _, _, _ = inspect.getfullargspec(self.predict) possible_keywords = [k for k in possible_keywords if k != 'self'] return possible_keywords, defaults @@ -39,10 +65,15 @@ def clean_fit_data(self): node = node.child_node def fit_traverse(self, **kwargs): - """Calls fit function of child nodes. + """ + Calls fit function of child nodes. The fit function can have different keyword arguments. All keywords have to be either defined in kwargs or in an fit output of a parent node. - + + The fit method of each node specifies a list of keyword arguments. + The fit method of each node returns a dictionary of values for keywords of follwing nodes. + + This method collects the results of each fit method call and calls the fit methods with the collected values. """ self.clean_fit_data() @@ -51,6 +82,7 @@ def fit_traverse(self, **kwargs): base = Node() base.fit_output = kwargs + # map all collected kwargs to node whose result the kwarg was available_kwargs = {key: base for key in kwargs.keys()} node = self @@ -58,10 +90,13 @@ def fit_traverse(self, **kwargs): while (node is not None): prev_node = node + # get necessary kwargs of current node possible_keywords, defaults = node.get_fit_argspec() last_required_keyword_index = len(possible_keywords) - len(defaults or []) required_kwargs = dict() + + # get the values to the necessary keywords if available. Use default if not. for index, keyword in enumerate(possible_keywords): if (keyword in available_kwargs): required_kwargs[keyword] = available_kwargs[keyword].fit_output[keyword] @@ -69,14 +104,19 @@ def fit_traverse(self, **kwargs): elif index >= last_required_keyword_index: required_kwargs[keyword] = defaults[index - last_required_keyword_index] - else: + else: # Neither default specified nor keyword available print ("Available keywords:", sorted(available_kwargs.keys())) raise ValueError('Node ' + str(type(node)) + ' requires keyword ' + str(keyword) + ' which is not available.') - + + if type(node) != Node: + self.logger.debug('Fit: ' + str(type(node).__name__)) + + # call fit method node.fit_output = node.fit(**required_kwargs) if (not isinstance(node.fit_output, dict)): raise ValueError('Node ' + str(type(node)) + ' does not return a dictionary.') + # collect resulting keyword-value pairs for keyword in node.fit_output.keys(): if keyword in available_kwargs: # delete old values @@ -92,13 +132,20 @@ def fit_traverse(self, **kwargs): def predict_traverse(self, **kwargs): """Calls predict function of child nodes. The predict function can have different keyword arguments. - All keywords have to be either defined in kwargs, in a predict output of a parent node or in the nodes own fit output + All keywords have to be either defined in kwargs, in a predict output of a parent node or in the nodes own fit output. + + The predict method of each node specifies a list of keyword arguments. + The predict method of each node returns a dictionary of values for keywords of follwing nodes. + + This method collects the results of each predict method call and calls the predict methods with the collected values. + For each node, the results of the fit call can also be passed to the predict method """ base = Node() base.predict_output = kwargs + # map all collected kwargs to node whose whose result the kwarg was available_kwargs = {key: base for key in kwargs.keys()} node = self @@ -115,10 +162,13 @@ def predict_traverse(self, **kwargs): while (node is not None): prev_node = node + # get necessary kwargs of current node possible_keywords, defaults = node.get_predict_argspec() last_required_keyword_index = len(possible_keywords) - len(defaults or []) required_kwargs = dict() + + # get the values to the necessary keywords if available. Use fit result or default if not. for index, keyword in enumerate(possible_keywords): if (keyword in available_kwargs): if (available_kwargs[keyword].predict_output is None): @@ -131,13 +181,14 @@ def predict_traverse(self, **kwargs): elif index >= last_required_keyword_index: required_kwargs[keyword] = defaults[index - last_required_keyword_index] - else: + else: # Neither default specified nor keyword available nor available in fit result of the node raise ValueError('Node ' + str(type(node)) + ' requires keyword ' + keyword + ' which is not available.') node.predict_output = node.predict(**required_kwargs) if (not isinstance(node.predict_output, dict)): raise ValueError('Node ' + str(type(node)) + ' does not return a dictionary.') + # collect keyword arguments for keyword in node.predict_output.keys(): if keyword in available_kwargs: # delete old values diff --git a/autoPyTorch/pipeline/base/pipeline.py b/autoPyTorch/pipeline/base/pipeline.py index a6bb9719d..af246f8a5 100644 --- a/autoPyTorch/pipeline/base/pipeline.py +++ b/autoPyTorch/pipeline/base/pipeline.py @@ -9,21 +9,25 @@ class Pipeline(): + """A machine learning pipeline""" + def __init__(self, pipeline_nodes=[]): + """Construct a Pipeline + + Keyword Arguments: + pipeline_nodes {list} -- The nodes of the pipeline (default: {[]}) + """ self.root = Node() self._pipeline_nodes = dict() - self.start_params = None self._parent_pipeline = None + # add all the given nodes to the pipeline last_node = self.root for node in pipeline_nodes: last_node.child_node = node self.add_pipeline_node(node) last_node = node - def _get_start_parameter(self): - return self.start_params - def __getitem__(self, key): return self._pipeline_nodes[key] @@ -37,7 +41,7 @@ def __contains__(self, key): def set_parent_pipeline(self, pipeline): """Set this pipeline as a child pipeline of the given pipeline. - This will allow the parent pipeline to access the pipeline nodes of its child pipelines + This will allow the parent pipeline to access the pipeline nodes of its child pipelines. Arguments: pipeline {Pipeline} -- parent pipeline @@ -80,33 +84,55 @@ def add_pipeline_node(self, pipeline_node): return pipeline_node def get_hyperparameter_search_space(self, dataset_info=None, **pipeline_config): + """Get the search space of the pipeline. + + Keyword Arguments: + dataset_info {DatasetInfo} -- Object describing the dataset. (default: {None}) + + Returns: + ConfigurationSpace -- The search space of the pipeline + """ pipeline_config = self.get_pipeline_config(**pipeline_config) + # check for hyperparameter search space updates and apply them if "hyperparameter_search_space_updates" in pipeline_config and pipeline_config["hyperparameter_search_space_updates"] is not None: assert isinstance(pipeline_config["hyperparameter_search_space_updates"], HyperparameterSearchSpaceUpdates) pipeline_config["hyperparameter_search_space_updates"].apply(self, pipeline_config) + # initialize the config space if "random_seed" in pipeline_config: cs = ConfigSpace.ConfigurationSpace(seed=pipeline_config["random_seed"]) else: cs = ConfigSpace.ConfigurationSpace() + # add the config space of each node for name, node in self._pipeline_nodes.items(): - config_space = node.get_hyperparameter_search_space(dataset_info=dataset_info, **pipeline_config) + #print("dataset_info" in pipeline_config.keys()) + config_space = node.get_hyperparameter_search_space(**pipeline_config) cs.add_configuration_space(prefix=name, configuration_space=config_space, delimiter=ConfigWrapper.delimiter) + # add the dependencies between the nodes for name, node in self._pipeline_nodes.items(): cs = node.insert_inter_node_hyperparameter_dependencies(cs, dataset_info=dataset_info, **pipeline_config) return cs def get_pipeline_config(self, throw_error_if_invalid=True, **pipeline_config): + """Get the full pipeline config given a partial pipeline config + + Keyword Arguments: + throw_error_if_invalid {bool} -- Throw an error if invalid config option is defined (default: {True}) + + Returns: + dict -- the full config for the pipeline, containing values for all options + """ options = self.get_pipeline_config_options() conditions = self.get_pipeline_config_conditions() parser = ConfigFileParser(options) pipeline_config = parser.set_defaults(pipeline_config, throw_error_if_invalid=throw_error_if_invalid) + # check the conditions e.g. max_budget > min_budget for c in conditions: try: c(pipeline_config) @@ -120,6 +146,11 @@ def get_pipeline_config(self, throw_error_if_invalid=True, **pipeline_config): def get_pipeline_config_options(self): + """Get all ConfigOptions of all nodes in the pipeline. + + Returns: + list -- A list of ConfigOptions. + """ if (self._parent_pipeline is not None): return self._parent_pipeline.get_pipeline_config_options() @@ -131,6 +162,11 @@ def get_pipeline_config_options(self): return options def get_pipeline_config_conditions(self): + """Get all ConfigConditions of all the nodes in the pipeline. + + Returns: + list -- A list of ConfigConditions + """ if (self._parent_pipeline is not None): return self._parent_pipeline.get_pipeline_config_options() @@ -140,46 +176,16 @@ def get_pipeline_config_conditions(self): conditions += node.get_pipeline_config_conditions() return conditions - - def print_config_space(self, **pipeline_config): - config_space = self.get_hyperparameter_search_space(**pipeline_config) - - if (len(config_space.get_hyperparameters()) == 0): - return - print(config_space) - - def print_config_space_per_node(self, **pipeline_config): - for name, node in self._pipeline_nodes.items(): - config_space = node.get_hyperparameter_search_space(**pipeline_config) - - if (len(config_space.get_hyperparameters()) == 0): - continue - print(name) - print(config_space) - - - def print_config_options(self): - for option in self.get_pipeline_config_options(): - print(str(option)) - - def print_config_options_per_node(self): - for name, node in self._pipeline_nodes.items(): - print(name) - for option in node.get_pipeline_config_options(): - print(" " + str(option)) - - def print_pipeline_nodes(self): - for name, node in self._pipeline_nodes.items(): - input_str = "[" - for edge in node.in_edges: - input_str += " (" + edge.out_idx + ", " + edge.target.get_name() + ", " + edge.kw + ") " - input_str += "]" - print(name + " \t\t Input: " + input_str) def clean(self): self.root.clean_fit_data() def clone(self): + """Clone the pipeline + + Returns: + Pipeline -- The cloned pipeline + """ pipeline_nodes = [] current_node = self.root.child_node diff --git a/autoPyTorch/pipeline/base/pipeline_node.py b/autoPyTorch/pipeline/base/pipeline_node.py index 6630acd1d..57f673c45 100644 --- a/autoPyTorch/pipeline/base/pipeline_node.py +++ b/autoPyTorch/pipeline/base/pipeline_node.py @@ -11,6 +11,8 @@ class PipelineNode(Node): + """ A node in the ML Pipeline""" + def __init__(self): """A pipeline node is a step in a pipeline. It can implement a fit function: @@ -34,6 +36,14 @@ def get_name(cls): return cls.__name__ def clone(self, skip=("pipeline", "fit_output", "predict_output", "child_node")): + """Clone a pipeline node + + Keyword Arguments: + skip {tuple} -- attributes that should not be cloned (default: {("pipeline", "fit_output", "predict_output", "child_node")}) + + Returns: + PipelineNode -- The cloned node + """ node_type = type(self) new_node = node_type.__new__(node_type) for key, value in self.__dict__.items(): @@ -43,32 +53,12 @@ def clone(self, skip=("pipeline", "fit_output", "predict_output", "child_node")) setattr(new_node, key, None) return new_node - # VIRTUAL - def fit(self, **kwargs): - """Fit pipeline node. - Each node computes its fit function in linear order. - All args have to be specified in a parent node fit output. - - Returns: - [dict] -- output values that will be passed to child nodes, if required - """ - - return dict() - - # VIRTUAL - def predict(self, **kwargs): - """Predict pipeline node. - Each node computes its predict function in linear order. - All args have to be specified in a parent node predict output or in the fit output of this node + def set_pipeline(self, pipeline): + """Set the pipeline of this node - Returns: - [dict] -- output values that will be passed to child nodes, if required + Arguments: + pipeline {Pipeline} -- The pipeline to set """ - - return dict() - - # VIRTUAL - def set_pipeline(self, pipeline): self.pipeline = pipeline # VIRTUAL @@ -124,30 +114,62 @@ def _apply_search_space_update(self, name, new_value_range, log=False): self._cs_updates[name] = tuple([new_value_range, log]) def _check_search_space_updates(self, *allowed_hps): + """Check if the given search space updates are valid. + + Arguments: + *allowed_hps: List of allowed hps. A list of lists, tuples or strings. + If a allowed hp is a string, hyperparameter updates with given string as name are allowed. + If allowed hp update is a star "*", all hyperparameter updates are allowed. + If allowed hp is a list, all elements in the list are allowed. + If allowed hp is a tuple, we join the values with the ConfigWrapper delimiter. + The elements here can also be lists or stars, with meaning explained above. + + Raises: + ValueError: The given search space updates are not valid. + """ + # process all allowed hps given and add them to this list exploded_allowed_hps = list() + + # iterate over all given allowed hps for allowed_hp in allowed_hps: - add = [list()] + add = [list()] # the list of allowed hps to add to exploded_allowed_hps. allowed_hp = (allowed_hp, ) if isinstance(allowed_hp, str) else allowed_hp + + # if tuple, iterate over all parts of allowed hp for part in allowed_hp: + # add the part to each element of add. Check if part is str or list. if isinstance(part, str): add = [x + [part] for x in add] else: add = [x + [p] for p in part for x in add] exploded_allowed_hps += add + + # join the allowed hps with ConfigWrapper delimiter exploded_allowed_hps = [ConfigWrapper.delimiter.join(x) for x in exploded_allowed_hps] + # Check given hyperparameter updates and raise exception if invalid hyperparameter update is given. for key in self._get_search_space_updates().keys(): if key not in exploded_allowed_hps and \ ConfigWrapper.delimiter.join(key.split(ConfigWrapper.delimiter)[:-1] + ["*"]) not in exploded_allowed_hps: raise ValueError("Invalid search space update given: %s" % key) def _get_search_space_updates(self, prefix=None): + """Get the search space updates with the given prefix + + Keyword Arguments: + prefix {str} -- Only return search space updates with given prefix (default: {None}) + + Returns: + dict -- Mapping of search space updates. Keys don't contain the prefix. + """ if prefix is None: return self._cs_updates if isinstance(prefix, tuple): prefix = ConfigWrapper.delimiter.join(prefix) result = dict() + + # iterate over all search space updates of this node and filter the ones out, that have the given prefix for key in self._cs_updates.keys(): if key.startswith(prefix + ConfigWrapper.delimiter): result[key[len(prefix + ConfigWrapper.delimiter):]] = self._cs_updates[key] - return result \ No newline at end of file + return result diff --git a/autoPyTorch/pipeline/base/sub_pipeline_node.py b/autoPyTorch/pipeline/base/sub_pipeline_node.py index f6272c372..c54b6d3df 100644 --- a/autoPyTorch/pipeline/base/sub_pipeline_node.py +++ b/autoPyTorch/pipeline/base/sub_pipeline_node.py @@ -3,7 +3,13 @@ from autoPyTorch.pipeline.base.pipeline import Pipeline class SubPipelineNode(PipelineNode): + """A Pipeline node that contains a sub-pipeline""" def __init__(self, sub_pipeline_nodes): + """Construct the node and the sub pipeline + + Arguments: + sub_pipeline_nodes {list} -- A list of nodes of the sub-pipeline + """ super(SubPipelineNode, self).__init__() self.sub_pipeline = Pipeline(sub_pipeline_nodes) diff --git a/autoPyTorch/pipeline/nodes/create_dataloader.py b/autoPyTorch/pipeline/nodes/create_dataloader.py index 1d51b529d..187332c9f 100644 --- a/autoPyTorch/pipeline/nodes/create_dataloader.py +++ b/autoPyTorch/pipeline/nodes/create_dataloader.py @@ -49,7 +49,7 @@ def fit(self, pipeline_config, hyperparameter_config, X, Y, train_indices, valid def predict(self, pipeline_config, X, batch_size): X = torch.from_numpy(to_dense(X)).float() - y_placeholder = torch.Tensor(X.size()[0]) + y_placeholder = torch.zeros(X.size()[0]) predict_loader = DataLoader(TensorDataset(X.float(), y_placeholder), batch_size) diff --git a/autoPyTorch/pipeline/nodes/cross_validation.py b/autoPyTorch/pipeline/nodes/cross_validation.py index 3f1ae7394..242b9b0e7 100644 --- a/autoPyTorch/pipeline/nodes/cross_validation.py +++ b/autoPyTorch/pipeline/nodes/cross_validation.py @@ -51,6 +51,28 @@ def __init__(self, train_pipeline_nodes): def fit(self, hyperparameter_config, pipeline_config, X_train, Y_train, X_valid, Y_valid, budget, budget_type, optimize_start_time, refit, rescore, dataset_info): + """Perform cross validation. + + Arguments: + hyperparameter_config {dict} -- The sampled hyperparameter config + pipeline_config {dict} -- The user specified configuration of the pipeline + X_train {data} -- The data. Cross Validation might split the data, + Y_train {data} -- The data. Cross Validation might split the data, + X_valid {data} -- The data. Cross Validation might split the data, + Y_valid {data} -- The data. Cross Validation might split the data, + budget {float} -- The budget for training. + budget_type {BaseTrainingTechnique} -- The type of budget. + optimize_start_time {float} -- Time when optimization has been started. + refit {bool} -- Whether we refit currently or not. + rescore {bool} -- Whether we refit in order to get the exact score of a hp-config during training. + dataset_info {DatasetInfo} -- Object containing information about the dataset. + + Raises: + Exception: Not a single CV split could be finished. + + Returns: + dict -- loss, info and additional results. + """ logger = logging.getLogger('autonet') loss = 0 infos = [] @@ -133,6 +155,22 @@ def clean_fit_data(self): self.sub_pipeline.root.clean_fit_data() def initialize_cross_validation(self, pipeline_config, budget, X_train, Y_train, X_valid, Y_valid, dataset_info, refit, logger): + """Initialize CV by computing split indices, + + Arguments: + pipeline_config {dict} -- User-defined configuration of the pipeline. + budget {float} -- The current budget. + X_train {array} -- The data + Y_train {array} -- The data + X_valid {array} -- The data + Y_valid {array} -- The data + dataset_info {DatasetInfo} -- Object describing the dataset + refit {bool} -- Wether we currently perform a refit. + logger {Logger} -- Logger to log stuff on the console. + + Returns: + tuple -- X, Y, number of splits, split indices, a penalty added to the loss, the budget for each cv split + """ budget_too_low_for_cv = budget < pipeline_config['min_budget_for_cv'] val_split = max(0, min(1, pipeline_config['validation_split'])) if refit: @@ -199,6 +237,19 @@ def remove_cross_validator(self, name): del self.cross_validators_adjust_y[name] def get_current_budget(self, cv_index, budget, budget_type, cv_start_time, num_cv_splits, logger): + """Get the budget for the current CV split. + + Arguments: + cv_index {int} -- The index of the current cv split. + budget {float} -- The current budget. + budget_type {BaseTrainingTechnique} -- The type of budget. + cv_start_time {float} -- Start time of cross validation. + num_cv_splits {int} -- total number of cv splits. + logger {Logger} -- A logger to log stuff on the console. + + Returns: + float -- The budget of the current + """ # adjust budget in case of budget type time if budget_type == BudgetTypeTime: remaining_budget = budget - (time.time() - cv_start_time) @@ -212,6 +263,22 @@ def get_current_budget(self, cv_index, budget, budget_type, cv_start_time, num_c return cur_budget def process_additional_results(self, additional_results, all_sub_pipeline_kwargs, X, Y, logger): + """Process additional results, like predictions for ensemble for example. + The data of additional results will be combined across the splits. + + Arguments: + additional_results {dict} -- Mapping from cv index to additional_results organized in a dictionary. This dictionary has the following structure: + {name1: {data1: ..., combinator1: }, name2: {data2: ..., combinator2: ...}, ...} + for each name, the given combinator should be identical across the splits. + for each name, the given combinator is called with a dictionary from split index to data + all_sub_pipeline_kwargs {dict} -- Mapping from cv index to kwargs with which the subpipeline has been called. + X {array} -- The full data, concatenation of training and validation. + Y {array} -- The target full data, concatenation of training and validation. + logger {Logger} -- a logger to print stuff on the console + + Returns: + dict -- mapping from name to combined data + """ combinators = dict() data = dict() result = dict() @@ -228,6 +295,15 @@ def process_additional_results(self, additional_results, all_sub_pipeline_kwargs @staticmethod def concat(upper, lower): + """Concatenate training and validation data + + Arguments: + upper {array} -- upper part of concatenated array + lower {array} -- lower part of concatenated value + + Returns: + array -- concatenated array + """ if (scipy.sparse.issparse(upper)): return scipy.sparse.vstack([upper, lower]) else: @@ -235,6 +311,18 @@ def concat(upper, lower): @staticmethod def shuffle_indices(indices, shuffle=True, seed=42): + """Shuffle the indices + + Arguments: + indices {array} -- The indices to shuffle + + Keyword Arguments: + shuffle {bool} -- Whether the indices should be shuffled (default: {True}) + seed {int} -- A random seed (default: {42}) + + Returns: + array -- Shuffled indices + """ rng = np.random.RandomState(42) if shuffle: rng.shuffle(indices) @@ -242,6 +330,21 @@ def shuffle_indices(indices, shuffle=True, seed=42): @staticmethod def get_validation_set_split_indices(pipeline_config, X_train, X_valid, Y_train, Y_valid, allow_shuffle=True): + """Get the indices for cv. + + Arguments: + pipeline_config {dict} -- The user specified configuration of the pipeline + X_train {array} -- The data + X_valid {array} -- The data + Y_train {array} -- The data + Y_valid {array} -- The data + + Keyword Arguments: + allow_shuffle {bool} -- shuffle data indices if it is specified in pipeline config and allow_shuffle is True (default: {True}) + + Returns: + tuple -- The concatenated data and the indices + """ train_indices = CrossValidation.shuffle_indices(np.array(list(range(X_train.shape[0]))), pipeline_config['shuffle'] and allow_shuffle, pipeline_config['random_seed']) valid_indices = CrossValidation.shuffle_indices(np.array(list(range(X_train.shape[0], X_train.shape[0] + X_valid.shape[0]))), @@ -252,4 +355,4 @@ def get_validation_set_split_indices(pipeline_config, X_train, X_valid, Y_train, return X, Y, (train_indices, valid_indices) def identity(x): - return x \ No newline at end of file + return x diff --git a/autoPyTorch/pipeline/nodes/ensemble.py b/autoPyTorch/pipeline/nodes/ensemble.py index af9975245..6fa16b3cd 100644 --- a/autoPyTorch/pipeline/nodes/ensemble.py +++ b/autoPyTorch/pipeline/nodes/ensemble.py @@ -6,9 +6,10 @@ from autoPyTorch.pipeline.base.pipeline_node import PipelineNode from autoPyTorch.utils.config.config_option import ConfigOption -from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector +from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector, AutoNetMetric, no_transform from autoPyTorch.pipeline.nodes import OneHotEncoding, OptimizationAlgorithm -from autoPyTorch.utils.ensemble import build_ensemble, read_ensemble_prediction_file, predictions_for_ensemble, combine_predictions, combine_test_predictions, \ +from autoPyTorch.pipeline.nodes.metric_selector import AutoNetMetric +from autoPyTorch.utils.ensemble import build_ensemble, read_ensemble_prediction_file, combine_predictions, combine_test_predictions, \ ensemble_logger, start_server from hpbandster.core.result import logged_results_to_HBS_result import json @@ -16,12 +17,20 @@ from hpbandster.core.nameserver import nic_name_to_host import time + +def predictions_for_ensemble(y_true, y_pred): + return y_pred + class EnableComputePredictionsForEnsemble(PipelineNode): """Put this Node in the training pipeline after the metric selector node""" def fit(self, pipeline_config, additional_metrics, refit, loss_penalty): if refit or pipeline_config["ensemble_size"] == 0 or loss_penalty > 0: return dict() - return {'additional_metrics': additional_metrics + [predictions_for_ensemble]} + return {'additional_metrics': additional_metrics + [ + AutoNetMetric(name="predictions_for_ensemble", + metric=predictions_for_ensemble, + loss_transform=no_transform, + ohe_transform=no_transform)]} class SavePredictionsForEnsemble(PipelineNode): @@ -64,25 +73,27 @@ def get_pipeline_config_options(self): class BuildEnsemble(PipelineNode): """Put this node after the optimization algorithm node""" - def fit(self, pipeline_config, final_metric_score, optimized_hyperparameter_config, budget, refit=None): + def fit(self, pipeline_config, optimized_hyperparameter_config, budget, loss, info, refit=None): if refit or pipeline_config["ensemble_size"] == 0 or pipeline_config["task_id"] not in [-1, 1]: - return {"final_metric_score": final_metric_score, "optimized_hyperparameter_config": optimized_hyperparameter_config, "budget": budget} + return {"optimized_hyperparameter_config": optimized_hyperparameter_config, "budget": budget} filename = os.path.join(pipeline_config["result_logger_dir"], 'predictions_for_ensemble.npy') - train_metric = self.pipeline[MetricSelector.get_name()].metrics[pipeline_config["train_metric"]] + optimize_metric = self.pipeline[MetricSelector.get_name()].metrics[pipeline_config["optimize_metric"]] y_transform = self.pipeline[OneHotEncoding.get_name()].complete_y_tranformation result = logged_results_to_HBS_result(pipeline_config["result_logger_dir"]) all_predictions, labels, model_identifiers, _ = read_ensemble_prediction_file(filename=filename, y_transform=y_transform) ensemble_selection, ensemble_configs = build_ensemble(result=result, - train_metric=train_metric, minimize=pipeline_config["minimize"], ensemble_size=pipeline_config["ensemble_size"], + optimize_metric=optimize_metric, ensemble_size=pipeline_config["ensemble_size"], all_predictions=all_predictions, labels=labels, model_identifiers=model_identifiers, only_consider_n_best=pipeline_config["ensemble_only_consider_n_best"], sorted_initialization_n_best=pipeline_config["ensemble_sorted_initialization_n_best"]) - return {"final_metric_score": final_metric_score, "optimized_hyperparameter_config": optimized_hyperparameter_config, "budget": budget, - "ensemble": ensemble_selection, "ensemble_final_metric_score": ensemble_selection.get_validation_performance(), - "ensemble_configs": ensemble_configs + return {"optimized_hyperparameter_config": optimized_hyperparameter_config, "budget": budget, + "ensemble": ensemble_selection, + "ensemble_configs": ensemble_configs, + "loss": loss, + "info": info } def predict(self, Y): @@ -112,4 +123,4 @@ def fit(self, pipeline_config, result_loggers, shutdownables, refit=False): shutdownables = shutdownables + [process] result_loggers = [ensemble_logger(directory=pipeline_config["result_logger_dir"], overwrite=True)] + result_loggers - return {"result_loggers": result_loggers, "shutdownables": shutdownables} \ No newline at end of file + return {"result_loggers": result_loggers, "shutdownables": shutdownables} diff --git a/autoPyTorch/pipeline/nodes/image/__init__.py b/autoPyTorch/pipeline/nodes/image/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/autoPyTorch/pipeline/nodes/image/autonet_settings_no_shuffle.py b/autoPyTorch/pipeline/nodes/image/autonet_settings_no_shuffle.py new file mode 100644 index 000000000..476fec091 --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/autonet_settings_no_shuffle.py @@ -0,0 +1,71 @@ +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + +import logging +import numpy as np +import sys, os +import pprint + +from autoPyTorch.pipeline.base.pipeline_node import PipelineNode +from autoPyTorch.utils.hyperparameter_search_space_update import parse_hyperparameter_search_space_updates + +from autoPyTorch.utils.config.config_option import ConfigOption, to_bool + +import random, torch + +class AutoNetSettingsNoShuffle(PipelineNode): + def __init__(self): + super(AutoNetSettingsNoShuffle, self).__init__() + + self.logger_settings = dict() + self.logger_settings['debug'] = logging.DEBUG + self.logger_settings['info'] = logging.INFO + self.logger_settings['warning'] = logging.WARNING + self.logger_settings['error'] = logging.ERROR + self.logger_settings['critical'] = logging.CRITICAL + + + def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid): + + autonet_logger = logging.getLogger('autonet') + hpbandster_logger = logging.getLogger('hpbandster') + + level = self.logger_settings[pipeline_config['log_level']] + autonet_logger.setLevel(level) + hpbandster_logger.setLevel(level) + + random.seed(pipeline_config['random_seed']) + torch.manual_seed(pipeline_config['random_seed']) + np.random.seed(pipeline_config['random_seed']) + + if 'result_logger_dir' in pipeline_config: + directory = os.path.join(pipeline_config['result_logger_dir'], "worker_logs_" + str(pipeline_config['task_id'])) + os.makedirs(directory, exist_ok=True) + + if level == logging.DEBUG: + self.addHandler([autonet_logger, hpbandster_logger], level, os.path.join(directory, 'autonet_debug.log')) + self.addHandler([autonet_logger, hpbandster_logger], logging.INFO, os.path.join(directory, 'autonet_info.log')) + else: + self.addHandler([autonet_logger, hpbandster_logger], level, os.path.join(directory, 'autonet.log')) + + autonet_logger.info("Start autonet with config:\n" + str(pprint.pformat(pipeline_config))) + + return { 'X_train': X_train, 'Y_train': Y_train, 'X_valid': X_valid, 'Y_valid': Y_valid } + + def get_pipeline_config_options(self): + options = [ + ConfigOption(name='log_level', default='warning', type=str, choices=list(self.logger_settings.keys())), + ConfigOption(name='random_seed', default=lambda c: abs(hash(c["run_id"])) % (2 ** 32), type=int, depends=True, info="Make sure to specify the same seed for all workers."), + ConfigOption(name='hyperparameter_search_space_updates', default=None, type=["directory", parse_hyperparameter_search_space_updates], + info="object of type HyperparameterSearchSpaceUpdates"), + ] + return options + + def addHandler(self, loggers, level, path): + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fh = logging.FileHandler(path) + fh.setLevel(level) + fh.setFormatter(formatter) + for logger in loggers: + logger.addHandler(fh) diff --git a/autoPyTorch/pipeline/nodes/image/create_dataset_info.py b/autoPyTorch/pipeline/nodes/image/create_dataset_info.py new file mode 100644 index 000000000..8c88e8875 --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/create_dataset_info.py @@ -0,0 +1,154 @@ +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + +import os +import numpy as np +import scipy.sparse + +from torchvision import datasets + +from autoPyTorch.pipeline.base.pipeline_node import PipelineNode + +from autoPyTorch.utils.config.config_option import ConfigOption, to_bool +from autoPyTorch.utils.config.config_file_parser import ConfigFileParser + + +class DataSetInfo(): + def __init__(self): + self.categorical_features = [] + self.x_shape = [] + self.y_shape = [] + self.is_sparse = False + self.default_dataset = None # could be set to CIFAR to download official CIFAR dataset from pytorch + +class CreateDatasetInfo(PipelineNode): + + default_datasets = { + # NAME # dataset # shape # classes + 'CIFAR10' : (datasets.CIFAR10, [50000, 3, 32, 32], 10), + 'CIFAR100' : (datasets.CIFAR100, [50000, 3, 32, 32], 10), + 'SVHN' : (datasets.SVHN, [70000, 3, 32, 32], 10), + 'MNIST' : (datasets.MNIST, [60000, 28, 28], 10), + } + + + def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid, dataset_path): + info = DataSetInfo() + info.is_sparse = scipy.sparse.issparse(X_train) + info.path = dataset_path + + if X_train[0] in self.default_datasets: + dataset_type, shape, classes = self.default_datasets[X_train[0]] + info.default_dataset = dataset_type + info.x_shape = shape + info.y_shape = [shape[0], classes] + X_train = np.array([X_train[0]]) + Y_train = np.array([]) + + elif len(X_train.shape) == 1: + if 'max_class_size' not in pipeline_config.keys(): + pipeline_config['max_class_size'] = None # backwards compatibility + + if "file_extensions" not in pipeline_config.keys(): + pipeline_config['file_extensions'] = ['.png', '.jpg', '.JPEG', '.pgm'] + + X_train, Y_train = self.add_subpaths(X_train, Y_train, + pipeline_config['images_root_folders'], pipeline_config['file_extensions'], pipeline_config['max_class_size'] or float('inf')) + X_valid, Y_valid = self.add_subpaths(X_valid, Y_valid, + pipeline_config['images_root_folders'], pipeline_config['file_extensions'], pipeline_config['max_class_size'] or float('inf')) + + info.x_shape = [X_train.shape[0]] + pipeline_config['images_shape'] + info.y_shape = Y_train.shape + + if len(info.y_shape) == 1 or info.y_shape[1] == 1: + info.y_shape = (info.y_shape[0], len(np.unique(Y_train))) + else: + info.x_shape = X_train.shape + info.y_shape = Y_train.shape + + return {'X_train' : X_train, 'Y_train' : Y_train, 'X_valid' : X_valid, 'Y_valid' : Y_valid, 'dataset_info' : info} + + + def predict(self, pipeline_config, X): + fit_res = self.fit(pipeline_config, X, np.zeros(X.shape[0]), None, None, pipeline_config) + return { 'X': fit_res['X_train'], 'dataset_info': fit_res['dataset_info'] } + + def get_pipeline_config_options(self): + options = [ + ConfigOption(name="file_extensions", default=['.png', '.jpg', '.JPEG', '.pgm'], type=str, list=True), + ConfigOption(name="images_shape", default=[3, 32, 32], type=int, list=True, info="Image size input to the networks, images will be rescaled to this."), + ConfigOption(name="images_root_folders", default=[ConfigFileParser.get_autonet_home()], type='directory', list=True, info="Directory relative to which image paths are given."), + ConfigOption(name="max_class_size", default=None, type=int), + ] + return options + + def add_subpaths(self, X, Y, root_folders, extensions, max_class_size): + if X is None or Y is None: + return None, None + + new_X, new_Y = [], [] + #for i, path in enumerate(X): + # for root in root_folders: + # tmp = os.path.join(root, path) + # if os.path.exists(tmp): + # path = tmp + # break + # if "."+path.split(".")[1] in extensions: + # new_X.append(X) + # new_Y = Y + # continue + # if not os.path.exists(path): + # print(path) + # raise Exception('Invalid path: ' + str(root_folders) + str(path)) + # if os.path.isfile(path) and os.path.splitext(path)[1] == '.h5': + # import h5py + # return h5py.File(path, 'r')['x'].value, h5py.File(os.path.join(root, Y[i]), 'r')['y'].value.squeeze() + # self.add_path(path, Y[i], new_X, new_Y, extensions, max_class_size) + + for i, path in enumerate(X): + for root in root_folders: + tmp = os.path.join(root, path) + if os.path.exists(tmp): + path = tmp + break + if not os.path.exists(path): + raise Exception('Invalid path: ' + str(root_folders) + str(path)) + if os.path.isfile(path) and os.path.splitext(path)[1] == '.h5': + import h5py + return h5py.File(path, 'r')['x'].value, h5py.File(os.path.join(root, Y[i]), 'r')['y'].value.squeeze() + self.add_path(path, Y[i], new_X, new_Y, extensions, max_class_size) + + if len(new_X) == 0: + raise Exception('Could not find any images in ' + str(root_folders) + '...' + str(extensions)) + return np.array(new_X), np.array(new_Y) + + def add_path(self, cur_X, cur_Y, new_X, new_Y, extensions, max_class_size): + is_file, max_class_size = self.add_file(cur_X, cur_Y, new_X, new_Y, extensions, max_class_size) + if is_file: + return + + for sub in os.listdir(cur_X): + if max_class_size <= 0: + return max_class_size + path = os.path.join(cur_X, sub) + is_file, max_class_size = self.add_file(path, cur_Y, new_X, new_Y, extensions, max_class_size) + + if not is_file: + max_class_size = self.add_path(path, cur_Y, new_X, new_Y, extensions, max_class_size) + + def add_file(self, cur_X, cur_Y, new_X, new_Y, extensions, max_class_size): + if not os.path.isfile(cur_X): + return False, max_class_size + if not os.path.splitext(cur_X)[1] in extensions: + return True, max_class_size + if os.path.getsize(cur_X) > 0: + new_X.append(cur_X) + new_Y.append(cur_Y) + max_class_size -= 1 + return True, max_class_size - 1 + else: + import logging + logging.getLogger('autonet').debug('Image is invalid! - size == 0:' + str(cur_X)) + return True, max_class_size + diff --git a/autoPyTorch/pipeline/nodes/image/create_image_dataloader.py b/autoPyTorch/pipeline/nodes/image/create_image_dataloader.py new file mode 100644 index 000000000..91f58d278 --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/create_image_dataloader.py @@ -0,0 +1,97 @@ +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + +import inspect +import logging +import numpy as np + +from autoPyTorch.pipeline.nodes.create_dataloader import CreateDataLoader +from autoPyTorch.utils.configspace_wrapper import ConfigWrapper +from autoPyTorch.utils.config.config_file_parser import ConfigFileParser +from autoPyTorch.utils.config.config_option import ConfigOption + +import torch +import scipy.sparse +from torch.utils.data import DataLoader, TensorDataset, Dataset +from autoPyTorch.data_management.image_loader import ImageFilelist, XYDataset +from torch.utils.data.sampler import SubsetRandomSampler +from torchvision import datasets, models, transforms + +class CreateImageDataLoader(CreateDataLoader): + + def fit(self, pipeline_config, hyperparameter_config, X, Y, train_indices, valid_indices, train_transform, valid_transform, dataset_info): + + # if len(X.shape) > 1: + # return super(CreateImageDataLoader, self).fit(pipeline_config, hyperparameter_config, X, Y, train_indices, valid_indices) + + torch.manual_seed(pipeline_config["random_seed"]) + hyperparameter_config = ConfigWrapper(self.get_name(), hyperparameter_config) + + if dataset_info.default_dataset: + train_dataset = dataset_info.default_dataset(root=pipeline_config['default_dataset_download_dir'], train=True, download=True, transform=train_transform) + if valid_indices is not None: + valid_dataset = dataset_info.default_dataset(root=pipeline_config['default_dataset_download_dir'], train=True, download=True, transform=valid_transform) + elif len(X.shape) > 1: + train_dataset = XYDataset(X, Y, transform=train_transform, target_transform=lambda y: y.astype(np.int64)) + valid_dataset = XYDataset(X, Y, transform=valid_transform, target_transform=lambda y: y.astype(np.int64)) + else: + train_dataset = ImageFilelist(X, Y, transform=train_transform, target_transform=lambda y: y.astype(np.int64), cache_size=pipeline_config['dataloader_cache_size_mb'] * 1000, image_size=dataset_info.x_shape[2:]) + if valid_indices is not None: + valid_dataset = ImageFilelist(X, Y, transform=valid_transform, target_transform=lambda y: y.astype(np.int64), cache_size=0, image_size=dataset_info.x_shape[2:]) + valid_dataset.cache = train_dataset.cache + + train_loader = DataLoader( + dataset=train_dataset, + batch_size=int(hyperparameter_config['batch_size']), + sampler=SubsetRandomSampler(train_indices), + drop_last=True, + pin_memory=True, + num_workers=pipeline_config['dataloader_worker']) + + valid_loader = None + if valid_indices is not None: + valid_loader = DataLoader( + dataset=valid_dataset, + batch_size=int(hyperparameter_config['batch_size']), + sampler=SubsetRandomSampler(valid_indices), + drop_last=False, + pin_memory=True, + num_workers=pipeline_config['dataloader_worker']) + + return {'train_loader': train_loader, 'valid_loader': valid_loader, 'batch_size': hyperparameter_config['batch_size']} + + def get_pipeline_config_options(self): + options = [ + ConfigOption("default_dataset_download_dir", default=ConfigFileParser.get_autonet_home(), type='directory', info="Directory default datasets will be downloaded to."), + ConfigOption("dataloader_worker", default=1, type=int), + ConfigOption("dataloader_cache_size_mb", default=0, type=int) + ] + return options + + def predict(self, pipeline_config, X, batch_size, predict_transform, dataset_info): + + if len(X.shape) > 1: + return super(CreateImageDataLoader, self).predict(pipeline_config, X, batch_size) + + + if dataset_info.default_dataset: + predict_dataset = dataset_info.default_dataset(root=pipeline_config['default_dataset_download_dir'], train=False, download=True, transform=predict_transform) + else: + try: + y_placeholder = torch.zeros(X.size()[0]) + except: + y_placeholder = torch.zeros(len(X)) + predict_dataset = ImageFilelist(X, y_placeholder, transform=predict_transform) + + predict_loader = DataLoader( + dataset=predict_dataset, + batch_size=int(batch_size), + shuffle=False, + pin_memory=True, + num_workers=pipeline_config['dataloader_worker']) + + return {'predict_loader': predict_loader} + + + diff --git a/autoPyTorch/pipeline/nodes/image/cross_validation_indices.py b/autoPyTorch/pipeline/nodes/image/cross_validation_indices.py new file mode 100644 index 000000000..d96493bf3 --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/cross_validation_indices.py @@ -0,0 +1,223 @@ +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + +import torch +import logging +import scipy.sparse +import numpy as np +import pandas as pd +import signal +import time +import math +import copy + +from sklearn.model_selection import StratifiedKFold +from autoPyTorch.pipeline.base.sub_pipeline_node import SubPipelineNode +from autoPyTorch.pipeline.base.pipeline import Pipeline + +from autoPyTorch.utils.config.config_option import ConfigOption, to_bool +from autoPyTorch.components.training.image.budget_types import BudgetTypeTime +from sklearn.model_selection import StratifiedShuffleSplit + +import time + +class CrossValidationIndices(SubPipelineNode): + def __init__(self, train_pipeline_nodes): + """CrossValidation pipeline node. + It will run the train_pipeline by providing different train and validation datasets given the cv_split value defined in the config. + Cross validation can be disabled by setting cv_splits to <= 1 in the config + This enables the validation_split config parameter which, if no validation data is provided, will split the train dataset according its value (percent of train dataset) + + Train: + The train_pipeline will receive the following inputs: + {hyperparameter_config, pipeline_config, X, Y, train_sampler, valid_sampler, budget, training_techniques, fit_start_time, categorical_features} + + Prediction: + The train_pipeline will receive the following inputs: + {pipeline_config, X} + + Arguments: + train_pipeline {Pipeline} -- training pipeline that will be computed cv_split times + train_result_node {PipelineNode} -- pipeline node that provides the results of the train_pipeline + """ + + super(CrossValidationIndices, self).__init__(train_pipeline_nodes) + + self.use_stratified_cv_split_default = False + self.logger = logging.getLogger('autonet') + + + def fit(self, hyperparameter_config, pipeline_config, X_train, Y_train, X_valid, Y_valid, budget, budget_type, dataset_info, config_id, working_directory): + + cv_splits = max(1, pipeline_config['cv_splits']) + val_split = max(0, min(1, pipeline_config['validation_split'])) + + budget_too_low_for_cv, cv_splits, loss_penalty = self.incorporate_num_cv_splits_in_budget(budget, pipeline_config, cv_splits) + + loss = 0 + infos = [] + + np.random.seed(pipeline_config['random_seed']) + + split_indices = [] + X = X_train + Y = Y_train + + if X_valid is not None and Y_valid is not None: + if cv_splits > 1: + self.logger.warning('CV splits are set to ' + str(cv_splits) + ' and validation set is specified, autonet will ignore cv splits and evaluate on given validation set') + if val_split > 0.0: + self.logger.warning('Validation split is set to ' + str(val_split) + ' and validation set is specified, autonet will ignore split and evaluate on given validation set') + + train_indices = self.shuffle_indices(list(range(X_train.shape[0])), pipeline_config['shuffle']) + valid_indices = self.shuffle_indices(list(range(X_train.shape[0], X_train.shape[0] + X_valid.shape[0])), pipeline_config['shuffle']) + + X = self.concat(X_train, X_valid) + Y = self.concat(Y_train, Y_valid) + + split_indices.append([train_indices, valid_indices]) + + elif cv_splits > 1: + if val_split > 0.0: + self.logger.warning('Validation split is set to ' + str(val_split) + ' and cv splits are specified, autonet will ignore validation split and evaluate on ' + str(cv_splits) + ' cv splits') + + if pipeline_config['use_stratified_cv_split'] and Y.shape[0] == dataset_info.x_shape[0]: + assert len(Y.shape) == 1 or Y.shape[1] == 1, "Y is in wrong shape for stratified CV split" + skf = StratifiedKFold(n_splits=cv_splits, shuffle=pipeline_config['shuffle']) + split_indices = list(skf.split(np.zeros(dataset_info.x_shape[0]), Y.reshape((-1, )))) + else: + indices = self.shuffle_indices(list(range(dataset_info.x_shape[0])), pipeline_config['shuffle']) + split_size = len(indices) / cv_splits + for split in range(cv_splits): + i1 = int(split*split_size) + i2 = int((split+1)*split_size) + train_indices, valid_indices = indices[:i1] + indices[i2:], indices[i1:i2] + split_indices.append([train_indices, valid_indices]) + + elif val_split > 0.0: + if pipeline_config['use_stratified_cv_split'] and Y.shape[0] == dataset_info.x_shape[0] and (len(Y.shape) == 1 or Y.shape[1] == 1): + sss = StratifiedShuffleSplit(n_splits=1, test_size=val_split, random_state=pipeline_config['random_seed']) + train, valid = list(sss.split(np.zeros(dataset_info.x_shape[0]), Y.reshape((-1, ))))[0] + split_indices.append([train.tolist(), valid.tolist()]) + + # samples = dataset_info.x_shape[0] + # skf = StratifiedKFold(n_splits=math.ceil(samples / (samples * val_split)), shuffle=pipeline_config['shuffle']) + # split_indices = [list(skf.split(np.zeros(dataset_info.x_shape[0]), Y.reshape((-1, ))))[0]] + else: + indices = self.shuffle_indices(list(range(dataset_info.x_shape[0])), pipeline_config['shuffle']) + split = int(len(indices) * (1-val_split)) + + train_indices, valid_indices = indices[:split], indices[split:] + split_indices.append([train_indices, valid_indices]) + else: + train_indices = self.shuffle_indices(list(range(dataset_info.x_shape[0])), pipeline_config['shuffle']) + split_indices.append([train_indices, []]) + + + + + if 'categorical_features' in pipeline_config and pipeline_config['categorical_features']: + categorical_features = pipeline_config['categorical_features'] + else: + categorical_features = [False] * dataset_info.x_shape[1] + + for i, split in enumerate(split_indices): + + self.logger.debug("CV split " + str(i)) + + train_indices = split[0] + valid_indices = split[1] if len(split[1]) > 0 else None + + if budget_too_low_for_cv: + cv_splits = 1 + + cur_budget = budget/cv_splits + + result = self.sub_pipeline.fit_pipeline( + hyperparameter_config=hyperparameter_config, pipeline_config=pipeline_config, + X=X, Y=Y, dataset_info=dataset_info, + train_indices=train_indices, valid_indices=valid_indices, + budget=cur_budget, budget_type=budget_type, + categorical_features=categorical_features, + config_id=config_id, + working_directory=working_directory) + + if result is not None: + loss += result['loss'] + infos.append(result['info']) + + if budget_too_low_for_cv: + break + + if (len(infos) == 0): + raise Exception("Could not finish a single cv split due to memory or time limitation") + + if len(infos) == 1: + info = infos[0] + else: + df = pd.DataFrame(infos) + info = dict(df.mean()) + + loss = loss / cv_splits + loss_penalty + + return {'loss': loss, 'info': info} + + def predict(self, pipeline_config, X, dataset_info): + return self.sub_pipeline.predict_pipeline(pipeline_config=pipeline_config, X=X, dataset_info=dataset_info) + + def get_pipeline_config_options(self): + options = [ + # percent/100 of train dataset used for validation if no validation and cv_splits == 1 + ConfigOption("validation_split", default=0.0, type=float, choices=[0, 1]), + # number of cross validation splits 1 -> no cv + ConfigOption("cv_splits", default=1, type=int), + ConfigOption("use_stratified_cv_split", default=self.use_stratified_cv_split_default, type=to_bool, choices=[True, False]), + # specify minimum budget for cv. If budget is smaller only evaluate a single fold. + ConfigOption("min_budget_for_cv", default=0, type=float), + # incorporate number of cv splits in budget: Use half the number of specified cv splits below given budget. + ConfigOption("half_num_cv_splits_below_budget", default=0, type=float), + # shuffle train and validation set + ConfigOption('shuffle', default=True, type=to_bool, choices=[True, False]), + ] + return options + + def split_cv(self, X_shape, split, max_splits): + split_size = X_shape[0] / max_splits + i1 = int(split*split_size) + i2 = int((split+1)*split_size) + + train_indices = list(range(0, i1)) + list(range(i2, X_shape[0])) + valid_indices = list(range(i1, i2)) + + return train_indices, valid_indices + + def concat(self, upper, lower): + if (scipy.sparse.issparse(upper)): + return scipy.sparse.vstack([upper, lower]) + else: + return np.concatenate([upper, lower]) + + def shuffle_indices(self, indices, shuffle): + if shuffle: + np.random.shuffle(indices) + return indices + + + def clean_fit_data(self): + super(CrossValidationIndices, self).clean_fit_data() + self.sub_pipeline.root.clean_fit_data() + + def incorporate_num_cv_splits_in_budget(self, budget, pipeline_config, cv_splits): + budget_too_low_for_cv = budget < pipeline_config["min_budget_for_cv"] and cv_splits > 1 + half_num_cv_splits = not budget_too_low_for_cv and budget < pipeline_config["half_num_cv_splits_below_budget"] and cv_splits > 1 + + if budget_too_low_for_cv: + self.logger.debug("Only evaluate a single fold of CV, since the budget is lower than the min_budget for cv") + return True, cv_splits, 1000 + + if half_num_cv_splits: + self.logger.debug("Using half number of cv splits since budget is lower than the budget you specified for half number of cv splits") + return False, int(math.ceil(cv_splits / 2)), 1000 + + return False, cv_splits, 0 diff --git a/autoPyTorch/pipeline/nodes/image/image_augmentation.py b/autoPyTorch/pipeline/nodes/image/image_augmentation.py new file mode 100644 index 000000000..9690ce206 --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/image_augmentation.py @@ -0,0 +1,223 @@ +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + +import inspect +import logging +import numpy as np + +from autoPyTorch.pipeline.base.pipeline_node import PipelineNode +from autoPyTorch.utils.configspace_wrapper import ConfigWrapper + +import torch +from torchvision import datasets, models, transforms +from autoPyTorch.components.preprocessing.image_preprocessing.transforms import Cutout, AutoAugment, FastAutoAugment + + +import time +from autoPyTorch.data_management.image_loader import ThreadCounter +class TimeCompose(object): + """Composes several transforms together. + Args: + transforms (list of ``Transform`` objects): list of transforms to compose. + Example: + >>> transforms.Compose([ + >>> transforms.CenterCrop(10), + >>> transforms.ToTensor(), + >>> ]) + """ + + def __init__(self, transforms): + self.transforms = transforms + self.counters = [ThreadCounter() for _ in transforms] + + def __call__(self, img): + for i, t in enumerate(self.transforms): + start_time = time.time() + img = t(img) + self.counters[i].add(time.time() - start_time) + return img + + def get_times(self): + return {str(t): self.counters[i].value() for i, t in enumerate(self.transforms) } + + def __repr__(self): + format_string = self.__class__.__name__ + '(' + for t in self.transforms: + format_string += '\n' + format_string += ' {0}'.format(t) + format_string += '\n)' + return format_string + +class ImageAugmentation(PipelineNode): + def __init__(self): + super(ImageAugmentation, self).__init__() + self.mean_std_cache = dict() + + def fit(self, pipeline_config, hyperparameter_config, dataset_info, X, Y, train_indices, valid_indices): + mean, std = self.compute_mean_std(pipeline_config, hyperparameter_config, X, Y, train_indices, dataset_info) #dataset_info.mean, dataset_info.std + + hyperparameter_config = ConfigWrapper(self.get_name(), hyperparameter_config) + + transform_list = [] + image_size = min(dataset_info.x_shape[-2], dataset_info.x_shape[-1]) + + if len(X.shape) > 1: + transform_list.append(transforms.ToPILImage()) + + if hyperparameter_config['augment']: + if hyperparameter_config['fastautoaugment'] and hyperparameter_config['autoaugment']: + # fast autoaugment and autoaugment + transform_list.extend([ + FastAutoAugment(), + AutoAugment(), + transforms.Resize(image_size), + transforms.RandomCrop(image_size, padding=4), + transforms.RandomHorizontalFlip() + ]) + elif hyperparameter_config['fastautoaugment']: + # fast autoaugment + transform_list.extend([ + FastAutoAugment(), + transforms.Resize(image_size), + transforms.RandomCrop(image_size, padding=4), + transforms.RandomHorizontalFlip() + ]) + elif hyperparameter_config['autoaugment']: + # autoaugment + transform_list.extend([ + AutoAugment(), + transforms.Resize(image_size), + transforms.RandomCrop(image_size, padding=4), + transforms.RandomHorizontalFlip() + ]) + else: + # default augment color, rotation, size + transform_list.extend([ + transforms.ColorJitter(brightness=0.196, saturation=0.196, hue=0.141), + transforms.RandomAffine(degrees=10, shear=0.1, fillcolor=127), + transforms.RandomResizedCrop(image_size, scale=(0.533, 1), ratio=(0.75, 1.25)), + transforms.RandomHorizontalFlip() + ]) + else: + transform_list.extend([ + transforms.Resize(image_size), + transforms.CenterCrop(image_size), + ]) + + + # grayscale if only one channel + if dataset_info.x_shape[1] == 1: + transform_list.append(transforms.Grayscale(1)) + + # normalize + transform_list.append(transforms.ToTensor()) + transform_list.append(transforms.Normalize(mean, std)) + + # cutout + if hyperparameter_config['cutout']: + n_holes = hyperparameter_config['cutout_holes'] + transform_list.append(Cutout(n_holes=1, length=hyperparameter_config['length'], probability=0.5)) + + + train_transform = transforms.Compose(transform_list) + + transform_list = [] + if len(X.shape) > 1: + transform_list.append(transforms.ToPILImage()) + + transform_list.extend([ + transforms.Resize(image_size), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + valid_transform = transforms.Compose([transforms.Grayscale(1)] + transform_list if dataset_info.x_shape[1] == 1 else transform_list) + + return { 'train_transform': train_transform, 'valid_transform': valid_transform, 'mean': mean, 'std': std } + + def predict(self, pipeline_config, mean, std): + + predict_transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + + return {'predict_transform': predict_transform} + + def get_hyperparameter_search_space(self, **pipeline_config): + import ConfigSpace as CS + import ConfigSpace.hyperparameters as CSH + cs = CS.ConfigurationSpace() + + augment = cs.add_hyperparameter(CSH.CategoricalHyperparameter('augment', [True, False])) + autoaugment = cs.add_hyperparameter(CSH.CategoricalHyperparameter('autoaugment', [True, False])) + fastautoaugment = cs.add_hyperparameter(CSH.CategoricalHyperparameter('fastautoaugment', [True, False])) + + cutout = cs.add_hyperparameter(CSH.CategoricalHyperparameter('cutout', [True, False])) + cutout_length = cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('length', lower=0, upper=20, log=False)) + cutout_holes = cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('cutout_holes', lower=1, upper=3, log=False)) + + cs.add_condition(CS.EqualsCondition(cutout_length, cutout, True)) + cs.add_condition(CS.EqualsCondition(cutout_holes, cutout, True)) + + cs.add_condition(CS.EqualsCondition(autoaugment, augment, True)) + cs.add_condition(CS.EqualsCondition(fastautoaugment, augment, True)) + + return cs + + def compute_mean_std(self, pipeline_config, hyperparameter_config, X, Y, train_indices, dataset_info): + log = logging.getLogger('autonet') + + if dataset_info.path in self.mean_std_cache: + mean, std = self.mean_std_cache[dataset_info.path] + log.debug('CACHED: MEAN: ' + str(mean) + ' -- STD: ' + str(std)) + return mean, std + + from autoPyTorch.pipeline.nodes.image.create_image_dataloader import CreateImageDataLoader + loader = CreateImageDataLoader() + + image_size = min(dataset_info.x_shape[-2], dataset_info.x_shape[-1]) + transform_list = [] + if len(X.shape) > 1: + transform_list.append(transforms.ToPILImage()) + transform_list.append(transforms.Resize(image_size)) + transform_list.append(transforms.CenterCrop(image_size)) + if dataset_info.x_shape[1] == 1: + transform_list.append(transforms.Grayscale(1)) + transform_list.append(transforms.ToTensor()) + train_transform = transforms.Compose(transform_list) + + cache_size = pipeline_config['dataloader_cache_size_mb'] + pipeline_config['dataloader_cache_size_mb'] = 0 + train_loader = loader.fit(pipeline_config, hyperparameter_config, X, Y, train_indices, None, train_transform, None, dataset_info)['train_loader'] + pipeline_config['dataloader_cache_size_mb'] = cache_size + + mean = 0. + std = 0. + nb_samples = 0. + + with torch.no_grad(): + for data, _ in train_loader: + + # import matplotlib.pyplot as plt + # img = plt.imshow(data.numpy()[0,1,:]) + # plt.show() + + batch_samples = data.size(0) + data = data.view(batch_samples, data.size(1), -1) + mean = mean + data.mean(2).sum(0) + std = std + data.std(2).sum(0) + nb_samples += batch_samples + + if nb_samples > 0.: + mean /= nb_samples + std /= nb_samples + mean, std = mean.numpy().tolist(), std.numpy().tolist() + else: + mean, std = [mean], [std] + + log.debug('MEAN: ' + str(mean) + ' -- STD: ' + str(std)) + + self.mean_std_cache[dataset_info.path] = [mean, std] + return mean, std diff --git a/autoPyTorch/pipeline/nodes/image/image_dataset_reader.py b/autoPyTorch/pipeline/nodes/image/image_dataset_reader.py new file mode 100644 index 000000000..85156d36a --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/image_dataset_reader.py @@ -0,0 +1,57 @@ +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + +import os +import numpy as np +import math + +from autoPyTorch.pipeline.base.pipeline_node import PipelineNode + +from autoPyTorch.utils.config.config_option import ConfigOption, to_bool +from autoPyTorch.utils.config.config_file_parser import ConfigFileParser + +from autoPyTorch.data_management.data_manager import ImageManager + +class ImageDatasetReader(PipelineNode): + def __init__(self): + super(ImageDatasetReader, self).__init__() + + def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid): + + if len(X_train.shape)==1 and len(X_train)==1: + X_train = X_train[0] + Y_train = 0 + + if X_valid is not None: + if len(X_valid.shape)==1 and len(X_valid)==1: + X_valid = X_valid[0] + Y_valid = None + + X_train, Y_train, path = self.read_data(X_train, Y_train) + X_valid, Y_valid, _ = self.read_data(X_valid, Y_valid) + + return { 'X_train': X_train, 'Y_train': Y_train, 'X_valid': X_valid, 'Y_valid': Y_valid, 'dataset_path': path } + + def get_pipeline_config_options(self): + options = [ + ] + return options + + def read_data(self, path, y): + if path is None: + return None, None, None + + if not isinstance(path, str): + return path, y, str(path)[0:300] + + if not os.path.isabs(path): + path = os.path.abspath(os.path.join(ConfigFileParser.get_autonet_home(), path)) + + if not os.path.exists(path): + raise ValueError('Path ' + str(path) + ' is not a valid path.') + + im = ImageManager() + im.read_data(path, is_classification=True) + + return im.X_train, im.Y_train, path diff --git a/autoPyTorch/pipeline/nodes/image/loss_module_selector_indices.py b/autoPyTorch/pipeline/nodes/image/loss_module_selector_indices.py new file mode 100644 index 000000000..be9741cb0 --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/loss_module_selector_indices.py @@ -0,0 +1,35 @@ +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + +import inspect +import numpy as np + +import ConfigSpace +import ConfigSpace.hyperparameters as CSH + +from autoPyTorch.pipeline.nodes.loss_module_selector import LossModuleSelector +from autoPyTorch.utils.configspace_wrapper import ConfigWrapper +from autoPyTorch.utils.config.config_option import ConfigOption + + +class LossModuleSelectorIndices(LossModuleSelector): + def fit(self, hyperparameter_config, pipeline_config, X, Y, train_indices, dataset_info): + + if Y.shape[0] == dataset_info.y_shape[0]: + return super(LossModuleSelectorIndices, self).fit(hyperparameter_config, pipeline_config, X=np.zeros((Y.shape[0], 1)), Y=Y, train_indices=train_indices) + + print(Y.shape[0], dataset_info.y_shape[0]) + + hyperparameter_config = ConfigWrapper(self.get_name(), hyperparameter_config) + + loss_module_name = hyperparameter_config["loss_module"] + loss_module = self.loss_modules[loss_module_name] + loss = loss_module.module + if inspect.isclass(loss): + loss = loss() + loss_module.set_loss_function(loss) + + return {'loss_function': loss_module} + + diff --git a/autoPyTorch/pipeline/nodes/image/multiple_datasets.py b/autoPyTorch/pipeline/nodes/image/multiple_datasets.py new file mode 100644 index 000000000..1b1adaf74 --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/multiple_datasets.py @@ -0,0 +1,115 @@ +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + +import os +import numpy as np +import math +import time +import pandas as pd +import logging +import random +import torch + +from autoPyTorch.pipeline.base.sub_pipeline_node import SubPipelineNode + +from autoPyTorch.utils.config.config_option import ConfigOption, to_bool +from autoPyTorch.utils.config.config_file_parser import ConfigFileParser + +class MultipleDatasets(SubPipelineNode): + + def __init__(self, sub_pipeline_nodes): + super(MultipleDatasets, self).__init__(sub_pipeline_nodes) + + self.logger = logging.getLogger('autonet') + + + def fit(self, hyperparameter_config, pipeline_config, X_train, Y_train, X_valid, Y_valid, budget, budget_type, config_id, working_directory): + if len(X_train.shape) > 1: + return self.sub_pipeline.fit_pipeline( hyperparameter_config=hyperparameter_config, + pipeline_config=pipeline_config, + X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid, + budget=budget, budget_type=budget_type, config_id=config_id, working_directory=working_directory) + + + max_datasets = X_train.shape[0] + max_steps = math.floor((math.log(pipeline_config['max_budget']) - math.log(pipeline_config['min_budget'])) / math.log(pipeline_config['eta'])) + current_step = max_steps - math.floor((math.log(pipeline_config['max_budget']) - math.log(budget)) / math.log(pipeline_config['eta'])) if budget > 1e-10 else 0 + n_datasets = math.floor(math.pow(max_datasets, current_step/max(1, max_steps)) + 1e-10) + + # refit can cause issues with different budget + if max_steps == 0 or n_datasets > max_datasets or not pipeline_config['increase_number_of_trained_datasets']: + n_datasets = max_datasets + + if X_valid is None or Y_valid is None: + X_valid = [None] * n_datasets + Y_valid = [None] * n_datasets + + if 'use_tensorboard_logger' in pipeline_config and pipeline_config['use_tensorboard_logger']: + import tensorboard_logger as tl + tl.log_value('Train/datasets', float(n_datasets), int(time.time())) + + infos = [] + loss = 0 + losses = [] + + self.logger.debug('Start fitting ' + str(n_datasets) + ' dataset(s). Current budget: ' + str(budget) + ' - Step: ' + str(current_step) + '/' + str(max_steps)) + + #dataset_order = list(range(n_datasets)) + #random.shuffle(dataset_order) + #if pipeline_config['dataset_order'] and len(pipeline_config['dataset_order']) == n_datasets: + # dataset_order = pipeline_config['dataset_order'] + # dataset_order = [i for i in dataset_order if i < n_datasets] + #X_train = X_train[dataset_order] + if np.any(pipeline_config['dataset_order']): + dataset_order = pipeline_config['dataset_order'] + else: + dataset_order = list(range(n_datasets)) + X_train = X_train[dataset_order] + + for dataset in range(n_datasets): + self.logger.info('Fit dataset (' + str(dataset+1) + '/' + str(n_datasets) + '): ' + str(X_train[dataset]) + ' for ' + str(round(budget / n_datasets)) + 's') + + result = self.sub_pipeline.fit_pipeline(hyperparameter_config=hyperparameter_config, + pipeline_config=pipeline_config, + X_train=X_train[dataset], Y_train=Y_train[dataset], X_valid=X_valid[dataset], Y_valid=Y_valid[dataset], + budget=budget / n_datasets, budget_type=budget_type, config_id=config_id, working_directory=working_directory) + + # copy/rename checkpoint - save one checkpoint for each trained dataset + if 'checkpoint' in result['info']: + src = result['info']['checkpoint'] + folder, file = os.path.split(src) + dest = os.path.join(folder, os.path.splitext(file)[0] + '_' + str(dataset) + '.pt') + import shutil + if dataset < n_datasets - 1: + shutil.copy(src, dest) + else: + os.rename(src, dest) + result['info']['checkpoint'] = dest + + result['info']['dataset_path'] = str(X_train[dataset]) + result['info']['dataset_id'] = dataset_order[dataset] + + infos.append(result['info']) + loss += result['loss'] + losses.append(result['loss']) + + if 'use_tensorboard_logger' in pipeline_config and pipeline_config['use_tensorboard_logger']: + import tensorboard_logger as tl + tl.log_value('Train/datasets', float(n_datasets), int(time.time())) + + loss = loss / n_datasets + + return {'loss': loss, 'losses': losses, 'info': infos} + + def predict(self, pipeline_config, X): + return self.sub_pipeline.predict_pipeline(pipeline_config=pipeline_config, X=X) + + def get_pipeline_config_options(self): + options = [ + ConfigOption('dataset_order', default=None, type=int, list=True, info="Order in which datasets are considered."), + + #autonet.refit sets this to false to avoid refit budget issues + ConfigOption('increase_number_of_trained_datasets', default=True, type=to_bool, info="Wether to increase the number of considered datasets with each successive halfing iteration.") + ] + return options diff --git a/autoPyTorch/pipeline/nodes/image/network_selector_datasetinfo.py b/autoPyTorch/pipeline/nodes/image/network_selector_datasetinfo.py new file mode 100644 index 000000000..4adf11c4b --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/network_selector_datasetinfo.py @@ -0,0 +1,35 @@ +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + + +from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector +from autoPyTorch.components.networks.base_net import BaseNet + +import torch.nn as nn +import ConfigSpace +import ConfigSpace.hyperparameters as CSH +from autoPyTorch.utils.configspace_wrapper import ConfigWrapper +from autoPyTorch.utils.config.config_option import ConfigOption +import torchvision.models as models + +class NetworkSelectorDatasetInfo(NetworkSelector): + def fit(self, hyperparameter_config, pipeline_config, dataset_info): + config = ConfigWrapper(self.get_name(), hyperparameter_config) + network_name = config['network'] + + network_type = self.networks[network_name] + network_config = ConfigWrapper(network_name, config) + activation = self.final_activations[pipeline_config["final_activation"]] + + in_features = dataset_info.x_shape[1:] + if len(in_features) == 1: + # feature data - otherwise image data (channels, width, height) + in_features = in_features[0] + + network = network_type( config=network_config, + in_features=in_features, out_features=dataset_info.y_shape[1], + final_activation=activation) + + # self.logger.debug('NETWORK:\n' + str(network)) + return {'network': network} diff --git a/autoPyTorch/pipeline/nodes/image/optimization_algorithm_no_timelimit.py b/autoPyTorch/pipeline/nodes/image/optimization_algorithm_no_timelimit.py new file mode 100644 index 000000000..30af87980 --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/optimization_algorithm_no_timelimit.py @@ -0,0 +1,363 @@ + +import numpy as np +import os +import time +import shutil +import netifaces +import traceback +import logging +import itertools +import random + + +import autoPyTorch.utils.thread_read_write as thread_read_write +import datetime + +from hpbandster.core.nameserver import NameServer, nic_name_to_host +from hpbandster.core.result import (json_result_logger, + logged_results_to_HBS_result) + +from autoPyTorch.pipeline.base.sub_pipeline_node import SubPipelineNode +from autoPyTorch.pipeline.base.pipeline import Pipeline +from autoPyTorch.utils.config.config_option import ConfigOption, to_bool + +from autoPyTorch.core.hpbandster_extensions.bohb_ext import BOHBExt +from autoPyTorch.core.hpbandster_extensions.hyperband_ext import HyperBandExt +from autoPyTorch.core.worker_no_timelimit import ModuleWorkerNoTimeLimit + +from autoPyTorch.components.training.image.budget_types import BudgetTypeTime, BudgetTypeEpochs +import copy + +from autoPyTorch.utils.modify_config_space import remove_constant_hyperparameter + +from autoPyTorch.utils.loggers import combined_logger, bohb_logger, tensorboard_logger + +import pprint + +tensorboard_logger_configured = False + +class OptimizationAlgorithmNoTimeLimit(SubPipelineNode): + def __init__(self, optimization_pipeline_nodes): + """OptimizationAlgorithm pipeline node. + It will run either the optimization algorithm (BOHB, Hyperband - defined in config) or start workers + Each worker will run the provided optimization_pipeline and will return the output + of the pipeline_result_node to the optimization algorithm + + Train: + The optimization_pipeline will get the following inputs: + {hyperparameter_config, pipeline_config, X_train, Y_train, X_valid, Y_valid, budget, budget_type} + The pipeline_result_node has to provide the following outputs: + - 'loss': the optimization value (minimize) + - 'info': dict containing info for the respective training process + + Predict: + The optimization_pipeline will get the following inputs: + {pipeline_config, X} + The pipeline_result_node has to provide the following outputs: + - 'Y': result of prediction for 'X' + Note: predict will not call the optimization algorithm + + Arguments: + optimization_pipeline {Pipeline} -- pipeline that will be optimized (hyperparamter) + pipeline_result_node {PipelineNode} -- pipeline node that provides the results of the optimization_pieline + """ + + super(OptimizationAlgorithmNoTimeLimit, self).__init__(optimization_pipeline_nodes) + + self.algorithms = dict() + self.algorithms["bohb"] = BOHBExt + self.algorithms["hyperband"] = HyperBandExt + + self.logger = logging.getLogger('autonet') + + self.n_datasets=1 + + def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid, refit=None): + res = None + + + config_space = self.pipeline.get_hyperparameter_search_space(**pipeline_config) + config_space, constants = remove_constant_hyperparameter(config_space) + config_space.seed(pipeline_config['random_seed']) + + self.n_datasets = X_train.shape[0] if X_train.shape[0]<10 else 1 + + #Get number of budgets + max_budget = pipeline_config["max_budget"] + min_budget = pipeline_config["min_budget"] + eta = pipeline_config["eta"] + max_SH_iter = -int(np.log(min_budget/max_budget)/np.log(eta)) + 1 + budgets = max_budget * np.power(eta, -np.linspace(max_SH_iter-1, 0, max_SH_iter)) + n_budgets = len(budgets) + + # Get permutations + self.permutations = self.get_permutations(n_budgets) + + self.logger.debug('BOHB-ConfigSpace:\n' + str(config_space)) + self.logger.debug('Constant Hyperparameter:\n' + str(pprint.pformat(constants))) + + run_id, task_id = pipeline_config['run_id'], pipeline_config['task_id'] + + + global tensorboard_logger_configured + if pipeline_config['use_tensorboard_logger'] and not tensorboard_logger_configured: + import tensorboard_logger as tl + directory = os.path.join(pipeline_config['result_logger_dir'], "worker_logs_" + str(task_id)) + os.makedirs(directory, exist_ok=True) + tl.configure(directory, flush_secs=60) + tensorboard_logger_configured = True + + if (refit is not None): + return self.run_refit(pipeline_config, refit, constants, X_train, Y_train, X_valid, Y_valid) + + try: + ns_credentials_dir, tmp_models_dir, network_interface_name = self.prepare_environment(pipeline_config) + + # start nameserver if not on cluster or on master node in cluster + if task_id in [1, -1]: + NS = self.get_nameserver(run_id, task_id, ns_credentials_dir, network_interface_name) + ns_host, ns_port = NS.start() + + self.run_worker(pipeline_config=pipeline_config, run_id=run_id, task_id=task_id, ns_credentials_dir=ns_credentials_dir, + network_interface_name=network_interface_name, X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid, + constant_hyperparameter=constants) + + # start BOHB if not on cluster or on master node in cluster + if task_id in [1, -1]: + self.run_optimization_algorithm(pipeline_config, config_space, constants, run_id, ns_host, ns_port, NS, task_id) + + + res = self.parse_results(pipeline_config["result_logger_dir"]) + + except Exception as e: + print(e) + traceback.print_exc() + finally: + self.clean_up(pipeline_config, ns_credentials_dir, tmp_models_dir) + + if (res): + return {'loss': res[0], 'optimized_hyperparameter_config': res[1], 'budget': res[2], 'info': dict()} + else: + return {'optimized_hyperparameter_config': dict(), 'budget': 0, 'loss': float('inf'), 'info': dict()} + + def predict(self, pipeline_config, X): + return self.sub_pipeline.predict_pipeline(pipeline_config=pipeline_config, X=X) + + def get_pipeline_config_options(self): + options = [ + ConfigOption("run_id", default="0", type=str, info="Unique id for each run."), + ConfigOption("task_id", default=-1, type=int, info="ID for each worker, if you run AutoNet on a cluster. Set to -1, if you run it locally. "), + ConfigOption("algorithm", default="bohb", type=str, choices=list(self.algorithms.keys())), + ConfigOption("budget_type", default="time", type=str, choices=['time', 'epochs']), + ConfigOption("min_budget", default=lambda c: 120 if c['budget_type'] == 'time' else 5, type=float, depends=True, info="Min budget for fitting configurations."), + ConfigOption("max_budget", default=lambda c: 6000 if c['budget_type'] == 'time' else 150, type=float, depends=True, info="Max budget for fitting configurations."), + ConfigOption("max_runtime", + default=lambda c: ((-int(np.log(c["min_budget"] / c["max_budget"]) / np.log(c["eta"])) + 1) * c["max_budget"]) + if c["budget_type"] == "time" else float("inf"), + type=float, depends=True, info="Total time for the run."), + ConfigOption("num_iterations", + default=lambda c: (-int(np.log(c["min_budget"] / c["max_budget"]) / np.log(c["eta"])) + 1) + if c["budget_type"] == "epochs" else float("inf"), + type=float, depends=True, info="Number of successive halving iterations"), + ConfigOption("eta", default=3, type=float, info='eta parameter of Hyperband.'), + ConfigOption("min_workers", default=1, type=int), + ConfigOption("working_dir", default=".", type="directory"), + ConfigOption("network_interface_name", default=self.get_default_network_interface_name(), type=str), + ConfigOption("memory_limit_mb", default=1000000, type=int), + ConfigOption("result_logger_dir", default=".", type="directory"), + ConfigOption("use_tensorboard_logger", default=False, type=to_bool), + ConfigOption("keep_only_incumbent_checkpoints", default=True, type=to_bool), + ConfigOption("global_results_dir", default=None, type='directory'), + ] + return options + + def get_default_network_interface_name(self): + try: + return netifaces.gateways()['default'][netifaces.AF_INET][1] + except: + return 'lo' + + def prepare_environment(self, pipeline_config): + if not os.path.exists(pipeline_config["working_dir"]) and pipeline_config['task_id'] in [1, -1]: + try: + os.mkdir(pipeline_config["working_dir"]) + except: + pass + tmp_models_dir = os.path.join(pipeline_config["working_dir"], "tmp_models_" + str(pipeline_config['run_id'])) + ns_credentials_dir = os.path.abspath(os.path.join(pipeline_config["working_dir"], "ns_credentials_" + str(pipeline_config['run_id']))) + network_interface_name = pipeline_config["network_interface_name"] or (netifaces.interfaces()[1] if len(netifaces.interfaces()) > 1 else "lo") + + if os.path.exists(tmp_models_dir) and pipeline_config['task_id'] in [1, -1]: + shutil.rmtree(tmp_models_dir) + if os.path.exists(ns_credentials_dir) and pipeline_config['task_id'] in [1, -1]: + shutil.rmtree(ns_credentials_dir) + return ns_credentials_dir, tmp_models_dir, network_interface_name + + def clean_up(self, pipeline_config, tmp_models_dir, ns_credentials_dir): + if pipeline_config['task_id'] in [1, -1]: + # Delete temporary files + if os.path.exists(tmp_models_dir): + shutil.rmtree(tmp_models_dir) + if os.path.exists(ns_credentials_dir): + shutil.rmtree(ns_credentials_dir) + + def get_nameserver(self, run_id, task_id, ns_credentials_dir, network_interface_name): + if not os.path.isdir(ns_credentials_dir): + try: + os.mkdir(ns_credentials_dir) + except: + pass + return NameServer(run_id=run_id, nic_name=network_interface_name, working_directory=ns_credentials_dir) + + def get_optimization_algorithm_instance(self, config_space, run_id, pipeline_config, ns_host, ns_port, result_logger, previous_result=None): + optimization_algorithm = self.algorithms[pipeline_config["algorithm"]] + + if pipeline_config["algorithm"]=="bohb_multi_kde": + hb = optimization_algorithm(configspace=config_space, run_id = run_id, + eta=pipeline_config["eta"], min_budget=pipeline_config["min_budget"], max_budget=pipeline_config["max_budget"], + host=ns_host, nameserver=ns_host, nameserver_port=ns_port, + result_logger=result_logger, + ping_interval=10**6, + working_directory=pipeline_config["working_dir"], + previous_result=previous_result, + n_kdes=self.n_datasets, + permutations=self.permutations) + else: + hb = optimization_algorithm(configspace=config_space, run_id = run_id, + eta=pipeline_config["eta"], min_budget=pipeline_config["min_budget"], max_budget=pipeline_config["max_budget"], + host=ns_host, nameserver=ns_host, nameserver_port=ns_port, + result_logger=result_logger, + ping_interval=10**6, + working_directory=pipeline_config["working_dir"], + previous_result=previous_result) + return hb + + + def parse_results(self, result_logger_dir): + res = logged_results_to_HBS_result(result_logger_dir) + id2config = res.get_id2config_mapping() + incumbent_trajectory = res.get_incumbent_trajectory(bigger_is_better=False, non_decreasing_budget=False) + + if (len(incumbent_trajectory['config_ids']) == 0): + return dict() + + final_config_id = incumbent_trajectory['config_ids'][-1] + return incumbent_trajectory['losses'][-1], id2config[final_config_id]['config'], incumbent_trajectory['budgets'][-1] + + + def run_worker(self, pipeline_config, constant_hyperparameter, run_id, task_id, ns_credentials_dir, network_interface_name, + X_train, Y_train, X_valid, Y_valid): + if not task_id == -1: + time.sleep(5) + while not os.path.isdir(ns_credentials_dir): + time.sleep(5) + host = nic_name_to_host(network_interface_name) + + worker = ModuleWorkerNoTimeLimit( pipeline=self.sub_pipeline, pipeline_config=pipeline_config, + constant_hyperparameter=constant_hyperparameter, + X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid, + budget_type=pipeline_config['budget_type'], + max_budget=pipeline_config["max_budget"], + host=host, run_id=run_id, + id=task_id, + working_directory=pipeline_config["result_logger_dir"], + permutations=self.permutations) + worker.load_nameserver_credentials(ns_credentials_dir) + # run in background if not on cluster + worker.run(background=(task_id <= 1)) + + + def run_optimization_algorithm(self, pipeline_config, config_space, constant_hyperparameter, run_id, ns_host, ns_port, nameserver, task_id): + self.logger.info("[AutoNet] Start " + pipeline_config["algorithm"]) + + # initialize optimization algorithm + + result_logger = self.get_result_logger(pipeline_config, constant_hyperparameter) + HB = self.get_optimization_algorithm_instance(config_space=config_space, run_id=run_id, + pipeline_config=pipeline_config, ns_host=ns_host, ns_port=ns_port, result_logger=result_logger) + + # start algorithm + min_num_workers = pipeline_config["min_workers"] if task_id != -1 else 1 + + reduce_runtime = pipeline_config["max_budget"] if pipeline_config["budget_type"] == "time" else 0 + + HB.wait_for_workers(min_num_workers) + self.logger.debug('Workers are ready!') + + thread_read_write.append('runs.log', "{0}: {1} | {2}-{3}\n".format( + str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")), + run_id, + pipeline_config['min_budget'], + pipeline_config['max_budget'])) + + HB.run_until(runtime=(pipeline_config["max_runtime"] - reduce_runtime), + n_iterations=pipeline_config["num_iterations"], + min_n_workers=min_num_workers) + + HB.shutdown(shutdown_workers=True) + nameserver.shutdown() + + + def clean_fit_data(self): + super(OptimizationAlgorithmNoTimeLimit, self).clean_fit_data() + self.sub_pipeline.root.clean_fit_data() + + def run_refit(self, pipeline_config, refit, constants, X_train, Y_train, X_valid, Y_valid): + start_time = time.time() + + result_logger = self.get_result_logger(pipeline_config, constants) + result_logger.new_config((0, 0, 0), refit["hyperparameter_config"], {'model_based_pick': False}) + + full_config = dict() + full_config.update(constants) + full_config.update(refit["hyperparameter_config"]) + + self.logger.debug('Refit-Config:\n' + str(pprint.pformat(full_config))) + + class Job(): + pass + job = Job() + job.id = (0, 0, 0) + job.kwargs = { + 'budget': refit['budget'], + 'config': refit["hyperparameter_config"], + } + + try: + res = self.sub_pipeline.fit_pipeline( + hyperparameter_config=full_config, pipeline_config=pipeline_config, + X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid, + budget=refit["budget"], budget_type=pipeline_config['budget_type'], config_id='refit', working_directory=pipeline_config['result_logger_dir']) + job.exception = None + except Exception as e: + self.logger.exception('Exception during refit') + res = None + job.exception = str(e) + + end_time = time.time() + + job.timestamps = {'submitted': start_time, 'started': start_time, 'finished': end_time} + job.result = res + + result_logger(job) + + return {'loss': res['loss'] if res else float('inf'), + 'optimized_hyperparameter_config': full_config, + 'budget': refit['budget'], + 'info': res['info'] if res else dict()} + + def get_result_logger(self, pipeline_config, constant_hyperparameter): + loggers = [bohb_logger(constant_hyperparameter=constant_hyperparameter, directory=pipeline_config["result_logger_dir"], overwrite=True)] + if pipeline_config['use_tensorboard_logger']: + loggers.append(tensorboard_logger(pipeline_config, constant_hyperparameter, pipeline_config['global_results_dir'])) + return combined_logger(*loggers) + + def get_permutations(self, n_budgets=1): + # Get permutations, since HB fits like this: b1 - b2 -b3 - b2 -b3, repeat them accordingly + idx = [i for i in range(self.n_datasets)] + permutations = np.array(list(itertools.permutations(idx))) + ret = [] + for perm in permutations: + for ind in range(n_budgets): + ret.append(perm) + return np.array(ret) diff --git a/autoPyTorch/pipeline/nodes/image/simple_scheduler_selector.py b/autoPyTorch/pipeline/nodes/image/simple_scheduler_selector.py new file mode 100644 index 000000000..59de24aa1 --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/simple_scheduler_selector.py @@ -0,0 +1,26 @@ +__author__ = "Michael Burkart" +__version__ = "0.0.1" +__license__ = "BSD" + + +from autoPyTorch.pipeline.base.pipeline_node import PipelineNode + +from autoPyTorch.pipeline.nodes.lr_scheduler_selector import LearningrateSchedulerSelector + +import ConfigSpace +import ConfigSpace.hyperparameters as CSH +from autoPyTorch.utils.configspace_wrapper import ConfigWrapper +from autoPyTorch.utils.config.config_option import ConfigOption + +class SimpleLearningrateSchedulerSelector(LearningrateSchedulerSelector): + + def fit(self, hyperparameter_config, pipeline_config, optimizer): + config = ConfigWrapper(self.get_name(), hyperparameter_config) + + scheduler_name = config['lr_scheduler'] + + lr_scheduler_type = self.lr_scheduler[scheduler_name] + lr_scheduler_config = ConfigWrapper(scheduler_name, config) + lr_scheduler = lr_scheduler_type(optimizer, lr_scheduler_config) + + return {'lr_scheduler': lr_scheduler} \ No newline at end of file diff --git a/autoPyTorch/pipeline/nodes/image/simple_train_node.py b/autoPyTorch/pipeline/nodes/image/simple_train_node.py new file mode 100644 index 000000000..cb88d26ea --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/simple_train_node.py @@ -0,0 +1,348 @@ +__author__ = "Max Dippel, Michael Burkart and Matthias Urban" +__version__ = "0.0.1" +__license__ = "BSD" + +import torch +import time +import logging + +import os, pprint +import scipy.sparse +import numpy as np +import torch.nn as nn +from torch.autograd import Variable +from torch.utils.data import DataLoader, TensorDataset + +from autoPyTorch.pipeline.base.pipeline_node import PipelineNode + +import ConfigSpace +import ConfigSpace.hyperparameters as CSH +from autoPyTorch.utils.configspace_wrapper import ConfigWrapper +from autoPyTorch.utils.config.config_option import ConfigOption, to_bool +from autoPyTorch.components.training.image.base_training import BaseTrainingTechnique, BaseBatchLossComputationTechnique + +from autoPyTorch.components.training.image.trainer import Trainer +from autoPyTorch.components.training.image.checkpoints.save_load import save_checkpoint, load_checkpoint, get_checkpoint_dir +from autoPyTorch.components.training.image.checkpoints.load_specific import load_model #, load_optimizer, load_scheduler + +torch.backends.cudnn.benchmark = True + +import signal + +class SimpleTrainNode(PipelineNode): + def __init__(self): + super(SimpleTrainNode, self).__init__() + self.default_minimize_value = True + self.logger = logging.getLogger('autonet') + self.training_techniques = dict() + self.batch_loss_computation_techniques = dict() + self.add_batch_loss_computation_technique("standard", BaseBatchLossComputationTechnique) + + def fit(self, hyperparameter_config, pipeline_config, + train_loader, valid_loader, + network, optimizer, lr_scheduler, + optimize_metric, additional_metrics, + log_functions, + budget, + loss_function, + budget_type, + config_id, working_directory, + train_indices, valid_indices): + + + if budget < 1e-5: + return {'loss': float('inf') if pipeline_config["minimize"] else -float('inf'), 'info': dict()} + + training_start_time = time.time() + # prepare + if not torch.cuda.is_available(): + pipeline_config["cuda"] = False + + device = torch.device('cuda' if pipeline_config['cuda'] else 'cpu') + + checkpoint_path = get_checkpoint_dir(working_directory) + checkpoint = None + if pipeline_config['save_checkpoints']: + checkpoint = load_checkpoint(checkpoint_path, config_id, budget) + + network = load_model(network, checkpoint) + + tensorboard_logging = 'use_tensorboard_logger' in pipeline_config and pipeline_config['use_tensorboard_logger'] + + # from torch.optim import SGD + # optimizer = SGD(network.parameters(), lr=0.3) + + # optimizer = load_optimizer(optimizer, checkpoint, device) + # lr_scheduler = load_scheduler(lr_scheduler, checkpoint) + + hyperparameter_config = ConfigWrapper(self.get_name(), hyperparameter_config) + + batch_loss_name = hyperparameter_config["batch_loss_computation_technique"] if "batch_loss_computation_technique" in hyperparameter_config else pipeline_config["batch_loss_computation_techniques"][0] + + batch_loss_computation_technique = self.batch_loss_computation_techniques[batch_loss_name]() + batch_loss_computation_technique.set_up( + pipeline_config, ConfigWrapper(batch_loss_name, hyperparameter_config), self.logger) + + + # Training loop + logs = [] + epoch = 0 + + optimize_metrics = [] + val_metrics = [optimize_metric] + additional_metrics + if pipeline_config['evaluate_on_train_data']: + optimize_metrics = val_metrics + elif valid_loader is None: + self.logger.warning('No valid data specified and train process should not evaluate on train data! Will ignore \"evaluate_on_train_data\" and evaluate on train data!') + optimize_metrics = val_metrics + + trainer = Trainer( + model=network, + loss_computation=batch_loss_computation_technique, + criterion=loss_function, + budget=budget, + optimizer=optimizer, + scheduler=lr_scheduler, + budget_type=budget_type, + device=device, + config_id=config_id, + checkpoint_path=checkpoint_path if pipeline_config['save_checkpoints'] else None, + images_to_plot=tensorboard_logging * pipeline_config['tensorboard_images_count']) + + model_params = self.count_parameters(network) + + start_up = time.time() - training_start_time + epoch_train_time = 0 + val_time = 0 + log_time = 0 + + # tmp = time.time() + # for _ in range(100): + # for _ in train_loader: + # pass + # time_used = time.time() - tmp + # self.logger.debug("Test time: " + str(time_used) + "s : \n" + str(pprint.pformat(train_loader.dataset.get_times('train_')))) + + self.logger.debug("Start train. Budget: " + str(budget)) + + last_log_time = time.time() + while True: + # prepare epoch + log = dict() + + # train + tmp = time.time() + optimize_metric_results, train_loss, stop_training = trainer.train(epoch + 1, train_loader, optimize_metrics) + + log['train_loss'] = train_loss + for i, metric in enumerate(optimize_metrics): + log['train_' + metric.name] = optimize_metric_results[i] + epoch_train_time += time.time() - tmp + + # evaluate + tmp = time.time() + if valid_loader is not None: + valid_metric_results = trainer.evaluate(valid_loader, val_metrics, epoch=epoch + 1) + + for i, metric in enumerate(val_metrics): + log['val_' + metric.name] = valid_metric_results[i] + val_time += time.time() - tmp + + # additional los - e.g. test evaluation + tmp = time.time() + for func in log_functions: + log[func.name] = func(network, epoch + 1) + log_time += time.time() - tmp + + log['epochs'] = epoch + 1 + log['model_parameters'] = model_params + log['learning_rate'] = optimizer.param_groups[0]['lr'] + + # log.update(train_loader.dataset.get_times('train_')) + # log.update(valid_loader.dataset.get_times('val_')) + + logs.append(log) + + epoch += 1 + + self.logger.debug("Epoch: " + str(epoch) + " : " + str(log)) + + if budget_type == 'epochs' and epoch + 1 >= budget: + break + + if stop_training: + break + + if tensorboard_logging and time.time() - last_log_time >= pipeline_config['tensorboard_min_log_interval']: + import tensorboard_logger as tl + worker_path = 'Train/' + tl.log_value(worker_path + 'budget', float(budget), epoch) + for name, value in log.items(): + tl.log_value(worker_path + name, float(value), epoch) + last_log_time = time.time() + + + # wrap up + wrap_up_start_time = time.time() + + self.logger.debug("Finished Training") + + opt_metric_name = 'train_' + optimize_metric.name + if valid_loader is not None: + opt_metric_name = 'val_' + optimize_metric.name + + if pipeline_config["minimize"]: + final_log = min(logs, key=lambda x:x[opt_metric_name]) + else: + final_log = max(logs, key=lambda x:x[opt_metric_name]) + + if tensorboard_logging: + import tensorboard_logger as tl + worker_path = 'Train/' + tl.log_value(worker_path + 'budget', float(budget), epoch) + for name, value in final_log.items(): + tl.log_value(worker_path + name, float(value), epoch) + + if trainer.latest_checkpoint: + final_log['checkpoint'] = trainer.latest_checkpoint + elif pipeline_config['save_checkpoints']: + path = save_checkpoint(checkpoint_path, config_id, budget, network, optimizer, lr_scheduler) + final_log['checkpoint'] = path + + final_log['train_datapoints'] = len(train_indices) + if valid_loader is not None: + final_log['val_datapoints'] = len(valid_indices) + + loss = final_log[opt_metric_name] * (1 if pipeline_config["minimize"] else -1) + + self.logger.info("Finished train with budget " + str(budget) + + "s, Training took " + str(int(wrap_up_start_time - training_start_time)) + + "s, Wrap up took " + str(int(time.time() - wrap_up_start_time)) + + "s, Init took " + str(int(start_up)) + + "s, Train took " + str(int(epoch_train_time)) + + "s, Validation took " + str(int(val_time)) + + "s, Log functions took " + str(int(log_time)) + + "s, Cumulative time " + str(int(trainer.cumulative_time)) + + "s.\nTotal time consumption in s: " + str(int(time.time() - training_start_time))) + + return {'loss': loss, 'info': final_log} + + def get_dataloader_times(self, dataloader): + read = dataloader.dataset.readTime.value() + read_avg = dataloader.dataset.readTime.avg() + augment = dataloader.dataset.augmentTime.value() + augment_avg = dataloader.dataset.augmentTime.avg() + return read, read_avg, augment, augment_avg + + @staticmethod + def count_parameters(model): + return sum(p.numel() for p in model.parameters() if p.requires_grad) + + def predict(self, pipeline_config, network, predict_loader, dataset_info, optimize_metric): + + if not torch.cuda.is_available(): + pipeline_config["cuda"] = False + else: + pipeline_config["cuda"] = True + + device = torch.device('cuda:0' if pipeline_config['cuda'] else 'cpu') + + if dataset_info.default_dataset: + metric_results = Trainer(None, network, None, None, None, None, None, device).evaluate(predict_loader, [optimize_metric]) + return { 'score': metric_results[0] } + else: + Y = predict(network, predict_loader, None, device) + return { 'Y': Y.detach().cpu().numpy() } + + def add_training_technique(self, name, training_technique): + if (not issubclass(training_technique, BaseTrainingTechnique)): + raise ValueError("training_technique type has to inherit from BaseTrainingTechnique") + self.training_techniques[name] = training_technique + + def remove_training_technique(self, name): + del self.training_techniques[name] + + def add_batch_loss_computation_technique(self, name, batch_loss_computation_technique): + if (not issubclass(batch_loss_computation_technique, BaseBatchLossComputationTechnique)): + raise ValueError("batch_loss_computation_technique type has to inherit from BaseBatchLossComputationTechnique, got " + str(batch_loss_computation_technique)) + self.batch_loss_computation_techniques[name] = batch_loss_computation_technique + + def remove_batch_loss_computation_technique(self, name, batch_loss_computation_technique): + del self.batch_loss_computation_techniques[name] + + def get_hyperparameter_search_space(self, **pipeline_config): + pipeline_config = self.pipeline.get_pipeline_config(**pipeline_config) + cs = ConfigSpace.ConfigurationSpace() + + hp_batch_loss_computation = cs.add_hyperparameter(CSH.CategoricalHyperparameter("batch_loss_computation_technique", list(self.batch_loss_computation_techniques.keys()))) + + for name, technique in self.batch_loss_computation_techniques.items(): + parent = {'parent': hp_batch_loss_computation, 'value': name} if hp_batch_loss_computation is not None else None + cs.add_configuration_space(prefix=name, configuration_space=technique.get_hyperparameter_search_space(**pipeline_config), + delimiter=ConfigWrapper.delimiter, parent_hyperparameter=parent) + + possible_loss_comps = sorted(set(pipeline_config["batch_loss_computation_techniques"]).intersection(self.batch_loss_computation_techniques.keys())) + + if 'batch_loss_computation_techniques' not in pipeline_config.keys(): + cs.add_hyperparameter(CSH.CategoricalHyperparameter("batch_loss_computation_technique", possible_loss_comps)) + self._check_search_space_updates() + + return cs + + def get_pipeline_config_options(self): + options = [ + ConfigOption(name="batch_loss_computation_techniques", default=list(self.batch_loss_computation_techniques.keys()), + type=str, list=True, choices=list(self.batch_loss_computation_techniques.keys())), + ConfigOption("minimize", default=self.default_minimize_value, type=to_bool, choices=[True, False]), + ConfigOption("cuda", default=True, type=to_bool, choices=[True, False]), + ConfigOption("save_checkpoints", default=False, type=to_bool, choices=[True, False], info="Wether to save state dicts as checkpoints."), + ConfigOption("tensorboard_min_log_interval", default=30, type=int), + ConfigOption("tensorboard_images_count", default=0, type=int), + ConfigOption("evaluate_on_train_data", default=True, type=to_bool), + ] + for name, technique in self.training_techniques.items(): + options += technique.get_pipeline_config_options() + for name, technique in self.batch_loss_computation_techniques.items(): + options += technique.get_pipeline_config_options() + return options + + +def predict(network, test_loader, metrics, device, move_network=True): + """ predict batchwise """ + # Build DataLoader + if move_network: + if torch.cuda.device_count() > 1: + network = nn.DataParallel(network) + network = network.to(device) + + # Batch prediction + network.eval() + if metrics is not None: + metric_results = [0] * len(metrics) + + N = 0.0 + for i, (X_batch, Y_batch) in enumerate(test_loader): + # Predict on batch + X_batch = Variable(X_batch).to(device) + batch_size = X_batch.size(0) + + Y_batch_pred = network(X_batch).detach().cpu() + + if metrics is None: + # Infer prediction shape + if i == 0: + Y_pred = Y_batch_pred + else: + # Add to prediction tensor + Y_pred = torch.cat((Y_pred, Y_batch_pred), 0) + else: + for i, metric in enumerate(metrics): + metric_results[i] += metric(Y_batch, Y_batch_pred) * batch_size + + N += batch_size + + if metrics is None: + return Y_pred + else: + return [res / N for res in metric_results] + diff --git a/autoPyTorch/pipeline/nodes/image/single_dataset.py b/autoPyTorch/pipeline/nodes/image/single_dataset.py new file mode 100644 index 000000000..0509518e5 --- /dev/null +++ b/autoPyTorch/pipeline/nodes/image/single_dataset.py @@ -0,0 +1,35 @@ +import logging + +from autoPyTorch.pipeline.base.sub_pipeline_node import SubPipelineNode + +from autoPyTorch.utils.config.config_option import ConfigOption, to_bool +from autoPyTorch.utils.config.config_file_parser import ConfigFileParser + +class SingleDataset(SubPipelineNode): + # Node for compatibility with MultipleDatasets model + + def __init__(self, sub_pipeline_nodes): + super(SingleDataset, self).__init__(sub_pipeline_nodes) + + self.logger = logging.getLogger('autonet') + + + def fit(self, hyperparameter_config, pipeline_config, X_train, Y_train, X_valid, Y_valid, budget, budget_type, config_id, working_directory): + return self.sub_pipeline.fit_pipeline(hyperparameter_config=hyperparameter_config, + pipeline_config=pipeline_config, + X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid, + budget=budget, budget_type=budget_type, config_id=config_id, working_directory=working_directory) + + + def predict(self, pipeline_config, X): + return self.sub_pipeline.predict_pipeline(pipeline_config=pipeline_config, X=X) + + def get_pipeline_config_options(self): + options = [ + ConfigOption('dataset_order', default=None, type=int, list=True, info="Only used for multiple datasets."), + + #autonet.refit sets this to false to avoid refit budget issues + ConfigOption('increase_number_of_trained_datasets', default=False, type=to_bool, info="Only used for multiple datasets.") + ] + return options + diff --git a/autoPyTorch/pipeline/nodes/log_functions_selector.py b/autoPyTorch/pipeline/nodes/log_functions_selector.py index 2462c8ed9..81e90caab 100644 --- a/autoPyTorch/pipeline/nodes/log_functions_selector.py +++ b/autoPyTorch/pipeline/nodes/log_functions_selector.py @@ -4,6 +4,7 @@ import inspect from autoPyTorch.pipeline.base.pipeline_node import PipelineNode +from autoPyTorch.pipeline.nodes.metric_selector import default_minimize_transform, no_transform from autoPyTorch.utils.config.config_option import ConfigOption @@ -16,7 +17,7 @@ def __init__(self): def fit(self, pipeline_config): return {'log_functions': [self.log_functions[log_function] for log_function in pipeline_config["additional_logs"]]} - def add_log_function(self, name, log_function): + def add_log_function(self, name, log_function, loss_transform=False): """Add a log function, will be called with the current trained network and the current training epoch Arguments: @@ -26,8 +27,11 @@ def add_log_function(self, name, log_function): if (not hasattr(log_function, '__call__')): raise ValueError("log function has to be a function") - self.log_functions[name] = log_function - log_function.__name__ = name + + if isinstance(loss_transform, bool): + loss_transform = default_minimize_transform if loss_transform else no_transform + + self.log_functions[name] = AutoNetLog(name, log_function, loss_transform) def remove_log_function(self, name): del self.log_functions[name] @@ -36,4 +40,17 @@ def get_pipeline_config_options(self): options = [ ConfigOption(name="additional_logs", default=[], type=str, list=True, choices=list(self.log_functions.keys())), ] - return options \ No newline at end of file + return options + + +class AutoNetLog(): + def __init__(self, name, log, loss_transform): + self.loss_transform = loss_transform + self.log = log + self.name = name + + def __call__(self, *args): + return self.log(*args) + + def get_loss_value(self, *args): + return self.loss_transform(self.__call__(*args)) diff --git a/autoPyTorch/pipeline/nodes/loss_module_selector.py b/autoPyTorch/pipeline/nodes/loss_module_selector.py index 730037ae1..4368554db 100644 --- a/autoPyTorch/pipeline/nodes/loss_module_selector.py +++ b/autoPyTorch/pipeline/nodes/loss_module_selector.py @@ -28,6 +28,7 @@ def fit(self, hyperparameter_config, pipeline_config, X, Y, train_indices): weights = loss_module.weight_strategy(pipeline_config, X[train_indices], Y[train_indices]) weights = torch.from_numpy(weights).float() + # pass weights to loss module loss = loss_module.module if "pos_weight" in inspect.getfullargspec(loss)[0] and weights is not None and inspect.isclass(loss): loss = loss(pos_weight=weights) @@ -92,4 +93,4 @@ def __call__(self, x, y): def to(self, device): result = AutoNetLossModule(self.module, self.weight_strategy, self.requires_target_class_labels) result.set_loss_function(self.function.to(device)) - return result \ No newline at end of file + return result diff --git a/autoPyTorch/pipeline/nodes/metric_selector.py b/autoPyTorch/pipeline/nodes/metric_selector.py index 8220571c2..24ff72bb7 100644 --- a/autoPyTorch/pipeline/nodes/metric_selector.py +++ b/autoPyTorch/pipeline/nodes/metric_selector.py @@ -6,48 +6,100 @@ from autoPyTorch.pipeline.base.pipeline_node import PipelineNode from autoPyTorch.utils.config.config_option import ConfigOption +import torch +import numpy as np + class MetricSelector(PipelineNode): def __init__(self): super(MetricSelector, self).__init__() self.metrics = dict() - self.default_train_metric = None + self.default_optimize_metric = None def fit(self, pipeline_config): - train_metric = self.metrics[pipeline_config["train_metric"]] - additional_metrics = [self.metrics[metric] for metric in pipeline_config["additional_metrics"] if metric != pipeline_config["train_metric"]] + optimize_metric = self.metrics[pipeline_config["optimize_metric"]] + additional_metrics = [self.metrics[metric] for metric in pipeline_config["additional_metrics"] if metric != pipeline_config["optimize_metric"]] + + return {'optimize_metric': optimize_metric, 'additional_metrics': additional_metrics} - return {'train_metric': train_metric, 'additional_metrics': additional_metrics} + def predict(self, optimize_metric): + return { 'optimize_metric': optimize_metric } - def add_metric(self, name, metric, is_default_train_metric=False): + def add_metric(self, name, metric, loss_transform=False, + requires_target_class_labels=False, is_default_optimize_metric=False): """Add a metric, this metric has to be a function that takes to arguments y_true and y_predict Arguments: name {string} -- name of metric for definition in config + loss_transform {callable / boolean} -- transform metric value to minimizable loss. If True: loss = 1 - metric_value metric {function} -- metric function takes y_true and y_pred - is_default_train_metric {bool} -- should the given metric be the default train metric if not specified in config + is_default_optimize_metric {bool} -- should the given metric be the default train metric if not specified in config """ if (not hasattr(metric, '__call__')): raise ValueError("Metric has to be a function") - self.metrics[name] = metric - metric.__name__ = name - if (not self.default_train_metric or is_default_train_metric): - self.default_train_metric = name + ohe_transform = undo_ohe if requires_target_class_labels else no_transform + if isinstance(loss_transform, bool): + loss_transform = default_minimize_transform if loss_transform else no_transform + + self.metrics[name] = AutoNetMetric(name=name, + metric=metric, + loss_transform=loss_transform, + ohe_transform=ohe_transform) + + if (not self.default_optimize_metric or is_default_optimize_metric): + self.default_optimize_metric = name def remove_metric(self, name): del self.metrics[name] - if (self.default_train_metric == name): + if (self.default_optimize_metric == name): if (len(self.metrics) > 0): - self.default_train_metric = list(self.metrics.keys())[0] + self.default_optimize_metric = list(self.metrics.keys())[0] else: - self.default_train_metric = None + self.default_optimize_metric = None def get_pipeline_config_options(self): options = [ - ConfigOption(name="train_metric", default=self.default_train_metric, type=str, choices=list(self.metrics.keys()), + ConfigOption(name="optimize_metric", default=self.default_optimize_metric, type=str, choices=list(self.metrics.keys()), info="This is the meta train metric BOHB will try to optimize."), ConfigOption(name="additional_metrics", default=[], type=str, list=True, choices=list(self.metrics.keys())) ] - return options \ No newline at end of file + return options + + +def default_minimize_transform(value): + return 1 - value + +def no_transform(value): + return value + +def ensure_numpy(y): + if type(y)==torch.Tensor: + return y.detach().cpu().numpy() + return y + +def undo_ohe(y): + if len(y.shape) == 1: + return(y) + return np.argmax(y, axis=1) + +class AutoNetMetric(): + def __init__(self, name, metric, loss_transform, ohe_transform): + self.loss_transform = loss_transform + self.metric = metric + self.ohe_transform = ohe_transform + self.name = name + + def __call__(self, Y_pred, Y_true): + + Y_pred = ensure_numpy(Y_pred) + Y_true = ensure_numpy(Y_true) + + if len(Y_pred.shape) != len(Y_true.shape): + Y_pred = undo_ohe(Y_pred) + Y_true = undo_ohe(Y_true) + return self.metric(self.ohe_transform(Y_true), self.ohe_transform(Y_pred)) + + def get_loss_value(self, Y_pred, Y_true): + return self.loss_transform(self.__call__(Y_pred, Y_true)) diff --git a/autoPyTorch/pipeline/nodes/optimization_algorithm.py b/autoPyTorch/pipeline/nodes/optimization_algorithm.py index 907b5aed2..095f8dc6e 100644 --- a/autoPyTorch/pipeline/nodes/optimization_algorithm.py +++ b/autoPyTorch/pipeline/nodes/optimization_algorithm.py @@ -12,14 +12,15 @@ from autoPyTorch.pipeline.base.sub_pipeline_node import SubPipelineNode from autoPyTorch.pipeline.base.pipeline import Pipeline +from autoPyTorch.pipeline.nodes import MetricSelector from autoPyTorch.utils.config.config_option import ConfigOption, to_bool from autoPyTorch.utils.config.config_condition import ConfigCondition from autoPyTorch.core.hpbandster_extensions.bohb_ext import BOHBExt from autoPyTorch.core.hpbandster_extensions.hyperband_ext import HyperBandExt -from autoPyTorch.core.worker import ModuleWorker +from autoPyTorch.core.worker import AutoNetWorker -from autoPyTorch.components.training.budget_types import BudgetTypeTime, BudgetTypeEpochs +from autoPyTorch.components.training.budget_types import BudgetTypeTime, BudgetTypeEpochs, BudgetTypeTrainingTime import copy class OptimizationAlgorithm(SubPipelineNode): @@ -56,32 +57,56 @@ def __init__(self, optimization_pipeline_nodes): self.budget_types = dict() self.budget_types["time"] = BudgetTypeTime self.budget_types["epochs"] = BudgetTypeEpochs + self.budget_types["training_time"] = BudgetTypeTrainingTime def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid, result_loggers, dataset_info, shutdownables, refit=None): + """Run the optimization algorithm. + + Arguments: + pipeline_config {dict} -- The configuration of the pipeline. + X_train {array} -- The data + Y_train {array} -- The data + X_valid {array} -- The data + Y_valid {array} -- The data + result_loggers {list} -- List of loggers that log the result + dataset_info {DatasetInfo} -- Object with information about the dataset + shutdownables {list} -- List of objects that need to shutdown when optimization is finished. + + Keyword Arguments: + refit {dict} -- dict containing information for refitting. None if optimization run should be started. (default: {None}) + + Returns: + dict -- Summary of optimization run. + """ logger = logging.getLogger('autonet') res = None run_id, task_id = pipeline_config['run_id'], pipeline_config['task_id'] + # Use tensorboard logger if pipeline_config['use_tensorboard_logger'] and not refit: import tensorboard_logger as tl directory = os.path.join(pipeline_config['result_logger_dir'], "worker_logs_" + str(task_id)) os.makedirs(directory, exist_ok=True) tl.configure(directory, flush_secs=5) + # Only do refitting if (refit is not None): logger.info("Start Refitting") - res = self.sub_pipeline.fit_pipeline( + + loss_info_dict = self.sub_pipeline.fit_pipeline( hyperparameter_config=refit["hyperparameter_config"], pipeline_config=pipeline_config, X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid, budget=refit["budget"], rescore=refit["rescore"], budget_type=self.budget_types[pipeline_config['budget_type']], optimize_start_time=time.time(), refit=True, hyperparameter_config_id=None, dataset_info=dataset_info) logger.info("Done Refitting") - return {'final_metric_score': res['loss'], - 'optimized_hyperparameter_config': refit["hyperparameter_config"], - 'budget': refit['budget']} + return {'optimized_hyperparameter_config': refit["hyperparameter_config"], + 'budget': refit['budget'], + 'loss': loss_info_dict['loss'], + 'info': loss_info_dict['info']} + # Start Optimization Algorithm try: ns_credentials_dir, tmp_models_dir, network_interface_name = self.prepare_environment(pipeline_config) @@ -103,7 +128,7 @@ def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid, result_logger dataset_info=dataset_info, logger=logger) - res = self.parse_results(pipeline_config["result_logger_dir"]) + res = self.parse_results(pipeline_config) except Exception as e: print(e) @@ -111,31 +136,40 @@ def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid, result_logger finally: self.clean_up(pipeline_config, ns_credentials_dir, tmp_models_dir) - if (res): - return {'final_metric_score': res[0], 'optimized_hyperparameter_config': res[1], 'budget': res[2]} - else: - return {'final_metric_score': None, 'optimized_hyperparameter_config': dict(), 'budget': 0} + if res: + return res + return {'optimized_hyperparameter_config': dict(), 'budget': 0, 'loss': float('inf'), 'info': dict()} def predict(self, pipeline_config, X): + """Run the predict pipeline. + + Arguments: + pipeline_config {dict} -- The configuration of the pipeline + X {array} -- The data + + Returns: + dict -- The predicted values in a dictionary + """ result = self.sub_pipeline.predict_pipeline(pipeline_config=pipeline_config, X=X) return {'Y': result['Y']} + # OVERRIDE def get_pipeline_config_options(self): options = [ ConfigOption("run_id", default="0", type=str, info="Unique id for each run."), ConfigOption("task_id", default=-1, type=int, info="ID for each worker, if you run AutoNet on a cluster. Set to -1, if you run it locally. "), - ConfigOption("algorithm", default="bohb", type=str, choices=list(self.algorithms.keys())), + ConfigOption("algorithm", default="bohb", type=str, choices=list(self.algorithms.keys()), info="Algorithm to use for config sampling."), ConfigOption("budget_type", default="time", type=str, choices=list(self.budget_types.keys())), - ConfigOption("min_budget", default=lambda c: self.budget_types[c["budget_type"]].default_min_budget, type=float, depends=True), - ConfigOption("max_budget", default=lambda c: self.budget_types[c["budget_type"]].default_max_budget, type=float, depends=True), + ConfigOption("min_budget", default=lambda c: self.budget_types[c["budget_type"]].default_min_budget, type=float, depends=True, info="Min budget for fitting configurations."), + ConfigOption("max_budget", default=lambda c: self.budget_types[c["budget_type"]].default_max_budget, type=float, depends=True, info="Max budget for fitting configurations."), ConfigOption("max_runtime", default=lambda c: ((-int(np.log(c["min_budget"] / c["max_budget"]) / np.log(c["eta"])) + 1) * c["max_budget"]) if c["budget_type"] == "time" else float("inf"), - type=float, depends=True), + type=float, depends=True, info="Total time for the run."), ConfigOption("num_iterations", default=lambda c: (-int(np.log(c["min_budget"] / c["max_budget"]) / np.log(c["eta"])) + 1) if c["budget_type"] == "epochs" else float("inf"), - type=float, depends=True), + type=float, depends=True, info="Number of successive halving iterations."), ConfigOption("eta", default=3, type=float, info='eta parameter of Hyperband.'), ConfigOption("min_workers", default=1, type=int), ConfigOption("working_dir", default=".", type="directory"), @@ -147,6 +181,7 @@ def get_pipeline_config_options(self): ] return options + # OVERRIDE def get_pipeline_config_conditions(self): def check_runtime(pipeline_config): return pipeline_config["budget_type"] != "time" or pipeline_config["max_runtime"] >= pipeline_config["max_budget"] @@ -158,12 +193,25 @@ def check_runtime(pipeline_config): def get_default_network_interface_name(self): + """Get the default network interface name + + Returns: + str -- The default network interface name + """ try: return netifaces.gateways()['default'][netifaces.AF_INET][1] except: return 'lo' def prepare_environment(self, pipeline_config): + """Create necessary folders and get network interface name + + Arguments: + pipeline_config {dict} -- The configuration of the pipeline + + Returns: + tuple -- path to created directories and network interface namei + """ if not os.path.exists(pipeline_config["working_dir"]) and pipeline_config['task_id'] in [1, -1]: try: os.mkdir(pipeline_config["working_dir"]) @@ -174,12 +222,19 @@ def prepare_environment(self, pipeline_config): network_interface_name = self.get_nic_name(pipeline_config) if os.path.exists(tmp_models_dir) and pipeline_config['task_id'] in [1, -1]: - shutil.rmtree(tmp_models_dir) + shutil.rmtree(tmp_models_dir) # not used right now if os.path.exists(ns_credentials_dir) and pipeline_config['task_id'] in [1, -1]: shutil.rmtree(ns_credentials_dir) return ns_credentials_dir, tmp_models_dir, network_interface_name def clean_up(self, pipeline_config, tmp_models_dir, ns_credentials_dir): + """Remove created folders + + Arguments: + pipeline_config {dict} -- The pipeline config + tmp_models_dir {[type]} -- The path to the temporary models (not used right now) + ns_credentials_dir {[type]} -- The path to the nameserver credentials + """ if pipeline_config['task_id'] in [1, -1]: # Delete temporary files if os.path.exists(tmp_models_dir): @@ -188,6 +243,17 @@ def clean_up(self, pipeline_config, tmp_models_dir, ns_credentials_dir): shutil.rmtree(ns_credentials_dir) def get_nameserver(self, run_id, task_id, ns_credentials_dir, network_interface_name): + """Get the namesever object + + Arguments: + run_id {str} -- The id of the run + task_id {int} -- An id for the worker + ns_credentials_dir {str} -- Path to ns credentials + network_interface_name {str} -- The network interface name + + Returns: + NameServer -- The NameServer object + """ if not os.path.isdir(ns_credentials_dir): try: os.mkdir(ns_credentials_dir) @@ -196,6 +262,22 @@ def get_nameserver(self, run_id, task_id, ns_credentials_dir, network_interface_ return NameServer(run_id=run_id, nic_name=network_interface_name, working_directory=ns_credentials_dir) def get_optimization_algorithm_instance(self, config_space, run_id, pipeline_config, ns_host, ns_port, loggers, previous_result=None): + """Get an instance of the optimization algorithm + + Arguments: + config_space {ConfigurationSpace} -- The config space to optimize. + run_id {str} -- An Id for the current run. + pipeline_config {dict} -- The configuration of the pipeline. + ns_host {str} -- Nameserver host. + ns_port {int} -- Nameserver port. + loggers {list} -- Loggers to log the results. + + Keyword Arguments: + previous_result {Result} -- A previous result to warmstart the search (default: {None}) + + Returns: + Master -- An optimization algorithm. + """ optimization_algorithm = self.algorithms[pipeline_config["algorithm"]] kwargs = {"configspace": config_space, "run_id": run_id, "eta": pipeline_config["eta"], "min_budget": pipeline_config["min_budget"], "max_budget": pipeline_config["max_budget"], @@ -208,9 +290,20 @@ def get_optimization_algorithm_instance(self, config_space, run_id, pipeline_con return hb - def parse_results(self, result_logger_dir): + def parse_results(self, pipeline_config): + """Parse the results of the optimization run + + Arguments: + pipeline_config {dict} -- The configuration of the pipeline. + + Raises: + RuntimeError: An Error occurred when parsing the results. + + Returns: + dict -- Dictionary summarizing the results + """ try: - res = logged_results_to_HBS_result(result_logger_dir) + res = logged_results_to_HBS_result(pipeline_config["result_logger_dir"]) id2config = res.get_id2config_mapping() incumbent_trajectory = res.get_incumbent_trajectory(bigger_is_better=False, non_decreasing_budget=False) except Exception as e: @@ -220,18 +313,38 @@ def parse_results(self, result_logger_dir): return dict() final_config_id = incumbent_trajectory['config_ids'][-1] - return incumbent_trajectory['losses'][-1], id2config[final_config_id]['config'], incumbent_trajectory['budgets'][-1] + final_budget = incumbent_trajectory['budgets'][-1] + best_run = [r for r in res.get_runs_by_id(final_config_id) if r.budget == final_budget][0] + return {'optimized_hyperparameter_config': id2config[final_config_id]['config'], + 'budget': final_budget, + 'loss': best_run.loss, + 'info': best_run.info} def run_worker(self, pipeline_config, run_id, task_id, ns_credentials_dir, network_interface_name, X_train, Y_train, X_valid, Y_valid, dataset_info, shutdownables): + """ Run the AutoNetWorker + + Arguments: + pipeline_config {dict} -- The configuration of the pipeline + run_id {str} -- An id for the run + task_id {int} -- An id for the worker + ns_credentials_dir {str} -- path to nameserver credentials + network_interface_name {str} -- the name of the network interface + X_train {array} -- The data + Y_train {array} -- The data + X_valid {array} -- The data + Y_valid {array} -- The data + dataset_info {DatasetInfo} -- Object describing the dataset + shutdownables {list} -- A list of objects that need to shutdown when the optimization is finished + """ if not task_id == -1: time.sleep(5) while not os.path.isdir(ns_credentials_dir): time.sleep(5) host = nic_name_to_host(network_interface_name) - worker = ModuleWorker(pipeline=self.sub_pipeline, pipeline_config=pipeline_config, + worker = AutoNetWorker(pipeline=self.sub_pipeline, pipeline_config=pipeline_config, X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid, dataset_info=dataset_info, budget_type=self.budget_types[pipeline_config['budget_type']], max_budget=pipeline_config["max_budget"], @@ -245,6 +358,19 @@ def run_worker(self, pipeline_config, run_id, task_id, ns_credentials_dir, netwo def run_optimization_algorithm(self, pipeline_config, run_id, ns_host, ns_port, nameserver, task_id, result_loggers, dataset_info, logger): + """ + + Arguments: + pipeline_config {dict} -- The configuration of the pipeline + run_id {str} -- An id for the run + ns_host {str} -- Nameserver host. + ns_port {int} -- Nameserver port. + nameserver {[type]} -- The nameserver. + task_id {int} -- An id for the worker + result_loggers {[type]} -- [description] + dataset_info {DatasetInfo} -- Object describing the dataset + logger {list} -- Loggers to log the results. + """ config_space = self.pipeline.get_hyperparameter_search_space(dataset_info=dataset_info, **pipeline_config) @@ -270,6 +396,7 @@ def run_optimization_algorithm(self, pipeline_config, run_id, ns_host, ns_port, @staticmethod def get_nic_name(pipeline_config): + """Get the nic name from the pipeline config""" return pipeline_config["network_interface_name"] or (netifaces.interfaces()[1] if len(netifaces.interfaces()) > 1 else "lo") @@ -278,8 +405,6 @@ def clean_fit_data(self): self.sub_pipeline.root.clean_fit_data() - - class tensorboard_logger(object): def __init__(self): self.start_time = time.time() diff --git a/autoPyTorch/pipeline/nodes/train_node.py b/autoPyTorch/pipeline/nodes/train_node.py index 021c79a32..25735e45b 100644 --- a/autoPyTorch/pipeline/nodes/train_node.py +++ b/autoPyTorch/pipeline/nodes/train_node.py @@ -20,13 +20,14 @@ from autoPyTorch.utils.config.config_option import ConfigOption, to_bool from autoPyTorch.components.training.base_training import BaseTrainingTechnique, BaseBatchLossComputationTechnique from autoPyTorch.components.training.trainer import Trainer -from autoPyTorch.components.training.lr_scheduling import LrScheduling -from autoPyTorch.components.training.budget_types import BudgetTypeTime, BudgetTypeEpochs import signal class TrainNode(PipelineNode): + """Training pipeline node. In this node, the network will be trained.""" + def __init__(self): + """Construct the node""" super(TrainNode, self).__init__() self.default_minimize_value = True self.training_techniques = dict() @@ -36,13 +37,34 @@ def __init__(self): def fit(self, hyperparameter_config, pipeline_config, train_loader, valid_loader, network, optimizer, - train_metric, additional_metrics, + optimize_metric, additional_metrics, log_functions, budget, loss_function, training_techniques, fit_start_time, refit): + """Train the network. + + Arguments: + hyperparameter_config {dict} -- The sampled hyperparameter config. + pipeline_config {dict} -- The user specified configuration of the pipeline + train_loader {DataLoader} -- Data for training. + valid_loader {DataLoader} -- Data for validation. + network {BaseNet} -- The neural network to be trained. + optimizer {AutoNetOptimizerBase} -- The selected optimizer. + optimize_metric {AutoNetMetric} -- The selected metric to optimize + additional_metrics {list} -- List of metrics, that should be logged + log_functions {list} -- List of AutoNetLofFunctions that can log additional stuff like test performance + budget {float} -- The budget for training + loss_function {_Loss} -- The selected PyTorch loss module + training_techniques {list} -- List of objects inheriting from BaseTrainingTechnique. + fit_start_time {float} -- Start time of fit + refit {bool} -- Whether training for refit or not. + + Returns: + dict -- loss and info reported to bohb + """ hyperparameter_config = ConfigWrapper(self.get_name(), hyperparameter_config) logger = logging.getLogger('autonet') logger.debug("Start train. Budget: " + str(budget)) @@ -53,7 +75,7 @@ def fit(self, hyperparameter_config, pipeline_config, trainer = Trainer( model=network, loss_computation=self.batch_loss_computation_techniques[hyperparameter_config["batch_loss_computation_technique"]](), - metrics=[train_metric] + additional_metrics, + metrics=[optimize_metric] + additional_metrics, log_functions=log_functions, criterion=loss_function, budget=budget, @@ -73,20 +95,20 @@ def fit(self, hyperparameter_config, pipeline_config, trainer.on_epoch_start(log=log, epoch=epoch) # training - train_metric_results, train_loss, stop_training = trainer.train(epoch + 1, train_loader) + optimize_metric_results, train_loss, stop_training = trainer.train(epoch + 1, train_loader) if valid_loader is not None and trainer.eval_valid_each_epoch: valid_metric_results = trainer.evaluate(valid_loader) # evaluate log['loss'] = train_loss for i, metric in enumerate(trainer.metrics): - log['train_' + metric.__name__] = train_metric_results[i] + log['train_' + metric.name] = optimize_metric_results[i] if valid_loader is not None and trainer.eval_valid_each_epoch: - log['val_' + metric.__name__] = valid_metric_results[i] + log['val_' + metric.name] = valid_metric_results[i] if trainer.eval_additional_logs_each_epoch: for additional_log in trainer.log_functions: - log[additional_log.__name__] = additional_log(trainer.model, epoch) + log[additional_log.name] = additional_log(trainer.model, epoch) # wrap up epoch stop_training = trainer.on_epoch_end(log=log, epoch=epoch) or stop_training @@ -96,7 +118,7 @@ def fit(self, hyperparameter_config, pipeline_config, log = {key: value for key, value in log.items() if not isinstance(value, np.ndarray)} logger.debug("Epoch: " + str(epoch) + " : " + str(log)) if 'use_tensorboard_logger' in pipeline_config and pipeline_config['use_tensorboard_logger']: - self.tensorboard_log(budget=budget, epoch=epoch, log=log) + self.tensorboard_log(budget=budget, epoch=epoch, log=log, logdir=pipeline_config["result_logger_dir"]) if stop_training: break @@ -105,7 +127,7 @@ def fit(self, hyperparameter_config, pipeline_config, torch.cuda.empty_cache() # wrap up - loss, final_log = self.wrap_up_training(trainer=trainer, logs=logs, epoch=epoch, minimize=pipeline_config['minimize'], + loss, final_log = self.wrap_up_training(trainer=trainer, logs=logs, epoch=epoch, train_loader=train_loader, valid_loader=valid_loader, budget=budget, training_start_time=training_start_time, fit_start_time=fit_start_time, best_over_epochs=pipeline_config['best_over_epochs'], refit=refit, logger=logger) @@ -113,6 +135,16 @@ def fit(self, hyperparameter_config, pipeline_config, def predict(self, pipeline_config, network, predict_loader): + """Predict using trained neural network + + Arguments: + pipeline_config {dict} -- The user specified configuration of the pipeline + network {BaseNet} -- The trained neural network. + predict_loader {DataLoader} -- The data to predict the labels for. + + Returns: + dict -- The predicted labels in a dict. + """ if pipeline_config["torch_num_threads"] > 0: torch.set_num_threads(pipeline_config["torch_num_threads"]) @@ -162,7 +194,6 @@ def get_pipeline_config_options(self): options = [ ConfigOption(name="batch_loss_computation_techniques", default=list(self.batch_loss_computation_techniques.keys()), type=str, list=True, choices=list(self.batch_loss_computation_techniques.keys())), - ConfigOption("minimize", default=self.default_minimize_value, type=to_bool, choices=[True, False]), ConfigOption("cuda", default=True, type=to_bool, choices=[True, False]), ConfigOption("torch_num_threads", default=1, type=int), ConfigOption("full_eval_each_epoch", default=False, type=to_bool, choices=[True, False], @@ -174,27 +205,49 @@ def get_pipeline_config_options(self): options += technique.get_pipeline_config_options() return options - def tensorboard_log(self, budget, epoch, log): + def tensorboard_log(self, budget, epoch, log, logdir): import tensorboard_logger as tl worker_path = 'Train/' - tl.log_value(worker_path + 'budget', float(budget), int(time.time())) + try: + tl.log_value(worker_path + 'budget', float(budget), int(time.time())) + except: + tl.configure(logdir) + tl.log_value(worker_path + 'budget', float(budget), int(time.time())) tl.log_value(worker_path + 'epoch', float(epoch + 1), int(time.time())) for name, value in log.items(): tl.log_value(worker_path + name, float(value), int(time.time())) - def wrap_up_training(self, trainer, logs, epoch, minimize, train_loader, valid_loader, budget, + def wrap_up_training(self, trainer, logs, epoch, train_loader, valid_loader, budget, training_start_time, fit_start_time, best_over_epochs, refit, logger): + """Wrap up and evaluate the training by computing missing log values + + Arguments: + trainer {Trainer} -- The trainer used for training. + logs {dict} -- The logs of the training + epoch {int} -- Number of Epochs trained + train_loader {DataLoader} -- The data for training + valid_loader {DataLoader} -- The data for validation + budget {float} -- Budget of training + training_start_time {float} -- Start time of training + fit_start_time {float} -- Start time of fit + best_over_epochs {bool} -- Whether best validation data over epochs should be used + refit {bool} -- Whether training was for refitting + logger {Logger} -- Logger for logging stuff to the console + + Returns: + tuple -- loss and selected final loss + """ wrap_up_start_time = time.time() trainer.model.epochs_trained = epoch trainer.model.logs = logs - train_metric = trainer.metrics[0] - opt_metric_name = 'train_' + train_metric.__name__ + optimize_metric = trainer.metrics[0] + opt_metric_name = 'train_' + optimize_metric.name if valid_loader is not None: - opt_metric_name = 'val_' + train_metric.__name__ + opt_metric_name = 'val_' + optimize_metric.name final_log = trainer.final_eval(opt_metric_name=opt_metric_name, - logs=logs, train_loader=train_loader, valid_loader=valid_loader, minimize=minimize, best_over_epochs=best_over_epochs, refit=refit) - loss = final_log[opt_metric_name] * (1 if minimize else -1) + logs=logs, train_loader=train_loader, valid_loader=valid_loader, best_over_epochs=best_over_epochs, refit=refit) + loss = trainer.metrics[0].loss_transform(final_log[opt_metric_name]) logger.info("Finished train with budget " + str(budget) + ": Preprocessing took " + str(int(training_start_time - fit_start_time)) + @@ -222,4 +275,4 @@ def predict(network, test_loader, device, move_network=True): Y_batch_pred = network(X_batch).detach().cpu() Y_batch_preds.append(Y_batch_pred) - return torch.cat(Y_batch_preds, 0) \ No newline at end of file + return torch.cat(Y_batch_preds, 0) diff --git a/autoPyTorch/utils/benchmarking/benchmark.py b/autoPyTorch/utils/benchmarking/benchmark.py index b2fa9e7bf..271e4ce62 100644 --- a/autoPyTorch/utils/benchmarking/benchmark.py +++ b/autoPyTorch/utils/benchmarking/benchmark.py @@ -8,6 +8,7 @@ ReadInstanceData, SaveResults, SetAutoNetConfig, + ApplyUserUpdates, SaveEnsembleLogs, SetEnsembleConfig) from autoPyTorch.utils.benchmarking.visualization_pipeline import (CollectAutoNetConfigTrajectories, @@ -46,12 +47,13 @@ def compute_ensemble_performance(self, **benchmark_config): def get_benchmark_pipeline(self): return Pipeline([ BenchmarkSettings(), - ForInstance([ #instance_file - ReadInstanceData(), #test_split, is_classification, instance + ForInstance([ # loop through instance files + ReadInstanceData(), # get test_split, is_classification, instance CreateAutoNet(), - ForAutoNetConfig([ #autonet_config_file - SetAutoNetConfig(), #use_dataset_metric, use_dataset_max_runtime - ForRun([ #num_runs, run_ids + #ApplyUserUpdates(), + ForAutoNetConfig([ # loop through autonet_config_file + SetAutoNetConfig(), # use_dataset_metric, use_dataset_max_runtime + ForRun([ # loop through num_runs, run_ids PrepareResultFolder(), FitAutoNet(), SaveResults(), diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/__init__.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/__init__.py index 972246d6f..c2975a56c 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/__init__.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/__init__.py @@ -9,4 +9,5 @@ from autoPyTorch.utils.benchmarking.benchmark_pipeline.save_results import SaveResults from autoPyTorch.utils.benchmarking.benchmark_pipeline.benchmark_settings import BenchmarkSettings from autoPyTorch.utils.benchmarking.benchmark_pipeline.save_ensemble_logs import SaveEnsembleLogs -from autoPyTorch.utils.benchmarking.benchmark_pipeline.set_ensemble_config import SetEnsembleConfig \ No newline at end of file +from autoPyTorch.utils.benchmarking.benchmark_pipeline.set_ensemble_config import SetEnsembleConfig +from autoPyTorch.utils.benchmarking.benchmark_pipeline.apply_user_updates import ApplyUserUpdates diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/apply_user_updates.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/apply_user_updates.py new file mode 100644 index 000000000..9fce0d4b7 --- /dev/null +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/apply_user_updates.py @@ -0,0 +1,71 @@ + +from autoPyTorch.utils.config.config_option import ConfigOption, to_bool +from autoPyTorch.pipeline.base.pipeline_node import PipelineNode +from autoPyTorch.utils.configspace_wrapper import ConfigWrapper + +import re +import os +import pandas as pd +import math +import numpy as np + + +class ApplyUserUpdates(PipelineNode): + + def fit(self, pipeline_config, autonet): + + path = pipeline_config['user_updates_config'] + if path is None: + return dict() + + if not os.path.exists(path): + raise ValueError('Invalid path: ' + path) + + data = np.array(pd.read_csv(path, header=None, sep=';')) + + for row in data: + name, value_range, is_log = row[0].strip(), self.string_to_list(str(row[1])), to_bool(row[2].strip()) + name_split = name.split(ConfigWrapper.delimiter) + autonet.pipeline[name_split[0]]._update_hyperparameter_range(ConfigWrapper.delimiter.join(name_split[1:]), value_range, is_log, check_validity=False) + + # print(autonet.get_hyperparameter_search_space()) + + return { 'autonet': autonet } + + + def get_pipeline_config_options(self): + options = [ + ConfigOption("user_updates_config", default=None, type='directory'), + ] + return options + + def string_to_list(self, string): + pattern = "\[(.*)\]" + match = re.search(pattern, string) + + if match is None: + # no list > make constant range + match = re.search(pattern, '[' + string + ',' + string + ']') + + if match is None: + raise ValueError('No valid range specified got: ' + string) + + lst = map(self.try_convert, match.group(1).split(',')) + return list(lst) + + def try_convert(self, string): + string = string.strip() + try: + return int(string) + except: + try: + return float(string) + except: + if string == 'True': + return True + if string == 'False': + return False + return string + + + diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/create_autonet.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/create_autonet.py index a6f0eb382..00dae6ede 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/create_autonet.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/create_autonet.py @@ -18,15 +18,19 @@ def fit(self, pipeline_config, data_manager): autonet_type = AutoNetMultilabel elif (data_manager.problem_type == ProblemType.FeatureClassification): autonet_type = AutoNetClassification + elif data_manager.problem_type == ProblemType.ImageClassification: + autonet = AutoNetImageClassification() + elif data_manager.problem_type == ProblemType.ImageClassificationMultipleDatasets: + autonet = AutoNetImageClassificationMultipleDatasets() else: raise ValueError('Problem type ' + str(data_manager.problem_type) + ' is not defined') autonet = autonet_type() if not pipeline_config["enable_ensemble"] else AutoNetEnsemble(autonet_type) test_logger = test_result if not pipeline_config["enable_ensemble"] else test_predictions_for_ensemble autonet.pipeline[autonet_nodes.LogFunctionsSelector.get_name()].add_log_function( - test_logger.__name__, test_logger(autonet, data_manager.X_test, data_manager.Y_test)) - - metrics = autonet.pipeline[autonet_nodes.MetricSelector.get_name()] + name=test_logger.__name__, + log_function=test_logger(autonet, data_manager.X_test, data_manager.Y_test), + loss_transform=(not pipeline_config["enable_ensemble"])) return { 'autonet': autonet } @@ -34,4 +38,4 @@ def get_pipeline_config_options(self): options = [ ConfigOption("enable_ensemble", default=False, type=to_bool) ] - return options \ No newline at end of file + return options diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/fit_autonet.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/fit_autonet.py index c6cff35a6..e53694975 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/fit_autonet.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/fit_autonet.py @@ -1,20 +1,136 @@ import time import logging from autoPyTorch.pipeline.base.pipeline_node import PipelineNode +from autoPyTorch.utils.config.config_option import ConfigOption +import json +import numpy as np class FitAutoNet(PipelineNode): - def fit(self, autonet, data_manager, **kwargs): + def __init__(self): + super(FitAutoNet, self).__init__() + + # if we have the required module 'resource' (not available on windows!) + self.guarantee_limits = module_exists("resource") and module_exists("pynisher") + + def fit(self, pipeline_config, autonet, data_manager, **kwargs): + start_time = time.time() + test_score = None + + if pipeline_config['refit_config'] is None: + # Start search + logging.getLogger('benchmark').debug("Fit autonet") + + # Email confirmation + if pipeline_config['confirmation_gmail_user']: + self.send_confirmation_mail(pipeline_config, autonet, data_manager) + + # Run fit + fit_result = self.fit_autonet(autonet, data_manager) + + if pipeline_config['refit_budget'] is not None: + # Refit + import os + import numpy as np + autonet_config = autonet.get_current_autonet_config() + from autoPyTorch.utils.loggers import get_refit_config + refit_config = get_refit_config(autonet_config['result_logger_dir']) + directory = os.path.join(autonet_config['result_logger_dir'], 'refit') + + autonet_config['result_logger_dir'] = directory + autonet_config['save_checkpoints'] = False + pipeline_config['refit_config'] = refit_config + + pipeline_config['refit_budget'] *= len(data_manager.X_train) + job_id = max(autonet_config['task_id'], 1) + if job_id == 1: + self.refit_autonet( + pipeline_config, autonet, autonet_config, + data_manager.X_train, data_manager.Y_train, + data_manager.X_valid, data_manager.Y_valid) - logging.getLogger('benchmark').debug("Fit autonet") + else: + # Refit + autonet_config= autonet.get_current_autonet_config() + fit_result = self.refit_autonet( + pipeline_config, autonet, autonet_config, + data_manager.X_train, data_manager.Y_train, + data_manager.X_valid, data_manager.Y_valid) - result = autonet.fit( - data_manager.X_train, data_manager.Y_train, - data_manager.X_valid, data_manager.Y_valid, - refit=False) - config, score = result[0], result[1] + if data_manager.X_test is not None: + # Score on test set + import numpy as np + test_score = autonet.score(data_manager.X_test, data_manager.Y_test.astype(np.int32)) return { 'fit_duration': int(time.time() - start_time), - 'incumbent_config': config, - 'final_score': score } \ No newline at end of file + 'fit_result': fit_result, + 'test_score': test_score} + + def fit_autonet(self, autonet, data_manager): + return autonet.fit( data_manager.X_train, data_manager.Y_train, + data_manager.X_valid, data_manager.Y_valid, + refit=False) + + def refit_autonet(self, pipeline_config, autonet, autonet_config, X_train, Y_train, X_valid, Y_valid): + logging.getLogger('benchmark').debug("Refit autonet") + + import torch + if torch.cuda.is_available(): + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + with open(pipeline_config['refit_config'], 'r') as f: + refit_config = json.load(f) + + if 'incumbent_config_path' in refit_config: + # > updates in set_autonet_config + with open(refit_config['incumbent_config_path'], 'r') as f: + config = json.load(f) + autonet_config['random_seed'] = refit_config['seed'] + autonet_config['dataset_order'] = refit_config['dataset_order'] + else: + config = refit_config + + fit_result = autonet.refit( + X_train, Y_train, + X_valid, Y_valid, + autonet_config=autonet_config, + hyperparameter_config=config, + budget=pipeline_config['refit_budget'] or autonet_config['max_budget']) + + logging.getLogger('benchmark').info("Result: " + str(fit_result)) + return fit_result + + def send_confirmation_mail(self, pipeline_config, autonet, data_manager): + user = pipeline_config['confirmation_gmail_user'] + import pprint + message = "\r\n".join(["Autonet run", + "Data:", + "%s", + "", + "Autonet Config:", + "%s" + "", + "", + "%s"]) % (pprint.pformat(data_manager.X_train.tolist()), pprint.pformat(autonet.get_current_autonet_config()), str(autonet.get_hyperparameter_search_space())) + user = user + '+benchmark@gmail.com' + from autoPyTorch.utils.mail import send_mail + send_mail(user, 'Benchmark Start', message) + + def get_pipeline_config_options(self): + options = [ + ConfigOption("refit_config", default=None, type='directory'), + ConfigOption("refit_budget", default=None, type=int), + ConfigOption("confirmation_gmail_user", default=None, type=str), + ] + return options + + +def module_exists(module_name): + try: + __import__(module_name) + except ImportError: + return False + else: + return True diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_autonet_config.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_autonet_config.py index 43352b59b..372895ec0 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_autonet_config.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_autonet_config.py @@ -18,6 +18,7 @@ def fit(self, pipeline_config, autonet, instance, data_manager, run_id, task_id) def get_pipeline_config_options(self): options = [ ConfigOption("autonet_configs", default=None, type='directory', list=True, required=True), + ConfigOption("autonet_config_root", default=ConfigFileParser.get_autonet_home(), type='directory'), ConfigOption("autonet_config_slice", default=None, type=str) ] return options @@ -25,9 +26,13 @@ def get_pipeline_config_options(self): @staticmethod def get_config_files(pipeline_config, parse_slice=True): config_files = pipeline_config['autonet_configs'] + if pipeline_config['autonet_config_root'] is not None: + config_files = [os.path.join(pipeline_config['autonet_config_root'], config) if not os.path.isabs(config) else config for config in config_files] + autonet_config_slice = ForAutoNetConfig.parse_slice(pipeline_config['autonet_config_slice']) if autonet_config_slice is not None and parse_slice: return config_files[autonet_config_slice] + return config_files @staticmethod @@ -48,4 +53,4 @@ def parse_slice(splice_string): start = int(split[0]) if split[0] != "" else 0 stop = int(split[1]) if split[1] != "" else None step = int(split[2]) if split[2] != "" else 1 - return slice(start, stop, step) \ No newline at end of file + return slice(start, stop, step) diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_instance.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_instance.py index 5ee1ab697..dd04fea54 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_instance.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_instance.py @@ -21,6 +21,7 @@ def get_pipeline_config_options(self): ConfigOption("instances", default=None, type='directory', required=True), ConfigOption("instance_slice", default=None, type=str), ConfigOption("dataset_root", default=ConfigFileParser.get_autonet_home(), type='directory'), + ConfigOption("multiple_datasets_indices", default=None, type=int, list=True), ] return options @@ -30,22 +31,28 @@ def get_instances(benchmark_config, instances_must_exist=True, instance_slice=No instances = [] if os.path.isfile(benchmark_config["instances"]): with open(benchmark_config["instances"], "r") as instances_file: - for line in instances_file: - if line.strip().startswith("openml"): - instances.append(line.strip()) - continue - - if line.strip().startswith("["): - instances.append([make_path(path, benchmark_config["dataset_root"]) for path in line.strip(' []').split(',')]) - continue + if os.path.splitext(benchmark_config['instances'])[1] == '.json': + import json + datasets = [make_path(path, benchmark_config["dataset_root"]) for path in json.load(instances_file)] + instances.append(datasets if benchmark_config['multiple_datasets_indices'] is None else [datasets[i] for i in benchmark_config['multiple_datasets_indices']]) + else: + for line in instances_file: + if line.strip().startswith("openml"): + instances.append(line.strip()) + continue - instance = os.path.abspath(os.path.join(benchmark_config["dataset_root"], line.strip())) - if os.path.isfile(instance) or os.path.isdir(instance): - instances.append(instance) - else: - if not instances_must_exist: + if line.strip().startswith("["): + datasets = [make_path(path, benchmark_config["dataset_root"]) for path in line.strip(' []\n').split(',')] + instances.append(datasets if benchmark_config['multiple_datasets_indices'] is None else [datasets[i] for i in benchmark_config['multiple_datasets_indices']]) + continue + + instance = os.path.abspath(os.path.join(benchmark_config["dataset_root"], line.strip())) + if os.path.isfile(instance) or os.path.isdir(instance): instances.append(instance) - logging.getLogger('benchmark').warning(str(instance) + " does not exist") + else: + if not instances_must_exist: + instances.append(instance) + logging.getLogger('benchmark').warning(str(instance) + " does not exist") elif os.path.isdir(benchmark_config["instances"]): for root, directories, filenames in os.walk(benchmark_config["instances"]): for filename in filenames: diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_run.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_run.py index 9119c6e30..69b57e34d 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_run.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/for_run.py @@ -4,13 +4,12 @@ import traceback class ForRun(SubPipelineNode): - def fit(self, pipeline_config, autonet, data_manager, instance, autonet_config_file, run_id, task_id): + def fit(self, pipeline_config, autonet, data_manager, instance, run_id, task_id): for run_number in self.parse_range(pipeline_config['run_number_range'], pipeline_config['num_runs']): try: logging.getLogger('benchmark').info("Start run " + str(run_id) + "_" + str(run_number)) self.sub_pipeline.fit_pipeline(pipeline_config=pipeline_config, - data_manager=data_manager, instance=instance, - autonet_config_file=autonet_config_file, autonet=autonet, + data_manager=data_manager, instance=instance, run_number=run_number, run_id=run_id, task_id=task_id) except Exception as e: print(e) @@ -42,4 +41,4 @@ def parse_range(range_string, fallback): start = int(split[0]) if split[0] != "" else 0 stop = int(split[1]) if split[1] != "" else fallback step = int(split[2]) if split[2] != "" else 1 - return range(start, stop, step) \ No newline at end of file + return range(start, stop, step) diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/prepare_result_folder.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/prepare_result_folder.py index 5391b4cc7..27a9ee627 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/prepare_result_folder.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/prepare_result_folder.py @@ -1,18 +1,19 @@ import os import logging -from ConfigSpace.read_and_write import json +from ConfigSpace.read_and_write import json as cs_json, pcs_new as cs_pcs from autoPyTorch.utils.config.config_option import ConfigOption, to_bool from autoPyTorch.pipeline.base.pipeline_node import PipelineNode from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates +from autoPyTorch.utils.modify_config_space import remove_constant_hyperparameter class PrepareResultFolder(PipelineNode): - def fit(self, pipeline_config, data_manager, instance, - autonet_config_file, autonet, run_number, run_id, task_id): + def fit(self, pipeline_config, data_manager, instance, + autonet, run_number, run_id, task_id): - instance_name, autonet_config_name, run_name = get_names(instance, autonet_config_file, run_id, run_number) - run_result_dir = get_run_result_dir(pipeline_config, instance, autonet_config_file, run_id, run_number) - instance_run_id = str(run_name) + "_" + str(instance_name) + "_" + str(autonet_config_name) + instance_name, run_name = get_names(instance, run_id, run_number) + run_result_dir = get_run_result_dir(pipeline_config, instance, run_id, run_number, autonet) + instance_run_id = str(run_name) + "-" + str(instance_name) instance_run_id = '_'.join(instance_run_id.split(':')) autonet.autonet_config = None #clean results of last fit @@ -37,16 +38,38 @@ def fit(self, pipeline_config, data_manager, instance, instance_info['instance_shape'] = data_manager.X_train.shape instance_info['categorical_features'] = data_manager.categorical_features - if autonet.get_current_autonet_config()["hyperparameter_search_space_updates"] is not None: - autonet.get_current_autonet_config()["hyperparameter_search_space_updates"].save_as_file( + autonet_config = autonet.get_current_autonet_config() + if autonet_config["hyperparameter_search_space_updates"] is not None: + autonet_config["hyperparameter_search_space_updates"].save_as_file( os.path.join(run_result_dir, "hyperparameter_search_space_updates.txt")) + if 'user_updates_config' in pipeline_config: + user_updates_config = pipeline_config['user_updates_config'] + if user_updates_config: + from shutil import copyfile + copyfile(user_updates_config, os.path.join(run_result_dir, 'user_updates_config.csv')) + self.write_config_to_file(run_result_dir, "instance.info", instance_info) self.write_config_to_file(run_result_dir, "benchmark.config", pipeline_config) - self.write_config_to_file(run_result_dir, "autonet.config", autonet.get_current_autonet_config()) - + self.write_config_to_file(run_result_dir, "autonet.config", autonet_config) + + # save refit config - add indent and sort keys + if 'refit_config' in pipeline_config and pipeline_config['refit_config'] is not None: + import json + with open(pipeline_config['refit_config'], 'r') as f: + refit_config = json.loads(f.read()) + with open(os.path.join(run_result_dir, 'refit_config.json'), 'w+') as f: + f.write(json.dumps(refit_config, indent=4, sort_keys=True)) + + # save search space + search_space = autonet.pipeline.get_hyperparameter_search_space(**autonet_config) with open(os.path.join(run_result_dir, "configspace.json"), "w") as f: - f.write(json.write(autonet.pipeline.get_hyperparameter_search_space(**autonet.get_current_autonet_config()))) + f.write(cs_json.write(search_space)) + + # save search space without constants - used by bohb - as pcs (simple) + simplified_search_space, _ = remove_constant_hyperparameter(search_space) + with open(os.path.join(run_result_dir, "configspace_simple.pcs"), "w") as f: + f.write(cs_pcs.write(simplified_search_space)) return { 'result_dir': run_result_dir } @@ -54,30 +77,41 @@ def fit(self, pipeline_config, data_manager, instance, def write_config_to_file(self, folder, filename, config): do_not_write = ["hyperparameter_search_space_updates"] with open(os.path.join(folder, filename), "w") as f: - f.write("\n".join([(key + '=' + str(value)) for key, value in config.items() if not key in do_not_write])) + f.write("\n".join([(key + '=' + str(value)) for (key, value) in sorted(config.items(), key=lambda x: x[0]) if not key in do_not_write])) def get_pipeline_config_options(self): options = [ - ConfigOption('result_dir', default=None, type='directory', required=True) + ConfigOption('result_dir', default=None, type='directory', required=True), + ConfigOption('name', default=None, type=str, required=True) ] return options -def get_names(instance, autonet_config_file, run_id, run_number): +def get_names(instance, run_id, run_number): if isinstance(instance, list): - instance_name = "_".join([os.path.split(p)[1].split(".")[0] for p in instance]) + for p in instance: + if not os.path.exists(p): + raise Exception('Invalid path: ' + str(p)) + instance_name = "-".join(sorted([os.path.split(p)[1].split(".")[0] for p in instance])) + if len(instance_name) > 40: + instance_name = "-".join(sorted([os.path.split(q)[1] for q in sorted(set(os.path.split(p)[0] for p in instance))] + [str(len(instance))])) else: instance_name = os.path.basename(instance).split(".")[0] - autonet_config_name = os.path.basename(autonet_config_file).split(".")[0] run_name = "run_" + str(run_id) + "_" + str(run_number) - return "_".join(instance_name.split(':')), autonet_config_name, run_name + return "_".join(instance_name.split(':')), run_name + +def get_run_result_dir(pipeline_config, instance, run_id, run_number, autonet): + instance_name, run_name = get_names(instance, run_id, run_number) + autonet_config = autonet.get_current_autonet_config() + benchmark_name = '_'.join(pipeline_config['name'].split(' ')) + + if 'refit_config' not in pipeline_config or pipeline_config['refit_config'] is None: + benchmark_name += "[{0}_{1}]".format(int(autonet_config['min_budget']), int(autonet_config['max_budget'])) + elif 'refit_budget' not in pipeline_config or pipeline_config['refit_budget'] is None: + benchmark_name += "[refit_{0}]".format(int(autonet_config['max_budget'])) + else: + benchmark_name += "[refit_{0}]".format(int(pipeline_config['refit_budget'])) -def get_run_result_dir(pipeline_config, instance, autonet_config_file, run_id, run_number): - instance_name, autonet_config_name, run_name = get_names(instance, autonet_config_file, run_id, run_number) - run_result_dir = os.path.join(pipeline_config['result_dir'], - pipeline_config["benchmark_name"], - instance_name, - autonet_config_name, - run_name) - return run_result_dir \ No newline at end of file + run_result_dir = os.path.join(pipeline_config['result_dir'], instance_name, benchmark_name, run_name) + return run_result_dir diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/read_instance_data.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/read_instance_data.py index f1f2734fe..3a74c3fce 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/read_instance_data.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/read_instance_data.py @@ -1,22 +1,45 @@ - +import numpy as np from autoPyTorch.utils.config.config_option import ConfigOption, to_bool from autoPyTorch.pipeline.base.pipeline_node import PipelineNode -from autoPyTorch.data_management.data_manager import DataManager +from autoPyTorch.data_management.data_manager import DataManager, ImageManager class ReadInstanceData(PipelineNode): def fit(self, pipeline_config, instance): - assert pipeline_config['problem_type'] in ['feature_classification', 'feature_multilabel', 'feature_regression'] - dm = DataManager(verbose=pipeline_config["data_manager_verbose"]) - dm.read_data(instance, - is_classification=(pipeline_config["problem_type"] in ['feature_classification', 'feature_multilabel']), - test_split=pipeline_config["test_split"]) + # Get data manager for train, val, test data + if pipeline_config['problem_type'] in ['feature_classification', 'feature_multilabel', 'feature_regression']: + dm = DataManager(verbose=pipeline_config["data_manager_verbose"]) + if pipeline_config['test_instances'] is not None: + dm_test = DataManager(verbose=pipeline_config["data_manager_verbose"]) + else: + dm = ImageManager(verbose=pipeline_config["data_manager_verbose"]) + if pipeline_config['test_instances'] is not None: + dm_test = ImageManager(verbose=pipeline_config["data_manager_verbose"]) + + # Read data + if pipeline_config['test_instances'] is not None: + # Use given test set + dm.read_data(instance, + is_classification=(pipeline_config["problem_type"] in ['feature_classification', 'feature_multilabel', 'image_classification']), + test_split=0.0) + dm_test.read_data(pipeline_config['test_instances'], + is_classification=(pipeline_config["problem_type"] in ['feature_classification', 'feature_multilabel', 'image_classification']), + test_split=0.0) + dm.X_test, dm.Y_test = dm_test.X_train, dm_test.Y_train.astype(np.int32) + + else: + # Use test split + dm.read_data(instance, + is_classification=(pipeline_config["problem_type"] in ['feature_classification', 'feature_multilabel', 'image_classification']), + test_split=pipeline_config["test_split"]) + return {"data_manager": dm} def get_pipeline_config_options(self): options = [ ConfigOption("test_split", default=0.0, type=float), - ConfigOption("problem_type", default='feature_classification', type=str, choices=['feature_classification', 'feature_multilabel', 'feature_regression']), + ConfigOption("problem_type", default='feature_classification', type=str, choices=['feature_classification', 'feature_multilabel', 'feature_regression', 'image_classification']), ConfigOption("data_manager_verbose", default=False, type=to_bool), + ConfigOption("test_instances", default=None, type=str) ] - return options \ No newline at end of file + return options diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/save_ensemble_logs.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/save_ensemble_logs.py index 65bb6cea9..964039454 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/save_ensemble_logs.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/save_ensemble_logs.py @@ -35,7 +35,7 @@ def save_ensemble_logs(pipeline_config, autonet, result_dir, ensemble_size=None, # prepare some variables autonet_config = autonet.get_current_autonet_config() metrics = autonet.pipeline[MetricSelector.get_name()].metrics - train_metric = metrics[autonet_config["train_metric"]] + optimize_metric = metrics[autonet_config["optimize_metric"]] y_transform = autonet.pipeline[OneHotEncoding.get_name()].complete_y_tranformation result = logged_results_to_HBS_result(result_dir) filename = os.path.join(result_dir, "predictions_for_ensemble.npy") @@ -82,7 +82,7 @@ def save_ensemble_logs(pipeline_config, autonet, result_dir, ensemble_size=None, # build an ensemble with current subset and size ensemble_start_time = time.time() ensemble, _ = build_ensemble(result=result, - train_metric=train_metric, minimize=autonet_config["minimize"], ensemble_size=ensemble_size or autonet_config["ensemble_size"], + optimize_metric=optimize_metric, ensemble_size=ensemble_size or autonet_config["ensemble_size"], all_predictions=subset_predictions, labels=labels, model_identifiers=subset_model_identifiers, only_consider_n_best=autonet_config["ensemble_only_consider_n_best"], sorted_initialization_n_best=autonet_config["ensemble_sorted_initialization_n_best"]) @@ -96,7 +96,7 @@ def save_ensemble_logs(pipeline_config, autonet, result_dir, ensemble_size=None, # evaluate the metrics metric_performances = dict() for metric_name, metric in metrics.items(): - if metric_name != autonet_config["train_metric"] and metric_name not in autonet_config["additional_metrics"]: + if metric_name != autonet_config["optimize_metric"] and metric_name not in autonet_config["additional_metrics"]: continue metric_performances[metric_name] = metric(ensemble_prediction, labels) if test_data_available: @@ -114,8 +114,7 @@ def save_ensemble_logs(pipeline_config, autonet, result_dir, ensemble_size=None, [ensemble.identifiers_[i] for i in ensemble.indices_], { "ensemble_size": ensemble.ensemble_size, - "metric": autonet_config["train_metric"], - "minimize": ensemble.minimize, + "metric": autonet_config["optimize_metric"], "sorted_initialization_n_best": ensemble.sorted_initialization_n_best, "only_consider_n_best": ensemble.only_consider_n_best, "bagging": ensemble.bagging, diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/save_results.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/save_results.py index d2e82a8ae..7891e80b5 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/save_results.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/save_results.py @@ -8,41 +8,28 @@ class SaveResults(PipelineNode): - def fit(self, result_dir, fit_duration, final_score, autonet, task_id): + def fit(self, result_dir, fit_duration, test_score, fit_result, autonet, task_id): if (task_id not in [-1, 1]): time.sleep(60) return dict() logging.getLogger('benchmark').info("Create and save summary") - autonet_config = autonet.autonet_config - res = logged_results_to_HBS_result(result_dir) - id2config = res.get_id2config_mapping() - incumbent_trajectory = res.get_incumbent_trajectory(bigger_is_better=False, non_decreasing_budget=False) - final_config_id = incumbent_trajectory['config_ids'][-1] - final_budget = incumbent_trajectory['budgets'][-1] - incumbent_config = id2config[final_config_id]['config'] - - final_info = [run["info"] for run in res.get_runs_by_id(final_config_id) if run["budget"] == final_budget][0] - - summary = dict() - summary["final_loss"] = final_score if autonet_config["minimize"] else -final_score - summary["incumbent_config"] = incumbent_config - summary["duration"] = fit_duration - for name in autonet_config['additional_metrics'] + [autonet_config["train_metric"]]: - try: - summary["final_" + name] = final_info["val_" + name] - except: - summary["final_" + name] = final_info["train_" + name] - - for name in autonet_config['additional_logs']: - try: - summary["final_" + name] = final_info[name] - except: - pass - + summary = { + "incumbent_config": fit_result["optimized_hyperparameter_config"], + "budget": fit_result["budget"], + "loss": fit_result["loss"], + "test_score": test_score, + "incumbent_config" : incumbent_config, + "info": fit_result["info"], + "duration": fit_duration, + } + + if "ensemble_configs" in fit_result: + summary["ensemble_configs"] = list(fit_result["ensemble_configs"].values()) + # write as json with open(os.path.join(result_dir, "summary.json"), "w") as f: json.dump(summary, f) - return dict() \ No newline at end of file + return dict() diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/set_autonet_config.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/set_autonet_config.py index d062ece70..b283bf613 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/set_autonet_config.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/set_autonet_config.py @@ -13,7 +13,7 @@ def fit(self, pipeline_config, autonet, autonet_config_file, data_manager, insta config['additional_logs'] = ['test_result' if not pipeline_config['enable_ensemble'] else 'test_predictions_for_ensemble'] if (pipeline_config['use_dataset_metric'] and data_manager.metric is not None): - config['train_metric'] = data_manager.metric + config['optimize_metric'] = data_manager.metric if (pipeline_config['use_dataset_max_runtime'] and data_manager.max_runtime is not None): config['max_runtime'] = data_manager.max_runtime @@ -27,7 +27,6 @@ def fit(self, pipeline_config, autonet, autonet_config_file, data_manager, insta if data_manager.categorical_features: config['categorical_features'] = data_manager.categorical_features - config['dataset_name'] = "_".join(os.path.basename(instance).split(":")) # Note: PrepareResultFolder will make a small run dependent update of the autonet_config autonet.update_autonet_config(**config) return dict() @@ -39,4 +38,4 @@ def get_pipeline_config_options(self): ConfigOption("working_dir", default=None, type='directory'), ConfigOption("network_interface_name", default=None, type=str) ] - return options \ No newline at end of file + return options diff --git a/autoPyTorch/utils/benchmarking/benchmark_pipeline/set_ensemble_config.py b/autoPyTorch/utils/benchmarking/benchmark_pipeline/set_ensemble_config.py index e34faa48f..6c1327cbe 100644 --- a/autoPyTorch/utils/benchmarking/benchmark_pipeline/set_ensemble_config.py +++ b/autoPyTorch/utils/benchmarking/benchmark_pipeline/set_ensemble_config.py @@ -24,7 +24,7 @@ def fit(self, pipeline_config, autonet, run_result_dir): autonet.autonet_config = autonet_config return {"result_dir": run_result_dir, - "train_metric": autonet_config["train_metric"], + "optimize_metric": autonet_config["optimize_metric"], "trajectories": []} def get_pipeline_config_options(self): diff --git a/autoPyTorch/utils/benchmarking/visualization_pipeline/collect_trajectories.py b/autoPyTorch/utils/benchmarking/visualization_pipeline/collect_trajectories.py index 1cb9efbac..8dcfe14c9 100644 --- a/autoPyTorch/utils/benchmarking/visualization_pipeline/collect_trajectories.py +++ b/autoPyTorch/utils/benchmarking/visualization_pipeline/collect_trajectories.py @@ -10,7 +10,7 @@ def fit(self, pipeline_config, run_id_range): instances = self.get_instances(pipeline_config, instance_slice=self.parse_slice(pipeline_config["instance_slice"])) result_trajectories = dict() - result_train_metrics = set() + result_optimize_metrics = set() for instance in instances: try: @@ -18,7 +18,7 @@ def fit(self, pipeline_config, run_id_range): # merge the trajectories into one dict instance_trajectories = pipeline_result["trajectories"] - train_metrics = pipeline_result["train_metrics"] + optimize_metrics = pipeline_result["optimize_metrics"] for metric, config_trajectories in instance_trajectories.items(): if metric not in result_trajectories: @@ -27,13 +27,13 @@ def fit(self, pipeline_config, run_id_range): if config not in result_trajectories[metric]: result_trajectories[metric][config] = dict() result_trajectories[metric][config][instance] = run_trajectories - result_train_metrics |= train_metrics + result_optimize_metrics |= optimize_metrics except Exception as e: print(e) traceback.print_exc() return {"trajectories": result_trajectories, - "train_metrics": result_train_metrics} + "optimize_metrics": result_optimize_metrics} class CollectAutoNetConfigTrajectories(ForAutoNetConfig): @@ -41,7 +41,7 @@ def fit(self, pipeline_config, instance, run_id_range): logging.getLogger('benchmark').info('Collecting data for dataset ' + instance) result_trajectories = dict() - result_train_metrics = set() + result_optimize_metrics = set() # iterate over all configs for config_file in self.get_config_files(pipeline_config): @@ -53,16 +53,16 @@ def fit(self, pipeline_config, instance, run_id_range): # merge the trajectories into one dict config_trajectories = pipeline_result["trajectories"] - train_metrics = pipeline_result["train_metrics"] + optimize_metrics = pipeline_result["optimize_metrics"] for metric, run_trajectories in config_trajectories.items(): if metric not in result_trajectories: result_trajectories[metric] = dict() result_trajectories[metric][autonet_config_name] = run_trajectories - result_train_metrics |= train_metrics + result_optimize_metrics |= optimize_metrics return {"trajectories": result_trajectories, - "train_metrics": result_train_metrics} + "optimize_metrics": result_optimize_metrics} class CollectRunTrajectories(ForRun): @@ -70,13 +70,13 @@ def fit(self, pipeline_config, instance, run_id_range, autonet_config_file): logging.getLogger('benchmark').info('Collecting data for autonet config ' + autonet_config_file) result_trajectories = dict() - train_metrics = set() + optimize_metrics = set() run_number_range = self.parse_range(pipeline_config['run_number_range'], pipeline_config['num_runs']) instance_result_dir = os.path.abspath(os.path.join(get_run_result_dir(pipeline_config, instance, autonet_config_file, "0", "0"), "..")) if not os.path.exists(instance_result_dir): logging.getLogger('benchmark').warn("Skipping %s because it no results exist" % instance_result_dir) - return {"trajectories": result_trajectories, "train_metrics": train_metrics} + return {"trajectories": result_trajectories, "optimize_metrics": optimize_metrics} run_result_dirs = next(os.walk(instance_result_dir))[1] # iterate over all run_numbers and run_ids @@ -97,7 +97,7 @@ def fit(self, pipeline_config, instance, run_id_range, autonet_config_file): autonet_config_file=autonet_config_file, run_result_dir=run_result_dir) run_trajectories = pipeline_result["trajectories"] - train_metric = pipeline_result["train_metric"] + optimize_metric = pipeline_result["optimize_metric"] # merge the trajectories into one dict for metric, trajectory in run_trajectories.items(): @@ -105,9 +105,9 @@ def fit(self, pipeline_config, instance, run_id_range, autonet_config_file): result_trajectories[metric] = list() result_trajectories[metric].append(trajectory) - if train_metric is not None: - train_metrics |= set([train_metric]) - return {"trajectories": result_trajectories, "train_metrics": train_metrics} + if optimize_metric is not None: + optimize_metrics |= set([optimize_metric]) + return {"trajectories": result_trajectories, "optimize_metrics": optimize_metrics} def parse_run_folder_name(run_folder_name): assert run_folder_name.startswith("run_") diff --git a/autoPyTorch/utils/benchmarking/visualization_pipeline/get_additional_trajectories.py b/autoPyTorch/utils/benchmarking/visualization_pipeline/get_additional_trajectories.py index 1ab7dafed..162537f84 100644 --- a/autoPyTorch/utils/benchmarking/visualization_pipeline/get_additional_trajectories.py +++ b/autoPyTorch/utils/benchmarking/visualization_pipeline/get_additional_trajectories.py @@ -4,7 +4,7 @@ class GetAdditionalTrajectories(PipelineNode): - def fit(self, pipeline_config, trajectories, train_metrics, instance): + def fit(self, pipeline_config, trajectories, optimize_metrics, instance): for additional_trajectory_path in pipeline_config["additional_trajectories"]: # open trajectory description file @@ -29,7 +29,7 @@ def fit(self, pipeline_config, trajectories, train_metrics, instance): print("Trajectory could not be loaded: %s. Skipping." % e) traceback.print_exc() return {"trajectories": trajectories, - "train_metrics": train_metrics} + "optimize_metrics": optimize_metrics} def get_pipeline_config_options(self): options = [ @@ -44,7 +44,8 @@ def csv_trajectory_loader(path, config_name, columns_description, trajectories): # parse the csv times_finished = list() - performances = dict() + performances = dict() + losses = dict() for row in reader: for i, col in enumerate(row): if i == columns_description["time_column"]: @@ -53,22 +54,26 @@ def csv_trajectory_loader(path, config_name, columns_description, trajectories): log_name = columns_description["metric_columns"][str(i)]["name"] transform = columns_description["metric_columns"][str(i)]["transform"] \ if "transform" in columns_description["metric_columns"][str(i)] else "x" + loss_transform = columns_description["metric_columns"][str(i)]["loss_transform"] \ + if "loss_transform" in columns_description["metric_columns"][str(i)] else "x" if log_name not in performances: performances[log_name] = list() + losses[log_name] = list() performances[log_name].append(eval_expr(transform.replace("x", col))) + losses[log_name].append(eval_expr(loss_transform.replace("x", col))) # add data to the other trajectories - for log_name, performance_list in performances.items(): + for log_name in performances.keys(): if log_name not in trajectories: trajectories[log_name] = dict() if config_name not in trajectories[log_name]: trajectories[log_name][config_name] = list() trajectories[log_name][config_name].append({ "times_finished": sorted(times_finished), - "losses": list(zip(*sorted(zip(times_finished, performance_list))))[1], - "flipped": False + "values": list(zip(*sorted(zip(times_finished, performances[log_name]))))[1], + "losses": list(zip(*sorted(zip(times_finished, losses[log_name]))))[1] }) trajectory_loaders = {"csv": csv_trajectory_loader} diff --git a/autoPyTorch/utils/benchmarking/visualization_pipeline/get_ensemble_trajectories.py b/autoPyTorch/utils/benchmarking/visualization_pipeline/get_ensemble_trajectories.py index af8d5a047..4f1bbd440 100644 --- a/autoPyTorch/utils/benchmarking/visualization_pipeline/get_ensemble_trajectories.py +++ b/autoPyTorch/utils/benchmarking/visualization_pipeline/get_ensemble_trajectories.py @@ -2,8 +2,7 @@ from autoPyTorch.pipeline.base.pipeline_node import PipelineNode from autoPyTorch.utils.config.config_option import ConfigOption, to_bool, to_list from autoPyTorch.utils.benchmarking.benchmark_pipeline.prepare_result_folder import get_run_result_dir -from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector -from autoPyTorch.pipeline.nodes import OneHotEncoding +from autoPyTorch.pipeline.nodes import OneHotEncoding, MetricSelector from autoPyTorch.pipeline.nodes.ensemble import read_ensemble_prediction_file from hpbandster.core.result import logged_results_to_HBS_result from copy import copy @@ -15,26 +14,27 @@ class GetEnsembleTrajectories(PipelineNode): - def fit(self, pipeline_config, run_result_dir, train_metric, trajectories): + def fit(self, pipeline_config, autonet, run_result_dir, optimize_metric, trajectories): ensemble_log_file = os.path.join(run_result_dir, "ensemble_log.json") test_log_file = os.path.join(run_result_dir, "test_result.json") - if not pipeline_config["enable_ensemble"] or train_metric is None or \ + if not pipeline_config["enable_ensemble"] or optimize_metric is None or \ (not os.path.exists(ensemble_log_file) and not os.path.exists(test_log_file)): - return {"trajectories": trajectories, "train_metric": train_metric} + return {"trajectories": trajectories, "optimize_metric": optimize_metric} try: started = logged_results_to_HBS_result(run_result_dir).HB_config["time_ref"] except: - return {"trajectories": trajectories, "train_metric": train_metric} + return {"trajectories": trajectories, "optimize_metric": optimize_metric} + metrics = autonet.pipeline[MetricSelector.get_name()].metrics ensemble_trajectories = dict() test_trajectories = dict() if os.path.exists(ensemble_log_file): - ensemble_trajectories = get_ensemble_trajectories(ensemble_log_file, started) + ensemble_trajectories = get_ensemble_trajectories(ensemble_log_file, started, metrics) if os.path.exists(test_log_file): - test_trajectories = get_ensemble_trajectories(test_log_file, started, prefix="", only_test=True) + test_trajectories = get_ensemble_trajectories(test_log_file, started, metrics, prefix="", only_test=True) - return {"trajectories": dict(trajectories, **ensemble_trajectories, **test_trajectories), "train_metric": train_metric} + return {"trajectories": dict(trajectories, **ensemble_trajectories, **test_trajectories), "optimize_metric": optimize_metric} def get_pipeline_config_options(self): options = [ @@ -42,7 +42,7 @@ def get_pipeline_config_options(self): ] return options -def get_ensemble_trajectories(ensemble_log_file, started, prefix="ensemble_", only_test=False): +def get_ensemble_trajectories(ensemble_log_file, started, metrics, prefix="ensemble_", only_test=False): ensemble_trajectories = dict() with open(ensemble_log_file) as f: for line in f: @@ -50,13 +50,21 @@ def get_ensemble_trajectories(ensemble_log_file, started, prefix="ensemble_", on finished = finished["finished"] if isinstance(finished, dict) else finished for metric_name, metric_value in metric_values.items(): - if only_test and not metric_name.startswith("test"): + if only_test and not metric_name.startswith("test_"): continue trajectory_name = prefix + metric_name + metric_obj = metrics[metric_name[5:]] if metric_name.startswith("test_") else metrics[metric_name] # save in trajectory if trajectory_name not in ensemble_trajectories: - ensemble_trajectories[trajectory_name] = {"times_finished": [], "losses": [], "flipped": False} + ensemble_trajectories[trajectory_name] = {"times_finished": [], "losses": [], "values": []} ensemble_trajectories[trajectory_name]["times_finished"].append(finished - started) - ensemble_trajectories[trajectory_name]["losses"].append(metric_value) - return ensemble_trajectories \ No newline at end of file + ensemble_trajectories[trajectory_name]["losses"].append(metric_obj.loss_transform(metric_value)) + ensemble_trajectories[trajectory_name]["values"].append(metric_value) + + for name, trajectory in ensemble_trajectories.items(): + for key, value_list in trajectory.items(): + if not isinstance(value_list, (list, tuple)): + continue + trajectory[key] = [value_list[0] if key != "times_finished" else 0] + value_list + return ensemble_trajectories diff --git a/autoPyTorch/utils/benchmarking/visualization_pipeline/get_run_trajectories.py b/autoPyTorch/utils/benchmarking/visualization_pipeline/get_run_trajectories.py index 686104c45..c3fae0c7d 100644 --- a/autoPyTorch/utils/benchmarking/visualization_pipeline/get_run_trajectories.py +++ b/autoPyTorch/utils/benchmarking/visualization_pipeline/get_run_trajectories.py @@ -2,6 +2,7 @@ from autoPyTorch.pipeline.base.pipeline_node import PipelineNode from autoPyTorch.utils.config.config_option import ConfigOption, to_bool from autoPyTorch.utils.benchmarking.benchmark_pipeline.prepare_result_folder import get_run_result_dir +from autoPyTorch.pipeline.nodes import MetricSelector, LogFunctionsSelector from copy import copy import os import logging @@ -11,16 +12,18 @@ class GetRunTrajectories(PipelineNode): def fit(self, pipeline_config, autonet, run_result_dir): parser = autonet.get_autonet_config_file_parser() autonet_config = parser.read(os.path.join(run_result_dir, "autonet.config")) + metrics = autonet.pipeline[MetricSelector.get_name()].metrics + log_functions = autonet.pipeline[LogFunctionsSelector.get_name()].log_functions if pipeline_config["only_finished_runs"] and not os.path.exists(os.path.join(run_result_dir, "summary.json")): logging.getLogger('benchmark').info('Skipping ' + run_result_dir + ' because the run is not finished yet') - return {"trajectories": dict(), "train_metric": None} + return {"trajectories": dict(), "optimize_metric": None} - trajectories = build_run_trajectories(run_result_dir, autonet_config) + trajectories = build_run_trajectories(run_result_dir, autonet_config, metrics, log_functions) if "test_result" in trajectories: - trajectories["test_%s" % autonet_config["train_metric"]] = trajectories["test_result"] + trajectories["test_%s" % autonet_config["optimize_metric"]] = trajectories["test_result"] return {"trajectories": trajectories, - "train_metric": autonet_config["train_metric"]} + "optimize_metric": autonet_config["optimize_metric"]} def get_pipeline_config_options(self): options = [ @@ -30,7 +33,7 @@ def get_pipeline_config_options(self): return options -def build_run_trajectories(results_folder, autonet_config): +def build_run_trajectories(results_folder, autonet_config, metrics, log_functions): # parse results try: res = logged_results_to_HBS_result(results_folder) @@ -40,31 +43,30 @@ def build_run_trajectories(results_folder, autonet_config): return dict() # prepare - metric_name = autonet_config["train_metric"] + metric_name = autonet_config["optimize_metric"] all_metrics = autonet_config["additional_metrics"] + [metric_name] - additional_metric_names = ["val_" + m for m in all_metrics] - additional_metric_names += ["train_" + m for m in all_metrics] - additional_metric_names += autonet_config["additional_logs"] + additional_metric_names = [("val_" + m, metrics[m]) for m in all_metrics] + additional_metric_names += [("train_" + m, metrics[m]) for m in all_metrics] + additional_metric_names += [(l, log_functions[l]) for l in autonet_config["additional_logs"]] # initialize incumbent trajectories incumbent_trajectories = dict() # save incumbent trajectories - incumbent_trajectories[metric_name] = incumbent_trajectory - incumbent_trajectory["flipped"] = not autonet_config["minimize"] - for name in additional_metric_names: + for name, obj in additional_metric_names: tj = copy(incumbent_trajectory) log_available = [name in run["info"] for config_id, budget in zip(tj["config_ids"], tj["budgets"]) for run in res.get_runs_by_id(config_id) if run["budget"] == budget] - tj["losses"] = [run["info"][name] for config_id, budget in zip(tj["config_ids"], tj["budgets"]) + tj["values"] = [run["info"][name] for config_id, budget in zip(tj["config_ids"], tj["budgets"]) for run in res.get_runs_by_id(config_id) if run["budget"] == budget and name in run["info"]] + tj["losses"] = [obj.loss_transform(x) for x in tj["values"]] + for key, value_list in tj.items(): - if key in ["losses", "flipped"]: + if key in ["losses"]: continue tj[key] = [value for i, value in enumerate(value_list) if log_available[i]] - tj["flipped"] = False if tj["losses"]: incumbent_trajectories[name] = tj @@ -75,4 +77,4 @@ def build_run_trajectories(results_folder, autonet_config): continue trajectory[key] = [value_list[0] if key != "times_finished" else 0] + value_list - return incumbent_trajectories \ No newline at end of file + return incumbent_trajectories diff --git a/autoPyTorch/utils/benchmarking/visualization_pipeline/plot_summary.py b/autoPyTorch/utils/benchmarking/visualization_pipeline/plot_summary.py index 70b5f7db7..8eb7e6f2d 100644 --- a/autoPyTorch/utils/benchmarking/visualization_pipeline/plot_summary.py +++ b/autoPyTorch/utils/benchmarking/visualization_pipeline/plot_summary.py @@ -8,11 +8,12 @@ import heapq class PlotSummary(PipelineNode): - def fit(self, pipeline_config, trajectories, train_metrics): + def fit(self, pipeline_config, trajectories, optimize_metrics): if not pipeline_config["skip_ranking_plot"]: - plot(pipeline_config, trajectories, train_metrics, "ranking", process_summary) + plot(dict(pipeline_config, plot_type="losses", y_scale="linear"), trajectories, optimize_metrics, "ranking", process_summary) if not pipeline_config["skip_average_plot"]: - plot(pipeline_config, trajectories, train_metrics, "average", trajectory_sampling) + plot(dict(pipeline_config, scale_uncertainty=0), trajectories, optimize_metrics, "average", process_summary) + plot(pipeline_config, trajectories, optimize_metrics, "sampled_average", trajectory_sampling) return dict() def get_pipeline_config_options(self): @@ -27,7 +28,7 @@ def get_ranking_plot_values(values, names, agglomeration): """ values = instance_name --> [((key=prefix + metric), value), ...] """ keys = {instance: set([key for key, _ in v]) for instance, v in values.items()} values = {instance: [(key, agglomeration([value for k, value in v if k == key])) for key in keys[instance]] for instance, v in values.items()} - sorted_values = {instance: sorted(map(lambda x: x[1], v), reverse=True) for instance, v in values.items()} # configs sorted by value + sorted_values = {instance: sorted(map(lambda x: x[1], v)) for instance, v in values.items()} # configs sorted by value ranks = {instance: {n: [sorted_values[instance].index(value) + 1 for config_name, value in v if config_name == n] for n in names} for instance, v in values.items()} ranks = to_dict([(n, r) for rank_dict in ranks.values() for n, r in rank_dict.items()]) @@ -51,7 +52,7 @@ def get_average_plot_values(values, names, agglomeration): "average": get_average_plot_values } -def process_summary(instance_name, metric_name, prefixes, trajectories, agglomeration, scale_uncertainty, cmap): +def process_summary(instance_name, metric_name, prefixes, trajectories, plot_type, agglomeration, scale_uncertainty, value_multiplier, cmap): assert instance_name in get_plot_values_funcs.keys() trajectory_names_to_prefix = {(("%s_%s" % (prefix, metric_name)) if prefix else metric_name): prefix for prefix in prefixes} @@ -89,12 +90,12 @@ def process_summary(instance_name, metric_name, prefixes, trajectories, agglomer # update trajectory values and pointers current_trajectory = trajectories[current_name][current_config][current_instance][trajectory_id] current_pointer = trajectory_pointers[(current_config, current_name)][current_instance][trajectory_id] - current_value = current_trajectory["losses"][current_pointer] + current_value = current_trajectory[plot_type][current_pointer] - trajectory_values[(current_config, current_name)][current_instance][trajectory_id] = current_value * (-1 if current_trajectory["flipped"] else 1) + trajectory_values[(current_config, current_name)][current_instance][trajectory_id] = current_value trajectory_pointers[(current_config, current_name)][current_instance][trajectory_id] += 1 - if trajectory_pointers[(current_config, current_name)][current_instance][trajectory_id] < len(current_trajectory["losses"]): + if trajectory_pointers[(current_config, current_name)][current_instance][trajectory_id] < len(current_trajectory[plot_type]): heapq.heappush(heap, (current_trajectory["times_finished"][trajectory_pointers[(current_config, current_name)][current_instance][trajectory_id]], current_config, current_name, current_instance, trajectory_id)) @@ -107,7 +108,7 @@ def process_summary(instance_name, metric_name, prefixes, trajectories, agglomer [x[k].pop() for x in [center, upper, lower] for k in x.keys()] # calculate ranks - values = to_dict([(instance, (config, name), value) + values = to_dict([(instance, (config, name), value * value_multiplier) for (config, name), instance_values in trajectory_values.items() for instance, values in instance_values.items() for value in values if value is not None]) @@ -157,7 +158,7 @@ def to_dict(tuple_list): return result -def trajectory_sampling(instance_name, metric_name, prefixes, trajectories, agglomeration, scale_uncertainty, cmap, num_samples=1000): +def trajectory_sampling(instance_name, metric_name, prefixes, trajectories, plot_type, agglomeration, scale_uncertainty, value_multiplier, cmap, num_samples=1000): averaged_trajectories = dict() # sample #num_samples average trajectories @@ -185,8 +186,10 @@ def trajectory_sampling(instance_name, metric_name, prefixes, trajectories, aggl metric_name=metric_name, prefixes=prefixes, trajectories=sampled_trajectories, + plot_type=plot_type, agglomeration=agglomeration, scale_uncertainty=0, + value_multiplier=value_multiplier, cmap=cmap ) @@ -205,8 +208,7 @@ def trajectory_sampling(instance_name, metric_name, prefixes, trajectories, aggl averaged_trajectories[trajectory_name][config].append({ "times_finished": d["finishing_times"], - "losses": d["center"], - "flipped": False + plot_type: d["center"], }) # compute mean and stddev over the averaged trajectories @@ -215,7 +217,9 @@ def trajectory_sampling(instance_name, metric_name, prefixes, trajectories, aggl metric_name=metric_name, prefixes=prefixes, trajectories=averaged_trajectories, + plot_type=plot_type, agglomeration="mean", scale_uncertainty=scale_uncertainty, + value_multiplier=1, cmap=cmap ) diff --git a/autoPyTorch/utils/benchmarking/visualization_pipeline/plot_trajectories.py b/autoPyTorch/utils/benchmarking/visualization_pipeline/plot_trajectories.py index 910742ac5..90a9f401e 100644 --- a/autoPyTorch/utils/benchmarking/visualization_pipeline/plot_trajectories.py +++ b/autoPyTorch/utils/benchmarking/visualization_pipeline/plot_trajectories.py @@ -8,10 +8,10 @@ class PlotTrajectories(PipelineNode): - def fit(self, pipeline_config, trajectories, train_metrics, instance): + def fit(self, pipeline_config, trajectories, optimize_metrics, instance): if not pipeline_config["skip_dataset_plots"]: - plot(pipeline_config, trajectories, train_metrics, instance, process_trajectory) - return {"trajectories": trajectories, "train_metrics": train_metrics} + plot(pipeline_config, trajectories, optimize_metrics, instance, process_trajectory) + return {"trajectories": trajectories, "optimize_metrics": optimize_metrics} def get_pipeline_config_options(self): @@ -26,22 +26,24 @@ def get_pipeline_config_options(self): ConfigOption('skip_dataset_plots', default=False, type=to_bool), ConfigOption('plot_markers', default=False, type=to_bool), ConfigOption('plot_individual', default=False, type=to_bool), + ConfigOption('plot_type', default="values", type=str, choices=["values", "losses"]), ConfigOption('xscale', default='log', type=str), ConfigOption('yscale', default='linear', type=str), ConfigOption('xmin', default=None, type=float), ConfigOption('xmax', default=None, type=float), ConfigOption('ymin', default=None, type=float), - ConfigOption('ymax', default=None, type=float) + ConfigOption('ymax', default=None, type=float), + ConfigOption('value_multiplier', default=1, type=float) ] return options -def plot(pipeline_config, trajectories, train_metrics, instance, process_fnc): +def plot(pipeline_config, trajectories, optimize_metrics, instance, process_fnc): import matplotlib.pyplot as plt from matplotlib.backends.backend_pdf import PdfPages extension = "pdf" - plot_logs = pipeline_config['plot_logs'] or train_metrics + plot_logs = pipeline_config['plot_logs'] or optimize_metrics output_folder = pipeline_config['output_folder'] instance_name = os.path.basename(instance).split(".")[0] @@ -62,8 +64,10 @@ def plot(pipeline_config, trajectories, train_metrics, instance, process_fnc): metric_name=metric_name, prefixes=pipeline_config["prefixes"], trajectories=trajectories, + plot_type=pipeline_config["plot_type"], agglomeration=pipeline_config["agglomeration"], scale_uncertainty=pipeline_config['scale_uncertainty'], + value_multiplier=pipeline_config['value_multiplier'], cmap=plt.get_cmap('jet')) if plot_empty: logging.getLogger('benchmark').warn('Not showing empty plot for ' + instance) @@ -77,7 +81,8 @@ def plot(pipeline_config, trajectories, train_metrics, instance, process_fnc): do_label_rename=pipeline_config['label_rename'], plt=plt, plot_individual=pipeline_config["plot_individual"], - plot_markers=pipeline_config["plot_markers"]) + plot_markers=pipeline_config["plot_markers"], + plot_type=pipeline_config["plot_type"]) plt.xscale(pipeline_config["xscale"]) plt.yscale(pipeline_config["yscale"]) @@ -95,7 +100,7 @@ def plot(pipeline_config, trajectories, train_metrics, instance, process_fnc): plt.close(figure) -def process_trajectory(instance_name, metric_name, prefixes, trajectories, agglomeration, scale_uncertainty, cmap): +def process_trajectory(instance_name, metric_name, prefixes, trajectories, plot_type, agglomeration, scale_uncertainty, value_multiplier, cmap): # iterate over the incumbent trajectories of the different runs linestyles = ['-', '--', '-.', ':'] plot_empty = True @@ -134,11 +139,11 @@ def process_trajectory(instance_name, metric_name, prefixes, trajectories, agglo current_trajectory = trajectory[trajectory_id] # update trajectory values and pointers - trajectory_values[trajectory_id] = current_trajectory["losses"][trajectory_pointers[trajectory_id]] + trajectory_values[trajectory_id] = current_trajectory[plot_type][trajectory_pointers[trajectory_id]] individual_trajectories[trajectory_id].append(trajectory_values[trajectory_id]) individual_times_finished[trajectory_id].append(times_finished) trajectory_pointers[trajectory_id] += 1 - if trajectory_pointers[trajectory_id] < len(current_trajectory["losses"]): + if trajectory_pointers[trajectory_id] < len(current_trajectory[plot_type]): heapq.heappush(heap, (trajectory[trajectory_id]["times_finished"][trajectory_pointers[trajectory_id]], trajectory_id) ) @@ -151,7 +156,7 @@ def process_trajectory(instance_name, metric_name, prefixes, trajectories, agglo continue if finishing_times and np.isclose(times_finished, finishing_times[-1]): [x.pop() for x in [center, upper, lower, finishing_times]] - values = [v * (-1 if current_trajectory["flipped"] else 1) for v in trajectory_values if v is not None] + values = [v * value_multiplier for v in trajectory_values if v is not None] if agglomeration == "median": center.append(np.median(values)) lower.append(np.percentile(values, int(50 - scale_uncertainty * 25))) @@ -176,7 +181,7 @@ def process_trajectory(instance_name, metric_name, prefixes, trajectories, agglo } return plot_empty, plot_data -def plot_trajectory(plot_data, instance_name, metric_name, font_size, do_label_rename, plt, plot_individual, plot_markers): +def plot_trajectory(plot_data, instance_name, metric_name, font_size, do_label_rename, plt, plot_individual, plot_markers, plot_type): for label, d in plot_data.items(): if do_label_rename: @@ -189,7 +194,7 @@ def plot_trajectory(plot_data, instance_name, metric_name, font_size, do_label_r plt.step(d["finishing_times"], d["center"], color=d["color"], label=label, where='post', linestyle=d["linestyle"], marker="o" if plot_markers else None) plt.fill_between(d["finishing_times"], d["lower"], d["upper"], step="post", color=[(d["color"][0], d["color"][1], d["color"][2], 0.5)]) plt.xlabel('wall clock time [s]', fontsize=font_size) - plt.ylabel('incumbent ' + metric_name, fontsize=font_size) + plt.ylabel('incumbent %s %s' % (metric_name, plot_type), fontsize=font_size) plt.legend(loc='best', prop={'size': font_size}) plt.title(instance_name, fontsize=font_size) diff --git a/autoPyTorch/utils/config/config_file_parser.py b/autoPyTorch/utils/config/config_file_parser.py index 0c685b224..e7d0acb59 100644 --- a/autoPyTorch/utils/config/config_file_parser.py +++ b/autoPyTorch/utils/config/config_file_parser.py @@ -39,7 +39,7 @@ def read_key_values_from_file(filename, delimiter='='): key_values[key] = value return key_values - def read(self, filename, key_values_dict=None): + def read(self, filename, key_values_dict=None, silent=False): """ Read a config file. @@ -58,6 +58,8 @@ def read(self, filename, key_values_dict=None): # open the config file for key, value in key_values_dict.items(): if (key not in self.config_options): + if silent: + continue raise ValueError("Config key '" + key + "' is not a valid autonet config option") option = self.config_options[key] @@ -204,4 +206,4 @@ def get_autonet_home(): """ Get the home directory of autonet """ if "AUTONET_HOME" in os.environ: return os.environ["AUTONET_HOME"] - return os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) \ No newline at end of file + return os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) diff --git a/autoPyTorch/utils/config_space_hyperparameter.py b/autoPyTorch/utils/config_space_hyperparameter.py index 218b74aa0..e3e0b115b 100644 --- a/autoPyTorch/utils/config_space_hyperparameter.py +++ b/autoPyTorch/utils/config_space_hyperparameter.py @@ -2,8 +2,7 @@ import ConfigSpace.hyperparameters as CSH -def get_hyperparameter(hyper_type, name, value_range): - log = False +def get_hyperparameter(hyper_type, name, value_range, log = False): if isinstance(value_range, tuple) and len(value_range) == 2 and isinstance(value_range[1], bool) and \ isinstance(value_range[0], (tuple, list)): value_range, log = value_range @@ -24,5 +23,5 @@ def get_hyperparameter(hyper_type, name, value_range): return CSH.UniformIntegerHyperparameter(name, lower=value_range[0], upper=value_range[1], log=log) raise ValueError('Unknown type: %s for hp %s' % (hyper_type, name) ) -def add_hyperparameter(cs, hyper_type, name, value_range): - return cs.add_hyperparameter(get_hyperparameter(hyper_type, name, value_range)) \ No newline at end of file +def add_hyperparameter(cs, hyper_type, name, value_range, log=False): + return cs.add_hyperparameter(get_hyperparameter(hyper_type, name, value_range, log)) diff --git a/autoPyTorch/utils/configspace_wrapper.py b/autoPyTorch/utils/configspace_wrapper.py index e09e73327..fc8561f2a 100644 --- a/autoPyTorch/utils/configspace_wrapper.py +++ b/autoPyTorch/utils/configspace_wrapper.py @@ -24,6 +24,12 @@ def __getitem__(self, key): pprint.pprint(self.config) return self.config[self.config_prefix + key] + def __iter__(self): + for k in self.config.__iter__(): + if not k.startswith(self.config_prefix): + continue + yield k[len(self.config_prefix):] + def __str__(self): return str(self.config) diff --git a/autoPyTorch/utils/ensemble.py b/autoPyTorch/utils/ensemble.py index da6842d5f..7ae7fe9fc 100644 --- a/autoPyTorch/utils/ensemble.py +++ b/autoPyTorch/utils/ensemble.py @@ -11,11 +11,11 @@ import logging from autoPyTorch.components.ensembles.ensemble_selection import EnsembleSelection -def build_ensemble(result, train_metric, minimize, +def build_ensemble(result, optimize_metric, ensemble_size, all_predictions, labels, model_identifiers, only_consider_n_best=0, sorted_initialization_n_best=0): id2config = result.get_id2config_mapping() - ensemble_selection = EnsembleSelection(ensemble_size, train_metric, minimize, + ensemble_selection = EnsembleSelection(ensemble_size, optimize_metric, only_consider_n_best=only_consider_n_best, sorted_initialization_n_best=sorted_initialization_n_best) # fit ensemble @@ -48,10 +48,6 @@ def read_ensemble_prediction_file(filename, y_transform): return all_predictions, labels, model_identifiers, all_timestamps -def predictions_for_ensemble(y_pred, y_true): - return y_pred - - class test_predictions_for_ensemble(): def __init__(self, autonet, X_test, Y_test): self.autonet = autonet diff --git a/autoPyTorch/utils/hyperparameter_search_space_update.py b/autoPyTorch/utils/hyperparameter_search_space_update.py index bfceaf0a2..52d670945 100644 --- a/autoPyTorch/utils/hyperparameter_search_space_update.py +++ b/autoPyTorch/utils/hyperparameter_search_space_update.py @@ -1,6 +1,7 @@ import ast import os + class HyperparameterSearchSpaceUpdate(): def __init__(self, node_name, hyperparameter, value_range, log=False): self.node_name = node_name diff --git a/autoPyTorch/utils/loggers.py b/autoPyTorch/utils/loggers.py new file mode 100644 index 000000000..b4ed1e65f --- /dev/null +++ b/autoPyTorch/utils/loggers.py @@ -0,0 +1,219 @@ +import time, os, shutil +from hpbandster.core.result import json_result_logger + +class bohb_logger(json_result_logger): + def __init__(self, constant_hyperparameter, directory, overwrite=False): + super(bohb_logger, self).__init__(directory, overwrite) + self.constants = constant_hyperparameter + + + def new_config(self, config_id, config, config_info): + import json + if not config_id in self.config_ids: + self.config_ids.add(config_id) + + full_config = dict() + full_config.update(self.constants) + full_config.update(config) + + with open(self.config_fn, 'a') as fh: + fh.write(json.dumps([config_id, full_config, config_info])) + fh.write('\n') + + +class tensorboard_logger(object): + def __init__(self, pipeline_config, constant_hyperparameter, global_results_dir): + self.start_time = time.time() + + b = pipeline_config['max_budget'] + budgets = [] + while b >= pipeline_config['min_budget']: + budgets.append(int(b)) + b /= pipeline_config['eta'] + + self.incumbent_results = {b: 0 for b in budgets} + self.mean_results = {b: [0, 0] for b in budgets} + + self.constants = constant_hyperparameter + self.results_logged = 0 + self.seed = pipeline_config['random_seed'] + self.max_budget = pipeline_config['max_budget'] + self.global_results_dir = global_results_dir + + self.keep_only_incumbent_checkpoints = pipeline_config['keep_only_incumbent_checkpoints'] + + self.incumbent_configs_dir = os.path.join(pipeline_config['result_logger_dir'], 'incumbents') + self.status_dir = pipeline_config['result_logger_dir'] + self.run_name = '-'.join(pipeline_config['run_id'].split('-')[1:]) + os.makedirs(self.incumbent_configs_dir, exist_ok=True) + + + def new_config(self, config_id, config, config_info): + pass + + def __call__(self, job): + import json + import tensorboard_logger as tl + + id = job.id + budget = int(job.kwargs['budget']) + config = job.kwargs['config'] + # timestamps = job.timestamps + result = job.result + # exception = job.exception + + if result is None: + return + + self.results_logged += 1 + + tl.log_value('BOHB/all_results', result['loss'] * -1, self.results_logged) + + if budget not in self.incumbent_results or result['loss'] < self.incumbent_results[budget]: + self.incumbent_results[budget] = result['loss'] + + full_config = dict() + full_config.update(self.constants) + full_config.update(config) + + refit_config = dict() + refit_config['budget'] = budget + refit_config['seed'] = self.seed + + refit_config['incumbent_config_path'] = os.path.join(self.incumbent_configs_dir, 'config_' + str(budget) + '.json') + with open(refit_config['incumbent_config_path'], 'w+') as f: + f.write(json.dumps(full_config, indent=4, sort_keys=True)) + + with open(os.path.join(self.incumbent_configs_dir, 'result_' + str(budget) + '.json'), 'w+') as f: + f.write(json.dumps([job.id, job.kwargs['budget'], job.timestamps, job.result, job.exception], indent=4, sort_keys=True)) + + checkpoints, refit_config['dataset_order'] = get_checkpoints(result['info']) or ([],None) + refit_config['incumbent_checkpoint_paths'] = [] + for i, checkpoint in enumerate(checkpoints): + dest = os.path.join(self.incumbent_configs_dir, 'checkpoint_' + str(budget) + '_' + str(i) + '.pt' if len(checkpoints) > 1 else 'checkpoint_' + str(budget) + '.pt') + if os.path.exists(dest): + os.remove(dest) + if self.keep_only_incumbent_checkpoints: + shutil.move(checkpoint, dest) + else: + shutil.copy(checkpoint, dest) + refit_config['incumbent_checkpoint_paths'].append(dest) + + refit_path = os.path.join(self.incumbent_configs_dir, 'refit_config_' + str(budget) + '.json') + with open(refit_path, 'w+') as f: + f.write(json.dumps(refit_config, indent=4, sort_keys=True)) + + if budget >= self.max_budget and self.global_results_dir is not None: + import autoPyTorch.utils.thread_read_write as thread_read_write + import datetime + + dataset_names = sorted([os.path.splitext(os.path.split(info['dataset_path'])[1])[0] for info in result['info']]) + suffix = '' + if len(result['info']) > 1: + suffix += '+[' + ', '.join(dataset_names) + ']' + if budget > self.max_budget: + suffix += '+Refit' + + for info in result['info']: + thread_read_write.update_results(self.global_results_dir, { + 'name': os.path.splitext(os.path.split(info['dataset_path'])[1])[0] + suffix, + 'result': round(info['val_top1'], 2), + 'seed': self.seed, + 'refit_config': refit_path, + 'text': "{0}/{1} -- {2}".format( + round(info['val_datapoints'] * (info['val_top1'] / 100)), + info['val_datapoints'], + round(budget / len(result['info']))) + }) + + if self.keep_only_incumbent_checkpoints and get_checkpoints(result['info']): + for checkpoint in get_checkpoints(result['info'])[0]: + if os.path.exists(checkpoint): + os.remove(checkpoint) + + if budget not in self.mean_results: + self.mean_results[budget] = [result['loss'], 1] + else: + self.mean_results[budget][0] += result['loss'] + self.mean_results[budget][1] += 1 + + for b, loss in self.incumbent_results.items(): + tl.log_value('BOHB/incumbent_results_' + str(b), loss * -1, self.mean_results[b][1]) + + for b, (loss, n) in self.mean_results.items(): + tl.log_value('BOHB/mean_results_' + str(b), loss * -1 / n if n > 0 else 0, n) + + status = dict() + for b, loss in self.incumbent_results.items(): + budget_status = dict() + budget_status['incumbent'] = loss * -1 + mean_res = self.mean_results[b] + budget_status['mean'] = mean_res[0] / mean_res[1] * -1 if mean_res[1] > 0 else 0 + budget_status['configs'] = mean_res[1] + status['budget: ' + str(b)] = budget_status + + import datetime + status["runtime"] = str(datetime.timedelta(seconds=time.time() - self.start_time)) + + with open(os.path.join(self.status_dir, 'bohb_status.json'), 'w+') as f: + f.write(json.dumps(status, indent=4, sort_keys=True)) + + +def get_checkpoints(info): + if not isinstance(info, list): + if 'checkpoint' in info: + return [info['checkpoint']] + return [] + + checkpoints = [] + dataset_order = [] + for subinfo in info: + if 'checkpoint' in subinfo: + checkpoints.append(subinfo['checkpoint']) + dataset_order.append(subinfo['dataset_id']) + return checkpoints, dataset_order + +class combined_logger(object): + def __init__(self, *loggers): + self.loggers = loggers + + def new_config(self, config_id, config, config_info): + for logger in self.loggers: + logger.new_config(config_id, config, config_info) + + def __call__(self, job): + for logger in self.loggers: + logger(job) + +def get_incumbents(directory): + + incumbents = os.path.join(directory, 'incumbents') + + if not os.path.exists(incumbents): + return None + + import re + file_re = [ + re.compile('config_([0-9]+).json'), + re.compile('refit_config_([0-9]+).json'), + re.compile('result_([0-9]+).json'), + re.compile('checkpoint_([0-9]+).*.pt'), + ] + + incumbent_files = [[] for _ in range(len(file_re))] + for filename in sorted(os.listdir(incumbents)): + for i, reg in enumerate(file_re): + match = reg.match(filename) + + if match: + budget = int(match.group(1)) + inc_file = os.path.join(incumbents, filename) + incumbent_files[i].append([budget, inc_file]) + + return incumbent_files + + +def get_refit_config(directory): + _, refit_configs, _, _ = get_incumbents(directory) + refit_config = max(refit_configs, key=lambda x: x[0]) #get config of max budget + return refit_config[1] diff --git a/autoPyTorch/utils/modify_config_space.py b/autoPyTorch/utils/modify_config_space.py new file mode 100644 index 000000000..a12335bc5 --- /dev/null +++ b/autoPyTorch/utils/modify_config_space.py @@ -0,0 +1,242 @@ +import ConfigSpace as CS +import ConfigSpace.hyperparameters as CSH +import copy + +def remove_constant_hyperparameter(cs): + constants = dict() + + hyperparameter_to_add = [] + for hyper in cs.get_hyperparameters(): + const, value = is_constant(hyper) + if const: + constants[hyper.name] = value + else: + hyperparameter_to_add.append(copy.copy(hyper)) + + for name in constants: + truncate_hyperparameter(cs, cs.get_hyperparameter(name)) + + cs._hyperparameter_idx = dict() + cs._idx_to_hyperparameter = dict() + cs._sort_hyperparameters() + cs._update_cache() + + return cs, constants + + +def is_constant(hyper): + if isinstance(hyper, CSH.Constant): + return True, hyper.value + + elif isinstance(hyper, CSH.UniformFloatHyperparameter) or isinstance(hyper, CSH.UniformIntegerHyperparameter): + if abs(hyper.upper - hyper.lower) < 1e-10: + return True, hyper.lower + + elif isinstance(hyper, CSH.CategoricalHyperparameter): + if len(hyper.choices) == 1: + return True, hyper.choices[0] + + return False, None + + +def override_hyperparameter(config_space, hyper): + import ConfigSpace.conditions as CSC + + for condition in config_space._children[hyper.name].values(): + subconditions = condition.components if isinstance(condition, CSC.AbstractConjunction) else [condition] + for subcondition in subconditions: + if subcondition.parent.name == hyper.name: + subcondition.parent = hyper + + for condition in config_space._parents[hyper.name].values(): + if condition is None: + continue # root + subconditions = condition.components if isinstance(condition, CSC.AbstractConjunction) else [condition] + for subcondition in subconditions: + if subcondition.child.name == hyper.name: + subcondition.child = hyper + + config_space._hyperparameters[hyper.name] = hyper + + +def update_conditions(config_space, parent): + import ConfigSpace.conditions as CSC + + if parent.name not in config_space._hyperparameters: + # already removed -> all condition already updated + return + + possible_values, is_value_range = get_hyperparameter_values(parent) + children = [config_space.get_hyperparameter(name) for name in config_space._children[parent.name]] + + for child in children: + if child.name not in config_space._children[parent.name]: + # already cut + continue + condition = config_space._children[parent.name][child.name] + + if isinstance(condition, CSC.AbstractConjunction): + is_and = isinstance(condition, CSC.AndConjunction) + state = 2 + + new_subconditions = [] + for subcondition in condition.components: + if subcondition.parent.name != parent.name: + new_subconditions.append(subcondition) + continue + substate = get_condition_state(subcondition, possible_values, is_value_range) + if substate == 0 and is_and and state == 2: + state = 0 + + if substate == 1 and not is_and and state == 2: + state = 1 + + if substate == 2: + new_subconditions.append(subcondition) + + else: + # condition is not relevant anymore + del config_space._children[parent.name][child.name] + del config_space._parents[child.name][parent.name] + for grand_parent, cond in config_space._parents[parent.name].items(): + if cond is None: + continue + cond_type = type(cond) + values, _ = get_hyperparameter_values(cond.parent) + # fake parent value first as it might be invalid atm and gets truncated later + new_condition = cond_type(child, cond.parent, values[0]) + new_condition.value = cond.value + config_space._children[grand_parent][child.name] = new_condition + config_space._parents[child.name][grand_parent] = new_condition + + if len(new_subconditions) == 0: + state = 1 if is_and else 0 # either everything was false or true + + if state == 2: + + if len(new_subconditions) == 1: + condition = new_subconditions[0] + config_space._children[condition.parent.name][child.name] = new_subconditions[0] + config_space._parents[child.name][condition.parent.name] = new_subconditions[0] + else: + condition.__init__(*new_subconditions) + + for subcondition in new_subconditions: + config_space._children[subcondition.parent.name][child.name] = condition + config_space._parents[child.name][subcondition.parent.name] = condition + + else: + state = get_condition_state(condition, possible_values, is_value_range) + + if state == 1: + del config_space._children[parent.name][child.name] + del config_space._parents[child.name][parent.name] + + for grand_parent, cond in config_space._parents[parent.name].items(): + if cond is None: + continue + cond_type = type(cond) + values, _ = get_hyperparameter_values(cond.parent) + # fake parent value first as it might be invalid atm and gets truncated later + new_condition = cond_type(child, cond.parent, values[0]) + new_condition.value = cond.value + config_space._children[grand_parent][child.name] = new_condition + config_space._parents[child.name][grand_parent] = new_condition + + if len(config_space._parents[child.name]) == 0: + config_space._conditionals.remove(child.name) + if state == 0: + truncate_hyperparameter(config_space, child) + + + + +def truncate_hyperparameter(config_space, hyper): + if hyper.name not in config_space._hyperparameters: + return + + parent_names = list(config_space._parents[hyper.name].keys()) + for parent_name in parent_names: + del config_space._children[parent_name][hyper.name] + + del config_space._parents[hyper.name] + del config_space._hyperparameters[hyper.name] + + if hyper.name in config_space._conditionals: + config_space._conditionals.remove(hyper.name) + + child_names = list(config_space._children[hyper.name].keys()) + for child_name in child_names: + truncate_hyperparameter(config_space, config_space.get_hyperparameter(child_name)) + + +def get_condition_state(condition, possible_values, is_range): + """ + 0: always false + 1: always true + 2: true or false + """ + import ConfigSpace.conditions as CSC + + c_val = condition.value + if isinstance(condition, CSC.EqualsCondition): + if is_range: + if approx(possible_values[0], possible_values[1]): + return 1 if approx(possible_values[0], c_val) else 0 + return 2 if c_val >= possible_values[0] and c_val <= possible_values[1] else 0 + else: + if len(possible_values) == 1: + return 1 if c_val == possible_values[0] else 0 + return 2 if c_val in possible_values else 0 + + if isinstance(condition, CSC.NotEqualsCondition): + if is_range: + if approx(possible_values[0], possible_values[1]): + return 0 if approx(possible_values[0], c_val) else 1 + return 2 if c_val >= possible_values[0] and c_val <= possible_values[1] else 1 + else: + if len(possible_values) == 1: + return 0 if c_val == possible_values[0] else 1 + return 2 if c_val in possible_values else 1 + + if isinstance(condition, CSC.GreaterThanCondition): # is_range has to be true + if c_val < possible_values[0]: + return 1 + if c_val >= possible_values[1]: + return 0 + return 2 + + if isinstance(condition, CSC.LessThanCondition): # is_range has to be true + if c_val <= possible_values[0]: + return 0 + if c_val > possible_values[1]: + return 1 + return 2 + + if isinstance(condition, CSC.InCondition): + inter = set(possible_values).intersection(set(c_val)) + if len(inter) == len(possible_values): + return 1 + if len(inter) == 0: + return 0 + return 2 + + +def approx(x, y): + return abs(x - y) < 1e-10 + +def get_hyperparameter_values(hyper): + """Returns list[choices/range] and bool[is value range] + """ + import ConfigSpace.hyperparameters as CSH + + if isinstance(hyper, CSH.CategoricalHyperparameter): + return hyper.choices, False + + if isinstance(hyper, CSH.NumericalHyperparameter): + return [hyper.lower, hyper.upper], True + + if isinstance(hyper, CSH.Constant): + return [hyper.value, hyper.value], True + + raise ValueError(str(type(hyper)) + ' is not supported') diff --git a/autoPyTorch/utils/thread_read_write.py b/autoPyTorch/utils/thread_read_write.py new file mode 100644 index 000000000..75d75f874 --- /dev/null +++ b/autoPyTorch/utils/thread_read_write.py @@ -0,0 +1,42 @@ + +import fasteners, json, os, threading + +thread_lock = threading.Lock() + +def write(filename, content): + with open(filename, 'w+') as f: + f.write(content) + +def read(filename): + content = '{}' + if os.path.exists(filename): + with open(filename, 'r') as f: + content = f.read() + return content + +def append(filename, content): + with fasteners.InterProcessLock('{0}.lock'.format(filename)): + with open(filename, 'a+') as f: + f.write(content) + +def update_results_thread(filename, info): + thread_lock.acquire() + with fasteners.InterProcessLock('{0}.lock'.format(filename)): + content = json.loads(read(filename)) + name = info['name'] + result = info['result'] + refit_config = info['refit_config'] + text = info['text'] + seed = str(info['seed']) + + infos = content[name] if name in content else dict() + infos[seed] = {'result': result, 'description': text, 'refit': refit_config} + content[name] = infos + + write(filename, json.dumps(content, indent=4, sort_keys=True)) + thread_lock.release() + + +def update_results(filename, info): + thread = threading.Thread(target = update_results_thread, args = (filename, info)) + thread.start() \ No newline at end of file diff --git a/configs/autonet/automl/bohb_cv_sparse.txt b/configs/autonet/automl/bohb_cv_sparse.txt index 6a653a1e9..031776098 100644 --- a/configs/autonet/automl/bohb_cv_sparse.txt +++ b/configs/autonet/automl/bohb_cv_sparse.txt @@ -3,10 +3,9 @@ min_budget=74 min_workers=4 max_runtime=92400 budget_type=time -train_metric=accuracy +optimize_metric=accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} -min_budget_for_cv=250 embeddings=[none] lr_scheduler=[cosine_annealing,plateau] memory_limit_mb=7000 diff --git a/configs/autonet/automl/cifar_example.txt b/configs/autonet/automl/cifar_example.txt new file mode 100644 index 000000000..39121280b --- /dev/null +++ b/configs/autonet/automl/cifar_example.txt @@ -0,0 +1,15 @@ +max_budget=1500 +min_budget=300 +min_workers=2 +max_runtime=3600 +budget_type=time +default_dataset_download_dir=./datasets/ +images_root_folders=./datasets/ +optimize_metric=accuracy +validation_split=0.1 +use_tensorboard_logger=True +networks=['resnet'] +lr_scheduler=['cosine_annealing'] +batch_loss_computation_techniques=['mixup'] +loss_modules=['cross_entropy'] +optimizer=['adamw'] diff --git a/configs/autonet/automl/hyperband_cv_sparse.txt b/configs/autonet/automl/hyperband_cv_sparse.txt index dd5c5ccaf..92ef6a5a5 100644 --- a/configs/autonet/automl/hyperband_cv_sparse.txt +++ b/configs/autonet/automl/hyperband_cv_sparse.txt @@ -4,7 +4,7 @@ min_workers=4 max_runtime=92400 budget_type=time algorithm=hyperband -train_metric=accuracy +optimize_metric=accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} min_budget_for_cv=250 diff --git a/configs/autonet/openml/autonet1.txt b/configs/autonet/openml/autonet1.txt index 09ce95754..0f688e5cd 100644 --- a/configs/autonet/openml/autonet1.txt +++ b/configs/autonet/openml/autonet1.txt @@ -3,7 +3,7 @@ min_budget=74 min_workers=1 max_runtime=92400 budget_type=time -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} memory_limit_mb=8000 diff --git a/configs/autonet/openml/full_cs.txt b/configs/autonet/openml/full_cs.txt index d04059e1f..3f5bc3fbd 100644 --- a/configs/autonet/openml/full_cs.txt +++ b/configs/autonet/openml/full_cs.txt @@ -3,7 +3,7 @@ min_budget=74 min_workers=1 max_runtime=92400 budget_type=time -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} memory_limit_mb=8000 diff --git a/configs/autonet/openml/gpu.txt b/configs/autonet/openml/gpu.txt index abc3a8701..0694f2e78 100644 --- a/configs/autonet/openml/gpu.txt +++ b/configs/autonet/openml/gpu.txt @@ -3,7 +3,7 @@ min_budget=74 min_workers=1 max_runtime=92400 budget_type=time -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} memory_limit_mb=8000 diff --git a/configs/autonet/openml/hyperband.txt b/configs/autonet/openml/hyperband.txt index cf9b44f21..5f88c8eb4 100644 --- a/configs/autonet/openml/hyperband.txt +++ b/configs/autonet/openml/hyperband.txt @@ -3,7 +3,7 @@ min_budget=74 min_workers=1 max_runtime=92400 budget_type=time -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} memory_limit_mb=8000 diff --git a/configs/autonet/openml/no_embeddings.txt b/configs/autonet/openml/no_embeddings.txt index 5a355e228..ef365e15c 100644 --- a/configs/autonet/openml/no_embeddings.txt +++ b/configs/autonet/openml/no_embeddings.txt @@ -3,7 +3,7 @@ min_budget=74 min_workers=1 max_runtime=92400 budget_type=time -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} memory_limit_mb=8000 diff --git a/configs/autonet/openml/no_hyperband.txt b/configs/autonet/openml/no_hyperband.txt index 2934083c1..87b1323fa 100644 --- a/configs/autonet/openml/no_hyperband.txt +++ b/configs/autonet/openml/no_hyperband.txt @@ -3,7 +3,7 @@ min_budget=6000 min_workers=1 max_runtime=92400 budget_type=time -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} memory_limit_mb=8000 diff --git a/configs/autonet/openml/restricted_cs.txt b/configs/autonet/openml/restricted_cs.txt index 63456ec10..db17fb7b2 100644 --- a/configs/autonet/openml/restricted_cs.txt +++ b/configs/autonet/openml/restricted_cs.txt @@ -3,7 +3,7 @@ min_budget=74 min_workers=1 max_runtime=92400 budget_type=time -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} memory_limit_mb=8000 diff --git a/configs/autonet/optim_alg_comparison/bohb.txt b/configs/autonet/optim_alg_comparison/bohb.txt index 16673b317..cf39239f0 100644 --- a/configs/autonet/optim_alg_comparison/bohb.txt +++ b/configs/autonet/optim_alg_comparison/bohb.txt @@ -3,7 +3,7 @@ min_budget=74 min_workers=4 max_runtime=92400 budget_type=time -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} batch_loss_computation_techniques=[mixup] @@ -15,4 +15,3 @@ normalization_strategies=[maxabs, standardize] over_sampling_methods=[smote] preprocessors=[none,truncated_svd] target_size_strategies=[none,upsample,median] -min_budget_for_cv=250 diff --git a/configs/autonet/optim_alg_comparison/bohb_tiny_cs.txt b/configs/autonet/optim_alg_comparison/bohb_tiny_cs.txt index 46c65d209..7e68a25cd 100644 --- a/configs/autonet/optim_alg_comparison/bohb_tiny_cs.txt +++ b/configs/autonet/optim_alg_comparison/bohb_tiny_cs.txt @@ -3,11 +3,10 @@ min_budget=74 min_workers=1 max_runtime=92400 budget_type=time -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} memory_limit_mb=5000 -min_budget_for_cv=250 embeddings=[none] lr_scheduler=[cosine_annealing] networks=[shapedresnet] diff --git a/configs/autonet/optim_alg_comparison/hyperband.txt b/configs/autonet/optim_alg_comparison/hyperband.txt index 3a3e42746..a3aa02389 100644 --- a/configs/autonet/optim_alg_comparison/hyperband.txt +++ b/configs/autonet/optim_alg_comparison/hyperband.txt @@ -4,7 +4,7 @@ min_workers=4 max_runtime=92400 budget_type=time algorithm=hyperband -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} batch_loss_computation_techniques=[mixup] @@ -16,4 +16,3 @@ normalization_strategies=[maxabs, standardize] over_sampling_methods=[smote] preprocessors=[none,truncated_svd] target_size_strategies=[none,upsample,median] -min_budget_for_cv=250 diff --git a/configs/autonet/optim_alg_comparison/hyperband_tiny_cs.txt b/configs/autonet/optim_alg_comparison/hyperband_tiny_cs.txt index 363eb6a7a..1e6b625a7 100644 --- a/configs/autonet/optim_alg_comparison/hyperband_tiny_cs.txt +++ b/configs/autonet/optim_alg_comparison/hyperband_tiny_cs.txt @@ -3,11 +3,10 @@ min_budget=74 min_workers=1 max_runtime=92400 budget_type=time -train_metric=balanced_accuracy +optimize_metric=balanced_accuracy cross_validator=k_fold cross_validator_args={"n_splits": 5} memory_limit_mb=5000 -min_budget_for_cv=250 algorithm=hyperband embeddings=[none] lr_scheduler=[cosine_annealing] diff --git a/configs/autonet/test/ensemble_test.txt b/configs/autonet/test/ensemble_test.txt index 42e8ece52..6b2ceb4c1 100644 --- a/configs/autonet/test/ensemble_test.txt +++ b/configs/autonet/test/ensemble_test.txt @@ -6,4 +6,4 @@ log_level=debug budget_type=epochs validation_split=0.2 ensemble_size=20 -train_metric=balanced_accuracy \ No newline at end of file +optimize_metric=balanced_accuracy \ No newline at end of file diff --git a/configs/autonet/test/test.txt b/configs/autonet/test/test.txt index 5e217a8a8..94748aa1a 100644 --- a/configs/autonet/test/test.txt +++ b/configs/autonet/test/test.txt @@ -6,4 +6,5 @@ log_level=debug budget_type=epochs cross_validator=k_fold cross_validator_args={"n_splits": 2} +networks=[mlpnet] hyperparameter_search_space_updates=configs/configspace/minimlp.txt \ No newline at end of file diff --git a/configs/benchmark/cifar_example.txt b/configs/benchmark/cifar_example.txt new file mode 100644 index 000000000..2fc4d8f74 --- /dev/null +++ b/configs/benchmark/cifar_example.txt @@ -0,0 +1,7 @@ +result_dir=benchmark_results +instances=configs/datasets/cifar.txt +autonet_configs=[configs/autonet/automl/cifar_example.txt] +problem_type=image_classification +log_level=info +test_split=0.1 +num_runs=1 diff --git a/configs/datasets/cifar.txt b/configs/datasets/cifar.txt new file mode 100644 index 000000000..d9099be33 --- /dev/null +++ b/configs/datasets/cifar.txt @@ -0,0 +1 @@ +[datasets/CIFAR10.csv] diff --git a/configs/datasets/openml_image.txt b/configs/datasets/openml_image.txt new file mode 100644 index 000000000..086605861 --- /dev/null +++ b/configs/datasets/openml_image.txt @@ -0,0 +1 @@ +openml:40927:3 \ No newline at end of file diff --git a/configs/refit/refit_example.json b/configs/refit/refit_example.json new file mode 100644 index 000000000..3f9b4f8bc --- /dev/null +++ b/configs/refit/refit_example.json @@ -0,0 +1,19 @@ +{ + "SimpleTrainNode:batch_loss_computation_technique": "mixup", + "SimpleTrainNode:mixup:alpha": 0.012, + "CreateImageDataLoader:batch_size": 147, + "NetworkSelectorDatasetInfo:network": "efficientnetb0", + "OptimizerSelector:optimizer": "adamw", + "OptimizerSelector:adamw:learning_rate": 0.012, + "OptimizerSelector:adamw:weight_decay": 0.000017, + "SimpleLearningrateSchedulerSelector:lr_scheduler": "cosine_annealing", + "SimpleLearningrateSchedulerSelector:cosine_annealing:T_max": 73, + "SimpleLearningrateSchedulerSelector:cosine_annealing:T_mult": 1.38, + "ImageAugmentation:augment": "True", + "ImageAugmentation:cutout": "True", + "ImageAugmentation:cutout_holes": 3, + "ImageAugmentation:autoaugment": "True", + "ImageAugmentation:fastautoaugment": "False", + "ImageAugmentation:length": 6, + "LossModuleSelectorIndices:loss_module": "cross_entropy" +} diff --git a/datasets/CIFAR10.csv b/datasets/CIFAR10.csv new file mode 100644 index 000000000..04ab00f08 --- /dev/null +++ b/datasets/CIFAR10.csv @@ -0,0 +1 @@ +CIFAR10, 0 \ No newline at end of file diff --git a/datasets/example.csv b/datasets/example.csv new file mode 100644 index 000000000..529464e61 --- /dev/null +++ b/datasets/example.csv @@ -0,0 +1,99 @@ +icebreaker_s_001689.png,8 +peke_s_000545.png,5 +convertible_s_000520.png,1 +domestic_dog_s_000455.png,5 +broodmare_s_000313.png,7 +capreolus_capreolus_s_001380.png,4 +true_cat_s_000886.png,3 +cruiser_s_000163.png,8 +ostrich_s_001561.png,2 +buckskin_s_000031.png,7 +cassowary_s_002024.png,2 +fighter_aircraft_s_001009.png,0 +convertible_s_000295.png,1 +lapdog_s_001489.png,5 +delivery_truck_s_001300.png,9 +rana_pipiens_s_000379.png,6 +ostrich_s_000026.png,2 +fighter_aircraft_s_000720.png,0 +supertanker_s_000275.png,8 +ostrich_s_000147.png,2 +male_horse_s_000742.png,7 +monoplane_s_000877.png,0 +fallow_deer_s_000351.png,4 +automobile_s_001645.png,1 +walking_horse_s_000071.png,7 +stallion_s_000015.png,7 +capreolus_capreolus_s_001283.png,4 +mule_deer_s_000357.png,4 +dumper_s_000805.png,9 +trailer_truck_s_001350.png,9 +green_frog_s_001384.png,6 +rhea_americana_s_000436.png,2 +capreolus_capreolus_s_001605.png,4 +auto_s_000800.png,1 +tailed_frog_s_000246.png,6 +cervus_elaphus_s_000903.png,4 +articulated_lorry_s_000916.png,9 +bullfrog_s_000797.png,6 +bullfrog_s_001028.png,6 +ladder_truck_s_001799.png,9 +toad_frog_s_001786.png,6 +wrecker_s_002395.png,9 +dump_truck_s_001363.png,9 +canis_familiaris_s_000450.png,5 +lipizzan_s_001223.png,7 +station_wagon_s_000464.png,1 +american_toad_s_001003.png,6 +dredger_s_000486.png,8 +wagtail_s_000747.png,2 +dump_truck_s_000163.png,9 +mutt_s_000997.png,5 +dump_truck_s_001097.png,9 +puppy_s_001045.png,5 +tabby_s_001593.png,3 +broodmare_s_000179.png,7 +car_s_000040.png,1 +domestic_cat_s_000913.png,3 +alley_cat_s_000843.png,3 +truck_s_000028.png,9 +estate_car_s_001092.png,1 +arabian_s_000782.png,7 +supertanker_s_000761.png,8 +garbage_truck_s_001211.png,9 +arabian_s_002303.png,7 +red_deer_s_001101.png,4 +tabby_cat_s_000069.png,3 +cervus_elaphus_s_001124.png,4 +trucking_rig_s_001247.png,9 +pekinese_s_000046.png,5 +police_boat_s_001118.png,8 +fallow_deer_s_001785.png,4 +camion_s_000599.png,9 +tabby_s_001774.png,3 +spring_frog_s_000407.png,6 +wagon_s_002463.png,1 +station_wagon_s_002537.png,1 +elk_s_001751.png,4 +house_cat_s_000064.png,3 +lorry_s_000562.png,9 +delivery_truck_s_001587.png,9 +wagon_s_000378.png,1 +trucking_rig_s_001431.png,9 +tractor_trailer_s_000653.png,9 +cassowary_s_000194.png,2 +fawn_s_001418.png,4 +mouser_s_000792.png,3 +bird_of_passage_s_000006.png,2 +sika_s_000337.png,4 +dawn_horse_s_001453.png,7 +police_cruiser_s_001385.png,1 +maltese_s_000562.png,5 +wagon_s_000572.png,1 +liberty_ship_s_001456.png,8 +western_toad_s_000622.png,6 +house_cat_s_002004.png,3 +bufo_bufo_s_002202.png,6 +tabby_cat_s_001983.png,3 +fallow_deer_s_001133.png,4 +red_deer_s_001719.png,4 diff --git a/datasets/example_images/alley_cat_s_000843.png b/datasets/example_images/alley_cat_s_000843.png new file mode 100644 index 000000000..bef5de531 Binary files /dev/null and b/datasets/example_images/alley_cat_s_000843.png differ diff --git a/datasets/example_images/american_toad_s_001003.png b/datasets/example_images/american_toad_s_001003.png new file mode 100644 index 000000000..c89f0bb36 Binary files /dev/null and b/datasets/example_images/american_toad_s_001003.png differ diff --git a/datasets/example_images/arabian_s_000782.png b/datasets/example_images/arabian_s_000782.png new file mode 100644 index 000000000..79b94674f Binary files /dev/null and b/datasets/example_images/arabian_s_000782.png differ diff --git a/datasets/example_images/arabian_s_002303.png b/datasets/example_images/arabian_s_002303.png new file mode 100644 index 000000000..3125066f4 Binary files /dev/null and b/datasets/example_images/arabian_s_002303.png differ diff --git a/datasets/example_images/articulated_lorry_s_000916.png b/datasets/example_images/articulated_lorry_s_000916.png new file mode 100644 index 000000000..9d5d1a140 Binary files /dev/null and b/datasets/example_images/articulated_lorry_s_000916.png differ diff --git a/datasets/example_images/auto_s_000800.png b/datasets/example_images/auto_s_000800.png new file mode 100644 index 000000000..ba42044dd Binary files /dev/null and b/datasets/example_images/auto_s_000800.png differ diff --git a/datasets/example_images/automobile_s_001645.png b/datasets/example_images/automobile_s_001645.png new file mode 100644 index 000000000..b8d317620 Binary files /dev/null and b/datasets/example_images/automobile_s_001645.png differ diff --git a/datasets/example_images/bird_of_passage_s_000006.png b/datasets/example_images/bird_of_passage_s_000006.png new file mode 100644 index 000000000..d9ef770b0 Binary files /dev/null and b/datasets/example_images/bird_of_passage_s_000006.png differ diff --git a/datasets/example_images/broodmare_s_000179.png b/datasets/example_images/broodmare_s_000179.png new file mode 100644 index 000000000..319f8a398 Binary files /dev/null and b/datasets/example_images/broodmare_s_000179.png differ diff --git a/datasets/example_images/broodmare_s_000313.png b/datasets/example_images/broodmare_s_000313.png new file mode 100644 index 000000000..87f106363 Binary files /dev/null and b/datasets/example_images/broodmare_s_000313.png differ diff --git a/datasets/example_images/buckskin_s_000031.png b/datasets/example_images/buckskin_s_000031.png new file mode 100644 index 000000000..276e335d0 Binary files /dev/null and b/datasets/example_images/buckskin_s_000031.png differ diff --git a/datasets/example_images/bufo_bufo_s_002202.png b/datasets/example_images/bufo_bufo_s_002202.png new file mode 100644 index 000000000..1eef4599a Binary files /dev/null and b/datasets/example_images/bufo_bufo_s_002202.png differ diff --git a/datasets/example_images/bullfrog_s_000797.png b/datasets/example_images/bullfrog_s_000797.png new file mode 100644 index 000000000..40c341c0f Binary files /dev/null and b/datasets/example_images/bullfrog_s_000797.png differ diff --git a/datasets/example_images/bullfrog_s_001028.png b/datasets/example_images/bullfrog_s_001028.png new file mode 100644 index 000000000..073e4522e Binary files /dev/null and b/datasets/example_images/bullfrog_s_001028.png differ diff --git a/datasets/example_images/camion_s_000599.png b/datasets/example_images/camion_s_000599.png new file mode 100644 index 000000000..86b77d960 Binary files /dev/null and b/datasets/example_images/camion_s_000599.png differ diff --git a/datasets/example_images/canis_familiaris_s_000450.png b/datasets/example_images/canis_familiaris_s_000450.png new file mode 100644 index 000000000..c01a7b840 Binary files /dev/null and b/datasets/example_images/canis_familiaris_s_000450.png differ diff --git a/datasets/example_images/capreolus_capreolus_s_001283.png b/datasets/example_images/capreolus_capreolus_s_001283.png new file mode 100644 index 000000000..b301a6c0c Binary files /dev/null and b/datasets/example_images/capreolus_capreolus_s_001283.png differ diff --git a/datasets/example_images/capreolus_capreolus_s_001380.png b/datasets/example_images/capreolus_capreolus_s_001380.png new file mode 100644 index 000000000..bf7c7366f Binary files /dev/null and b/datasets/example_images/capreolus_capreolus_s_001380.png differ diff --git a/datasets/example_images/capreolus_capreolus_s_001605.png b/datasets/example_images/capreolus_capreolus_s_001605.png new file mode 100644 index 000000000..5e207d74c Binary files /dev/null and b/datasets/example_images/capreolus_capreolus_s_001605.png differ diff --git a/datasets/example_images/car_s_000040.png b/datasets/example_images/car_s_000040.png new file mode 100644 index 000000000..5c4261e68 Binary files /dev/null and b/datasets/example_images/car_s_000040.png differ diff --git a/datasets/example_images/cassowary_s_000194.png b/datasets/example_images/cassowary_s_000194.png new file mode 100644 index 000000000..046b033a7 Binary files /dev/null and b/datasets/example_images/cassowary_s_000194.png differ diff --git a/datasets/example_images/cassowary_s_002024.png b/datasets/example_images/cassowary_s_002024.png new file mode 100644 index 000000000..0e0195107 Binary files /dev/null and b/datasets/example_images/cassowary_s_002024.png differ diff --git a/datasets/example_images/cervus_elaphus_s_000903.png b/datasets/example_images/cervus_elaphus_s_000903.png new file mode 100644 index 000000000..ac4d5a00d Binary files /dev/null and b/datasets/example_images/cervus_elaphus_s_000903.png differ diff --git a/datasets/example_images/cervus_elaphus_s_001124.png b/datasets/example_images/cervus_elaphus_s_001124.png new file mode 100644 index 000000000..09f486ee6 Binary files /dev/null and b/datasets/example_images/cervus_elaphus_s_001124.png differ diff --git a/datasets/example_images/convertible_s_000295.png b/datasets/example_images/convertible_s_000295.png new file mode 100644 index 000000000..98fbf9f3c Binary files /dev/null and b/datasets/example_images/convertible_s_000295.png differ diff --git a/datasets/example_images/convertible_s_000520.png b/datasets/example_images/convertible_s_000520.png new file mode 100644 index 000000000..ee73ee4b8 Binary files /dev/null and b/datasets/example_images/convertible_s_000520.png differ diff --git a/datasets/example_images/cruiser_s_000163.png b/datasets/example_images/cruiser_s_000163.png new file mode 100644 index 000000000..a3d79e112 Binary files /dev/null and b/datasets/example_images/cruiser_s_000163.png differ diff --git a/datasets/example_images/dawn_horse_s_001453.png b/datasets/example_images/dawn_horse_s_001453.png new file mode 100644 index 000000000..db4fa7089 Binary files /dev/null and b/datasets/example_images/dawn_horse_s_001453.png differ diff --git a/datasets/example_images/delivery_truck_s_001300.png b/datasets/example_images/delivery_truck_s_001300.png new file mode 100644 index 000000000..94031550a Binary files /dev/null and b/datasets/example_images/delivery_truck_s_001300.png differ diff --git a/datasets/example_images/delivery_truck_s_001587.png b/datasets/example_images/delivery_truck_s_001587.png new file mode 100644 index 000000000..a86fb263d Binary files /dev/null and b/datasets/example_images/delivery_truck_s_001587.png differ diff --git a/datasets/example_images/domestic_cat_s_000913.png b/datasets/example_images/domestic_cat_s_000913.png new file mode 100644 index 000000000..305c57841 Binary files /dev/null and b/datasets/example_images/domestic_cat_s_000913.png differ diff --git a/datasets/example_images/domestic_dog_s_000455.png b/datasets/example_images/domestic_dog_s_000455.png new file mode 100644 index 000000000..03cbdc0d3 Binary files /dev/null and b/datasets/example_images/domestic_dog_s_000455.png differ diff --git a/datasets/example_images/dredger_s_000486.png b/datasets/example_images/dredger_s_000486.png new file mode 100644 index 000000000..6876e9574 Binary files /dev/null and b/datasets/example_images/dredger_s_000486.png differ diff --git a/datasets/example_images/dump_truck_s_000163.png b/datasets/example_images/dump_truck_s_000163.png new file mode 100644 index 000000000..bb343b384 Binary files /dev/null and b/datasets/example_images/dump_truck_s_000163.png differ diff --git a/datasets/example_images/dump_truck_s_001097.png b/datasets/example_images/dump_truck_s_001097.png new file mode 100644 index 000000000..5fffe2a4e Binary files /dev/null and b/datasets/example_images/dump_truck_s_001097.png differ diff --git a/datasets/example_images/dump_truck_s_001363.png b/datasets/example_images/dump_truck_s_001363.png new file mode 100644 index 000000000..a852d9a24 Binary files /dev/null and b/datasets/example_images/dump_truck_s_001363.png differ diff --git a/datasets/example_images/dumper_s_000805.png b/datasets/example_images/dumper_s_000805.png new file mode 100644 index 000000000..3c4d3410f Binary files /dev/null and b/datasets/example_images/dumper_s_000805.png differ diff --git a/datasets/example_images/elk_s_001751.png b/datasets/example_images/elk_s_001751.png new file mode 100644 index 000000000..5aa450e9c Binary files /dev/null and b/datasets/example_images/elk_s_001751.png differ diff --git a/datasets/example_images/estate_car_s_001092.png b/datasets/example_images/estate_car_s_001092.png new file mode 100644 index 000000000..b5b7fd952 Binary files /dev/null and b/datasets/example_images/estate_car_s_001092.png differ diff --git a/datasets/example_images/fallow_deer_s_000351.png b/datasets/example_images/fallow_deer_s_000351.png new file mode 100644 index 000000000..8583e73a7 Binary files /dev/null and b/datasets/example_images/fallow_deer_s_000351.png differ diff --git a/datasets/example_images/fallow_deer_s_001133.png b/datasets/example_images/fallow_deer_s_001133.png new file mode 100644 index 000000000..3e8fd5969 Binary files /dev/null and b/datasets/example_images/fallow_deer_s_001133.png differ diff --git a/datasets/example_images/fallow_deer_s_001785.png b/datasets/example_images/fallow_deer_s_001785.png new file mode 100644 index 000000000..9abf685b9 Binary files /dev/null and b/datasets/example_images/fallow_deer_s_001785.png differ diff --git a/datasets/example_images/fawn_s_001418.png b/datasets/example_images/fawn_s_001418.png new file mode 100644 index 000000000..e004bc8f5 Binary files /dev/null and b/datasets/example_images/fawn_s_001418.png differ diff --git a/datasets/example_images/fighter_aircraft_s_000720.png b/datasets/example_images/fighter_aircraft_s_000720.png new file mode 100644 index 000000000..46cb393e4 Binary files /dev/null and b/datasets/example_images/fighter_aircraft_s_000720.png differ diff --git a/datasets/example_images/fighter_aircraft_s_001009.png b/datasets/example_images/fighter_aircraft_s_001009.png new file mode 100644 index 000000000..7c7e5dcf4 Binary files /dev/null and b/datasets/example_images/fighter_aircraft_s_001009.png differ diff --git a/datasets/example_images/garbage_truck_s_001211.png b/datasets/example_images/garbage_truck_s_001211.png new file mode 100644 index 000000000..a0cedebee Binary files /dev/null and b/datasets/example_images/garbage_truck_s_001211.png differ diff --git a/datasets/example_images/green_frog_s_001384.png b/datasets/example_images/green_frog_s_001384.png new file mode 100644 index 000000000..63b604143 Binary files /dev/null and b/datasets/example_images/green_frog_s_001384.png differ diff --git a/datasets/example_images/house_cat_s_000064.png b/datasets/example_images/house_cat_s_000064.png new file mode 100644 index 000000000..cae1fef87 Binary files /dev/null and b/datasets/example_images/house_cat_s_000064.png differ diff --git a/datasets/example_images/house_cat_s_002004.png b/datasets/example_images/house_cat_s_002004.png new file mode 100644 index 000000000..79e064548 Binary files /dev/null and b/datasets/example_images/house_cat_s_002004.png differ diff --git a/datasets/example_images/icebreaker_s_001689.png b/datasets/example_images/icebreaker_s_001689.png new file mode 100644 index 000000000..b5b9e8e3c Binary files /dev/null and b/datasets/example_images/icebreaker_s_001689.png differ diff --git a/datasets/example_images/ladder_truck_s_001799.png b/datasets/example_images/ladder_truck_s_001799.png new file mode 100644 index 000000000..58a0c8401 Binary files /dev/null and b/datasets/example_images/ladder_truck_s_001799.png differ diff --git a/datasets/example_images/lapdog_s_001489.png b/datasets/example_images/lapdog_s_001489.png new file mode 100644 index 000000000..259d9b2b9 Binary files /dev/null and b/datasets/example_images/lapdog_s_001489.png differ diff --git a/datasets/example_images/liberty_ship_s_001456.png b/datasets/example_images/liberty_ship_s_001456.png new file mode 100644 index 000000000..de42766d5 Binary files /dev/null and b/datasets/example_images/liberty_ship_s_001456.png differ diff --git a/datasets/example_images/lipizzan_s_001223.png b/datasets/example_images/lipizzan_s_001223.png new file mode 100644 index 000000000..09a79f7b1 Binary files /dev/null and b/datasets/example_images/lipizzan_s_001223.png differ diff --git a/datasets/example_images/lorry_s_000562.png b/datasets/example_images/lorry_s_000562.png new file mode 100644 index 000000000..b53d1befd Binary files /dev/null and b/datasets/example_images/lorry_s_000562.png differ diff --git a/datasets/example_images/male_horse_s_000742.png b/datasets/example_images/male_horse_s_000742.png new file mode 100644 index 000000000..991b3ab87 Binary files /dev/null and b/datasets/example_images/male_horse_s_000742.png differ diff --git a/datasets/example_images/maltese_s_000562.png b/datasets/example_images/maltese_s_000562.png new file mode 100644 index 000000000..0699958ae Binary files /dev/null and b/datasets/example_images/maltese_s_000562.png differ diff --git a/datasets/example_images/monoplane_s_000877.png b/datasets/example_images/monoplane_s_000877.png new file mode 100644 index 000000000..fbfaa7682 Binary files /dev/null and b/datasets/example_images/monoplane_s_000877.png differ diff --git a/datasets/example_images/mouser_s_000792.png b/datasets/example_images/mouser_s_000792.png new file mode 100644 index 000000000..03927b4dd Binary files /dev/null and b/datasets/example_images/mouser_s_000792.png differ diff --git a/datasets/example_images/mule_deer_s_000357.png b/datasets/example_images/mule_deer_s_000357.png new file mode 100644 index 000000000..5fe5f2d39 Binary files /dev/null and b/datasets/example_images/mule_deer_s_000357.png differ diff --git a/datasets/example_images/mutt_s_000997.png b/datasets/example_images/mutt_s_000997.png new file mode 100644 index 000000000..93f2c74c6 Binary files /dev/null and b/datasets/example_images/mutt_s_000997.png differ diff --git a/datasets/example_images/ostrich_s_000026.png b/datasets/example_images/ostrich_s_000026.png new file mode 100644 index 000000000..320f5e502 Binary files /dev/null and b/datasets/example_images/ostrich_s_000026.png differ diff --git a/datasets/example_images/ostrich_s_000147.png b/datasets/example_images/ostrich_s_000147.png new file mode 100644 index 000000000..01e375468 Binary files /dev/null and b/datasets/example_images/ostrich_s_000147.png differ diff --git a/datasets/example_images/ostrich_s_001561.png b/datasets/example_images/ostrich_s_001561.png new file mode 100644 index 000000000..2d1ebb0d6 Binary files /dev/null and b/datasets/example_images/ostrich_s_001561.png differ diff --git a/datasets/example_images/peke_s_000545.png b/datasets/example_images/peke_s_000545.png new file mode 100644 index 000000000..fd75a564d Binary files /dev/null and b/datasets/example_images/peke_s_000545.png differ diff --git a/datasets/example_images/pekinese_s_000046.png b/datasets/example_images/pekinese_s_000046.png new file mode 100644 index 000000000..ecaf53565 Binary files /dev/null and b/datasets/example_images/pekinese_s_000046.png differ diff --git a/datasets/example_images/police_boat_s_001118.png b/datasets/example_images/police_boat_s_001118.png new file mode 100644 index 000000000..7adad01ee Binary files /dev/null and b/datasets/example_images/police_boat_s_001118.png differ diff --git a/datasets/example_images/police_cruiser_s_001385.png b/datasets/example_images/police_cruiser_s_001385.png new file mode 100644 index 000000000..6b2e00a0b Binary files /dev/null and b/datasets/example_images/police_cruiser_s_001385.png differ diff --git a/datasets/example_images/puppy_s_001045.png b/datasets/example_images/puppy_s_001045.png new file mode 100644 index 000000000..3d5c52f73 Binary files /dev/null and b/datasets/example_images/puppy_s_001045.png differ diff --git a/datasets/example_images/rana_pipiens_s_000379.png b/datasets/example_images/rana_pipiens_s_000379.png new file mode 100644 index 000000000..7bcb45394 Binary files /dev/null and b/datasets/example_images/rana_pipiens_s_000379.png differ diff --git a/datasets/example_images/red_deer_s_001101.png b/datasets/example_images/red_deer_s_001101.png new file mode 100644 index 000000000..3ec29f469 Binary files /dev/null and b/datasets/example_images/red_deer_s_001101.png differ diff --git a/datasets/example_images/red_deer_s_001719.png b/datasets/example_images/red_deer_s_001719.png new file mode 100644 index 000000000..6379d6af7 Binary files /dev/null and b/datasets/example_images/red_deer_s_001719.png differ diff --git a/datasets/example_images/rhea_americana_s_000436.png b/datasets/example_images/rhea_americana_s_000436.png new file mode 100644 index 000000000..a5de92cf6 Binary files /dev/null and b/datasets/example_images/rhea_americana_s_000436.png differ diff --git a/datasets/example_images/sika_s_000337.png b/datasets/example_images/sika_s_000337.png new file mode 100644 index 000000000..22be9cc44 Binary files /dev/null and b/datasets/example_images/sika_s_000337.png differ diff --git a/datasets/example_images/spring_frog_s_000407.png b/datasets/example_images/spring_frog_s_000407.png new file mode 100644 index 000000000..86f323a5f Binary files /dev/null and b/datasets/example_images/spring_frog_s_000407.png differ diff --git a/datasets/example_images/stallion_s_000015.png b/datasets/example_images/stallion_s_000015.png new file mode 100644 index 000000000..4db4e59cc Binary files /dev/null and b/datasets/example_images/stallion_s_000015.png differ diff --git a/datasets/example_images/station_wagon_s_000464.png b/datasets/example_images/station_wagon_s_000464.png new file mode 100644 index 000000000..a74f1d98b Binary files /dev/null and b/datasets/example_images/station_wagon_s_000464.png differ diff --git a/datasets/example_images/station_wagon_s_002537.png b/datasets/example_images/station_wagon_s_002537.png new file mode 100644 index 000000000..b24969db7 Binary files /dev/null and b/datasets/example_images/station_wagon_s_002537.png differ diff --git a/datasets/example_images/supertanker_s_000275.png b/datasets/example_images/supertanker_s_000275.png new file mode 100644 index 000000000..c01c4370a Binary files /dev/null and b/datasets/example_images/supertanker_s_000275.png differ diff --git a/datasets/example_images/supertanker_s_000761.png b/datasets/example_images/supertanker_s_000761.png new file mode 100644 index 000000000..8b4a3ac00 Binary files /dev/null and b/datasets/example_images/supertanker_s_000761.png differ diff --git a/datasets/example_images/tabby_cat_s_000069.png b/datasets/example_images/tabby_cat_s_000069.png new file mode 100644 index 000000000..78a62654d Binary files /dev/null and b/datasets/example_images/tabby_cat_s_000069.png differ diff --git a/datasets/example_images/tabby_cat_s_001983.png b/datasets/example_images/tabby_cat_s_001983.png new file mode 100644 index 000000000..cb423e861 Binary files /dev/null and b/datasets/example_images/tabby_cat_s_001983.png differ diff --git a/datasets/example_images/tabby_s_001593.png b/datasets/example_images/tabby_s_001593.png new file mode 100644 index 000000000..608e56f4a Binary files /dev/null and b/datasets/example_images/tabby_s_001593.png differ diff --git a/datasets/example_images/tabby_s_001774.png b/datasets/example_images/tabby_s_001774.png new file mode 100644 index 000000000..a7a54b3d9 Binary files /dev/null and b/datasets/example_images/tabby_s_001774.png differ diff --git a/datasets/example_images/tailed_frog_s_000246.png b/datasets/example_images/tailed_frog_s_000246.png new file mode 100644 index 000000000..2ba29dcab Binary files /dev/null and b/datasets/example_images/tailed_frog_s_000246.png differ diff --git a/datasets/example_images/toad_frog_s_001786.png b/datasets/example_images/toad_frog_s_001786.png new file mode 100644 index 000000000..f31dda3d4 Binary files /dev/null and b/datasets/example_images/toad_frog_s_001786.png differ diff --git a/datasets/example_images/tractor_trailer_s_000653.png b/datasets/example_images/tractor_trailer_s_000653.png new file mode 100644 index 000000000..3f1b7bf97 Binary files /dev/null and b/datasets/example_images/tractor_trailer_s_000653.png differ diff --git a/datasets/example_images/trailer_truck_s_001350.png b/datasets/example_images/trailer_truck_s_001350.png new file mode 100644 index 000000000..e20f68e34 Binary files /dev/null and b/datasets/example_images/trailer_truck_s_001350.png differ diff --git a/datasets/example_images/truck_s_000028.png b/datasets/example_images/truck_s_000028.png new file mode 100644 index 000000000..975c4da6c Binary files /dev/null and b/datasets/example_images/truck_s_000028.png differ diff --git a/datasets/example_images/trucking_rig_s_001247.png b/datasets/example_images/trucking_rig_s_001247.png new file mode 100644 index 000000000..eae266b15 Binary files /dev/null and b/datasets/example_images/trucking_rig_s_001247.png differ diff --git a/datasets/example_images/trucking_rig_s_001431.png b/datasets/example_images/trucking_rig_s_001431.png new file mode 100644 index 000000000..7674c0595 Binary files /dev/null and b/datasets/example_images/trucking_rig_s_001431.png differ diff --git a/datasets/example_images/true_cat_s_000886.png b/datasets/example_images/true_cat_s_000886.png new file mode 100644 index 000000000..badf1aeab Binary files /dev/null and b/datasets/example_images/true_cat_s_000886.png differ diff --git a/datasets/example_images/wagon_s_000378.png b/datasets/example_images/wagon_s_000378.png new file mode 100644 index 000000000..e53f30df6 Binary files /dev/null and b/datasets/example_images/wagon_s_000378.png differ diff --git a/datasets/example_images/wagon_s_000572.png b/datasets/example_images/wagon_s_000572.png new file mode 100644 index 000000000..a8df0244b Binary files /dev/null and b/datasets/example_images/wagon_s_000572.png differ diff --git a/datasets/example_images/wagon_s_002463.png b/datasets/example_images/wagon_s_002463.png new file mode 100644 index 000000000..ae436205b Binary files /dev/null and b/datasets/example_images/wagon_s_002463.png differ diff --git a/datasets/example_images/wagtail_s_000747.png b/datasets/example_images/wagtail_s_000747.png new file mode 100644 index 000000000..e0fee91b9 Binary files /dev/null and b/datasets/example_images/wagtail_s_000747.png differ diff --git a/datasets/example_images/walking_horse_s_000071.png b/datasets/example_images/walking_horse_s_000071.png new file mode 100644 index 000000000..a25e7c505 Binary files /dev/null and b/datasets/example_images/walking_horse_s_000071.png differ diff --git a/datasets/example_images/western_toad_s_000622.png b/datasets/example_images/western_toad_s_000622.png new file mode 100644 index 000000000..44b973d7b Binary files /dev/null and b/datasets/example_images/western_toad_s_000622.png differ diff --git a/datasets/example_images/wrecker_s_002395.png b/datasets/example_images/wrecker_s_002395.png new file mode 100644 index 000000000..e67b841ea Binary files /dev/null and b/datasets/example_images/wrecker_s_002395.png differ diff --git a/examples/basics/Auto-PyTorch Tutorial.ipynb b/examples/basics/Auto-PyTorch Tutorial.ipynb new file mode 100644 index 000000000..c0e6887eb --- /dev/null +++ b/examples/basics/Auto-PyTorch Tutorial.ipynb @@ -0,0 +1,667 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction\n", + "\n", + "This tutorial introduces the basic Auto-PyTorch API together with the classes for featurized and image data.\n", + "So far, Auto-PyTorch covers classification and regression on featurized data as well as classification on image data.\n", + "For installing Auto-PyTorch, please refer to the github page.\n", + "\n", + "**Disclaimer**: In this notebook, data will be downloaded from the openml project for featurized tasks and CIFAR10 will be downloaded for image classification. Hence, an internet connection is required." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# API\n", + "\n", + "There are classes for featurized tasks (classification, multi-label classification, regression) and image tasks (classification). You can import them via:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from autoPyTorch import (AutoNetClassification,\n", + " AutoNetMultilabel,\n", + " AutoNetRegression,\n", + " AutoNetImageClassification,\n", + " AutoNetImageClassificationMultipleDatasets)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Other imports for later usage\n", + "import pandas as pd\n", + "import numpy as np\n", + "import os as os\n", + "import openml\n", + "import json" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Upon initialization of a class, you can specify its configuration. Later, you can override its configuration in each fit call. The *config_preset* allows to constrain the search space to one of *tiny_cs, medium_cs* or *full_cs*. These presets can be seen in *core/presets/*." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "autonet = AutoNetClassification(config_preset=\"tiny_cs\", result_logger_dir=\"logs/\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here are some useful methods provided by the API:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# Get the current configuration as dict\n", + "current_configuration = autonet.get_current_autonet_config()\n", + "\n", + "# Get the ConfigSpace object with all hyperparameters, conditions, default values and default ranges\n", + "hyperparameter_search_space = autonet.get_hyperparameter_search_space()\n", + "\n", + "# Print all possible configuration options \n", + "#autonet.print_help()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "scrolled": true + }, + "source": [ + "The most important methods for using Auto-PyTorch are ***fit***, ***refit***, ***score*** and ***predict***.\n", + "\n", + "First, we get some data:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# Get data from the openml task \"Supervised Classification on credit-g (https://www.openml.org/t/31)\"\n", + "task = openml.tasks.get_task(task_id=31)\n", + "X, y = task.get_X_and_y()\n", + "ind_train, ind_test = task.get_train_test_split_indices()\n", + "X_train, Y_train = X[ind_train], y[ind_train]\n", + "X_test, Y_test = X[ind_test], y[ind_test]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***fit*** is used to search for a good configuration by fitting configurations chosen by the algorithm (by default BOHB). The incumbent configuration is then returned and stored in the class.\n", + "\n", + "We recommend to have a look at the possible configuration options first. Some of the most important options allow you to set the budget type (epochs or time), run id and task id for cluster usage, tensorboard logging, seed and more.\n", + "\n", + "Here we search for a configuration for 300 seconds with 60-100 s time for fitting each individual configuration.\n", + "Use the *validation_split* parameter to specify a split size. You can also pass your own validation set\n", + "via *X_val* and *Y_val*. Use *log_level=\"info\"* or *log_level=\"debug\"* for more detailed output." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "autonet = AutoNetClassification(config_preset=\"tiny_cs\", result_logger_dir=\"logs/\")\n", + "# Fit (note that the settings are for demonstration, you might need larger budgets)\n", + "results_fit = autonet.fit(X_train=X_train,\n", + " Y_train=Y_train,\n", + " validation_split=0.3,\n", + " max_runtime=300,\n", + " min_budget=60,\n", + " max_budget=100,\n", + " refit=True)\n", + "\n", + "# Save fit results as json\n", + "with open(\"logs/results_fit.json\", \"w\") as file:\n", + " json.dump(results_fit, file)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***refit*** allows you to fit a configuration of your choice for a defined time. By default, the incumbent configuration is refitted during a *fit* call using the *max_budget*. However, *refit* might be useful if you want to fit on the full dataset or even another dataset or if you just want to fit a model without searching.\n", + "\n", + "You can specify a hyperparameter configuration to fit (if you do not specify a configuration the incumbent configuration from the last fit call will be used):" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[INFO] [13:07:21:autonet] Start autonet with config:\n", + "{'embeddings': ['none'], 'lr_scheduler': ['cosine_annealing', 'plateau'], 'networks': ['shapedresnet'], 'over_sampling_methods': ['smote'], 'preprocessors': ['none', 'truncated_svd', 'power_transformer'], 'target_size_strategies': ['none', 'upsample', 'median'], 'result_logger_dir': 'logs/', 'budget_type': 'epochs', 'log_level': 'info', 'use_tensorboard_logger': True, 'validation_split': 0.0, 'hyperparameter_search_space_updates': None, 'categorical_features': None, 'dataset_name': None, 'run_id': '0', 'task_id': -1, 'algorithm': 'bohb', 'eta': 3, 'min_workers': 1, 'working_dir': '.', 'network_interface_name': 'eth0', 'memory_limit_mb': 1000000, 'run_worker_on_master_node': True, 'use_pynisher': True, 'refit_validation_split': 0.0, 'cross_validator': 'none', 'cross_validator_args': {}, 'min_budget_for_cv': 0, 'shuffle': True, 'imputation_strategies': ['mean', 'median', 'most_frequent'], 'normalization_strategies': ['none', 'minmax', 'standardize', 'maxabs'], 'under_sampling_methods': ['none', 'random'], 'final_activation': 'softmax', 'initialization_methods': ['default', 'sparse'], 'initializer': 'simple_initializer', 'optimizer': ['adam', 'adamw', 'sgd', 'rmsprop'], 'additional_logs': [], 'optimize_metric': 'accuracy', 'additional_metrics': [], 'loss_modules': ['cross_entropy', 'cross_entropy_weighted'], 'batch_loss_computation_techniques': ['standard', 'mixup'], 'cuda': True, 'torch_num_threads': 1, 'full_eval_each_epoch': False, 'best_over_epochs': False, 'early_stopping_patience': inf, 'early_stopping_reset_parameters': False, 'random_seed': 1103059814, 'min_budget': 5, 'max_budget': 150, 'max_runtime': inf, 'num_iterations': 4, 'cv_splits': 1, 'increase_number_of_trained_datasets': False}\n", + "[INFO] [13:07:21:autonet] Start Refitting\n", + "[INFO] [13:07:21:autonet] [AutoNet] No validation set given and either no cross validator given or budget too low for CV. Continue by splitting 0 of training data.\n", + "[INFO] [13:07:21:autonet] [AutoNet] CV split 0 of 1\n", + "[INFO] [13:07:25:autonet] Finished train with budget 50.0: Preprocessing took 0s, Training took 4s, Wrap up took 0s. Total time consumption in s: 4\n", + "[INFO] [13:07:25:autonet] [AutoNet] Done with current split!\n", + "[INFO] [13:07:25:autonet] Aggregate the results across the splits\n", + "[INFO] [13:07:25:autonet] Process 1 additional result(s)\n", + "[INFO] [13:07:25:autonet] Done Refitting\n" + ] + } + ], + "source": [ + "# Create an autonet\n", + "autonet_config = {\n", + " \"result_logger_dir\" : \"logs/\",\n", + " \"budget_type\" : \"epochs\",\n", + " \"log_level\" : \"info\", \n", + " \"use_tensorboard_logger\" : True,\n", + " \"validation_split\" : 0.0\n", + " }\n", + "autonet = AutoNetClassification(**autonet_config)\n", + "\n", + "# Sample a random hyperparameter configuration as an example\n", + "hyperparameter_config = autonet.get_hyperparameter_search_space().sample_configuration().get_dictionary()\n", + "\n", + "# Refit with sampled hyperparameter config for 120 s. This time on the full dataset.\n", + "results_refit = autonet.refit(X_train=X_train,\n", + " Y_train=Y_train,\n", + " X_valid=None,\n", + " Y_valid=None,\n", + " hyperparameter_config=hyperparameter_config,\n", + " autonet_config=autonet.get_current_autonet_config(),\n", + " budget=50)\n", + "\n", + "# Save json\n", + "with open(\"logs/results_refit.json\", \"w\") as file:\n", + " json.dump(results_refit, file)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***pred*** returns the predictions of the incumbent model. ***score*** can be used to evaluate the model on a test set. " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model prediction: [[0.]\n", + " [0.]\n", + " [0.]\n", + " [0.]\n", + " [0.]\n", + " [0.]\n", + " [0.]\n", + " [0.]\n", + " [0.]\n", + " [1.]]\n", + "Accuracy score 74.0\n" + ] + } + ], + "source": [ + "# See how the random configuration performs (often it just predicts 0)\n", + "score = autonet.score(X_test=X_test, Y_test=Y_test)\n", + "pred = autonet.predict(X=X_test)\n", + "\n", + "print(\"Model prediction:\", pred[0:10])\n", + "print(\"Accuracy score\", score)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, you can also get the incumbent model as PyTorch Sequential model via" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sequential(\n", + " (0): Linear(in_features=20, out_features=34, bias=True)\n", + " (1): Sequential(\n", + " (0): ResBlock(\n", + " (shortcut): Linear(in_features=34, out_features=48, bias=True)\n", + " (start_norm): Sequential(\n", + " (0): BatchNorm1d(34, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (1): ReLU()\n", + " )\n", + " (layers): Sequential(\n", + " (0): Linear(in_features=34, out_features=48, bias=True)\n", + " (1): BatchNorm1d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (2): ReLU()\n", + " (3): Dropout(p=0.4477955154159557, inplace=False)\n", + " (4): Linear(in_features=48, out_features=48, bias=True)\n", + " )\n", + " )\n", + " (1): ResBlock(\n", + " (layers): Sequential(\n", + " (0): BatchNorm1d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (1): ReLU()\n", + " (2): Linear(in_features=48, out_features=48, bias=True)\n", + " (3): BatchNorm1d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (4): ReLU()\n", + " (5): Dropout(p=0.4477955154159557, inplace=False)\n", + " (6): Linear(in_features=48, out_features=48, bias=True)\n", + " )\n", + " )\n", + " )\n", + " (2): Sequential(\n", + " (0): ResBlock(\n", + " (shortcut): Linear(in_features=48, out_features=62, bias=True)\n", + " (start_norm): Sequential(\n", + " (0): BatchNorm1d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (1): ReLU()\n", + " )\n", + " (layers): Sequential(\n", + " (0): Linear(in_features=48, out_features=62, bias=True)\n", + " (1): BatchNorm1d(62, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (2): ReLU()\n", + " (3): Dropout(p=0.22389775770797785, inplace=False)\n", + " (4): Linear(in_features=62, out_features=62, bias=True)\n", + " )\n", + " )\n", + " (1): ResBlock(\n", + " (layers): Sequential(\n", + " (0): BatchNorm1d(62, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (1): ReLU()\n", + " (2): Linear(in_features=62, out_features=62, bias=True)\n", + " (3): BatchNorm1d(62, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (4): ReLU()\n", + " (5): Dropout(p=0.22389775770797785, inplace=False)\n", + " (6): Linear(in_features=62, out_features=62, bias=True)\n", + " )\n", + " )\n", + " )\n", + " (3): Sequential(\n", + " (0): ResBlock(\n", + " (shortcut): Linear(in_features=62, out_features=79, bias=True)\n", + " (start_norm): Sequential(\n", + " (0): BatchNorm1d(62, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (1): ReLU()\n", + " )\n", + " (layers): Sequential(\n", + " (0): Linear(in_features=62, out_features=79, bias=True)\n", + " (1): BatchNorm1d(79, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (2): ReLU()\n", + " (3): Dropout(p=0.0, inplace=False)\n", + " (4): Linear(in_features=79, out_features=79, bias=True)\n", + " )\n", + " )\n", + " (1): ResBlock(\n", + " (layers): Sequential(\n", + " (0): BatchNorm1d(79, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (1): ReLU()\n", + " (2): Linear(in_features=79, out_features=79, bias=True)\n", + " (3): BatchNorm1d(79, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (4): ReLU()\n", + " (5): Dropout(p=0.0, inplace=False)\n", + " (6): Linear(in_features=79, out_features=79, bias=True)\n", + " )\n", + " )\n", + " )\n", + " (4): BatchNorm1d(79, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (5): ReLU()\n", + " (6): Linear(in_features=79, out_features=2, bias=True)\n", + ")\n" + ] + } + ], + "source": [ + "pytorch_model = autonet.get_pytorch_model()\n", + "print(pytorch_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Featurized Data\n", + "\n", + "All classes for featurized data (*AutoNetClassification*, *AutoNetMultilabel*, *AutoNetRegression*) can be used as in the example above. The only difference is the type of labels they accept." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Image Data\n", + "\n", + "Auto-PyTorch provides two classes for image data. *autonet_image_classification* can be used for classification for images. The *autonet_multi_image_classification* class allows to search for configurations for image classification across multiple datasets. This means Auto-PyTorch will try to choose a configuration that works well on all given datasets." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Could not find BOHB_Multi_KDE, replacing with object\n" + ] + } + ], + "source": [ + "# Load classes\n", + "autonet_image_classification = AutoNetImageClassification(config_preset=\"full_cs\", result_logger_dir=\"logs/\")\n", + "autonet_multi_image_classification = AutoNetImageClassificationMultipleDatasets(config_preset=\"tiny_cs\", result_logger_dir=\"logs/\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For passing your image data, you have two options (note that arrays are expected):\n", + "\n", + "I) Via a path to a comma-separated value file, which in turn contains the paths to the images and the image labels (note header is assumed to be None):" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "csv_dir = os.path.abspath(\"../../datasets/example.csv\")\n", + "\n", + "X_train = np.array([csv_dir])\n", + "Y_train = np.array([0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "II) directly passing the paths to the images and the labels" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_csv(csv_dir, header=None)\n", + "X_train = df.values[:,0]\n", + "Y_train = df.values[:,1]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "scrolled": false + }, + "source": [ + "Make sure you specify *image_root_folders* if the paths to the images are not specified from your current working directory. You can also specify *images_shape* to up- or downscale images.\n", + "\n", + "Using the flag *save_checkpoints=True* will save checkpoints to the result directory:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'loss': -84.48275862068965,\n", + " 'optimized_hyperparameter_config': {'NetworkSelectorDatasetInfo:darts:auxiliary': False,\n", + " 'NetworkSelectorDatasetInfo:darts:drop_path_prob': 0.1,\n", + " 'NetworkSelectorDatasetInfo:darts:init_channels': 36,\n", + " 'NetworkSelectorDatasetInfo:darts:layers': 20,\n", + " 'CreateImageDataLoader:batch_size': 58,\n", + " 'ImageAugmentation:augment': True,\n", + " 'ImageAugmentation:cutout': True,\n", + " 'LossModuleSelectorIndices:loss_module': 'cross_entropy',\n", + " 'NetworkSelectorDatasetInfo:network': 'densenet',\n", + " 'OptimizerSelector:optimizer': 'adam',\n", + " 'SimpleLearningrateSchedulerSelector:lr_scheduler': 'adapt',\n", + " 'SimpleTrainNode:batch_loss_computation_technique': 'standard',\n", + " 'ImageAugmentation:autoaugment': True,\n", + " 'ImageAugmentation:cutout_holes': 2,\n", + " 'ImageAugmentation:fastautoaugment': True,\n", + " 'ImageAugmentation:length': 17,\n", + " 'NetworkSelectorDatasetInfo:densenet:blocks': 4,\n", + " 'NetworkSelectorDatasetInfo:densenet:growth_rate': 28,\n", + " 'NetworkSelectorDatasetInfo:densenet:layer_in_block_1': 8,\n", + " 'NetworkSelectorDatasetInfo:densenet:layer_in_block_2': 16,\n", + " 'NetworkSelectorDatasetInfo:densenet:layer_in_block_3': 49,\n", + " 'NetworkSelectorDatasetInfo:densenet:use_dropout': False,\n", + " 'OptimizerSelector:adam:learning_rate': 0.00012377327234853046,\n", + " 'OptimizerSelector:adam:weight_decay': 0.06147134718475827,\n", + " 'SimpleLearningrateSchedulerSelector:adapt:T_max': 450,\n", + " 'SimpleLearningrateSchedulerSelector:adapt:T_mult': 1.189428111774201,\n", + " 'SimpleLearningrateSchedulerSelector:adapt:patience': 4,\n", + " 'SimpleLearningrateSchedulerSelector:adapt:threshold': 0.02209366315824298,\n", + " 'NetworkSelectorDatasetInfo:densenet:layer_in_block_4': 63},\n", + " 'budget': 400.0,\n", + " 'info': {}}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "autonet_image_classification.fit(X_train=X_train,\n", + " Y_train=Y_train,\n", + " images_shape=[3,32,32],\n", + " min_budget=200,\n", + " max_budget=400,\n", + " max_runtime=600,\n", + " save_checkpoints=True,\n", + " images_root_folders=[os.path.abspath(\"../../datasets/example_images\")])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Auto-PyTorch also supports some common datasets. By passing a comma-separated value file with just one line, e.g. \"CIFAR10, 0\" and specifying *default_dataset_download_dir* it will automatically download the data and use it for searching. Supported datasets are CIFAR10, CIFAR100, SVHN and MNIST." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 50000\n", + "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./datasets/cifar-10-python.tar.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100.0%" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ./datasets/cifar-10-python.tar.gz to ./datasets\n", + "Files already downloaded and verified\n", + "0 50000\n", + "Files already downloaded and verified\n", + "Files already downloaded and verified\n" + ] + }, + { + "data": { + "text/plain": [ + "{'loss': -31.03144184934964,\n", + " 'optimized_hyperparameter_config': {'NetworkSelectorDatasetInfo:darts:auxiliary': False,\n", + " 'NetworkSelectorDatasetInfo:darts:drop_path_prob': 0.1,\n", + " 'NetworkSelectorDatasetInfo:darts:init_channels': 36,\n", + " 'NetworkSelectorDatasetInfo:darts:layers': 20,\n", + " 'CreateImageDataLoader:batch_size': 55,\n", + " 'ImageAugmentation:augment': True,\n", + " 'ImageAugmentation:cutout': False,\n", + " 'LossModuleSelectorIndices:loss_module': 'cross_entropy',\n", + " 'NetworkSelectorDatasetInfo:network': 'darts',\n", + " 'OptimizerSelector:optimizer': 'sgd',\n", + " 'SimpleLearningrateSchedulerSelector:lr_scheduler': 'adapt',\n", + " 'SimpleTrainNode:batch_loss_computation_technique': 'mixup',\n", + " 'ImageAugmentation:autoaugment': False,\n", + " 'ImageAugmentation:fastautoaugment': True,\n", + " 'NetworkSelectorDatasetInfo:darts:edge_normal_0': 'avg_pool_3x3',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_normal_1': 'dil_conv_5x5',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_reduce_0': 'avg_pool_3x3',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_reduce_1': 'dil_conv_5x5',\n", + " 'NetworkSelectorDatasetInfo:darts:inputs_node_normal_3': '1_2',\n", + " 'NetworkSelectorDatasetInfo:darts:inputs_node_normal_4': '0_2',\n", + " 'NetworkSelectorDatasetInfo:darts:inputs_node_normal_5': '3_4',\n", + " 'NetworkSelectorDatasetInfo:darts:inputs_node_reduce_3': '0_1',\n", + " 'NetworkSelectorDatasetInfo:darts:inputs_node_reduce_4': '0_1',\n", + " 'NetworkSelectorDatasetInfo:darts:inputs_node_reduce_5': '0_1',\n", + " 'OptimizerSelector:sgd:learning_rate': 0.056126455704317305,\n", + " 'OptimizerSelector:sgd:momentum': 0.2554615131836397,\n", + " 'OptimizerSelector:sgd:weight_decay': 0.0064933176160527645,\n", + " 'SimpleLearningrateSchedulerSelector:adapt:T_max': 815,\n", + " 'SimpleLearningrateSchedulerSelector:adapt:T_mult': 1.2809915617006586,\n", + " 'SimpleLearningrateSchedulerSelector:adapt:patience': 3,\n", + " 'SimpleLearningrateSchedulerSelector:adapt:threshold': 0.36553138862173745,\n", + " 'SimpleTrainNode:mixup:alpha': 0.9499156113310157,\n", + " 'NetworkSelectorDatasetInfo:darts:edge_normal_12': 'sep_conv_3x3',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_normal_13': 'sep_conv_3x3',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_normal_3': 'sep_conv_5x5',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_normal_4': 'skip_connect',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_normal_5': 'max_pool_3x3',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_normal_7': 'avg_pool_3x3',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_reduce_10': 'sep_conv_5x5',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_reduce_2': 'sep_conv_5x5',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_reduce_3': 'max_pool_3x3',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_reduce_5': 'avg_pool_3x3',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_reduce_6': 'sep_conv_3x3',\n", + " 'NetworkSelectorDatasetInfo:darts:edge_reduce_9': 'skip_connect'},\n", + " 'budget': 900.0,\n", + " 'info': {}}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "path_to_cifar_csv = os.path.abspath(\"../../datasets/CIFAR10.csv\")\n", + "\n", + "autonet_image_classification.fit(X_train=np.array([path_to_cifar_csv]),\n", + " Y_train=np.array([0]),\n", + " min_budget=600,\n", + " max_budget=900,\n", + " max_runtime=1800,\n", + " default_dataset_download_dir=\"./datasets\",\n", + " images_root_folders=[\"./datasets\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For searching across multiple datasets, pass multiple csv files to the corresponding Auto-PyTorch class. Make sure your specify *images_root_folders* for each of them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "autonet_multi_image_classification.fit(X_train=np.array([path_to_cifar_csv, csv_dir]),\n", + " Y_train=np.array([0]),\n", + " min_budget=1500,\n", + " max_budget=2000,\n", + " max_runtime=4000,\n", + " default_dataset_download_dir=\"./datasets\",\n", + " images_root_folders=[\"./datasets\", \"./datasets/example_images\"])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/basics/ensemble.py b/examples/basics/ensemble.py index 93d70f31f..d52ed054f 100644 --- a/examples/basics/ensemble.py +++ b/examples/basics/ensemble.py @@ -11,7 +11,7 @@ dm = DataManager() dm.generate_classification(num_classes=3, num_features=21, num_samples=1500) -# Note: every parameter has a default value, you do not have to specify anything. The given parameter allow a fast test. +# Note: every parameter has a default value, you do not have to specify anything. The given parameters allow for a fast test. autonet = AutoNetEnsemble(AutoNetClassification, budget_type='epochs', min_budget=1, max_budget=9, num_iterations=1, log_level='debug') @@ -19,4 +19,4 @@ ensemble_only_consider_n_best=3) print(res) -print("Score:", autonet.score(X_test=dm.X_train, Y_test=dm.Y_train)) \ No newline at end of file +print("Score:", autonet.score(X_test=dm.X_train, Y_test=dm.Y_train)) diff --git a/examples/basics/modify_pipeline.py b/examples/basics/modify_pipeline.py index 5155a652d..bf7a81a86 100644 --- a/examples/basics/modify_pipeline.py +++ b/examples/basics/modify_pipeline.py @@ -20,7 +20,7 @@ autonet = AutoNetClassification(budget_type='epochs', min_budget=1, max_budget=9, num_iterations=1, log_level='info') logs = autonet.pipeline[LogFunctionsSelector.get_name()] -logs.add_log_function('test_result', test_result(autonet, dm.X_test, dm.Y_test)) +logs.add_log_function('test_result', test_result(autonet, dm.X_test, dm.Y_test), True) res = autonet.fit(X_train=dm.X, Y_train=dm.Y, X_valid=dm.X_train, Y_valid=dm.Y_train) diff --git a/examples/real_data/advanced_classification.py b/examples/real_data/advanced_classification.py index 7d13a2fe6..cc7b58897 100644 --- a/examples/real_data/advanced_classification.py +++ b/examples/real_data/advanced_classification.py @@ -54,13 +54,13 @@ autonet = AutoNetClassification() if TEST_CASE != 3 else AutoNetMultilabel() # add metrics and test_result to pipeline -autonet.pipeline[autonet_nodes.LogFunctionsSelector.get_name()].add_log_function('test_result', test_result(autonet, dm.X_test, dm.Y_test)) +autonet.pipeline[autonet_nodes.LogFunctionsSelector.get_name()].add_log_function('test_result', test_result(autonet, dm.X_test, dm.Y_test), True) # Fit autonet using train data res = autonet.fit(min_budget=300, max_budget=900, max_runtime=1800, budget_type='time', normalization_strategies=['maxabs'], - train_metric=metric, + optimize_metric=metric, additional_metrics=additional_metrices, cross_validator='stratified_k_fold', cross_validator_args={'n_splits': 3}, diff --git a/examples/real_data/openml_task.py b/examples/real_data/openml_task.py new file mode 100644 index 000000000..799d9267b --- /dev/null +++ b/examples/real_data/openml_task.py @@ -0,0 +1,30 @@ +import openml +from pprint import pprint +from autoPyTorch import AutoNetClassification +from sklearn.metrics import accuracy_score + + +# get OpenML task by its ID +task = openml.tasks.get_task(task_id=32) +X, y = task.get_X_and_y() +ind_train, ind_test = task.get_train_test_split_indices() + + +# run Auto-PyTorch +autoPyTorch = AutoNetClassification("tiny_cs", # config preset + log_level='info', + max_runtime=300, + min_budget=30, + max_budget=90) + +autoPyTorch.fit(X[ind_train], y[ind_train], validation_split=0.3) + + +# predict +y_pred = autoPyTorch.predict(X[ind_test]) + +print("Accuracy score", accuracy_score(y[ind_test], y_pred)) + + +# print network configuration +pprint(autoPyTorch.fit_result["optimized_hyperparameter_config"]) diff --git a/optional-requirements.txt b/optional-requirements.txt index e9b4bd0af..400f72608 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,4 +1,2 @@ SimpleITK -openml matplotlib -tensorboard_logger \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6b46729cb..eaa9db14d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,9 @@ scikit-learn>=0.20.0 imblearn ConfigSpace pynisher -hpbandster \ No newline at end of file +hpbandster +fasteners +torch +torchvision +tensorboard_logger +openml diff --git a/scripts/run_benchmark_cluster.py b/scripts/run_benchmark_cluster.py index d199a8f2b..d5173436f 100644 --- a/scripts/run_benchmark_cluster.py +++ b/scripts/run_benchmark_cluster.py @@ -36,6 +36,11 @@ host_config_orig = [l[13:] for l in runscript_template if l.startswith("#HOST_CONFIG ")][0].strip() host_config_file = os.path.join(autonet_home, host_config_orig) if not os.path.isabs(host_config_orig) else host_config_orig + # parse define statements + for i in range(len(runscript_template)): + if runscript_template[i].startswith("#DEFINE"): + runscript_template[i] = "%s=%s\n" % (runscript_template[i].split()[1], " ".join(runscript_template[i].split()[2:])) + # parse template args runscript_template_args = [l[19:].strip().split() for l in runscript_template if l.startswith("#TEMPLATE_ARGUMENT ")] parsed_template_args = dict() @@ -129,7 +134,6 @@ autonet_config = autonet.get_current_autonet_config() # add autonet config specific stuff to replacement dict - replacement_dict["NUM_WORKERS"] = autonet_config["min_workers"] replacement_dict["NUM_NODES"] = autonet_config["min_workers"] + (0 if autonet_config["run_worker_on_master_node"] else 1) replacement_dict["MEMORY_LIMIT_MB"] = autonet_config["memory_limit_mb"] + args.memory_bonus time_limit_base = autonet_config["max_runtime"] if autonet_config["max_runtime"] < float("inf") else (benchmark_config["time_limit"] - max(args.time_bonus)) diff --git a/scripts/run_benchmark_cluster_condensed.py b/scripts/run_benchmark_cluster_condensed.py new file mode 100644 index 000000000..b03ceb99b --- /dev/null +++ b/scripts/run_benchmark_cluster_condensed.py @@ -0,0 +1,228 @@ +import os, sys, re, shutil +import subprocess +import json +from math import ceil +from copy import copy +sys.path.append(os.path.abspath(os.path.join(__file__, "..", ".."))) + +from autoPyTorch.utils.config.config_file_parser import ConfigFileParser +from autoPyTorch.utils.benchmarking.benchmark import Benchmark +from autoPyTorch.utils.benchmarking.benchmark_pipeline import ForInstance, ForAutoNetConfig, ForRun, CreateAutoNet, SetAutoNetConfig +from autoPyTorch.data_management.data_manager import DataManager, ProblemType + +import argparse + +__author__ = "Max Dippel, Michael Burkart and Matthias Urban" +__version__ = "0.0.1" +__license__ = "BSD" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Run benchmarks for autonet.') + parser.add_argument("--partial_benchmark", default=None, nargs="+", help="Only run a part of the benchmark. Run other parts later or in parallel. 3-tuple: instance_slice, autonet_config_slice, run_number_range.") + parser.add_argument("--time_bonus", default=[7200, 8200, 10800], type=int, nargs="+", help="Give the job some more time.") + parser.add_argument("--memory_bonus", default=1000, type=int, help="Give the job some more memory. Unit: MB.") + parser.add_argument("--result_dir", default=None, help="The dir to save the results") + parser.add_argument("--output_dir", default=None, help="The dir to save the outputs") + parser.add_argument("--template_args", default=[], nargs="+", type=str, help="Additional args specified in template") + parser.add_argument("--num_condense", default=100, type=int, help="How many jobs should be condensed to single job array") + parser.add_argument("runscript", help="The script template used to submit job on cluster.") + parser.add_argument('benchmark', help='The benchmark to run') + args = parser.parse_args() + + # parse the runscript template + with open(args.runscript, "r") as f: + runscript_template = list(f) + runscript_name = os.path.basename(args.runscript if not args.runscript.endswith(".template") else args.runscript[:-9]) + autonet_home = ConfigFileParser.get_autonet_home() + host_config_orig = [l[13:] for l in runscript_template if l.startswith("#HOST_CONFIG ")][0].strip() + host_config_file = os.path.join(autonet_home, host_config_orig) if not os.path.isabs(host_config_orig) else host_config_orig + + # parse template args + runscript_template_args = [l[19:].strip().split() for l in runscript_template if l.startswith("#TEMPLATE_ARGUMENT ")] + parsed_template_args = dict() + for variable_name, default in runscript_template_args: + try: + value = [a.split("=")[1] for a in args.template_args if a.split("=")[0] == variable_name][0] + except IndexError: + value = default + parsed_template_args[variable_name] = value + + # get benchmark config + benchmark_config_file = args.benchmark + + benchmark = Benchmark() + config_parser = benchmark.get_benchmark_config_file_parser() + + benchmark_config = config_parser.read(benchmark_config_file) + benchmark_config.update(config_parser.read(host_config_file)) + config_parser.set_defaults(benchmark_config) + + # get ranges of runs, autonet_configs and instances + all_configs = ForAutoNetConfig.get_config_files(benchmark_config, parse_slice=False) + all_instances = ForInstance.get_instances(benchmark_config, instances_must_exist=True) + + runs_range = list(range(benchmark_config["num_runs"])) + configs_range = list(range(len(all_configs))) + instances_range = list(range(len(all_instances))) + + if args.partial_benchmark: + if len(args.partial_benchmark) > 0: + instances_range = instances_range[ForInstance.parse_slice(args.partial_benchmark[0])] + if len(args.partial_benchmark) > 1: + configs_range = configs_range[ForAutoNetConfig.parse_slice(args.partial_benchmark[1])] + if len(args.partial_benchmark) > 2: + runs_range = list(ForRun.parse_range(args.partial_benchmark[2], benchmark_config["num_runs"])) + + # set up dict used used to make replacements in runscript + base_dir = os.getcwd() + result_dir = os.path.abspath(args.result_dir) if args.result_dir is not None else benchmark_config["result_dir"] + outputs_folder = os.path.abspath(args.output_dir) if args.output_dir is not None else os.path.join(base_dir, "outputs") + benchmark = args.benchmark if os.path.isabs(args.benchmark) else os.path.join(base_dir, args.benchmark) + output_base_dir = os.path.join(outputs_folder, os.path.basename(benchmark).split(".")[0]) + replacement_dict = { + "BASE_DIR": base_dir, + "OUTPUTS_FOLDER": outputs_folder, + "OUTPUT_BASE_DIR": output_base_dir, + "AUTONET_HOME": autonet_home, + "BENCHMARK": benchmark, + "BENCHMARK_NAME": os.path.basename(benchmark).split(".")[0], + "HOST_CONFIG": host_config_file, + "ORIG_HOST_CONFIG": host_config_orig, + "ORIG_BENCHMARK": args.benchmark, + "RESULT_DIR": result_dir + } + replacement_dict.update(parsed_template_args) + + # create directories + if os.path.exists(output_base_dir) and input("%s exists. Delete? (y/n)" %output_base_dir).startswith("y"): + shutil.rmtree(output_base_dir) + if not os.path.exists(outputs_folder): + os.mkdir(outputs_folder) + if not os.path.exists(output_base_dir): + os.mkdir(output_base_dir) + if not os.path.exists(result_dir): + os.mkdir(result_dir) + + # divide script + divided_runscript_template = [[]] + for line in runscript_template: + if line.startswith("#JOBSCRIPT START"): + assert len(divided_runscript_template) == 1 + divided_runscript_template += [[]] + + divided_runscript_template[-1].append(line) + + if line.startswith("#JOBSCRIPT END"): + assert len(divided_runscript_template) == 2 + divided_runscript_template += [[]] + replacement_dicts = list() + replacement_dict_keys = set() + + + # iterate over all runs + for run_number in runs_range: + replacement_dict["RUN_NUMBER"] = run_number + + for config_id in configs_range: + replacement_dict["CONFIG_ID"] = config_id + replacement_dict["CONFIG_FILE"] = all_configs[config_id] + + # get autonet + dm = DataManager() + dm.problem_type = { + "feature_classification": ProblemType.FeatureClassification, + "feature_multilabel": ProblemType.FeatureMultilabel, + "feature_regression": ProblemType.FeatureRegression + }[benchmark_config["problem_type"]] + autonet = CreateAutoNet().fit(benchmark_config, dm)["autonet"] + autonet_config_file = benchmark_config["autonet_configs"][config_id] + + for instance_id in instances_range: + replacement_dict["INSTANCE_ID"] = instance_id + replacement_dict["INSTANCE_FILE"] = all_instances[instance_id] + + # read autonet config + SetAutoNetConfig().fit(benchmark_config, autonet, autonet_config_file, dm, all_instances[instance_id]) + autonet_config = autonet.get_current_autonet_config() + + # add autonet config specific stuff to replacement dict + replacement_dict["NUM_NODES"] = autonet_config["min_workers"] + (0 if autonet_config["run_worker_on_master_node"] else 1) + replacement_dict["MEMORY_LIMIT_MB"] = autonet_config["memory_limit_mb"] + args.memory_bonus + time_limit_base = autonet_config["max_runtime"] if autonet_config["max_runtime"] < float("inf") else (benchmark_config["time_limit"] - max(args.time_bonus)) + replacement_dict.update({("TIME_LIMIT[%s]" % i): int(t + time_limit_base) for i, t in enumerate(args.time_bonus)}) + replacement_dict["NUM_PROCESSES"] = max(autonet_config["torch_num_threads"], int(ceil(replacement_dict["MEMORY_LIMIT_MB"] / benchmark_config["memory_per_core"]))) + + replacement_dicts.append(copy(replacement_dict)) + replacement_dict_keys |= set(replacement_dict.keys()) + + # build final runscript + for k in range(ceil(len(replacement_dicts) / args.num_condense)): + output_dir = os.path.join(output_base_dir, "part_%s" % k) + if os.path.exists(output_dir) and input("%s exists. Delete? (y/n)" % output_dir).startswith("y"): + shutil.rmtree(output_dir) + os.mkdir(output_dir) + replacement_dicts_split = replacement_dicts[k * args.num_condense : (k + 1) * args.num_condense] + + # unify replacement dict + unified_replacement_dict = {"OUTPUT_DIR": output_dir} + for key in replacement_dict_keys: + all_values = [replacement_dict[key] for replacement_dict in replacement_dicts_split] + + if key == "NUM_NODES": + unified_replacement_dict[key] = sum(map(int, all_values)) + elif key in ["NUM_PROCESSES", "MEMORY_LIMIT_MB"] or key.startswith("TIME_LIMIT"): + unified_replacement_dict[key] = max(map(int, all_values)) + elif all(all_values[0] == v for v in all_values): + unified_replacement_dict[key] = all_values[0] + + final_runscript = [] + for i, part in enumerate(divided_runscript_template): + if i != 1: + pattern = re.compile("|".join(map(lambda x: re.escape("$${" + x + "}"), unified_replacement_dict.keys()))) + runscript = [pattern.sub(lambda x: str(unified_replacement_dict[x.group()[3:-1]]), l) for l in part] + + # DEFINE STATEMENTS + for j in range(len(runscript)): + if runscript[j].startswith("#DEFINE"): + runscript[j] = "GLOBAL_%s=%s\n" % (runscript[j].split()[1], " ".join(runscript[j].split()[2:])) + final_runscript.extend(runscript) + continue + + final_runscript += ["TASK_ID=$GLOBAL_TASK_ID\n"] + for j, replacement_dict in enumerate(replacement_dicts_split): + replacement_dict["OUTPUT_DIR"] = output_dir + runscript = [ + "if [ $TASK_ID -gt 0 ]; then\n", + "if [ $TASK_ID -le %s ]; then\n" % replacement_dict["NUM_NODES"], + "RUN_ID=\"${GLOBAL_RUN_ID}_[%s]\"\n" % j] + pattern = re.compile("|".join(map(lambda x: re.escape("$${" + x + "}"), replacement_dict.keys()))) + runscript += [pattern.sub(lambda x: str(replacement_dict[x.group()[3:-1]]), l) for l in part] + runscript += ["fi\n", "fi\n", "TASK_ID=`expr $TASK_ID - %s`\n" % replacement_dict["NUM_NODES"], "\n"] + final_runscript.extend(runscript) + + command = [l[9:] for l in final_runscript if l.startswith("#COMMAND ")][0].strip() + + # save runscript + with open(os.path.join(output_dir, runscript_name), "w") as f: + f.writelines(final_runscript) + + # submit job + os.chdir(output_dir) + print("Calling %s in %s" % (command, os.getcwd())) + try: + command_output = subprocess.check_output(command, shell=True) + except subprocess.CalledProcessError as e: + print("Warning: %s" % e) + command_output = str(e).encode("utf-8") + if not input("Continue (y/n)? ").startswith("y"): + raise + os.chdir(base_dir) + + # save output and info data + with open(os.path.join(output_dir, "call.info"), "w") as f: + print(command, file=f) + json.dump([unified_replacement_dict, replacement_dicts_split], f) + print("", file=f) + with open(os.path.join(output_dir, "call.info"), "ba") as f: + f.write(command_output) diff --git a/scripts/run_nemo_singularity.moab.template b/scripts/run_nemo_singularity.moab.template index 73ff89023..e8bf9a7d7 100644 --- a/scripts/run_nemo_singularity.moab.template +++ b/scripts/run_nemo_singularity.moab.template @@ -9,24 +9,29 @@ cp $${BASE_DIR}/$${IMAGE} $TMPDIR/image.simg cp $${BENCHMARK} $TMPDIR/$${BENCHMARK_NAME}.txt cp $${HOST_CONFIG} $TMPDIR/host_config.txt module load tools/singularity/2.6 - cd $TMPDIR -JOBID=(${MOAB_JOBID//[/ }) -COMMAND="python /data/Auto-PyTorch/scripts/run_benchmark.py /tmp/$${BENCHMARK_NAME}.txt --partial_benchmark $${INSTANCE_ID} $${CONFIG_ID} $${RUN_NUMBER} --host_config /tmp/host_config.txt --result_dir /tmp/benchmark_results --run_id $JOBID --task_id $MOAB_JOBARRAYINDEX" + +#DEFINE TASK_ID $MOAB_JOBARRAYINDEX +#DEFINE RUN_ID (${MOAB_JOBID//[/ }) + +#JOBSCRIPT START +COMMAND="python /data/Auto-PyTorch/scripts/run_benchmark.py /tmp/$${BENCHMARK_NAME}.txt --partial_benchmark $${INSTANCE_ID} $${CONFIG_ID} $${RUN_NUMBER} --host_config /tmp/host_config.txt --result_dir /tmp/benchmark_results --run_id $RUN_ID --task_id $TASK_ID" COMMAND="singularity exec -B $${AUTONET_HOME}:/external_autonet_home/ -B $TMPDIR:/tmp image.simg $COMMAND" echo "Run benchmark: $COMMAND" timeout -k $${TIME_LIMIT[1]} $${TIME_LIMIT[0]} $COMMAND 1> $TMPDIR/stdout.txt 2> $TMPDIR/stderr.txt -echo "Job finished. Copy output to $${OUTPUT_DIR}" -cp $TMPDIR/stdout.txt $${OUTPUT_DIR}/stdout_${MOAB_JOBARRAYINDEX}.txt -cp $TMPDIR/stderr.txt $${OUTPUT_DIR}/stderr_${MOAB_JOBARRAYINDEX}.txt - -if [ $MOAB_JOBARRAYINDEX -eq 1 ] +if [ $TASK_ID -eq 1 ] then echo "Copy benchmark results" cd $TMPDIR cp -r benchmark_results/* $${RESULT_DIR} fi +#JOBSCRIPT END + +echo "Job finished. Copy output to $${OUTPUT_DIR}" +cd $TMPDIR +cp stdout.txt $${OUTPUT_DIR}/stdout_${MOAB_JOBARRAYINDEX}.txt +cp stderr.txt $${OUTPUT_DIR}/stderr_${MOAB_JOBARRAYINDEX}.txt #COMMAND msub run_nemo_singularity.moab #HOST_CONFIG configs/hosts/nemo_singularity.txt diff --git a/scripts/visualize_benchmark.py b/scripts/visualize_benchmark.py index 8ad4c458c..bff95001b 100644 --- a/scripts/visualize_benchmark.py +++ b/scripts/visualize_benchmark.py @@ -33,12 +33,14 @@ parser.add_argument("--skip_average_plot", action="store_true", help="Whether the average plot should be skipped") parser.add_argument("--plot_markers", action="store_true", help="Whether markers should be plotted") parser.add_argument("--plot_individual", action="store_true", help="Whether the individual trajectories should be plotted") + parser.add_argument("--plot_type", default="values", help="Whether to plot metric values or losses") parser.add_argument("--xscale", default="log", type=str, help="Whether x should be in logscale") parser.add_argument("--yscale", default="linear", help="Whether x should be in logscale") parser.add_argument("--xmin", default=None, type=float, help="Limit the x axis") parser.add_argument("--xmax", default=None, type=float, help="Limit the x axis") parser.add_argument("--ymin", default=None, type=float, help="Limit the y axis") parser.add_argument("--ymax", default=None, type=float, help="Limit the y axis") + parser.add_argument("--value_multiplier", default=1, type=float, help="Multiply each value") parser.add_argument('benchmark', help='The benchmark to visualize') args = parser.parse_args() @@ -91,7 +93,8 @@ benchmark_config["ymin"] = args.ymin benchmark_config["ymax"] = args.ymax benchmark_config["plot_individual"] = args.plot_individual - benchmark_config["plot_markers"] = args.plot_markers - + benchmark_config["plot_markers"] = args.plot_markers + benchmark_config["plot_type"] = args.plot_type + benchmark_config["value_multiplier"] = args.value_multiplier benchmark.visualize_benchmark(**benchmark_config) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..b88034e41 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md diff --git a/setup.py b/setup.py index b72b0bcec..10c84cdae 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,9 @@ import os import setuptools +with open("README.md", "r") as f: + long_description = f.read() + requirements = [] with open('requirements.txt', 'r') as f: for line in f: @@ -18,21 +21,24 @@ setuptools.setup( name="autoPyTorch", - version="0.0.1", + version="0.0.2", author="AutoML Freiburg", - author_email="urbanm@informatik.uni-freiburg.de", + author_email="zimmerl@informatik.uni-freiburg.de", description=("Auto-PyTorch searches neural architectures using BO-HB"), + long_description=long_description, + url="https://github.com/automl/Auto-PyTorch", + long_description_content_type="text/markdown", license="3-clause BSD", keywords="machine learning algorithm configuration hyperparameter " "optimization tuning neural architecture deep learning", - url="", packages=setuptools.find_packages(), classifiers=[ "Development Status :: 3 - Alpha", "Topic :: Utilities", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Artificial Intelligence", - "License :: OSI Approved :: 3-clause BSD", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: BSD License", ], python_requires='>=3', platforms=['Linux'], diff --git a/test/test_pipeline/test_log_selector.py b/test/test_pipeline/test_log_selector.py index 36b1a4bcf..7a0cac701 100644 --- a/test/test_pipeline/test_log_selector.py +++ b/test/test_pipeline/test_log_selector.py @@ -28,7 +28,7 @@ def log_fnc2(network, epoch): log_functions = selector.fit_output['log_functions'] - self.assertListEqual(log_functions, [log_fnc2]) + self.assertListEqual([x.log for x in log_functions], [log_fnc2]) diff --git a/test/test_pipeline/test_metric_selector.py b/test/test_pipeline/test_metric_selector.py index a41113f26..188e6fbec 100644 --- a/test/test_pipeline/test_metric_selector.py +++ b/test/test_pipeline/test_metric_selector.py @@ -20,14 +20,14 @@ def test_selector(self): selector.add_metric("accuracy", accuracy) selector.add_metric("mean", mean_distance) - pipeline_config = pipeline.get_pipeline_config(train_metric="accuracy", additional_metrics=['auc', 'mean']) + pipeline_config = pipeline.get_pipeline_config(optimize_metric="accuracy", additional_metrics=['auc', 'mean']) pipeline.fit_pipeline(pipeline_config=pipeline_config) - selected_train_metric = selector.fit_output['train_metric'] + selected_optimize_metric = selector.fit_output['optimize_metric'] selected_additional_metrics = selector.fit_output['additional_metrics'] - self.assertEqual(selected_train_metric, accuracy) - self.assertSetEqual(set(selected_additional_metrics), set([auc_metric, mean_distance])) + self.assertEqual(selected_optimize_metric.metric, accuracy) + self.assertSetEqual(set(x.metric for x in selected_additional_metrics), set([auc_metric, mean_distance])) diff --git a/test/test_pipeline/test_optimization_algorithm.py b/test/test_pipeline/test_optimization_algorithm.py index f996ce765..fc1151f81 100644 --- a/test/test_pipeline/test_optimization_algorithm.py +++ b/test/test_pipeline/test_optimization_algorithm.py @@ -25,7 +25,7 @@ def test_optimizer(self): class ResultNode(PipelineNode): def fit(self, X_train, Y_train): - return {'loss': X_train.shape[1], 'info': {'a': X_train.shape[1], 'b': Y_train.shape[1]}} + return {'loss': X_train.shape[1], 'info': {'train_a': X_train.shape[1], 'train_b': Y_train.shape[1]}} def get_hyperparameter_search_space(self, **pipeline_config): cs = CS.ConfigurationSpace() @@ -34,7 +34,8 @@ def get_hyperparameter_search_space(self, **pipeline_config): def get_pipeline_config_options(self): return [ - ConfigOption("result_logger_dir", default=".", type="directory") + ConfigOption("result_logger_dir", default=".", type="directory"), + ConfigOption("optimize_metric", default="a", type=str), ] logger = logging.getLogger('hpbandster') @@ -55,4 +56,4 @@ def get_pipeline_config_options(self): result_of_opt_pipeline = pipeline[OptimizationAlgorithm.get_name()].fit_output['optimized_hyperparameter_config'] print(pipeline[OptimizationAlgorithm.get_name()].fit_output) - self.assertIn(result_of_opt_pipeline[ResultNode.get_name() + ConfigWrapper.delimiter + 'hyper'], list(range(0, 31))) \ No newline at end of file + self.assertIn(result_of_opt_pipeline[ResultNode.get_name() + ConfigWrapper.delimiter + 'hyper'], list(range(0, 31)))