[Feature] support instance_aware_colorization inference

.circleci/test.yml

@@ -62,6 +62,7 @@ jobs:
             pip install git+https://github.com/open-mmlab/mmengine.git@main
             pip install -U openmim
             mim install 'mmcv >= 2.0.0rc1'
+            mim install 'mmdet >= 3.0.0rc2'
             pip install -r requirements/tests.txt
       - run:
           name: Build and install
@@ -98,13 +99,14 @@ jobs:
           name: Build Docker image
           command: |
             docker build .circleci/docker -t mmedit:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >>
-            docker run --gpus all -t -d -v /home/circleci/project:/mmedit -v /home/circleci/mmengine:/mmengine -w /mmedit --name mmedit mmedit:gpu
+            docker run --gpus all -t -d -v /home/circleci/project:/mmedit -v /home/circleci/mmengine:/mmengine -v /home/circleci/mmdetection:/mmdetection -w /mmedit --name mmedit mmedit:gpu
       - run:
           name: Install mmedit dependencies
           command: |
             docker exec mmedit pip install -e /mmengine
             docker exec mmedit pip install -U openmim
             docker exec mmedit mim install 'mmcv >= 2.0.0rc1'
+            docker exec mmedit mim install 'mmdet >= 3.0.0rc2'
             docker exec mmedit pip install -r requirements/tests.txt
       - run:
           name: Build and install

.github/workflows/merge_stage_test.yml

@@ -41,10 +41,11 @@ jobs:
         run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
       - name: Install MMEngine
         run: pip install git+https://github.com/open-mmlab/mmengine.git@main
-      - name: Install MMCV
+      - name: Install MMCV and MMDet
         run: |
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
+          mim install 'mmdet >= 3.0.0rc2'
       - name: Install other dependencies
         run: pip install -r requirements/tests.txt
       - name: Build and install
@@ -88,10 +89,11 @@ jobs:
         run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
       - name: Install MMEngine
         run: pip install git+https://github.com/open-mmlab/mmengine.git@main
-      - name: Install MMCV
+      - name: Install MMCV and MMDet
         run: |
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
+          mim install 'mmdet >= 3.0.0rc2'
       - name: Install other dependencies
         run: pip install -r requirements/tests.txt
       - name: Build and install
@@ -177,6 +179,7 @@ jobs:
           python -m pip install git+https://github.com/open-mmlab/mmengine.git@main
           python -m pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
+          mim install 'mmdet >= 3.0.0rc2'
           python -m pip install -r requirements/tests.txt
       - name: Build and install
         run: |

.github/workflows/pr_stage_test.yml

@@ -35,10 +35,11 @@ jobs:
         run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
       - name: Install MMEngine
         run: pip install git+https://github.com/open-mmlab/mmengine.git@main
-      - name: Install MMCV
+      - name: Install MMCV and MMDet
         run: |
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
+          mim install 'mmdet >= 3.0.0rc2'
       - name: Install other dependencies
         run: pip install -r requirements/tests.txt
       - name: Build and install
@@ -94,6 +95,7 @@ jobs:
           pip install git+https://github.com/open-mmlab/mmengine.git@main
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
+          mim install 'mmdet >= 3.0.0rc2'
           pip install -r requirements/tests.txt
       - name: Build and install
         run: |
@@ -127,6 +129,7 @@ jobs:
           python -m pip install git+https://github.com/open-mmlab/mmengine.git@main
           python -m pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
+          mim install 'mmdet >= 3.0.0rc2'
           python -m pip install -r requirements/tests.txt
       - name: Build and install
         run: |

README.md

@@ -104,22 +104,26 @@ hope MMEditing could provide better experience.
 
 ## What's New
 
-- \[2022-09-13\] 🎉[MMGeneration](https://github.com/open-mmlab/mmgeneration/tree/1.x) was merged into MMEditing! And we are calling for your [suggestion](https://github.com/open-mmlab/mmediting/discussions/1108)!
-- \[2022-08-31\] v1.0.0rc0 was released. This release introduced a brand new and flexible training & test engine, but it's still in progress. Welcome
-  to try according to [the documentation](https://mmediting.readthedocs.io/en/1.x/).
-- \[2022-06-01\] v0.15.0 was released.
-  - Support FLAVR
-  - Support AOT-GAN
-  - Support CAIN with ReduceLROnPlateau Scheduler
-- \[2022-04-01\] v0.14.0 was released.
-  - Support TOFlow in video frame interpolation
-- \[2022-03-01\] v0.13.0 was released.
-  - Support CAIN
-  - Support EDVR-L
-  - Support running in Windows
-- \[2022-02-11\] Switch to **PyTorch 1.5+**. The compatibility to earlier versions of PyTorch will no longer be guaranteed.
-
-Please refer to [changelog.md](docs/en/notes/3_changelog.md) for details and release history.
+### 🌟 Preview of 1.x version
+
+A brand new version of [**MMEditing v1.0.0rc2**](https://github.com/open-mmlab/mmediting/releases/tag/v1.0.0rc2) was released in 02/11/2022:
+
+- Support all the tasks, models, metrics, and losses in [MMGeneration](https://github.com/open-mmlab/mmgeneration) 😍。
+- Unifies interfaces of all components based on [MMEngine](https://github.com/open-mmlab/mmengine).
+- Support patch-based and slider-based image and video comparison viewer.
+- Support image colorization.
+
+Find more new features in [1.x branch](https://github.com/open-mmlab/mmediting/tree/1.x). Issues and PRs are welcome!
+
+### 💎 Stable version
+
+**0.16.0** was released in 31/10/2022:
+
+- `VisualizationHook` is deprecated. Users should use `MMEditVisualizationHook` instead.
+- Fix FLAVR register.
+- Fix the number of channels in RDB.
+
+Please refer to [changelog.md](docs/en/changelog.md) for details and release history.
 
 ## Installation
 
@@ -215,6 +219,13 @@ Supported algorithms:
 
 </details>
 
+<details open>
+<summary>Image Colorization</summary>
+
+- ✅ [InstColorization](configs/inst_colorization/README.md) (CVPR'2020)
+
+</details>
+
 <details open>
 <summary>Unconditional GANs</summary>
 

README_zh-CN.md

@@ -37,7 +37,7 @@
 
 [English](/README.md) | 简体中文
 
-## Introduction
+## 介绍
 
 MMEditing 是基于 PyTorch 的图像&视频编辑和生成开源工具箱。是 [OpenMMLab](https://openmmlab.com/) 项目的成员之一。
 
@@ -101,24 +101,28 @@ https://user-images.githubusercontent.com/12756472/158972813-d8d0f19c-f49c-4618-
 需要注意的是 **MMSR** 已作为 MMEditing 的一部分并入本仓库。
 MMEditing 缜密地设计新的框架并将其精心实现，希望能够为您带来更好的体验。
 
-## 最新消息
-
-- \[2022-09-13\] 🎉 [MMGeneration](<(https://github.com/open-mmlab/mmgeneration/tree/1.x)>) 合入 MMEditing! 对于该合入计划，我们期待您的 [建议](https://github.com/open-mmlab/mmediting/discussions/1108)!
-- \[2022-08-31\] v1.0.0rc0 版本发布
-  这个版本引入一个全新的，可扩展性强的训练和测试引擎，但目前仍在开发中。欢迎根据[文档](https://mmediting.readthedocs.io/en/1.x/)进行试用。
-- \[2022-06-01\] v0.15.0 版本发布
-  - 支持 FLAVR
-  - 支持 AOT-GAN
-  - 新版 CAIN，支持 ReduceLROnPlateau 策略
-- \[2022-04-01\] v0.14.0 版本发布
-  - 支持视频插帧算法 TOFlow
-- \[2022-03-01\] v0.13.0 版本发布
-  - 支持 CAIN
-  - 支持 EDVR-L
-  - 支持在 Windows 系统中运行
-- \[2022-02-11\] 切换到 **PyTorch 1.5+**. 将不再保证与早期版本的 PyTorch 的兼容性
-
-请查看 [changelog.md](docs/zh_cn/notes/3_changelog.md) 以获取更多细节与发版记录
+## 最新进展
+
+### 🌟 1.x 预览版本
+
+全新的 [**MMEditing v1.0.0rc2**](https://github.com/open-mmlab/mmediting/releases/tag/v1.0.0rc2) 已经在 02/11/2022 发布:
+
+- 支持[MMGeneration](https://github.com/open-mmlab/mmgeneration)中的全量任务、模型、优化函数和评价指标 😍。
+- 基于[MMEngine](https://github.com/open-mmlab/mmengine)统一了各组件接口。
+- 支持基于图像子块以及滑动条的图像和视频比较可视化工具。
+- 支持图像上色任务。
+
+在[1.x 分支](https://github.com/open-mmlab/mmediting/tree/1.x)中发现更多特性！欢迎提 Issues 和 PRs！
+
+### 💎 稳定版本
+
+最新的 **0.16.0** 版本已经在 31/10/2022 发布：
+
+- `VisualizationHook` 将被启用，建议用户使用 `MMEditVisualizationHook`。
+- 修复 FLAVR 的注册问题。
+- 修正 RDB 模型中的通道数。
+
+如果像了解更多版本更新细节和历史信息，请阅读[更新日志](docs/en/changelog.md)。
 
 ## 安装
 
@@ -213,6 +217,13 @@ pip3 install -e .
 
 </details>
 
+<details open>
+<summary>图像上色</summary>
+
+- ✅ [InstColorization](configs/inst_colorization/README.md) (CVPR'2020)
+
+</details>
+
 <details open>
 <summary>Unconditional GANs</summary>
 

configs/inst_colorization/README.md

@@ -0,0 +1,55 @@
+# Instance-aware Image Colorization (CVPR'2020)
+
+> [Instance-Aware Image Colorization](https://openaccess.thecvf.com/content_CVPR_2020/html/Su_Instance-Aware_Image_Colorization_CVPR_2020_paper.html)
+
+> **Task**: Colorization
+
+<!-- [ALGORITHM] -->
+
+## Abstract
+
+<!-- [ABSTRACT] -->
+
+Image colorization is inherently an ill-posed problem with multi-modal uncertainty. Previous methods leverage the deep neural network to map input grayscale images to plausible color outputs directly. Although these learning-based methods have shown impressive performance, they usually fail on the input images that contain multiple objects. The leading cause is that existing models perform learning and colorization on the entire image. In the absence of a clear figure-ground separation, these models cannot effectively locate and learn meaningful object-level semantics. In this paper, we propose a method for achieving instance-aware colorization. Our network architecture leverages an off-the-shelf object detector to obtain cropped object images and uses an instance colorization network to extract object-level features. We use a similar network to extract the full-image features and apply a fusion module to full object-level and image-level features to predict the final colors. Both colorization networks and fusion modules are learned from a large-scale dataset. Experimental results show that our work outperforms existing methods on different quality metrics and achieves state-of-the-art performance on image colorization.
+
+<!-- [IMAGE] -->
+
+<div align=center >
+ <img src="https://github.com/ericsujw/InstColorization/blob/master/imgs/teaser.png?raw=true" width="400"/>
+</div >
+
+## Results and models
+
+|                                               Method                                                |                                               Download                                                |
+| :-------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------: |
+| [instance_aware_colorization_officiial](/configs/inst_colorization/inst-colorizatioon_full_official_cocostuff-256x256.py) | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmediting/inst_colorization/inst-colorizatioon_full_official_cocostuff-256x256-5b9d4eee.pth) |
+
+## Quick Start
+
+<details>
+<summary>Colorization demo</summary>
+
+You can use the following commands to colorize an image.
+
+```shell
+
+python demo/colorization_demo.py configs/inst_colorization/inst-colorizatioon_full_official_cocostuff-256x256.py https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmediting/inst_colorization/inst-colorizatioon_full_official_cocostuff-256x256-5b9d4eee.pth input.jpg output.jpg
+```
+
+For more demos, you can refer to [Tutorial 3: inference with pre-trained models](https://mmediting.readthedocs.io/en/1.x/user_guides/3_inference.html).
+
+</details>
+
+<details>
+<summary align="right">Instance-aware Image Colorization (CVPR'2020)</summary>
+
+```bibtex
+@inproceedings{Su-CVPR-2020,
+  author = {Su, Jheng-Wei and Chu, Hung-Kuo and Huang, Jia-Bin},
+  title = {Instance-aware Image Colorization},
+  booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
+  year = {2020}
+}
+```
+
+</details>

configs/inst_colorization/README_zh-CN.md

@@ -0,0 +1,54 @@
+# Instance-aware Image Colorization (CVPR'2020)
+
+> [Instance-Aware Image Colorization](https://openaccess.thecvf.com/content_CVPR_2020/html/Su_Instance-Aware_Image_Colorization_CVPR_2020_paper.html)
+
+> **任务**: 图像上色
+
+<!-- [ALGORITHM] -->
+
+## 摘要
+
+<!-- [ABSTRACT] -->
+
+Image colorization is inherently an ill-posed problem with multi-modal uncertainty. Previous methods leverage the deep neural network to map input grayscale images to plausible color outputs directly. Although these learning-based methods have shown impressive performance, they usually fail on the input images that contain multiple objects. The leading cause is that existing models perform learning and colorization on the entire image. In the absence of a clear figure-ground separation, these models cannot effectively locate and learn meaningful object-level semantics. In this paper, we propose a method for achieving instance-aware colorization. Our network architecture leverages an off-the-shelf object detector to obtain cropped object images and uses an instance colorization network to extract object-level features. We use a similar network to extract the full-image features and apply a fusion module to full object-level and image-level features to predict the final colors. Both colorization networks and fusion modules are learned from a large-scale dataset. Experimental results show that our work outperforms existing methods on different quality metrics and achieves state-of-the-art performance on image colorization.
+
+<!-- [IMAGE] -->
+
+<div align=center >
+ <img src="https://github.com/ericsujw/InstColorization/blob/master/imgs/teaser.png?raw=true" width="400"/>
+</div >
+
+## 结果和模型
+
+|                                               Method                                                |                                               Download                                                |
+| :-------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------: |
+| [instance_aware_colorization_officiial](/configs/inst_colorization/inst-colorizatioon_full_official_cocostuff-256x256.py) | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmediting/inst_colorization/inst-colorizatioon_full_official_cocostuff-256x256-5b9d4eee.pth) |
+
+## 快速开始
+
+<details>
+<summary>图像上色模型</summary>
+
+您可以使用以下命令来对一张图像进行上色。
+
+```shell
+python demo/colorization_demo.py configs/inst_colorization/inst-colorizatioon_full_official_cocostuff-256x256.py https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmediting/inst_colorization/inst-colorizatioon_full_official_cocostuff-256x256-5b9d4eee.pth input.jpg output.jpg
+```
+
+更多细节可以参考 [Tutorial 3: inference with pre-trained models](https://mmediting.readthedocs.io/en/1.x/user_guides/3_inference.html)。
+
+</details>
+
+<details>
+<summary align="right">Instance-aware Image Colorization (CVPR'2020)</summary>
+
+```bibtex
+@inproceedings{Su-CVPR-2020,
+  author = {Su, Jheng-Wei and Chu, Hung-Kuo and Huang, Jia-Bin},
+  title = {Instance-aware Image Colorization},
+  booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
+  year = {2020}
+}
+```
+
+</details>

configs/inst_colorization/inst-colorizatioon_full_official_cocostuff-256x256.py

@@ -0,0 +1,59 @@
+_base_ = ['../_base_/default_runtime.py']
+
+experiment_name = 'inst-colorization_full_official_cocostuff_256x256'
+work_dir = f'./work_dirs/{experiment_name}'
+save_dir = './work_dirs/'
+
+stage = 'full'
+
+model = dict(
+    type='InstColorization',
+    data_preprocessor=dict(
+        type='EditDataPreprocessor',
+        mean=[127.5],
+        std=[127.5],
+    ),
+    image_model=dict(
+        type='ColorizationNet', input_nc=4, output_nc=2, norm_type='batch'),
+    instance_model=dict(
+        type='ColorizationNet', input_nc=4, output_nc=2, norm_type='batch'),
+    fusion_model=dict(
+        type='FusionNet', input_nc=4, output_nc=2, norm_type='batch'),
+    color_data_opt=dict(
+        ab_thresh=0,
+        p=1.0,
+        sample_PS=[
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+        ],
+        ab_norm=110,
+        ab_max=110.,
+        ab_quant=10.,
+        l_norm=100.,
+        l_cent=50.,
+        mask_cent=0.5),
+    which_direction='AtoB',
+    loss=dict(type='HuberLoss', delta=.01))
+
+# yapf: disable
+test_pipeline = [
+    dict(type='LoadImageFromFile', key='img', channel_order='rgb'),
+    dict(
+        type='InstanceCrop',
+        config_file='mmdet::mask_rcnn/mask-rcnn_x101-32x8d_fpn_ms-poly-3x_coco.py',  # noqa
+        finesize=256,
+        box_num_upbound=5),
+    dict(
+        type='Resize',
+        keys=['img', 'cropped_img'],
+        scale=(256, 256),
+        keep_ratio=False),
+    dict(type='PackEditInputs'),
+]