From 68a764ca8651f2cdbefb01a8753166301912d891 Mon Sep 17 00:00:00 2001 From: Ben Swinney Date: Wed, 17 Jul 2019 23:21:05 +1000 Subject: [PATCH 1/5] Ensure all images loaded into DataLoader have 3 channnels --- fasterai/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fasterai/dataset.py b/fasterai/dataset.py index 7586e37..9183c8e 100644 --- a/fasterai/dataset.py +++ b/fasterai/dataset.py @@ -9,7 +9,7 @@ from .augs import noisify def get_colorize_data(sz:int, bs:int, crappy_path:Path, good_path:Path, random_seed:int=None, keep_pct:float=1.0, num_workers:int=8, xtra_tfms=[])->ImageDataBunch: - src = (ImageImageList.from_folder(crappy_path) + src = (ImageImageList.from_folder(crappy_path, convert_mode='RGB') .use_partial_data(sample_pct=keep_pct, seed=random_seed) .split_by_rand_pct(0.1, seed=random_seed)) From 5080166fcd4102739ff15f757b65b762f5c9ef10 Mon Sep 17 00:00:00 2001 From: Ben Swinney Date: Wed, 17 Jul 2019 23:34:48 +1000 Subject: [PATCH 2/5] .vscode config --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 02551cb..ebab98f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,4 @@ { - "python.pythonPath": "/home/jason/anaconda3/envs/fastaiv1/bin/python", + "python.pythonPath": "/usr/local/bin/python3", "python.linting.enabled": true } \ No newline at end of file From 1b457ce22d0964ac54216c0025b9d68db109cd5c Mon Sep 17 00:00:00 2001 From: Ben Swinney Date: Wed, 17 Jul 2019 23:35:32 +1000 Subject: [PATCH 3/5] Added PyTorch Large Model Support and increased training to 256px --- ColorizeTrainingStable.ipynb | 162 ++++++++++++++++++++++++++++++++--- 1 file changed, 151 insertions(+), 11 deletions(-) diff --git a/ColorizeTrainingStable.ipynb b/ColorizeTrainingStable.ipynb index 1963ab9..e67e14e 100644 --- a/ColorizeTrainingStable.ipynb +++ b/ColorizeTrainingStable.ipynb @@ -53,29 +53,115 @@ "## Setup" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Activate Large Model Support for PyTorch\n", + "This will allow us to fit the model within a GPU with smaller memory capacity (e.g. GTX 1070 8Gb)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Large Model Support (LMS) is a feature provided in IBM Watson Machine Learning Community Edition (WML-CE) PyTorch V1.1.0 that allows the successful training of deep learning models that would otherwise exhaust GPU memory and abort with “out-of-memory” errors. LMS manages this oversubscription of GPU memory by temporarily swapping tensors to host memory when they are not needed. One or more elements of a deep learning model can lead to GPU memory exhaustion.\n", + "\n", + "Requires the use of IBM WML-CE (Available here: https://www.ibm.com/support/knowledgecenter/en/SS5SF7_1.6.1/welcome/welcome.html)\n", + "\n", + "Further Reading on PyTorch with Large Model Support: https://www.ibm.com/support/knowledgecenter/en/SS5SF7_1.6.1/welcome/welcome.html" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ + "import shutil" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Set limit of GPU used before swapping to tensors to host memory\n", + "max_gpu_mem = 7\n", + "\n", + "def gb_to_bytes(gb):\n", + " return gb*1024*1024*1024\n", + "\n", + "# Enable PyTorch LMS\n", + "torch.cuda.set.enabled_lms(True)\n", + "# Set LMS limit\n", + "torch.cuda.set_limit_lms(gb_to_bytes(max_gpu_memory))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check LMS is enabled\n", + "torch.cuda.get_enabled_lms()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check LMS Limit has been set\n", + "torch.cuda.get_limit_lms()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Path to Training Data\n", "path = Path('data/imagenet/ILSVRC/Data/CLS-LOC')\n", "path_hr = path\n", - "path_lr = path/'bandw'\n", "\n", + "# Path to Black and White images\n", + "path_bandw = Path('/training/DeOldify')\n", + "path_lr = path_bandw/'bandw'\n", + "\n", + "# Name of Model\n", "proj_id = 'StableModel'\n", "\n", + "# Name of Generator\n", "gen_name = proj_id + '_gen'\n", "pre_gen_name = gen_name + '_0'\n", + "\n", + "# Name of Critic\n", "crit_name = proj_id + '_crit'\n", "\n", + "# Name of Generated Images folder, located within the Black and White folder\n", "name_gen = proj_id + '_image_gen'\n", "path_gen = path/name_gen\n", "\n", + "# Path to tensorboard data\n", "TENSORBOARD_PATH = Path('data/tensorboard/' + proj_id)\n", "\n", "nf_factor = 2\n", - "pct_start = 1e-8" + "pct_start = 1e-8\n", + "\n", + "# Number of workers for DataLoader\n", + "num_works = 2" ] }, { @@ -86,7 +172,7 @@ "source": [ "def get_data(bs:int, sz:int, keep_pct:float):\n", " return get_colorize_data(sz=sz, bs=bs, crappy_path=path_lr, good_path=path_hr, \n", - " random_seed=None, keep_pct=keep_pct)\n", + " random_seed=None, keep_pct=keep_pct, num_workers=num_works)\n", "\n", "def get_crit_data(classes, bs, sz):\n", " src = ImageList.from_folder(path, include=classes, recurse=True).random_split_by_pct(0.1, seed=42)\n", @@ -172,7 +258,7 @@ "metadata": {}, "outputs": [], "source": [ - "bs=88\n", + "bs=88 # This can be increased if using PyTorch LMS, training could be slower.\n", "sz=64\n", "keep_pct=1.0" ] @@ -262,7 +348,7 @@ "metadata": {}, "outputs": [], "source": [ - "bs=20\n", + "bs=40 # This can be increased if using PyTorch LMS, training could be slower.\n", "sz=128\n", "keep_pct=1.0" ] @@ -316,7 +402,7 @@ "metadata": {}, "outputs": [], "source": [ - "bs=8\n", + "bs=16 # This can be increased if using PyTorch LMS, training could be slower.\n", "sz=192\n", "keep_pct=0.50" ] @@ -357,6 +443,60 @@ "learn_gen.save(pre_gen_name)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 256px" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bs=8 # This can be increased if using PyTorch LMS, training could be slower.\n", + "sz=256\n", + "keep_pct=0.50" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn_gen.data = get_data(sz=sz, bs=bs, keep_pct=keep_pct)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn_gen.unfreeze()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn_gen.fit_one_cycle(1, pct_start=pct_start, max_lr=slice(5e-8,5e-5))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn_gen.save(pre_gen_name)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -400,7 +540,7 @@ "outputs": [], "source": [ "bs=8\n", - "sz=192" + "sz=256" ] }, { @@ -460,8 +600,8 @@ "metadata": {}, "outputs": [], "source": [ - "bs=16\n", - "sz=192" + "bs=8\n", + "sz=256" ] }, { @@ -543,7 +683,7 @@ "outputs": [], "source": [ "lr=2e-5\n", - "sz=192\n", + "sz=256\n", "bs=5" ] }, @@ -624,7 +764,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.0" + "version": "3.7.3" } }, "nbformat": 4, From 91a7438c12cf678d427de8905cb7f0f684d2099a Mon Sep 17 00:00:00 2001 From: Ben Swinney Date: Wed, 17 Jul 2019 23:52:56 +1000 Subject: [PATCH 4/5] Fixed typo on max_gpu_mem --- ColorizeTrainingStable.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ColorizeTrainingStable.ipynb b/ColorizeTrainingStable.ipynb index e67e14e..a9daf36 100644 --- a/ColorizeTrainingStable.ipynb +++ b/ColorizeTrainingStable.ipynb @@ -96,7 +96,7 @@ "# Enable PyTorch LMS\n", "torch.cuda.set.enabled_lms(True)\n", "# Set LMS limit\n", - "torch.cuda.set_limit_lms(gb_to_bytes(max_gpu_memory))" + "torch.cuda.set_limit_lms(gb_to_bytes(max_gpu_mem))" ] }, { From 684fbabea8acc3466c652ff270444bec6cf30d81 Mon Sep 17 00:00:00 2001 From: Ben Swinney Date: Wed, 17 Jul 2019 23:54:13 +1000 Subject: [PATCH 5/5] Delete settings.json --- .vscode/settings.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index ebab98f..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "python.pythonPath": "/usr/local/bin/python3", - "python.linting.enabled": true -} \ No newline at end of file