mirror of
https://github.com/jantic/DeOldify.git
synced 2026-08-30 18:02:24 +08:00
Adding fork of fastai- life is easier like this
This commit is contained in:
Jason Antic
@@ -0,0 +1 @@
|
||||
weights/
|
||||
@@ -0,0 +1,60 @@
|
||||
Apache License, Version 2.0 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License.
|
||||
|
||||
Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License.
|
||||
|
||||
Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution.
|
||||
|
||||
You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
|
||||
|
||||
You must give any other recipients of the Work or Derivative Works a copy of this License; and You must cause any modified files to carry prominent notices stating that You changed the files; and You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions.
|
||||
|
||||
Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks.
|
||||
|
||||
This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty.
|
||||
|
||||
Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability.
|
||||
|
||||
In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability.
|
||||
|
||||
While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
from .lm_rnn import *
|
||||
|
||||
class AdaptiveSoftmax(nn.Module):
|
||||
def __init__(self, input_size, cutoff):
|
||||
super().__init__()
|
||||
self.input_size,self.cutoff = input_size,cutoff
|
||||
self.output_size = cutoff[0] + len(cutoff) - 1
|
||||
self.head = nn.Linear(input_size, self.output_size)
|
||||
self.tail = nn.ModuleList()
|
||||
for i in range(len(cutoff) - 1):
|
||||
seq = nn.Sequential(nn.Linear(input_size, input_size // 4 ** i, False),
|
||||
nn.Linear(input_size // 4 ** i, cutoff[i + 1] - cutoff[i], False))
|
||||
self.tail.append(seq)
|
||||
|
||||
def reset(self):
|
||||
nn.init.xavier_normal(self.head.weight)
|
||||
for tail in self.tail:
|
||||
nn.init.xavier_normal(tail[0].weight)
|
||||
nn.init.xavier_normal(tail[1].weight)
|
||||
|
||||
def set_target(self, target):
|
||||
self.id = []
|
||||
for i in range(len(self.cutoff) - 1):
|
||||
mask = target.ge(self.cutoff[i]).mul(target.lt(self.cutoff[i + 1]))
|
||||
if mask.sum() > 0:
|
||||
self.id.append(Variable(mask.float().nonzero().squeeze(1)))
|
||||
else: self.id.append(None)
|
||||
|
||||
def forward(self, input):
|
||||
output = [self.head(input)]
|
||||
for i in range(len(self.id)):
|
||||
if self.id[i] is not None:
|
||||
output.append(self.tail[i](input.index_select(0, self.id[i])))
|
||||
else: output.append(None)
|
||||
return output
|
||||
|
||||
def log_prob(self, input):
|
||||
lsm = nn.LogSoftmax().cuda()
|
||||
head_out = self.head(input)
|
||||
batch_size = head_out.size(0)
|
||||
prob = torch.zeros(batch_size, self.cutoff[-1]).cuda()
|
||||
lsm_head = lsm(head_out)
|
||||
prob.narrow(1, 0, self.output_size).add_(lsm_head.narrow(1, 0, self.output_size).data)
|
||||
for i in range(len(self.tail)):
|
||||
pos = self.cutoff[i]
|
||||
i_size = self.cutoff[i + 1] - pos
|
||||
buffer = lsm_head.narrow(1, self.cutoff[0] + i, 1)
|
||||
buffer = buffer.expand(batch_size, i_size)
|
||||
lsm_tail = lsm(self.tail[i](input))
|
||||
prob.narrow(1, pos, i_size).copy_(buffer.data).add_(lsm_tail.data)
|
||||
return prob
|
||||
|
||||
|
||||
class AdaptiveLoss(nn.Module):
|
||||
def __init__(self, cutoff):
|
||||
super().__init__()
|
||||
self.cutoff = cutoff
|
||||
self.criterions = nn.ModuleList([nn.CrossEntropyLoss(size_average=False) for i in self.cutoff])
|
||||
|
||||
def remap_target(self, target):
|
||||
new_target = [target.clone()]
|
||||
for i in range(len(self.cutoff) - 1):
|
||||
mask = target.ge(self.cutoff[i]).mul(target.lt(self.cutoff[i + 1]))
|
||||
new_target[0][mask] = self.cutoff[0] + i
|
||||
if mask.sum() > 0: new_target.append(target[mask].add(-self.cutoff[i]))
|
||||
else: new_target.append(None)
|
||||
return new_target
|
||||
|
||||
def forward(self, input, target):
|
||||
batch_size = input[0].size(0)
|
||||
target = self.remap_target(target.data)
|
||||
output = 0.0
|
||||
for i in range(len(input)):
|
||||
if input[i] is not None:
|
||||
assert(target[i].min() >= 0 and target[i].max() <= input[i].size(1))
|
||||
criterion = self.criterions[i]
|
||||
output += criterion(input[i], Variable(target[i]))
|
||||
output /= batch_size
|
||||
return output
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
from .dataset import *
|
||||
from .learner import *
|
||||
|
||||
|
||||
class PassthruDataset(Dataset):
|
||||
def __init__(self,*args, is_reg=True, is_multi=False):
|
||||
*xs,y=args
|
||||
self.xs,self.y = xs,y
|
||||
self.is_reg = is_reg
|
||||
self.is_multi = is_multi
|
||||
|
||||
def __len__(self): return len(self.y)
|
||||
def __getitem__(self, idx): return [o[idx] for o in self.xs] + [self.y[idx]]
|
||||
|
||||
@classmethod
|
||||
def from_data_frame(cls, df, cols_x, col_y, is_reg=True, is_multi=False):
|
||||
cols = [df[o] for o in cols_x+[col_y]]
|
||||
return cls(*cols, is_reg=is_reg, is_multi=is_multi)
|
||||
|
||||
|
||||
class ColumnarDataset(Dataset):
|
||||
def __init__(self, cats, conts, y, is_reg, is_multi):
|
||||
n = len(cats[0]) if cats else len(conts[0])
|
||||
self.cats = np.stack(cats, 1).astype(np.int64) if cats else np.zeros((n,1))
|
||||
self.conts = np.stack(conts, 1).astype(np.float32) if conts else np.zeros((n,1))
|
||||
self.y = np.zeros((n,1)) if y is None else y
|
||||
if is_reg:
|
||||
self.y = self.y[:,None]
|
||||
self.is_reg = is_reg
|
||||
self.is_multi = is_multi
|
||||
|
||||
def __len__(self): return len(self.y)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return [self.cats[idx], self.conts[idx], self.y[idx]]
|
||||
|
||||
@classmethod
|
||||
def from_data_frames(cls, df_cat, df_cont, y=None, is_reg=True, is_multi=False):
|
||||
cat_cols = [c.values for n,c in df_cat.items()]
|
||||
cont_cols = [c.values for n,c in df_cont.items()]
|
||||
return cls(cat_cols, cont_cols, y, is_reg, is_multi)
|
||||
|
||||
@classmethod
|
||||
def from_data_frame(cls, df, cat_flds, y=None, is_reg=True, is_multi=False):
|
||||
return cls.from_data_frames(df[cat_flds], df.drop(cat_flds, axis=1), y, is_reg, is_multi)
|
||||
|
||||
|
||||
class ColumnarModelData(ModelData):
|
||||
def __init__(self, path, trn_ds, val_ds, bs, test_ds=None, shuffle=True):
|
||||
test_dl = DataLoader(test_ds, bs, shuffle=False, num_workers=1) if test_ds is not None else None
|
||||
super().__init__(path, DataLoader(trn_ds, bs, shuffle=shuffle, num_workers=1),
|
||||
DataLoader(val_ds, bs*2, shuffle=False, num_workers=1), test_dl)
|
||||
|
||||
@classmethod
|
||||
def from_arrays(cls, path, val_idxs, xs, y, is_reg=True, is_multi=False, bs=64, test_xs=None, shuffle=True):
|
||||
((val_xs, trn_xs), (val_y, trn_y)) = split_by_idx(val_idxs, xs, y)
|
||||
test_ds = PassthruDataset(*(test_xs.T), [0] * len(test_xs), is_reg=is_reg, is_multi=is_multi) if test_xs is not None else None
|
||||
return cls(path, PassthruDataset(*(trn_xs.T), trn_y, is_reg=is_reg, is_multi=is_multi),
|
||||
PassthruDataset(*(val_xs.T), val_y, is_reg=is_reg, is_multi=is_multi),
|
||||
bs=bs, shuffle=shuffle, test_ds=test_ds)
|
||||
|
||||
@classmethod
|
||||
def from_data_frames(cls, path, trn_df, val_df, trn_y, val_y, cat_flds, bs, is_reg, is_multi, test_df=None, shuffle=True):
|
||||
trn_ds = ColumnarDataset.from_data_frame(trn_df, cat_flds, trn_y, is_reg, is_multi)
|
||||
val_ds = ColumnarDataset.from_data_frame(val_df, cat_flds, val_y, is_reg, is_multi)
|
||||
test_ds = ColumnarDataset.from_data_frame(test_df, cat_flds, None, is_reg, is_multi) if test_df is not None else None
|
||||
return cls(path, trn_ds, val_ds, bs, test_ds=test_ds, shuffle=shuffle)
|
||||
|
||||
@classmethod
|
||||
def from_data_frame(cls, path, val_idxs, df, y, cat_flds, bs, is_reg=True, is_multi=False, test_df=None, shuffle=True):
|
||||
((val_df, trn_df), (val_y, trn_y)) = split_by_idx(val_idxs, df, y)
|
||||
return cls.from_data_frames(path, trn_df, val_df, trn_y, val_y, cat_flds, bs, is_reg, is_multi, test_df=test_df, shuffle=shuffle)
|
||||
|
||||
def get_learner(self, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
|
||||
y_range=None, use_bn=False, **kwargs):
|
||||
model = MixedInputModel(emb_szs, n_cont, emb_drop, out_sz, szs, drops, y_range, use_bn, self.is_reg, self.is_multi)
|
||||
return StructuredLearner(self, StructuredModel(to_gpu(model)), opt_fn=optim.Adam, **kwargs)
|
||||
|
||||
|
||||
def emb_init(x):
|
||||
x = x.weight.data
|
||||
sc = 2/(x.size(1)+1)
|
||||
x.uniform_(-sc,sc)
|
||||
|
||||
|
||||
class MixedInputModel(nn.Module):
|
||||
def __init__(self, emb_szs, n_cont, emb_drop, out_sz, szs, drops,
|
||||
y_range=None, use_bn=False, is_reg=True, is_multi=False):
|
||||
super().__init__()
|
||||
self.embs = nn.ModuleList([nn.Embedding(c, s) for c,s in emb_szs])
|
||||
for emb in self.embs: emb_init(emb)
|
||||
n_emb = sum(e.embedding_dim for e in self.embs)
|
||||
self.n_emb, self.n_cont=n_emb, n_cont
|
||||
|
||||
szs = [n_emb+n_cont] + szs
|
||||
self.lins = nn.ModuleList([
|
||||
nn.Linear(szs[i], szs[i+1]) for i in range(len(szs)-1)])
|
||||
self.bns = nn.ModuleList([
|
||||
nn.BatchNorm1d(sz) for sz in szs[1:]])
|
||||
for o in self.lins: kaiming_normal(o.weight.data)
|
||||
self.outp = nn.Linear(szs[-1], out_sz)
|
||||
kaiming_normal(self.outp.weight.data)
|
||||
|
||||
self.emb_drop = nn.Dropout(emb_drop)
|
||||
self.drops = nn.ModuleList([nn.Dropout(drop) for drop in drops])
|
||||
self.bn = nn.BatchNorm1d(n_cont)
|
||||
self.use_bn,self.y_range = use_bn,y_range
|
||||
self.is_reg = is_reg
|
||||
self.is_multi = is_multi
|
||||
|
||||
def forward(self, x_cat, x_cont):
|
||||
if self.n_emb != 0:
|
||||
x = [e(x_cat[:,i]) for i,e in enumerate(self.embs)]
|
||||
x = torch.cat(x, 1)
|
||||
x = self.emb_drop(x)
|
||||
if self.n_cont != 0:
|
||||
x2 = self.bn(x_cont)
|
||||
x = torch.cat([x, x2], 1) if self.n_emb != 0 else x2
|
||||
for l,d,b in zip(self.lins, self.drops, self.bns):
|
||||
x = F.relu(l(x))
|
||||
if self.use_bn: x = b(x)
|
||||
x = d(x)
|
||||
x = self.outp(x)
|
||||
if not self.is_reg:
|
||||
if self.is_multi:
|
||||
x = F.sigmoid(x)
|
||||
else:
|
||||
x = F.log_softmax(x)
|
||||
elif self.y_range:
|
||||
x = F.sigmoid(x)
|
||||
x = x*(self.y_range[1] - self.y_range[0])
|
||||
x = x+self.y_range[0]
|
||||
return x
|
||||
|
||||
|
||||
class StructuredLearner(Learner):
|
||||
def __init__(self, data, models, **kwargs):
|
||||
super().__init__(data, models, **kwargs)
|
||||
|
||||
def _get_crit(self, data): return F.mse_loss if data.is_reg else F.binary_cross_entropy if data.is_multi else F.nll_loss
|
||||
|
||||
def summary(self):
|
||||
x = [torch.ones(3, self.data.trn_ds.cats.shape[1], dtype=torch.int64), torch.rand(3, self.data.trn_ds.conts.shape[1])]
|
||||
return model_summary(self.model, x)
|
||||
|
||||
|
||||
class StructuredModel(BasicModel):
|
||||
def get_layer_groups(self):
|
||||
m=self.model
|
||||
return [m.embs, children(m.lins)+children(m.bns), m.outp]
|
||||
|
||||
|
||||
class CollabFilterDataset(Dataset):
|
||||
def __init__(self, path, user_col, item_col, ratings):
|
||||
self.ratings,self.path = ratings.values.astype(np.float32),path
|
||||
self.n = len(ratings)
|
||||
(self.users,self.user2idx,self.user_col,self.n_users) = self.proc_col(user_col)
|
||||
(self.items,self.item2idx,self.item_col,self.n_items) = self.proc_col(item_col)
|
||||
self.min_score,self.max_score = min(ratings),max(ratings)
|
||||
self.cols = [self.user_col,self.item_col,self.ratings]
|
||||
|
||||
@classmethod
|
||||
def from_data_frame(cls, path, df, user_name, item_name, rating_name):
|
||||
return cls(path, df[user_name], df[item_name], df[rating_name])
|
||||
|
||||
@classmethod
|
||||
def from_csv(cls, path, csv, user_name, item_name, rating_name):
|
||||
df = pd.read_csv(os.path.join(path,csv))
|
||||
return cls.from_data_frame(path, df, user_name, item_name, rating_name)
|
||||
|
||||
def proc_col(self,col):
|
||||
uniq = col.unique()
|
||||
name2idx = {o:i for i,o in enumerate(uniq)}
|
||||
return (uniq, name2idx, np.array([name2idx[x] for x in col]), len(uniq))
|
||||
|
||||
def __len__(self): return self.n
|
||||
def __getitem__(self, idx): return [o[idx] for o in self.cols]
|
||||
|
||||
def get_data(self, val_idxs, bs):
|
||||
val, trn = zip(*split_by_idx(val_idxs, *self.cols))
|
||||
return ColumnarModelData(self.path, PassthruDataset(*trn), PassthruDataset(*val), bs)
|
||||
|
||||
def get_model(self, n_factors):
|
||||
model = EmbeddingDotBias(n_factors, self.n_users, self.n_items, self.min_score, self.max_score)
|
||||
return CollabFilterModel(to_gpu(model))
|
||||
|
||||
def get_learner(self, n_factors, val_idxs, bs, **kwargs):
|
||||
return CollabFilterLearner(self.get_data(val_idxs, bs), self.get_model(n_factors), **kwargs)
|
||||
|
||||
|
||||
def get_emb(ni,nf):
|
||||
e = nn.Embedding(ni, nf)
|
||||
e.weight.data.uniform_(-0.05,0.05)
|
||||
return e
|
||||
|
||||
|
||||
class EmbeddingDotBias(nn.Module):
|
||||
def __init__(self, n_factors, n_users, n_items, min_score, max_score):
|
||||
super().__init__()
|
||||
self.min_score,self.max_score = min_score,max_score
|
||||
(self.u, self.i, self.ub, self.ib) = [get_emb(*o) for o in [
|
||||
(n_users, n_factors), (n_items, n_factors), (n_users,1), (n_items,1)
|
||||
]]
|
||||
|
||||
def forward(self, users, items):
|
||||
um = self.u(users)* self.i(items)
|
||||
res = um.sum(1) + self.ub(users).squeeze() + self.ib(items).squeeze()
|
||||
return F.sigmoid(res) * (self.max_score-self.min_score) + self.min_score
|
||||
|
||||
|
||||
class CollabFilterLearner(Learner):
|
||||
def __init__(self, data, models, **kwargs):
|
||||
super().__init__(data, models, **kwargs)
|
||||
|
||||
def _get_crit(self, data): return F.mse_loss
|
||||
|
||||
def summary(self): return model_summary(self.model, [torch.ones(3, dtype=torch.int64), torch.ones(3, dtype=torch.int64)])
|
||||
|
||||
|
||||
class CollabFilterModel(BasicModel):
|
||||
def get_layer_groups(self): return self.model
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
from .core import *
|
||||
from .layers import *
|
||||
from .learner import *
|
||||
from .initializers import *
|
||||
|
||||
model_meta = {
|
||||
resnet18:[8,6], resnet34:[8,6], resnet50:[8,6], resnet101:[8,6], resnet152:[8,6],
|
||||
vgg16:[0,22], vgg19:[0,22],
|
||||
resnext50:[8,6], resnext101:[8,6], resnext101_64:[8,6],
|
||||
wrn:[8,6], inceptionresnet_2:[-2,9], inception_4:[-1,9],
|
||||
dn121:[0,7], dn161:[0,7], dn169:[0,7], dn201:[0,7],
|
||||
}
|
||||
model_features = {inception_4: 3072, dn121: 2048, dn161: 4416,} # nasnetalarge: 4032*2}
|
||||
|
||||
class ConvnetBuilder():
|
||||
"""Class representing a convolutional network.
|
||||
|
||||
Arguments:
|
||||
f: a model creation function (e.g. resnet34, vgg16, etc)
|
||||
c (int): size of the last layer
|
||||
is_multi (bool): is multilabel classification?
|
||||
(def here http://scikit-learn.org/stable/modules/multiclass.html)
|
||||
is_reg (bool): is a regression?
|
||||
ps (float or array of float): dropout parameters
|
||||
xtra_fc (list of ints): list of hidden layers with # hidden neurons
|
||||
xtra_cut (int): # layers earlier than default to cut the model, default is 0
|
||||
custom_head : add custom model classes that are inherited from nn.modules at the end of the model
|
||||
that is mentioned on Argument 'f'
|
||||
"""
|
||||
|
||||
def __init__(self, f, c, is_multi, is_reg, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, pretrained=True):
|
||||
self.f,self.c,self.is_multi,self.is_reg,self.xtra_cut = f,c,is_multi,is_reg,xtra_cut
|
||||
if xtra_fc is None: xtra_fc = [512]
|
||||
if ps is None: ps = [0.25]*len(xtra_fc) + [0.5]
|
||||
self.ps,self.xtra_fc = ps,xtra_fc
|
||||
|
||||
if f in model_meta: cut,self.lr_cut = model_meta[f]
|
||||
else: cut,self.lr_cut = 0,0
|
||||
cut-=xtra_cut
|
||||
layers = cut_model(f(pretrained), cut)
|
||||
self.nf = model_features[f] if f in model_features else (num_features(layers)*2)
|
||||
if not custom_head: layers += [AdaptiveConcatPool2d(), Flatten()]
|
||||
self.top_model = nn.Sequential(*layers)
|
||||
|
||||
n_fc = len(self.xtra_fc)+1
|
||||
if not isinstance(self.ps, list): self.ps = [self.ps]*n_fc
|
||||
|
||||
if custom_head: fc_layers = [custom_head]
|
||||
else: fc_layers = self.get_fc_layers()
|
||||
self.n_fc = len(fc_layers)
|
||||
self.fc_model = to_gpu(nn.Sequential(*fc_layers))
|
||||
if not custom_head: apply_init(self.fc_model, kaiming_normal)
|
||||
self.model = to_gpu(nn.Sequential(*(layers+fc_layers)))
|
||||
|
||||
@property
|
||||
def name(self): return f'{self.f.__name__}_{self.xtra_cut}'
|
||||
|
||||
def create_fc_layer(self, ni, nf, p, actn=None):
|
||||
res=[nn.BatchNorm1d(num_features=ni)]
|
||||
if p: res.append(nn.Dropout(p=p))
|
||||
res.append(nn.Linear(in_features=ni, out_features=nf))
|
||||
if actn: res.append(actn)
|
||||
return res
|
||||
|
||||
def get_fc_layers(self):
|
||||
res=[]
|
||||
ni=self.nf
|
||||
for i,nf in enumerate(self.xtra_fc):
|
||||
res += self.create_fc_layer(ni, nf, p=self.ps[i], actn=nn.ReLU())
|
||||
ni=nf
|
||||
final_actn = nn.Sigmoid() if self.is_multi else nn.LogSoftmax()
|
||||
if self.is_reg: final_actn = None
|
||||
res += self.create_fc_layer(ni, self.c, p=self.ps[-1], actn=final_actn)
|
||||
return res
|
||||
|
||||
def get_layer_groups(self, do_fc=False):
|
||||
if do_fc:
|
||||
return [self.fc_model]
|
||||
idxs = [self.lr_cut]
|
||||
c = children(self.top_model)
|
||||
if len(c)==3: c = children(c[0])+c[1:]
|
||||
lgs = list(split_by_idxs(c,idxs))
|
||||
return lgs+[self.fc_model]
|
||||
|
||||
|
||||
class ConvLearner(Learner):
|
||||
"""
|
||||
Class used to train a chosen supported covnet model. Eg. ResNet-34, etc.
|
||||
Arguments:
|
||||
data: training data for model
|
||||
models: model architectures to base learner
|
||||
precompute: bool to reuse precomputed activations
|
||||
**kwargs: parameters from Learner() class
|
||||
"""
|
||||
def __init__(self, data, models, precompute=False, **kwargs):
|
||||
self.precompute = False
|
||||
super().__init__(data, models, **kwargs)
|
||||
if hasattr(data, 'is_multi') and not data.is_reg and self.metrics is None:
|
||||
self.metrics = [accuracy_thresh(0.5)] if self.data.is_multi else [accuracy]
|
||||
if precompute: self.save_fc1()
|
||||
self.freeze()
|
||||
self.precompute = precompute
|
||||
|
||||
def _get_crit(self, data):
|
||||
if not hasattr(data, 'is_multi'): return super()._get_crit(data)
|
||||
|
||||
return F.l1_loss if data.is_reg else F.binary_cross_entropy if data.is_multi else F.nll_loss
|
||||
|
||||
@classmethod
|
||||
def pretrained(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
|
||||
pretrained=True, **kwargs):
|
||||
models = ConvnetBuilder(f, data.c, data.is_multi, data.is_reg,
|
||||
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=pretrained)
|
||||
return cls(data, models, precompute, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def lsuv_learner(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
|
||||
needed_std=1.0, std_tol=0.1, max_attempts=10, do_orthonorm=False, **kwargs):
|
||||
models = ConvnetBuilder(f, data.c, data.is_multi, data.is_reg,
|
||||
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=False)
|
||||
convlearn=cls(data, models, precompute, **kwargs)
|
||||
convlearn.lsuv_init()
|
||||
return convlearn
|
||||
|
||||
@property
|
||||
def model(self): return self.models.fc_model if self.precompute else self.models.model
|
||||
|
||||
def half(self):
|
||||
if self.fp16: return
|
||||
self.fp16 = True
|
||||
if type(self.model) != FP16: self.models.model = FP16(self.model)
|
||||
if not isinstance(self.models.fc_model, FP16): self.models.fc_model = FP16(self.models.fc_model)
|
||||
def float(self):
|
||||
if not self.fp16: return
|
||||
self.fp16 = False
|
||||
if type(self.models.model) == FP16: self.models.model = self.model.module.float()
|
||||
if type(self.models.fc_model) == FP16: self.models.fc_model = self.models.fc_model.module.float()
|
||||
|
||||
@property
|
||||
def data(self): return self.fc_data if self.precompute else self.data_
|
||||
|
||||
def create_empty_bcolz(self, n, name):
|
||||
return bcolz.carray(np.zeros((0,n), np.float32), chunklen=1, mode='w', rootdir=name)
|
||||
|
||||
def set_data(self, data, precompute=False):
|
||||
super().set_data(data)
|
||||
if precompute:
|
||||
self.unfreeze()
|
||||
self.save_fc1()
|
||||
self.freeze()
|
||||
self.precompute = True
|
||||
else:
|
||||
self.freeze()
|
||||
|
||||
def get_layer_groups(self):
|
||||
return self.models.get_layer_groups(self.precompute)
|
||||
|
||||
def summary(self):
|
||||
precompute = self.precompute
|
||||
self.precompute = False
|
||||
res = super().summary()
|
||||
self.precompute = precompute
|
||||
return res
|
||||
|
||||
def get_activations(self, force=False):
|
||||
tmpl = f'_{self.models.name}_{self.data.sz}.bc'
|
||||
# TODO: Somehow check that directory names haven't changed (e.g. added test set)
|
||||
names = [os.path.join(self.tmp_path, p+tmpl) for p in ('x_act', 'x_act_val', 'x_act_test')]
|
||||
if os.path.exists(names[0]) and not force:
|
||||
self.activations = [bcolz.open(p) for p in names]
|
||||
else:
|
||||
self.activations = [self.create_empty_bcolz(self.models.nf,n) for n in names]
|
||||
|
||||
def save_fc1(self):
|
||||
self.get_activations()
|
||||
act, val_act, test_act = self.activations
|
||||
m=self.models.top_model
|
||||
if len(self.activations[0])!=len(self.data.trn_ds):
|
||||
predict_to_bcolz(m, self.data.fix_dl, act)
|
||||
if len(self.activations[1])!=len(self.data.val_ds):
|
||||
predict_to_bcolz(m, self.data.val_dl, val_act)
|
||||
if self.data.test_dl and (len(self.activations[2])!=len(self.data.test_ds)):
|
||||
if self.data.test_dl: predict_to_bcolz(m, self.data.test_dl, test_act)
|
||||
|
||||
self.fc_data = ImageClassifierData.from_arrays(self.data.path,
|
||||
(act, self.data.trn_y), (val_act, self.data.val_y), self.data.bs, classes=self.data.classes,
|
||||
test = test_act if self.data.test_dl else None, num_workers=8)
|
||||
|
||||
def freeze(self):
|
||||
""" Freeze all but the very last layer.
|
||||
|
||||
Make all layers untrainable (i.e. frozen) except for the last layer.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
self.freeze_to(-1)
|
||||
|
||||
def unfreeze(self):
|
||||
""" Unfreeze all layers.
|
||||
|
||||
Make all layers trainable by unfreezing. This will also set the `precompute` to `False` since we can
|
||||
no longer pre-calculate the activation of frozen layers.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
self.freeze_to(0)
|
||||
self.precompute = False
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
|
||||
def sum_geom(a,r,n): return a*n if r==1 else math.ceil(a*(1-r**n)/(1-r))
|
||||
|
||||
def is_listy(x): return isinstance(x, (list,tuple))
|
||||
def is_iter(x): return isinstance(x, collections.Iterable)
|
||||
def map_over(x, f): return [f(o) for o in x] if is_listy(x) else f(x)
|
||||
def map_none(x, f): return None if x is None else f(x)
|
||||
def delistify(x): return x[0] if is_listy(x) else x
|
||||
def listify(x, y):
|
||||
if not is_iter(x): x=[x]
|
||||
n = y if type(y)==int else len(y)
|
||||
if len(x)==1: x = x * n
|
||||
return x
|
||||
|
||||
conv_dict = {np.dtype('int8'): torch.LongTensor, np.dtype('int16'): torch.LongTensor,
|
||||
np.dtype('int32'): torch.LongTensor, np.dtype('int64'): torch.LongTensor,
|
||||
np.dtype('float32'): torch.FloatTensor, np.dtype('float64'): torch.FloatTensor}
|
||||
|
||||
def A(*a):
|
||||
"""convert iterable object into numpy array"""
|
||||
return np.array(a[0]) if len(a)==1 else [np.array(o) for o in a]
|
||||
|
||||
def T(a, half=False, cuda=True):
|
||||
"""
|
||||
Convert numpy array into a pytorch tensor.
|
||||
if Cuda is available and USE_GPU=True, store resulting tensor in GPU.
|
||||
"""
|
||||
if not torch.is_tensor(a):
|
||||
a = np.array(np.ascontiguousarray(a))
|
||||
if a.dtype in (np.int8, np.int16, np.int32, np.int64):
|
||||
a = torch.LongTensor(a.astype(np.int64))
|
||||
elif a.dtype in (np.float32, np.float64):
|
||||
a = torch.cuda.HalfTensor(a) if half else torch.FloatTensor(a)
|
||||
else: raise NotImplementedError(a.dtype)
|
||||
if cuda: a = to_gpu(a, async=True)
|
||||
return a
|
||||
|
||||
def create_variable(x, volatile, requires_grad=False):
|
||||
if type (x) != Variable:
|
||||
if IS_TORCH_04: x = Variable(T(x), requires_grad=requires_grad)
|
||||
else: x = Variable(T(x), requires_grad=requires_grad, volatile=volatile)
|
||||
return x
|
||||
|
||||
def V_(x, requires_grad=False, volatile=False):
|
||||
'''equivalent to create_variable, which creates a pytorch tensor'''
|
||||
return create_variable(x, volatile=volatile, requires_grad=requires_grad)
|
||||
def V(x, requires_grad=False, volatile=False):
|
||||
'''creates a single or a list of pytorch tensors, depending on input x. '''
|
||||
return map_over(x, lambda o: V_(o, requires_grad, volatile))
|
||||
|
||||
def VV_(x):
|
||||
'''creates a volatile tensor, which does not require gradients. '''
|
||||
return create_variable(x, True)
|
||||
|
||||
def VV(x):
|
||||
'''creates a single or a list of pytorch tensors, depending on input x. '''
|
||||
return map_over(x, VV_)
|
||||
|
||||
def to_np(v):
|
||||
'''returns an np.array object given an input of np.array, list, tuple, torch variable or tensor.'''
|
||||
if isinstance(v, (np.ndarray, np.generic)): return v
|
||||
if isinstance(v, (list,tuple)): return [to_np(o) for o in v]
|
||||
if isinstance(v, Variable): v=v.data
|
||||
if isinstance(v, torch.cuda.HalfTensor): v=v.float()
|
||||
return v.cpu().numpy()
|
||||
|
||||
IS_TORCH_04 = LooseVersion(torch.__version__) >= LooseVersion('0.4')
|
||||
USE_GPU = torch.cuda.is_available()
|
||||
def to_gpu(x, *args, **kwargs):
|
||||
'''puts pytorch variable to gpu, if cuda is available and USE_GPU is set to true. '''
|
||||
return x.cuda(*args, **kwargs) if USE_GPU else x
|
||||
|
||||
def noop(*args, **kwargs): return
|
||||
|
||||
def split_by_idxs(seq, idxs):
|
||||
'''A generator that returns sequence pieces, seperated by indexes specified in idxs. '''
|
||||
last = 0
|
||||
for idx in idxs:
|
||||
if not (-len(seq) <= idx < len(seq)):
|
||||
raise KeyError(f'Idx {idx} is out-of-bounds')
|
||||
yield seq[last:idx]
|
||||
last = idx
|
||||
yield seq[last:]
|
||||
|
||||
def trainable_params_(m):
|
||||
'''Returns a list of trainable parameters in the model m. (i.e., those that require gradients.)'''
|
||||
return [p for p in m.parameters() if p.requires_grad]
|
||||
|
||||
def chain_params(p):
|
||||
if is_listy(p):
|
||||
return list(chain(*[trainable_params_(o) for o in p]))
|
||||
return trainable_params_(p)
|
||||
|
||||
def set_trainable_attr(m,b):
|
||||
m.trainable=b
|
||||
for p in m.parameters(): p.requires_grad=b
|
||||
|
||||
def apply_leaf(m, f):
|
||||
c = children(m)
|
||||
if isinstance(m, nn.Module): f(m)
|
||||
if len(c)>0:
|
||||
for l in c: apply_leaf(l,f)
|
||||
|
||||
def set_trainable(l, b):
|
||||
apply_leaf(l, lambda m: set_trainable_attr(m,b))
|
||||
|
||||
def SGD_Momentum(momentum):
|
||||
return lambda *args, **kwargs: optim.SGD(*args, momentum=momentum, **kwargs)
|
||||
|
||||
def one_hot(a,c): return np.eye(c)[a]
|
||||
|
||||
def partition(a, sz):
|
||||
"""splits iterables a in equal parts of size sz"""
|
||||
return [a[i:i+sz] for i in range(0, len(a), sz)]
|
||||
|
||||
def partition_by_cores(a):
|
||||
return partition(a, len(a)//num_cpus() + 1)
|
||||
|
||||
def num_cpus():
|
||||
try:
|
||||
return len(os.sched_getaffinity(0))
|
||||
except AttributeError:
|
||||
return os.cpu_count()
|
||||
|
||||
|
||||
class BasicModel():
|
||||
def __init__(self,model,name='unnamed'): self.model,self.name = model,name
|
||||
def get_layer_groups(self, do_fc=False): return children(self.model)
|
||||
|
||||
class SingleModel(BasicModel):
|
||||
def get_layer_groups(self): return [self.model]
|
||||
|
||||
class SimpleNet(nn.Module):
|
||||
def __init__(self, layers):
|
||||
super().__init__()
|
||||
self.layers = nn.ModuleList([
|
||||
nn.Linear(layers[i], layers[i + 1]) for i in range(len(layers) - 1)])
|
||||
|
||||
def forward(self, x):
|
||||
x = x.view(x.size(0), -1)
|
||||
for l in self.layers:
|
||||
l_x = l(x)
|
||||
x = F.relu(l_x)
|
||||
return F.log_softmax(l_x, dim=-1)
|
||||
|
||||
|
||||
def save(fn, a):
|
||||
"""Utility function that savess model, function, etc as pickle"""
|
||||
pickle.dump(a, open(fn,'wb'))
|
||||
def load(fn):
|
||||
"""Utility function that loads model, function, etc as pickle"""
|
||||
return pickle.load(open(fn,'rb'))
|
||||
def load2(fn):
|
||||
"""Utility funciton allowing model piclking across Python2 and Python3"""
|
||||
return pickle.load(open(fn,'rb'), encoding='iso-8859-1')
|
||||
|
||||
def load_array(fname):
|
||||
'''
|
||||
Load array using bcolz, which is based on numpy, for fast array saving and loading operations.
|
||||
https://github.com/Blosc/bcolz
|
||||
'''
|
||||
return bcolz.open(fname)[:]
|
||||
|
||||
|
||||
def chunk_iter(iterable, chunk_size):
|
||||
'''A generator that yields chunks of iterable, chunk_size at a time. '''
|
||||
while True:
|
||||
chunk = []
|
||||
try:
|
||||
for _ in range(chunk_size): chunk.append(next(iterable))
|
||||
yield chunk
|
||||
except StopIteration:
|
||||
if chunk: yield chunk
|
||||
break
|
||||
|
||||
def set_grad_enabled(mode): return torch.set_grad_enabled(mode) if IS_TORCH_04 else contextlib.suppress()
|
||||
|
||||
def no_grad_context(): return torch.no_grad() if IS_TORCH_04 else contextlib.suppress()
|
||||
@@ -0,0 +1,90 @@
|
||||
import torch, queue
|
||||
from torch.utils.data.sampler import SequentialSampler, RandomSampler, BatchSampler
|
||||
from .imports import *
|
||||
from .core import *
|
||||
import collections,sys,traceback,threading
|
||||
|
||||
string_classes = (str, bytes)
|
||||
|
||||
|
||||
def get_tensor(batch, pin, half=False):
|
||||
if isinstance(batch, (np.ndarray, np.generic)):
|
||||
batch = T(batch, half=half, cuda=False).contiguous()
|
||||
if pin: batch = batch.pin_memory()
|
||||
return to_gpu(batch)
|
||||
elif isinstance(batch, string_classes):
|
||||
return batch
|
||||
elif isinstance(batch, collections.Mapping):
|
||||
return {k: get_tensor(sample, pin, half) for k, sample in batch.items()}
|
||||
elif isinstance(batch, collections.Sequence):
|
||||
return [get_tensor(sample, pin, half) for sample in batch]
|
||||
raise TypeError(f"batch must contain numbers, dicts or lists; found {type(batch)}")
|
||||
|
||||
|
||||
class DataLoader(object):
|
||||
def __init__(self, dataset, batch_size=1, shuffle=False, sampler=None, batch_sampler=None, pad_idx=0,
|
||||
num_workers=None, pin_memory=False, drop_last=False, pre_pad=True, half=False,
|
||||
transpose=False, transpose_y=False):
|
||||
self.dataset,self.batch_size,self.num_workers = dataset,batch_size,num_workers
|
||||
self.pin_memory,self.drop_last,self.pre_pad = pin_memory,drop_last,pre_pad
|
||||
self.transpose,self.transpose_y,self.pad_idx,self.half = transpose,transpose_y,pad_idx,half
|
||||
|
||||
if batch_sampler is not None:
|
||||
if batch_size > 1 or shuffle or sampler is not None or drop_last:
|
||||
raise ValueError('batch_sampler is mutually exclusive with '
|
||||
'batch_size, shuffle, sampler, and drop_last')
|
||||
|
||||
if sampler is not None and shuffle:
|
||||
raise ValueError('sampler is mutually exclusive with shuffle')
|
||||
|
||||
if batch_sampler is None:
|
||||
if sampler is None:
|
||||
sampler = RandomSampler(dataset) if shuffle else SequentialSampler(dataset)
|
||||
batch_sampler = BatchSampler(sampler, batch_size, drop_last)
|
||||
|
||||
if num_workers is None:
|
||||
self.num_workers = num_cpus()
|
||||
|
||||
self.sampler = sampler
|
||||
self.batch_sampler = batch_sampler
|
||||
|
||||
def __len__(self): return len(self.batch_sampler)
|
||||
|
||||
def jag_stack(self, b):
|
||||
if len(b[0].shape) not in (1,2): return np.stack(b)
|
||||
ml = max(len(o) for o in b)
|
||||
if min(len(o) for o in b)==ml: return np.stack(b)
|
||||
res = np.zeros((len(b), ml), dtype=b[0].dtype) + self.pad_idx
|
||||
for i,o in enumerate(b):
|
||||
if self.pre_pad: res[i, -len(o):] = o
|
||||
else: res[i, :len(o)] = o
|
||||
return res
|
||||
|
||||
def np_collate(self, batch):
|
||||
b = batch[0]
|
||||
if isinstance(b, (np.ndarray, np.generic)): return self.jag_stack(batch)
|
||||
elif isinstance(b, (int, float)): return np.array(batch)
|
||||
elif isinstance(b, string_classes): return batch
|
||||
elif isinstance(b, collections.Mapping):
|
||||
return {key: self.np_collate([d[key] for d in batch]) for key in b}
|
||||
elif isinstance(b, collections.Sequence):
|
||||
return [self.np_collate(samples) for samples in zip(*batch)]
|
||||
raise TypeError(("batch must contain numbers, dicts or lists; found {}".format(type(b))))
|
||||
|
||||
def get_batch(self, indices):
|
||||
res = self.np_collate([self.dataset[i] for i in indices])
|
||||
if self.transpose: res[0] = res[0].T
|
||||
if self.transpose_y: res[1] = res[1].T
|
||||
return res
|
||||
|
||||
def __iter__(self):
|
||||
if self.num_workers==0:
|
||||
for batch in map(self.get_batch, iter(self.batch_sampler)):
|
||||
yield get_tensor(batch, self.pin_memory, self.half)
|
||||
else:
|
||||
with ThreadPoolExecutor(max_workers=self.num_workers) as e:
|
||||
# avoid py3.6 issue where queue is infinite and can result in memory exhaustion
|
||||
for c in chunk_iter(iter(self.batch_sampler), self.num_workers*10):
|
||||
for batch in e.map(self.get_batch, c):
|
||||
yield get_tensor(batch, self.pin_memory, self.half)
|
||||
|
||||
@@ -0,0 +1,529 @@
|
||||
import csv
|
||||
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
from .core import *
|
||||
from .transforms import *
|
||||
from .layer_optimizer import *
|
||||
from .dataloader import DataLoader
|
||||
|
||||
def get_cv_idxs(n, cv_idx=0, val_pct=0.2, seed=42):
|
||||
""" Get a list of index values for Validation set from a dataset
|
||||
|
||||
Arguments:
|
||||
n : int, Total number of elements in the data set.
|
||||
cv_idx : int, starting index [idx_start = cv_idx*int(val_pct*n)]
|
||||
val_pct : (int, float), validation set percentage
|
||||
seed : seed value for RandomState
|
||||
|
||||
Returns:
|
||||
list of indexes
|
||||
"""
|
||||
np.random.seed(seed)
|
||||
n_val = int(val_pct*n)
|
||||
idx_start = cv_idx*n_val
|
||||
idxs = np.random.permutation(n)
|
||||
return idxs[idx_start:idx_start+n_val]
|
||||
|
||||
def resize_img(fname, targ, path, new_path):
|
||||
"""
|
||||
Enlarge or shrink a single image to scale, such that the smaller of the height or width dimension is equal to targ.
|
||||
"""
|
||||
dest = os.path.join(path,new_path,str(targ),fname)
|
||||
if os.path.exists(dest): return
|
||||
im = Image.open(os.path.join(path, fname)).convert('RGB')
|
||||
r,c = im.size
|
||||
ratio = targ/min(r,c)
|
||||
sz = (scale_to(r, ratio, targ), scale_to(c, ratio, targ))
|
||||
os.makedirs(os.path.split(dest)[0], exist_ok=True)
|
||||
im.resize(sz, Image.LINEAR).save(dest)
|
||||
|
||||
def resize_imgs(fnames, targ, path, new_path):
|
||||
"""
|
||||
Enlarge or shrink a set of images in the same directory to scale, such that the smaller of the height or width dimension is equal to targ.
|
||||
Note:
|
||||
-- This function is multithreaded for efficiency.
|
||||
-- When destination file or folder already exist, function exists without raising an error.
|
||||
"""
|
||||
if not os.path.exists(os.path.join(path,new_path,str(targ),fnames[0])):
|
||||
with ThreadPoolExecutor(8) as e:
|
||||
ims = e.map(lambda x: resize_img(x, targ, path, new_path), fnames)
|
||||
for x in tqdm(ims, total=len(fnames), leave=False): pass
|
||||
return os.path.join(path,new_path,str(targ))
|
||||
|
||||
def read_dir(path, folder):
|
||||
""" Returns a list of relative file paths to `path` for all files within `folder` """
|
||||
full_path = os.path.join(path, folder)
|
||||
fnames = glob(f"{full_path}/*.*")
|
||||
directories = glob(f"{full_path}/*/")
|
||||
if any(fnames):
|
||||
return [os.path.relpath(f,path) for f in fnames]
|
||||
elif any(directories):
|
||||
raise FileNotFoundError("{} has subdirectories but contains no files. Is your directory structure is correct?".format(full_path))
|
||||
else:
|
||||
raise FileNotFoundError("{} folder doesn't exist or is empty".format(full_path))
|
||||
|
||||
def read_dirs(path, folder):
|
||||
'''
|
||||
Fetches name of all files in path in long form, and labels associated by extrapolation of directory names.
|
||||
'''
|
||||
lbls, fnames, all_lbls = [], [], []
|
||||
full_path = os.path.join(path, folder)
|
||||
for lbl in sorted(os.listdir(full_path)):
|
||||
if lbl not in ('.ipynb_checkpoints','.DS_Store'):
|
||||
all_lbls.append(lbl)
|
||||
for fname in os.listdir(os.path.join(full_path, lbl)):
|
||||
if fname not in ('.DS_Store'):
|
||||
fnames.append(os.path.join(folder, lbl, fname))
|
||||
lbls.append(lbl)
|
||||
return fnames, lbls, all_lbls
|
||||
|
||||
def n_hot(ids, c):
|
||||
'''
|
||||
one hot encoding by index. Returns array of length c, where all entries are 0, except for the indecies in ids
|
||||
'''
|
||||
res = np.zeros((c,), dtype=np.float32)
|
||||
res[ids] = 1
|
||||
return res
|
||||
|
||||
def folder_source(path, folder):
|
||||
"""
|
||||
Returns the filenames and labels for a folder within a path
|
||||
|
||||
Returns:
|
||||
-------
|
||||
fnames: a list of the filenames within `folder`
|
||||
all_lbls: a list of all of the labels in `folder`, where the # of labels is determined by the # of directories within `folder`
|
||||
lbl_arr: a numpy array of the label indices in `all_lbls`
|
||||
"""
|
||||
fnames, lbls, all_lbls = read_dirs(path, folder)
|
||||
lbl2idx = {lbl:idx for idx,lbl in enumerate(all_lbls)}
|
||||
idxs = [lbl2idx[lbl] for lbl in lbls]
|
||||
lbl_arr = np.array(idxs, dtype=int)
|
||||
return fnames, lbl_arr, all_lbls
|
||||
|
||||
def parse_csv_labels(fn, skip_header=True, cat_separator = ' '):
|
||||
"""Parse filenames and label sets from a CSV file.
|
||||
|
||||
This method expects that the csv file at path :fn: has two columns. If it
|
||||
has a header, :skip_header: should be set to True. The labels in the
|
||||
label set are expected to be space separated.
|
||||
|
||||
Arguments:
|
||||
fn: Path to a CSV file.
|
||||
skip_header: A boolean flag indicating whether to skip the header.
|
||||
|
||||
Returns:
|
||||
a two-tuple of (
|
||||
image filenames,
|
||||
a dictionary of filenames and corresponding labels
|
||||
)
|
||||
.
|
||||
:param cat_separator: the separator for the categories column
|
||||
"""
|
||||
df = pd.read_csv(fn, index_col=0, header=0 if skip_header else None, dtype=str)
|
||||
fnames = df.index.values
|
||||
df.iloc[:,0] = df.iloc[:,0].str.split(cat_separator)
|
||||
return fnames, list(df.to_dict().values())[0]
|
||||
|
||||
def nhot_labels(label2idx, csv_labels, fnames, c):
|
||||
|
||||
all_idx = {k: n_hot([label2idx[o] for o in ([] if type(v) == float else v)], c)
|
||||
for k,v in csv_labels.items()}
|
||||
return np.stack([all_idx[o] for o in fnames])
|
||||
|
||||
def csv_source(folder, csv_file, skip_header=True, suffix='', continuous=False, cat_separator=' '):
|
||||
fnames,csv_labels = parse_csv_labels(csv_file, skip_header, cat_separator)
|
||||
return dict_source(folder, fnames, csv_labels, suffix, continuous)
|
||||
|
||||
def dict_source(folder, fnames, csv_labels, suffix='', continuous=False):
|
||||
all_labels = sorted(list(set(p for o in csv_labels.values() for p in ([] if type(o) == float else o))))
|
||||
full_names = [os.path.join(folder,str(fn)+suffix) for fn in fnames]
|
||||
if continuous:
|
||||
label_arr = np.array([np.array(csv_labels[i]).astype(np.float32)
|
||||
for i in fnames])
|
||||
else:
|
||||
label2idx = {v:k for k,v in enumerate(all_labels)}
|
||||
label_arr = nhot_labels(label2idx, csv_labels, fnames, len(all_labels))
|
||||
is_single = np.all(label_arr.sum(axis=1)==1)
|
||||
if is_single: label_arr = np.argmax(label_arr, axis=1)
|
||||
return full_names, label_arr, all_labels
|
||||
|
||||
class BaseDataset(Dataset):
|
||||
"""An abstract class representing a fastai dataset. Extends torch.utils.data.Dataset."""
|
||||
def __init__(self, transform=None):
|
||||
self.transform = transform
|
||||
self.n = self.get_n()
|
||||
self.c = self.get_c()
|
||||
self.sz = self.get_sz()
|
||||
|
||||
def get1item(self, idx):
|
||||
x,y = self.get_x(idx),self.get_y(idx)
|
||||
return self.get(self.transform, x, y)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
if isinstance(idx,slice):
|
||||
xs,ys = zip(*[self.get1item(i) for i in range(*idx.indices(self.n))])
|
||||
return np.stack(xs),ys
|
||||
return self.get1item(idx)
|
||||
|
||||
def __len__(self): return self.n
|
||||
|
||||
def get(self, tfm, x, y):
|
||||
return (x,y) if tfm is None else tfm(x,y)
|
||||
|
||||
@abstractmethod
|
||||
def get_n(self):
|
||||
"""Return number of elements in the dataset == len(self)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_c(self):
|
||||
"""Return number of classes in a dataset."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_sz(self):
|
||||
"""Return maximum size of an image in a dataset."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_x(self, i):
|
||||
"""Return i-th example (image, wav, etc)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_y(self, i):
|
||||
"""Return i-th label."""
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def is_multi(self):
|
||||
"""Returns true if this data set contains multiple labels per sample."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def is_reg(self):
|
||||
"""True if the data set is used to train regression models."""
|
||||
return False
|
||||
|
||||
def open_image(fn):
|
||||
""" Opens an image using OpenCV given the file path.
|
||||
|
||||
Arguments:
|
||||
fn: the file path of the image
|
||||
|
||||
Returns:
|
||||
The image in RGB format as numpy array of floats normalized to range between 0.0 - 1.0
|
||||
"""
|
||||
flags = cv2.IMREAD_UNCHANGED+cv2.IMREAD_ANYDEPTH+cv2.IMREAD_ANYCOLOR
|
||||
if not os.path.exists(fn) and not str(fn).startswith("http"):
|
||||
raise OSError('No such file or directory: {}'.format(fn))
|
||||
elif os.path.isdir(fn) and not str(fn).startswith("http"):
|
||||
raise OSError('Is a directory: {}'.format(fn))
|
||||
else:
|
||||
#res = np.array(Image.open(fn), dtype=np.float32)/255
|
||||
#if len(res.shape)==2: res = np.repeat(res[...,None],3,2)
|
||||
#return res
|
||||
try:
|
||||
if str(fn).startswith("http"):
|
||||
req = urllib.urlopen(str(fn))
|
||||
image = np.asarray(bytearray(req.read()), dtype="uint8")
|
||||
im = cv2.imdecode(image, flags).astype(np.float32)/255
|
||||
else:
|
||||
im = cv2.imread(str(fn), flags).astype(np.float32)/255
|
||||
if im is None: raise OSError(f'File not recognized by opencv: {fn}')
|
||||
return cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
|
||||
except Exception as e:
|
||||
raise OSError('Error handling image at: {}'.format(fn)) from e
|
||||
|
||||
class FilesDataset(BaseDataset):
|
||||
def __init__(self, fnames, transform, path):
|
||||
self.path,self.fnames = path,fnames
|
||||
super().__init__(transform)
|
||||
def get_sz(self): return self.transform.sz
|
||||
def get_x(self, i): return open_image(os.path.join(self.path, self.fnames[i]))
|
||||
def get_n(self): return len(self.fnames)
|
||||
|
||||
def resize_imgs(self, targ, new_path):
|
||||
dest = resize_imgs(self.fnames, targ, self.path, new_path)
|
||||
return self.__class__(self.fnames, self.y, self.transform, dest)
|
||||
|
||||
def denorm(self,arr):
|
||||
"""Reverse the normalization done to a batch of images.
|
||||
|
||||
Arguments:
|
||||
arr: of shape/size (N,3,sz,sz)
|
||||
"""
|
||||
if type(arr) is not np.ndarray: arr = to_np(arr)
|
||||
if len(arr.shape)==3: arr = arr[None]
|
||||
return self.transform.denorm(np.rollaxis(arr,1,4))
|
||||
|
||||
|
||||
class FilesArrayDataset(FilesDataset):
|
||||
def __init__(self, fnames, y, transform, path):
|
||||
self.y=y
|
||||
assert(len(fnames)==len(y))
|
||||
super().__init__(fnames, transform, path)
|
||||
def get_y(self, i): return self.y[i]
|
||||
def get_c(self):
|
||||
return self.y.shape[1] if len(self.y.shape)>1 else 0
|
||||
|
||||
class FilesIndexArrayDataset(FilesArrayDataset):
|
||||
def get_c(self): return int(self.y.max())+1
|
||||
|
||||
|
||||
class FilesNhotArrayDataset(FilesArrayDataset):
|
||||
@property
|
||||
def is_multi(self): return True
|
||||
|
||||
|
||||
class FilesIndexArrayRegressionDataset(FilesArrayDataset):
|
||||
def is_reg(self): return True
|
||||
|
||||
class ArraysDataset(BaseDataset):
|
||||
def __init__(self, x, y, transform):
|
||||
self.x,self.y=x,y
|
||||
assert(len(x)==len(y))
|
||||
super().__init__(transform)
|
||||
def get_x(self, i): return self.x[i]
|
||||
def get_y(self, i): return self.y[i]
|
||||
def get_n(self): return len(self.y)
|
||||
def get_sz(self): return self.x.shape[1]
|
||||
|
||||
|
||||
class ArraysIndexDataset(ArraysDataset):
|
||||
def get_c(self): return int(self.y.max())+1
|
||||
def get_y(self, i): return self.y[i]
|
||||
|
||||
|
||||
class ArraysIndexRegressionDataset(ArraysIndexDataset):
|
||||
def is_reg(self): return True
|
||||
|
||||
|
||||
class ArraysNhotDataset(ArraysDataset):
|
||||
def get_c(self): return self.y.shape[1]
|
||||
@property
|
||||
def is_multi(self): return True
|
||||
|
||||
|
||||
class ModelData():
|
||||
"""Encapsulates DataLoaders and Datasets for training, validation, test. Base class for fastai *Data classes."""
|
||||
def __init__(self, path, trn_dl, val_dl, test_dl=None):
|
||||
self.path,self.trn_dl,self.val_dl,self.test_dl = path,trn_dl,val_dl,test_dl
|
||||
|
||||
@classmethod
|
||||
def from_dls(cls, path,trn_dl,val_dl,test_dl=None):
|
||||
#trn_dl,val_dl = DataLoader(trn_dl),DataLoader(val_dl)
|
||||
#if test_dl: test_dl = DataLoader(test_dl)
|
||||
return cls(path, trn_dl, val_dl, test_dl)
|
||||
|
||||
@property
|
||||
def is_reg(self): return self.trn_ds.is_reg
|
||||
@property
|
||||
def is_multi(self): return self.trn_ds.is_multi
|
||||
@property
|
||||
def trn_ds(self): return self.trn_dl.dataset
|
||||
@property
|
||||
def val_ds(self): return self.val_dl.dataset
|
||||
@property
|
||||
def test_ds(self): return self.test_dl.dataset
|
||||
@property
|
||||
def trn_y(self): return self.trn_ds.y
|
||||
@property
|
||||
def val_y(self): return self.val_ds.y
|
||||
|
||||
|
||||
class ImageData(ModelData):
|
||||
def __init__(self, path, datasets, bs, num_workers, classes):
|
||||
trn_ds,val_ds,fix_ds,aug_ds,test_ds,test_aug_ds = datasets
|
||||
self.path,self.bs,self.num_workers,self.classes = path,bs,num_workers,classes
|
||||
self.trn_dl,self.val_dl,self.fix_dl,self.aug_dl,self.test_dl,self.test_aug_dl = [
|
||||
self.get_dl(ds,shuf) for ds,shuf in [
|
||||
(trn_ds,True),(val_ds,False),(fix_ds,False),(aug_ds,False),
|
||||
(test_ds,False),(test_aug_ds,False)
|
||||
]
|
||||
]
|
||||
|
||||
def get_dl(self, ds, shuffle):
|
||||
if ds is None: return None
|
||||
return DataLoader(ds, batch_size=self.bs, shuffle=shuffle,
|
||||
num_workers=self.num_workers, pin_memory=False)
|
||||
|
||||
@property
|
||||
def sz(self): return self.trn_ds.sz
|
||||
@property
|
||||
def c(self): return self.trn_ds.c
|
||||
|
||||
def resized(self, dl, targ, new_path):
|
||||
return dl.dataset.resize_imgs(targ,new_path) if dl else None
|
||||
|
||||
def resize(self, targ_sz, new_path='tmp'):
|
||||
new_ds = []
|
||||
dls = [self.trn_dl,self.val_dl,self.fix_dl,self.aug_dl]
|
||||
if self.test_dl: dls += [self.test_dl, self.test_aug_dl]
|
||||
else: dls += [None,None]
|
||||
t = tqdm_notebook(dls)
|
||||
for dl in t: new_ds.append(self.resized(dl, targ_sz, new_path))
|
||||
t.close()
|
||||
return self.__class__(new_ds[0].path, new_ds, self.bs, self.num_workers, self.classes)
|
||||
|
||||
@staticmethod
|
||||
def get_ds(fn, trn, val, tfms, test=None, **kwargs):
|
||||
res = [
|
||||
fn(trn[0], trn[1], tfms[0], **kwargs), # train
|
||||
fn(val[0], val[1], tfms[1], **kwargs), # val
|
||||
fn(trn[0], trn[1], tfms[1], **kwargs), # fix
|
||||
fn(val[0], val[1], tfms[0], **kwargs) # aug
|
||||
]
|
||||
if test is not None:
|
||||
if isinstance(test, tuple):
|
||||
test_lbls = test[1]
|
||||
test = test[0]
|
||||
else:
|
||||
if len(trn[1].shape) == 1:
|
||||
test_lbls = np.zeros((len(test),1))
|
||||
else:
|
||||
test_lbls = np.zeros((len(test),trn[1].shape[1]))
|
||||
res += [
|
||||
fn(test, test_lbls, tfms[1], **kwargs), # test
|
||||
fn(test, test_lbls, tfms[0], **kwargs) # test_aug
|
||||
]
|
||||
else: res += [None,None]
|
||||
return res
|
||||
|
||||
|
||||
class ImageClassifierData(ImageData):
|
||||
@classmethod
|
||||
def from_arrays(cls, path, trn, val, bs=64, tfms=(None,None), classes=None, num_workers=4, test=None, continuous=False):
|
||||
""" Read in images and their labels given as numpy arrays
|
||||
|
||||
Arguments:
|
||||
path: a root path of the data (used for storing trained models, precomputed values, etc)
|
||||
trn: a tuple of training data matrix and target label/classification array (e.g. `trn=(x,y)` where `x` has the
|
||||
shape of `(5000, 784)` and `y` has the shape of `(5000,)`)
|
||||
val: a tuple of validation data matrix and target label/classification array.
|
||||
bs: batch size
|
||||
tfms: transformations (for data augmentations). e.g. output of `tfms_from_model`
|
||||
classes: a list of all labels/classifications
|
||||
num_workers: a number of workers
|
||||
test: a matrix of test data (the shape should match `trn[0]`)
|
||||
|
||||
Returns:
|
||||
ImageClassifierData
|
||||
"""
|
||||
f = ArraysIndexRegressionDataset if continuous else ArraysIndexDataset
|
||||
datasets = cls.get_ds(f, trn, val, tfms, test=test)
|
||||
return cls(path, datasets, bs, num_workers, classes=classes)
|
||||
|
||||
@classmethod
|
||||
def from_paths(cls, path, bs=64, tfms=(None,None), trn_name='train', val_name='valid', test_name=None, test_with_labels=False, num_workers=8):
|
||||
""" Read in images and their labels given as sub-folder names
|
||||
|
||||
Arguments:
|
||||
path: a root path of the data (used for storing trained models, precomputed values, etc)
|
||||
bs: batch size
|
||||
tfms: transformations (for data augmentations). e.g. output of `tfms_from_model`
|
||||
trn_name: a name of the folder that contains training images.
|
||||
val_name: a name of the folder that contains validation images.
|
||||
test_name: a name of the folder that contains test images.
|
||||
num_workers: number of workers
|
||||
|
||||
Returns:
|
||||
ImageClassifierData
|
||||
"""
|
||||
assert not(tfms[0] is None or tfms[1] is None), "please provide transformations for your train and validation sets"
|
||||
trn,val = [folder_source(path, o) for o in (trn_name, val_name)]
|
||||
if test_name:
|
||||
test = folder_source(path, test_name) if test_with_labels else read_dir(path, test_name)
|
||||
else: test = None
|
||||
datasets = cls.get_ds(FilesIndexArrayDataset, trn, val, tfms, path=path, test=test)
|
||||
return cls(path, datasets, bs, num_workers, classes=trn[2])
|
||||
|
||||
@classmethod
|
||||
def from_csv(cls, path, folder, csv_fname, bs=64, tfms=(None,None),
|
||||
val_idxs=None, suffix='', test_name=None, continuous=False, skip_header=True, num_workers=8, cat_separator=' '):
|
||||
""" Read in images and their labels given as a CSV file.
|
||||
|
||||
This method should be used when training image labels are given in an CSV file as opposed to
|
||||
sub-directories with label names.
|
||||
|
||||
Arguments:
|
||||
path: a root path of the data (used for storing trained models, precomputed values, etc)
|
||||
folder: a name of the folder in which training images are contained.
|
||||
csv_fname: a name of the CSV file which contains target labels.
|
||||
bs: batch size
|
||||
tfms: transformations (for data augmentations). e.g. output of `tfms_from_model`
|
||||
val_idxs: index of images to be used for validation. e.g. output of `get_cv_idxs`.
|
||||
If None, default arguments to get_cv_idxs are used.
|
||||
suffix: suffix to add to image names in CSV file (sometimes CSV only contains the file name without file
|
||||
extension e.g. '.jpg' - in which case, you can set suffix as '.jpg')
|
||||
test_name: a name of the folder which contains test images.
|
||||
continuous: TODO
|
||||
skip_header: skip the first row of the CSV file.
|
||||
num_workers: number of workers
|
||||
cat_separator: Labels category separator
|
||||
|
||||
Returns:
|
||||
ImageClassifierData
|
||||
"""
|
||||
assert not (tfms[0] is None or tfms[1] is None), "please provide transformations for your train and validation sets"
|
||||
assert not (os.path.isabs(folder)), "folder needs to be a relative path"
|
||||
fnames,y,classes = csv_source(folder, csv_fname, skip_header, suffix, continuous=continuous, cat_separator=cat_separator)
|
||||
return cls.from_names_and_array(path, fnames, y, classes, val_idxs, test_name,
|
||||
num_workers=num_workers, suffix=suffix, tfms=tfms, bs=bs, continuous=continuous)
|
||||
|
||||
@classmethod
|
||||
def from_path_and_array(cls, path, folder, y, classes=None, val_idxs=None, test_name=None,
|
||||
num_workers=8, tfms=(None,None), bs=64):
|
||||
""" Read in images given a sub-folder and their labels given a numpy array
|
||||
|
||||
Arguments:
|
||||
path: a root path of the data (used for storing trained models, precomputed values, etc)
|
||||
folder: a name of the folder in which training images are contained.
|
||||
y: numpy array which contains target labels ordered by filenames.
|
||||
bs: batch size
|
||||
tfms: transformations (for data augmentations). e.g. output of `tfms_from_model`
|
||||
val_idxs: index of images to be used for validation. e.g. output of `get_cv_idxs`.
|
||||
If None, default arguments to get_cv_idxs are used.
|
||||
test_name: a name of the folder which contains test images.
|
||||
num_workers: number of workers
|
||||
|
||||
Returns:
|
||||
ImageClassifierData
|
||||
"""
|
||||
assert not (tfms[0] is None or tfms[1] is None), "please provide transformations for your train and validation sets"
|
||||
assert not (os.path.isabs(folder)), "folder needs to be a relative path"
|
||||
fnames = np.core.defchararray.add(f'{folder}/', sorted(os.listdir(f'{path}{folder}')))
|
||||
return cls.from_names_and_array(path, fnames, y, classes, val_idxs, test_name,
|
||||
num_workers=num_workers, tfms=tfms, bs=bs)
|
||||
|
||||
@classmethod
|
||||
def from_names_and_array(cls, path, fnames, y, classes, val_idxs=None, test_name=None,
|
||||
num_workers=8, suffix='', tfms=(None,None), bs=64, continuous=False):
|
||||
val_idxs = get_cv_idxs(len(fnames)) if val_idxs is None else val_idxs
|
||||
((val_fnames,trn_fnames),(val_y,trn_y)) = split_by_idx(val_idxs, np.array(fnames), y)
|
||||
|
||||
test_fnames = read_dir(path, test_name) if test_name else None
|
||||
if continuous: f = FilesIndexArrayRegressionDataset
|
||||
else:
|
||||
f = FilesIndexArrayDataset if len(trn_y.shape)==1 else FilesNhotArrayDataset
|
||||
datasets = cls.get_ds(f, (trn_fnames,trn_y), (val_fnames,val_y), tfms,
|
||||
path=path, test=test_fnames)
|
||||
return cls(path, datasets, bs, num_workers, classes=classes)
|
||||
|
||||
def split_by_idx(idxs, *a):
|
||||
"""
|
||||
Split each array passed as *a, to a pair of arrays like this (elements selected by idxs, the remaining elements)
|
||||
This can be used to split multiple arrays containing training data to validation and training set.
|
||||
|
||||
:param idxs [int]: list of indexes selected
|
||||
:param a list: list of np.array, each array should have same amount of elements in the first dimension
|
||||
:return: list of tuples, each containing a split of corresponding array from *a.
|
||||
First element of each tuple is an array composed from elements selected by idxs,
|
||||
second element is an array of remaining elements.
|
||||
"""
|
||||
mask = np.zeros(len(a[0]),dtype=bool)
|
||||
mask[np.array(idxs)] = True
|
||||
return [(o[mask],o[~mask]) for o in a]
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import collections
|
||||
import itertools
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import time
|
||||
|
||||
class LazyThreadPoolExecutor(ThreadPoolExecutor):
|
||||
def map(self, fn, *iterables, timeout=None, chunksize=1, prefetch=None):
|
||||
"""
|
||||
Collects iterables lazily, rather than immediately.
|
||||
Docstring same as parent: https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Executor
|
||||
Implmentation taken from this PR: https://github.com/python/cpython/pull/707
|
||||
"""
|
||||
if timeout is not None: end_time = timeout + time.time()
|
||||
if prefetch is None: prefetch = self._max_workers
|
||||
if prefetch < 0: raise ValueError("prefetch count may not be negative")
|
||||
argsiter = zip(*iterables)
|
||||
fs = collections.deque(self.submit(fn, *args) for args in itertools.islice(argsiter, self._max_workers+prefetch))
|
||||
# Yield must be hidden in closure so that the futures are submitted before the first iterator value is required.
|
||||
def result_iterator():
|
||||
nonlocal argsiter
|
||||
try:
|
||||
while fs:
|
||||
res = fs[0].result() if timeout is None else fs[0].result(end_time-time.time())
|
||||
# Got a result, future needn't be cancelled
|
||||
del fs[0]
|
||||
# Dispatch next task before yielding to keep pipeline full
|
||||
if argsiter:
|
||||
try:
|
||||
args = next(argsiter)
|
||||
except StopIteration:
|
||||
argsiter = None
|
||||
else:
|
||||
fs.append(self.submit(fn, *args))
|
||||
yield res
|
||||
finally:
|
||||
for future in fs: future.cancel()
|
||||
return result_iterator()
|
||||
@@ -0,0 +1,73 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from .core import trainable_params_
|
||||
from .torch_imports import *
|
||||
|
||||
IS_TORCH_04 = LooseVersion(torch.__version__) >= LooseVersion('0.4')
|
||||
|
||||
class FP16(nn.Module):
|
||||
def __init__(self, module):
|
||||
super().__init__()
|
||||
self.module = batchnorm_to_fp32(module.half())
|
||||
|
||||
def forward(self, input):
|
||||
if is_float(input): input = input.half()
|
||||
return self.module(input)
|
||||
|
||||
def load_state_dict(self, *inputs, **kwargs):
|
||||
self.module.load_state_dict(*inputs, **kwargs)
|
||||
|
||||
def state_dict(self, *inputs, **kwargs):
|
||||
return self.module.state_dict(*inputs, **kwargs)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.module[idx]
|
||||
|
||||
def is_float(tensor):
|
||||
if IS_TORCH_04: return tensor.is_floating_point()
|
||||
if isinstance(tensor, Variable): tensor = tensor.data
|
||||
return isinstance(tensor, torch.cuda.FloatTensor)
|
||||
|
||||
def batchnorm_to_fp32(module):
|
||||
'''
|
||||
BatchNorm layers to have parameters in single precision.
|
||||
Find all layers and convert them back to float. This can't
|
||||
be done with built in .apply as that function will apply
|
||||
fn to all modules, parameters, and buffers. Thus we wouldn't
|
||||
be able to guard the float conversion based on the module type.
|
||||
'''
|
||||
if isinstance(module, nn.modules.batchnorm._BatchNorm):
|
||||
module.float()
|
||||
for child in module.children():
|
||||
batchnorm_to_fp32(child)
|
||||
return module
|
||||
|
||||
def copy_model_to_fp32(m, optim):
|
||||
""" Creates a fp32 copy of model parameters and sets optimizer parameters
|
||||
"""
|
||||
fp32_params = [m_param.clone().type(torch.cuda.FloatTensor).detach() for m_param in trainable_params_(m)]
|
||||
optim_groups = [group['params'] for group in optim.param_groups]
|
||||
iter_fp32_params = iter(fp32_params)
|
||||
for group_params in optim_groups:
|
||||
for i in range(len(group_params)):
|
||||
if not group_params[i].requires_grad: continue # only update trainable_params_
|
||||
fp32_param = next(iter_fp32_params)
|
||||
assert(fp32_param.shape == group_params[i].shape)
|
||||
fp32_param.requires_grad = group_params[i].requires_grad
|
||||
group_params[i] = fp32_param
|
||||
return fp32_params
|
||||
|
||||
def copy_fp32_to_model(m, fp32_params):
|
||||
m_params = trainable_params_(m)
|
||||
assert(len(m_params) == len(fp32_params))
|
||||
for fp32_param, m_param in zip(fp32_params, m_params):
|
||||
m_param.data.copy_(fp32_param.data)
|
||||
|
||||
def update_fp32_grads(fp32_params, m):
|
||||
m_params = trainable_params_(m)
|
||||
assert(len(m_params) == len(fp32_params))
|
||||
for fp32_param, m_param in zip(fp32_params, m_params):
|
||||
if fp32_param.grad is None:
|
||||
fp32_param.grad = nn.Parameter(fp32_param.data.new().resize_(*fp32_param.data.size()))
|
||||
fp32_param.grad.data.copy_(m_param.grad.data)
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
from IPython.lib.deepreload import reload as dreload
|
||||
import PIL, os, numpy as np, math, collections, threading, json, bcolz, random, scipy, cv2
|
||||
import pandas as pd, pickle, sys, itertools, string, sys, re, datetime, time, shutil, copy
|
||||
import seaborn as sns, matplotlib
|
||||
import IPython, graphviz, sklearn_pandas, sklearn, warnings, pdb
|
||||
import contextlib
|
||||
from abc import abstractmethod
|
||||
from glob import glob, iglob
|
||||
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
|
||||
from itertools import chain
|
||||
from functools import partial
|
||||
from collections import Iterable, Counter, OrderedDict
|
||||
from isoweek import Week
|
||||
from pandas_summary import DataFrameSummary
|
||||
from IPython.lib.display import FileLink
|
||||
from PIL import Image, ImageEnhance, ImageOps
|
||||
from sklearn import metrics, ensemble, preprocessing
|
||||
from operator import itemgetter, attrgetter
|
||||
from pathlib import Path
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
from matplotlib import pyplot as plt, rcParams, animation
|
||||
from ipywidgets import interact, interactive, fixed, widgets
|
||||
matplotlib.rc('animation', html='html5')
|
||||
np.set_printoptions(precision=5, linewidth=110, suppress=True)
|
||||
|
||||
from ipykernel.kernelapp import IPKernelApp
|
||||
def in_notebook(): return IPKernelApp.initialized()
|
||||
|
||||
def in_ipynb():
|
||||
try:
|
||||
#cls = get_ipython().__class__.__name__
|
||||
#return cls == 'ZMQInteractiveShell'
|
||||
return False
|
||||
except NameError:
|
||||
return False
|
||||
|
||||
import tqdm as tq
|
||||
from tqdm import tqdm_notebook, tnrange
|
||||
|
||||
def clear_tqdm():
|
||||
inst = getattr(tq.tqdm, '_instances', None)
|
||||
if not inst: return
|
||||
try:
|
||||
for i in range(len(inst)): inst.pop().close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if in_notebook():
|
||||
def tqdm(*args, **kwargs):
|
||||
clear_tqdm()
|
||||
return tq.tqdm(*args, file=sys.stdout, **kwargs)
|
||||
def trange(*args, **kwargs):
|
||||
clear_tqdm()
|
||||
return tq.trange(*args, file=sys.stdout, **kwargs)
|
||||
else:
|
||||
from tqdm import tqdm, trange
|
||||
tnrange=trange
|
||||
tqdm_notebook=tqdm
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
|
||||
def cond_init(m, init_fn):
|
||||
if not isinstance(m, (nn.BatchNorm1d,nn.BatchNorm2d,nn.BatchNorm3d)):
|
||||
if hasattr(m, 'weight'): init_fn(m.weight)
|
||||
if hasattr(m, 'bias'): m.bias.data.fill_(0.)
|
||||
|
||||
def apply_init(m, init_fn):
|
||||
m.apply(lambda x: cond_init(x, init_fn))
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
|
||||
import gzip
|
||||
from urllib.request import urlretrieve
|
||||
from tqdm import tqdm
|
||||
|
||||
class TqdmUpTo(tqdm):
|
||||
def update_to(self, b=1, bsize=1, tsize=None):
|
||||
if tsize is not None: self.total = tsize
|
||||
self.update(b * bsize - self.n)
|
||||
|
||||
def get_data(url, filename):
|
||||
if not os.path.exists(filename):
|
||||
|
||||
dirname = os.path.dirname(filename)
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
|
||||
with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
|
||||
urlretrieve(url, filename, reporthook=t.update_to)
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
from .core import *
|
||||
|
||||
def opt_params(parm, lr, wd):
|
||||
return {'params': chain_params(parm), 'lr':lr, 'weight_decay':wd}
|
||||
|
||||
class LayerOptimizer():
|
||||
def __init__(self, opt_fn, layer_groups, lrs, wds=None):
|
||||
if not isinstance(layer_groups, (list,tuple)): layer_groups=[layer_groups]
|
||||
if not isinstance(lrs, Iterable): lrs=[lrs]
|
||||
if len(lrs)==1: lrs=lrs*len(layer_groups)
|
||||
if wds is None: wds=0.
|
||||
if not isinstance(wds, Iterable): wds=[wds]
|
||||
if len(wds)==1: wds=wds*len(layer_groups)
|
||||
self.layer_groups,self.lrs,self.wds = layer_groups,lrs,wds
|
||||
self.opt = opt_fn(self.opt_params())
|
||||
|
||||
def opt_params(self):
|
||||
assert(len(self.layer_groups) == len(self.lrs))
|
||||
assert(len(self.layer_groups) == len(self.wds))
|
||||
params = list(zip(self.layer_groups,self.lrs,self.wds))
|
||||
return [opt_params(*p) for p in params]
|
||||
|
||||
@property
|
||||
def lr(self): return self.lrs[-1]
|
||||
|
||||
@property
|
||||
def mom(self):
|
||||
if 'betas' in self.opt.param_groups[0]:
|
||||
return self.opt.param_groups[0]['betas'][0]
|
||||
else:
|
||||
return self.opt.param_groups[0]['momentum']
|
||||
|
||||
def set_lrs(self, lrs):
|
||||
if not isinstance(lrs, Iterable): lrs=[lrs]
|
||||
if len(lrs)==1: lrs=lrs*len(self.layer_groups)
|
||||
set_lrs(self.opt, lrs)
|
||||
self.lrs=lrs
|
||||
|
||||
def set_wds_out(self, wds):
|
||||
if not isinstance(wds, Iterable): wds=[wds]
|
||||
if len(wds)==1: wds=wds*len(self.layer_groups)
|
||||
set_wds_out(self.opt, wds)
|
||||
set_wds(self.opt, [0] * len(self.layer_groups))
|
||||
self.wds=wds
|
||||
|
||||
def set_wds(self, wds):
|
||||
if not isinstance(wds, Iterable): wds=[wds]
|
||||
if len(wds)==1: wds=wds*len(self.layer_groups)
|
||||
set_wds(self.opt, wds)
|
||||
set_wds_out(self.opt, [0] * len(self.layer_groups))
|
||||
self.wds=wds
|
||||
|
||||
def set_mom(self,momentum):
|
||||
if 'betas' in self.opt.param_groups[0]:
|
||||
for pg in self.opt.param_groups: pg['betas'] = (momentum, pg['betas'][1])
|
||||
else:
|
||||
for pg in self.opt.param_groups: pg['momentum'] = momentum
|
||||
|
||||
def set_beta(self,beta):
|
||||
if 'betas' in self.opt.param_groups[0]:
|
||||
for pg in self.opt.param_groups: pg['betas'] = (pg['betas'][0],beta)
|
||||
elif 'alpha' in self.opt.param_groups[0]:
|
||||
for pg in self.opt.param_groups: pg['alpha'] = beta
|
||||
|
||||
def set_opt_fn(self, opt_fn):
|
||||
if type(self.opt) != type(opt_fn(self.opt_params())):
|
||||
self.opt = opt_fn(self.opt_params())
|
||||
|
||||
def zip_strict_(l, r):
|
||||
assert(len(l) == len(r))
|
||||
return zip(l, r)
|
||||
|
||||
def set_lrs(opt, lrs):
|
||||
if not isinstance(lrs, Iterable): lrs=[lrs]
|
||||
if len(lrs)==1: lrs=lrs*len(opt.param_groups)
|
||||
for pg,lr in zip_strict_(opt.param_groups,lrs): pg['lr'] = lr
|
||||
|
||||
def set_wds_out(opt, wds):
|
||||
if not isinstance(wds, Iterable): wds=[wds]
|
||||
if len(wds)==1: wds=wds*len(opt.param_groups)
|
||||
assert(len(opt.param_groups) == len(wds))
|
||||
for pg,wd in zip_strict_(opt.param_groups,wds): pg['wd'] = wd
|
||||
|
||||
def set_wds(opt, wds):
|
||||
if not isinstance(wds, Iterable): wds=[wds]
|
||||
if len(wds)==1: wds=wds*len(opt.param_groups)
|
||||
assert(len(opt.param_groups) == len(wds))
|
||||
for pg,wd in zip_strict_(opt.param_groups,wds): pg['weight_decay'] = wd
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
class AdaptiveConcatPool2d(nn.Module):
|
||||
def __init__(self, sz=None):
|
||||
super().__init__()
|
||||
sz = sz or (1,1)
|
||||
self.ap = nn.AdaptiveAvgPool2d(sz)
|
||||
self.mp = nn.AdaptiveMaxPool2d(sz)
|
||||
def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)
|
||||
|
||||
class Lambda(nn.Module):
|
||||
def __init__(self, f): super().__init__(); self.f=f
|
||||
def forward(self, x): return self.f(x)
|
||||
|
||||
class Flatten(nn.Module):
|
||||
def __init__(self): super().__init__()
|
||||
def forward(self, x): return x.view(x.size(0), -1)
|
||||
|
||||
@@ -0,0 +1,458 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
from .core import *
|
||||
from .transforms import *
|
||||
from .model import *
|
||||
from .dataset import *
|
||||
from .sgdr import *
|
||||
from .layer_optimizer import *
|
||||
from .layers import *
|
||||
from .metrics import *
|
||||
from .losses import *
|
||||
from .swa import *
|
||||
from .fp16 import *
|
||||
from .lsuv_initializer import apply_lsuv_init
|
||||
import time
|
||||
|
||||
|
||||
class Learner():
|
||||
def __init__(self, data, models, opt_fn=None, tmp_name='tmp', models_name='models', metrics=None, clip=None, crit=None):
|
||||
"""
|
||||
Combines a ModelData object with a nn.Module object, such that you can train that
|
||||
module.
|
||||
data (ModelData): An instance of ModelData.
|
||||
models(module): chosen neural architecture for solving a supported problem.
|
||||
opt_fn(function): optimizer function, uses SGD with Momentum of .9 if none.
|
||||
tmp_name(str): output name of the directory containing temporary files from training process
|
||||
models_name(str): output name of the directory containing the trained model
|
||||
metrics(list): array of functions for evaluating a desired metric. Eg. accuracy.
|
||||
clip(float): gradient clip chosen to limit the change in the gradient to prevent exploding gradients Eg. .3
|
||||
"""
|
||||
self.data_,self.models,self.metrics = data,models,metrics
|
||||
self.sched=None
|
||||
self.wd_sched = None
|
||||
self.clip = None
|
||||
self.opt_fn = opt_fn or SGD_Momentum(0.9)
|
||||
self.tmp_path = tmp_name if os.path.isabs(tmp_name) else os.path.join(self.data.path, tmp_name)
|
||||
self.models_path = models_name if os.path.isabs(models_name) else os.path.join(self.data.path, models_name)
|
||||
os.makedirs(self.tmp_path, exist_ok=True)
|
||||
os.makedirs(self.models_path, exist_ok=True)
|
||||
self.crit = crit if crit else self._get_crit(data)
|
||||
self.reg_fn = None
|
||||
self.fp16 = False
|
||||
|
||||
@classmethod
|
||||
def from_model_data(cls, m, data, **kwargs):
|
||||
self = cls(data, BasicModel(to_gpu(m)), **kwargs)
|
||||
self.unfreeze()
|
||||
return self
|
||||
|
||||
def __getitem__(self,i): return self.children[i]
|
||||
|
||||
@property
|
||||
def children(self): return children(self.model)
|
||||
|
||||
@property
|
||||
def model(self): return self.models.model
|
||||
|
||||
@property
|
||||
def data(self): return self.data_
|
||||
|
||||
def summary(self): return model_summary(self.model, [torch.rand(3, 3, self.data.sz,self.data.sz)])
|
||||
|
||||
def __repr__(self): return self.model.__repr__()
|
||||
|
||||
def lsuv_init(self, needed_std=1.0, std_tol=0.1, max_attempts=10, do_orthonorm=False):
|
||||
x = V(next(iter(self.data.trn_dl))[0])
|
||||
self.models.model=apply_lsuv_init(self.model, x, needed_std=needed_std, std_tol=std_tol,
|
||||
max_attempts=max_attempts, do_orthonorm=do_orthonorm,
|
||||
cuda=USE_GPU and torch.cuda.is_available())
|
||||
|
||||
def set_bn_freeze(self, m, do_freeze):
|
||||
if hasattr(m, 'running_mean'): m.bn_freeze = do_freeze
|
||||
|
||||
def bn_freeze(self, do_freeze):
|
||||
apply_leaf(self.model, lambda m: self.set_bn_freeze(m, do_freeze))
|
||||
|
||||
def freeze_to(self, n):
|
||||
c=self.get_layer_groups()
|
||||
for l in c: set_trainable(l, False)
|
||||
for l in c[n:]: set_trainable(l, True)
|
||||
|
||||
def freeze_all_but(self, n):
|
||||
c=self.get_layer_groups()
|
||||
for l in c: set_trainable(l, False)
|
||||
set_trainable(c[n], True)
|
||||
|
||||
def freeze_groups(self, groups):
|
||||
c = self.get_layer_groups()
|
||||
self.unfreeze()
|
||||
for g in groups:
|
||||
set_trainable(c[g], False)
|
||||
|
||||
def unfreeze_groups(self, groups):
|
||||
c = self.get_layer_groups()
|
||||
for g in groups:
|
||||
set_trainable(c[g], True)
|
||||
|
||||
def unfreeze(self): self.freeze_to(0)
|
||||
|
||||
def get_model_path(self, name): return os.path.join(self.models_path,name)+'.h5'
|
||||
|
||||
def save(self, name):
|
||||
save_model(self.model, self.get_model_path(name))
|
||||
if hasattr(self, 'swa_model'): save_model(self.swa_model, self.get_model_path(name)[:-3]+'-swa.h5')
|
||||
|
||||
def load(self, name):
|
||||
load_model(self.model, self.get_model_path(name))
|
||||
if hasattr(self, 'swa_model'): load_model(self.swa_model, self.get_model_path(name)[:-3]+'-swa.h5')
|
||||
|
||||
def set_data(self, data): self.data_ = data
|
||||
|
||||
def get_cycle_end(self, name):
|
||||
if name is None: return None
|
||||
return lambda sched, cycle: self.save_cycle(name, cycle)
|
||||
|
||||
def save_cycle(self, name, cycle): self.save(f'{name}_cyc_{cycle}')
|
||||
def load_cycle(self, name, cycle): self.load(f'{name}_cyc_{cycle}')
|
||||
|
||||
def half(self):
|
||||
if self.fp16: return
|
||||
self.fp16 = True
|
||||
if type(self.model) != FP16: self.models.model = FP16(self.model)
|
||||
def float(self):
|
||||
if not self.fp16: return
|
||||
self.fp16 = False
|
||||
if type(self.model) == FP16: self.models.model = self.model.module
|
||||
self.model.float()
|
||||
|
||||
def fit_gen(self, model, data, layer_opt, n_cycle, cycle_len=None, cycle_mult=1, cycle_save_name=None, best_save_name=None,
|
||||
use_clr=None, use_clr_beta=None, metrics=None, callbacks=None, use_wd_sched=False, norm_wds=False,
|
||||
wds_sched_mult=None, use_swa=False, swa_start=1, swa_eval_freq=5, **kwargs):
|
||||
|
||||
"""Method does some preparation before finally delegating to the 'fit' method for
|
||||
fitting the model. Namely, if cycle_len is defined, it adds a 'Cosine Annealing'
|
||||
scheduler for varying the learning rate across iterations.
|
||||
|
||||
Method also computes the total number of epochs to fit based on provided 'cycle_len',
|
||||
'cycle_mult', and 'n_cycle' parameters.
|
||||
|
||||
Args:
|
||||
model (Learner): Any neural architecture for solving a supported problem.
|
||||
Eg. ResNet-34, RNN_Learner etc.
|
||||
|
||||
data (ModelData): An instance of ModelData.
|
||||
|
||||
layer_opt (LayerOptimizer): An instance of the LayerOptimizer class
|
||||
|
||||
n_cycle (int): number of cycles
|
||||
|
||||
cycle_len (int): number of cycles before lr is reset to the initial value.
|
||||
E.g if cycle_len = 3, then the lr is varied between a maximum
|
||||
and minimum value over 3 epochs.
|
||||
|
||||
cycle_mult (int): additional parameter for influencing how the lr resets over
|
||||
the cycles. For an intuitive explanation, please see
|
||||
https://github.com/fastai/fastai/blob/master/courses/dl1/lesson1.ipynb
|
||||
|
||||
cycle_save_name (str): use to save the weights at end of each cycle
|
||||
|
||||
best_save_name (str): use to save weights of best model during training.
|
||||
|
||||
metrics (function): some function for evaluating a desired metric. Eg. accuracy.
|
||||
|
||||
callbacks (list(Callback)): callbacks to apply during the training.
|
||||
|
||||
use_wd_sched (bool, optional): set to True to enable weight regularization using
|
||||
the technique mentioned in https://arxiv.org/abs/1711.05101. When this is True
|
||||
alone (see below), the regularization is detached from gradient update and
|
||||
applied directly to the weights.
|
||||
|
||||
norm_wds (bool, optional): when this is set to True along with use_wd_sched, the
|
||||
regularization factor is normalized with each training cycle.
|
||||
|
||||
wds_sched_mult (function, optional): when this is provided along with use_wd_sched
|
||||
as True, the value computed by this function is multiplied with the regularization
|
||||
strength. This function is passed the WeightDecaySchedule object. And example
|
||||
function that can be passed is:
|
||||
f = lambda x: np.array(x.layer_opt.lrs) / x.init_lrs
|
||||
|
||||
use_swa (bool, optional): when this is set to True, it will enable the use of
|
||||
Stochastic Weight Averaging (https://arxiv.org/abs/1803.05407). The learner will
|
||||
include an additional model (in the swa_model attribute) for keeping track of the
|
||||
average weights as described in the paper. All testing of this technique so far has
|
||||
been in image classification, so use in other contexts is not guaranteed to work.
|
||||
|
||||
swa_start (int, optional): if use_swa is set to True, then this determines the epoch
|
||||
to start keeping track of the average weights. It is 1-indexed per the paper's
|
||||
conventions.
|
||||
|
||||
swa_eval_freq (int, optional): if use_swa is set to True, this determines the frequency
|
||||
at which to evaluate the performance of the swa_model. This evaluation can be costly
|
||||
for models using BatchNorm (requiring a full pass through the data), which is why the
|
||||
default is not to evaluate after each epoch.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
|
||||
if callbacks is None: callbacks=[]
|
||||
if metrics is None: metrics=self.metrics
|
||||
|
||||
if use_wd_sched:
|
||||
# This needs to come before CosAnneal() because we need to read the initial learning rate from
|
||||
# layer_opt.lrs - but CosAnneal() alters the layer_opt.lrs value initially (divides by 100)
|
||||
if np.sum(layer_opt.wds) == 0:
|
||||
print('fit() warning: use_wd_sched is set to True, but weight decay(s) passed are 0. Use wds to '
|
||||
'pass weight decay values.')
|
||||
batch_per_epoch = len(data.trn_dl)
|
||||
cl = cycle_len if cycle_len else 1
|
||||
self.wd_sched = WeightDecaySchedule(layer_opt, batch_per_epoch, cl, cycle_mult, n_cycle,
|
||||
norm_wds, wds_sched_mult)
|
||||
callbacks += [self.wd_sched]
|
||||
|
||||
if use_clr is not None:
|
||||
clr_div,cut_div = use_clr[:2]
|
||||
moms = use_clr[2:] if len(use_clr) > 2 else None
|
||||
cycle_end = self.get_cycle_end(cycle_save_name)
|
||||
self.sched = CircularLR(layer_opt, len(data.trn_dl)*cycle_len, on_cycle_end=cycle_end, div=clr_div, cut_div=cut_div,
|
||||
momentums=moms)
|
||||
elif use_clr_beta is not None:
|
||||
div,pct = use_clr_beta[:2]
|
||||
moms = use_clr_beta[2:] if len(use_clr_beta) > 3 else None
|
||||
cycle_end = self.get_cycle_end(cycle_save_name)
|
||||
self.sched = CircularLR_beta(layer_opt, len(data.trn_dl)*cycle_len, on_cycle_end=cycle_end, div=div,
|
||||
pct=pct, momentums=moms)
|
||||
elif cycle_len:
|
||||
cycle_end = self.get_cycle_end(cycle_save_name)
|
||||
cycle_batches = len(data.trn_dl)*cycle_len
|
||||
self.sched = CosAnneal(layer_opt, cycle_batches, on_cycle_end=cycle_end, cycle_mult=cycle_mult)
|
||||
elif not self.sched: self.sched=LossRecorder(layer_opt)
|
||||
callbacks+=[self.sched]
|
||||
|
||||
if best_save_name is not None:
|
||||
callbacks+=[SaveBestModel(self, layer_opt, metrics, best_save_name)]
|
||||
|
||||
if use_swa:
|
||||
# make a copy of the model to track average weights
|
||||
self.swa_model = copy.deepcopy(model)
|
||||
callbacks+=[SWA(model, self.swa_model, swa_start)]
|
||||
|
||||
n_epoch = int(sum_geom(cycle_len if cycle_len else 1, cycle_mult, n_cycle))
|
||||
return fit(model, data, n_epoch, layer_opt.opt, self.crit,
|
||||
metrics=metrics, callbacks=callbacks, reg_fn=self.reg_fn, clip=self.clip, fp16=self.fp16,
|
||||
swa_model=self.swa_model if use_swa else None, swa_start=swa_start,
|
||||
swa_eval_freq=swa_eval_freq, **kwargs)
|
||||
|
||||
def get_layer_groups(self): return self.models.get_layer_groups()
|
||||
|
||||
def get_layer_opt(self, lrs, wds):
|
||||
|
||||
"""Method returns an instance of the LayerOptimizer class, which
|
||||
allows for setting differential learning rates for different
|
||||
parts of the model.
|
||||
|
||||
An example of how a model maybe differentiated into different parts
|
||||
for application of differential learning rates and weight decays is
|
||||
seen in ../.../courses/dl1/fastai/conv_learner.py, using the dict
|
||||
'model_meta'. Currently, this seems supported only for convolutional
|
||||
networks such as VGG-19, ResNet-XX etc.
|
||||
|
||||
Args:
|
||||
lrs (float or list(float)): learning rate(s) for the model
|
||||
|
||||
wds (float or list(float)): weight decay parameter(s).
|
||||
|
||||
Returns:
|
||||
An instance of a LayerOptimizer
|
||||
"""
|
||||
return LayerOptimizer(self.opt_fn, self.get_layer_groups(), lrs, wds)
|
||||
|
||||
def fit(self, lrs, n_cycle, wds=None, **kwargs):
|
||||
|
||||
"""Method gets an instance of LayerOptimizer and delegates to self.fit_gen(..)
|
||||
|
||||
Note that one can specify a list of learning rates which, when appropriately
|
||||
defined, will be applied to different segments of an architecture. This seems
|
||||
mostly relevant to ImageNet-trained models, where we want to alter the layers
|
||||
closest to the images by much smaller amounts.
|
||||
|
||||
Likewise, a single or list of weight decay parameters can be specified, which
|
||||
if appropriate for a model, will apply variable weight decay parameters to
|
||||
different segments of the model.
|
||||
|
||||
Args:
|
||||
lrs (float or list(float)): learning rate for the model
|
||||
|
||||
n_cycle (int): number of cycles (or iterations) to fit the model for
|
||||
|
||||
wds (float or list(float)): weight decay parameter(s).
|
||||
|
||||
kwargs: other arguments
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
self.sched = None
|
||||
layer_opt = self.get_layer_opt(lrs, wds)
|
||||
return self.fit_gen(self.model, self.data, layer_opt, n_cycle, **kwargs)
|
||||
|
||||
def warm_up(self, lr, wds=None):
|
||||
layer_opt = self.get_layer_opt(lr/4, wds)
|
||||
self.sched = LR_Finder(layer_opt, len(self.data.trn_dl), lr, linear=True)
|
||||
return self.fit_gen(self.model, self.data, layer_opt, 1)
|
||||
|
||||
def lr_find(self, start_lr=1e-5, end_lr=10, wds=None, linear=False, **kwargs):
|
||||
"""Helps you find an optimal learning rate for a model.
|
||||
|
||||
It uses the technique developed in the 2015 paper
|
||||
`Cyclical Learning Rates for Training Neural Networks`, where
|
||||
we simply keep increasing the learning rate from a very small value,
|
||||
until the loss starts decreasing.
|
||||
|
||||
Args:
|
||||
start_lr (float/numpy array) : Passing in a numpy array allows you
|
||||
to specify learning rates for a learner's layer_groups
|
||||
end_lr (float) : The maximum learning rate to try.
|
||||
wds (iterable/float)
|
||||
|
||||
Examples:
|
||||
As training moves us closer to the optimal weights for a model,
|
||||
the optimal learning rate will be smaller. We can take advantage of
|
||||
that knowledge and provide lr_find() with a starting learning rate
|
||||
1000x smaller than the model's current learning rate as such:
|
||||
|
||||
>> learn.lr_find(lr/1000)
|
||||
|
||||
>> lrs = np.array([ 1e-4, 1e-3, 1e-2 ])
|
||||
>> learn.lr_find(lrs / 1000)
|
||||
|
||||
Notes:
|
||||
lr_find() may finish before going through each batch of examples if
|
||||
the loss decreases enough.
|
||||
|
||||
.. _Cyclical Learning Rates for Training Neural Networks:
|
||||
http://arxiv.org/abs/1506.01186
|
||||
|
||||
"""
|
||||
self.save('tmp')
|
||||
layer_opt = self.get_layer_opt(start_lr, wds)
|
||||
self.sched = LR_Finder(layer_opt, len(self.data.trn_dl), end_lr, linear=linear)
|
||||
self.fit_gen(self.model, self.data, layer_opt, 1, **kwargs)
|
||||
self.load('tmp')
|
||||
|
||||
def lr_find2(self, start_lr=1e-5, end_lr=10, num_it = 100, wds=None, linear=False, stop_dv=True, **kwargs):
|
||||
"""A variant of lr_find() that helps find the best learning rate. It doesn't do
|
||||
an epoch but a fixed num of iterations (which may be more or less than an epoch
|
||||
depending on your data).
|
||||
At each step, it computes the validation loss and the metrics on the next
|
||||
batch of the validation data, so it's slower than lr_find().
|
||||
|
||||
Args:
|
||||
start_lr (float/numpy array) : Passing in a numpy array allows you
|
||||
to specify learning rates for a learner's layer_groups
|
||||
end_lr (float) : The maximum learning rate to try.
|
||||
num_it : the number of iterations you want it to run
|
||||
wds (iterable/float)
|
||||
stop_dv : stops (or not) when the losses starts to explode.
|
||||
"""
|
||||
self.save('tmp')
|
||||
layer_opt = self.get_layer_opt(start_lr, wds)
|
||||
self.sched = LR_Finder2(layer_opt, num_it, end_lr, linear=linear, metrics=self.metrics, stop_dv=stop_dv)
|
||||
self.fit_gen(self.model, self.data, layer_opt, num_it//len(self.data.trn_dl) + 1, all_val=True, **kwargs)
|
||||
self.load('tmp')
|
||||
|
||||
def predict(self, is_test=False, use_swa=False):
|
||||
dl = self.data.test_dl if is_test else self.data.val_dl
|
||||
m = self.swa_model if use_swa else self.model
|
||||
return predict(m, dl)
|
||||
|
||||
def predict_with_targs(self, is_test=False, use_swa=False):
|
||||
dl = self.data.test_dl if is_test else self.data.val_dl
|
||||
m = self.swa_model if use_swa else self.model
|
||||
return predict_with_targs(m, dl)
|
||||
|
||||
def predict_dl(self, dl): return predict_with_targs(self.model, dl)[0]
|
||||
|
||||
def predict_array(self, arr):
|
||||
self.model.eval()
|
||||
return to_np(self.model(to_gpu(V(T(arr)))))
|
||||
|
||||
def TTA(self, n_aug=4, is_test=False):
|
||||
""" Predict with Test Time Augmentation (TTA)
|
||||
|
||||
Additional to the original test/validation images, apply image augmentation to them
|
||||
(just like for training images) and calculate the mean of predictions. The intent
|
||||
is to increase the accuracy of predictions by examining the images using multiple
|
||||
perspectives.
|
||||
|
||||
Args:
|
||||
n_aug: a number of augmentation images to use per original image
|
||||
is_test: indicate to use test images; otherwise use validation images
|
||||
|
||||
Returns:
|
||||
(tuple): a tuple containing:
|
||||
|
||||
log predictions (numpy.ndarray): log predictions (i.e. `np.exp(log_preds)` will return probabilities)
|
||||
targs (numpy.ndarray): target values when `is_test==False`; zeros otherwise.
|
||||
"""
|
||||
dl1 = self.data.test_dl if is_test else self.data.val_dl
|
||||
dl2 = self.data.test_aug_dl if is_test else self.data.aug_dl
|
||||
preds1,targs = predict_with_targs(self.model, dl1)
|
||||
preds1 = [preds1]*math.ceil(n_aug/4)
|
||||
preds2 = [predict_with_targs(self.model, dl2)[0] for i in tqdm(range(n_aug), leave=False)]
|
||||
return np.stack(preds1+preds2), targs
|
||||
|
||||
def fit_opt_sched(self, phases, cycle_save_name=None, best_save_name=None, stop_div=False, data_list=None, callbacks=None,
|
||||
cut = None, use_swa=False, swa_start=1, swa_eval_freq=5, **kwargs):
|
||||
"""Wraps us the content of phases to send them to model.fit(..)
|
||||
|
||||
This will split the training in several parts, each with their own learning rates/
|
||||
wds/momentums/optimizer detailed in phases.
|
||||
|
||||
Additionaly we can add a list of different data objets in data_list to train
|
||||
on different datasets (to change the size for instance) for each of these groups.
|
||||
|
||||
Args:
|
||||
phases: a list of TrainingPhase objects
|
||||
stop_div: when True, stops the training if the loss goes too high
|
||||
data_list: a list of different Data objects.
|
||||
kwargs: other arguments
|
||||
use_swa (bool, optional): when this is set to True, it will enable the use of
|
||||
Stochastic Weight Averaging (https://arxiv.org/abs/1803.05407). The learner will
|
||||
include an additional model (in the swa_model attribute) for keeping track of the
|
||||
average weights as described in the paper. All testing of this technique so far has
|
||||
been in image classification, so use in other contexts is not guaranteed to work.
|
||||
swa_start (int, optional): if use_swa is set to True, then this determines the epoch
|
||||
to start keeping track of the average weights. It is 1-indexed per the paper's
|
||||
conventions.
|
||||
swa_eval_freq (int, optional): if use_swa is set to True, this determines the frequency
|
||||
at which to evaluate the performance of the swa_model. This evaluation can be costly
|
||||
for models using BatchNorm (requiring a full pass through the data), which is why the
|
||||
default is not to evaluate after each epoch.
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
if data_list is None: data_list=[]
|
||||
if callbacks is None: callbacks=[]
|
||||
layer_opt = LayerOptimizer(phases[0].opt_fn, self.get_layer_groups(), 1e-2, phases[0].wds)
|
||||
if len(data_list) == 0: nb_batches = [len(self.data.trn_dl)] * len(phases)
|
||||
else: nb_batches = [len(data.trn_dl) for data in data_list]
|
||||
self.sched = OptimScheduler(layer_opt, phases, nb_batches, stop_div)
|
||||
callbacks.append(self.sched)
|
||||
metrics = self.metrics
|
||||
if best_save_name is not None:
|
||||
callbacks+=[SaveBestModel(self, layer_opt, metrics, best_save_name)]
|
||||
if use_swa:
|
||||
# make a copy of the model to track average weights
|
||||
self.swa_model = copy.deepcopy(self.model)
|
||||
callbacks+=[SWA(self.model, self.swa_model, swa_start)]
|
||||
n_epochs = [phase.epochs for phase in phases] if cut is None else cut
|
||||
if len(data_list)==0: data_list = [self.data]
|
||||
return fit(self.model, data_list, n_epochs,layer_opt, self.crit,
|
||||
metrics=metrics, callbacks=callbacks, reg_fn=self.reg_fn, clip=self.clip, fp16=self.fp16,
|
||||
swa_model=self.swa_model if use_swa else None, swa_start=swa_start,
|
||||
swa_eval_freq=swa_eval_freq, **kwargs)
|
||||
|
||||
def _get_crit(self, data): return F.mse_loss
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
import warnings
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
from .rnn_reg import LockedDropout,WeightDrop,EmbeddingDropout
|
||||
from .model import Stepper
|
||||
from .core import set_grad_enabled
|
||||
|
||||
IS_TORCH_04 = LooseVersion(torch.__version__) >= LooseVersion('0.4')
|
||||
|
||||
def seq2seq_reg(output, xtra, loss, alpha=0, beta=0):
|
||||
hs,dropped_hs = xtra
|
||||
if alpha: # Activation Regularization
|
||||
loss = loss + (alpha * dropped_hs[-1].pow(2).mean()).sum()
|
||||
if beta: # Temporal Activation Regularization (slowness)
|
||||
h = hs[-1]
|
||||
if len(h)>1: loss = loss + (beta * (h[1:] - h[:-1]).pow(2).mean()).sum()
|
||||
return loss
|
||||
|
||||
|
||||
def repackage_var(h):
|
||||
"""Wraps h in new Variables, to detach them from their history."""
|
||||
if IS_TORCH_04: return h.detach() if type(h) == torch.Tensor else tuple(repackage_var(v) for v in h)
|
||||
else: return Variable(h.data) if type(h) == Variable else tuple(repackage_var(v) for v in h)
|
||||
|
||||
|
||||
class RNN_Encoder(nn.Module):
|
||||
|
||||
"""A custom RNN encoder network that uses
|
||||
- an embedding matrix to encode input,
|
||||
- a stack of LSTM or QRNN layers to drive the network, and
|
||||
- variational dropouts in the embedding and LSTM/QRNN layers
|
||||
|
||||
The architecture for this network was inspired by the work done in
|
||||
"Regularizing and Optimizing LSTM Language Models".
|
||||
(https://arxiv.org/pdf/1708.02182.pdf)
|
||||
"""
|
||||
|
||||
initrange=0.1
|
||||
|
||||
def __init__(self, ntoken, emb_sz, n_hid, n_layers, pad_token, bidir=False,
|
||||
dropouth=0.3, dropouti=0.65, dropoute=0.1, wdrop=0.5, qrnn=False):
|
||||
""" Default constructor for the RNN_Encoder class
|
||||
|
||||
Args:
|
||||
bs (int): batch size of input data
|
||||
ntoken (int): number of vocabulary (or tokens) in the source dataset
|
||||
emb_sz (int): the embedding size to use to encode each token
|
||||
n_hid (int): number of hidden activation per LSTM layer
|
||||
n_layers (int): number of LSTM layers to use in the architecture
|
||||
pad_token (int): the int value used for padding text.
|
||||
dropouth (float): dropout to apply to the activations going from one LSTM layer to another
|
||||
dropouti (float): dropout to apply to the input layer.
|
||||
dropoute (float): dropout to apply to the embedding layer.
|
||||
wdrop (float): dropout used for a LSTM's internal (or hidden) recurrent weights.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
|
||||
super().__init__()
|
||||
self.ndir = 2 if bidir else 1
|
||||
self.bs, self.qrnn = 1, qrnn
|
||||
self.encoder = nn.Embedding(ntoken, emb_sz, padding_idx=pad_token)
|
||||
self.encoder_with_dropout = EmbeddingDropout(self.encoder)
|
||||
if self.qrnn:
|
||||
#Using QRNN requires cupy: https://github.com/cupy/cupy
|
||||
from .torchqrnn.qrnn import QRNNLayer
|
||||
self.rnns = [QRNNLayer(emb_sz if l == 0 else n_hid, (n_hid if l != n_layers - 1 else emb_sz)//self.ndir,
|
||||
save_prev_x=True, zoneout=0, window=2 if l == 0 else 1, output_gate=True) for l in range(n_layers)]
|
||||
if wdrop:
|
||||
for rnn in self.rnns:
|
||||
rnn.linear = WeightDrop(rnn.linear, wdrop, weights=['weight'])
|
||||
else:
|
||||
self.rnns = [nn.LSTM(emb_sz if l == 0 else n_hid, (n_hid if l != n_layers - 1 else emb_sz)//self.ndir,
|
||||
1, bidirectional=bidir) for l in range(n_layers)]
|
||||
if wdrop: self.rnns = [WeightDrop(rnn, wdrop) for rnn in self.rnns]
|
||||
self.rnns = torch.nn.ModuleList(self.rnns)
|
||||
self.encoder.weight.data.uniform_(-self.initrange, self.initrange)
|
||||
|
||||
self.emb_sz,self.n_hid,self.n_layers,self.dropoute = emb_sz,n_hid,n_layers,dropoute
|
||||
self.dropouti = LockedDropout(dropouti)
|
||||
self.dropouths = nn.ModuleList([LockedDropout(dropouth) for l in range(n_layers)])
|
||||
|
||||
def forward(self, input):
|
||||
""" Invoked during the forward propagation of the RNN_Encoder module.
|
||||
Args:
|
||||
input (Tensor): input of shape (sentence length x batch_size)
|
||||
|
||||
Returns:
|
||||
raw_outputs (tuple(list (Tensor), list(Tensor)): list of tensors evaluated from each RNN layer without using
|
||||
dropouth, list of tensors evaluated from each RNN layer using dropouth,
|
||||
"""
|
||||
sl,bs = input.size()
|
||||
if bs!=self.bs:
|
||||
self.bs=bs
|
||||
self.reset()
|
||||
with set_grad_enabled(self.training):
|
||||
emb = self.encoder_with_dropout(input, dropout=self.dropoute if self.training else 0)
|
||||
emb = self.dropouti(emb)
|
||||
raw_output = emb
|
||||
new_hidden,raw_outputs,outputs = [],[],[]
|
||||
for l, (rnn,drop) in enumerate(zip(self.rnns, self.dropouths)):
|
||||
current_input = raw_output
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
raw_output, new_h = rnn(raw_output, self.hidden[l])
|
||||
new_hidden.append(new_h)
|
||||
raw_outputs.append(raw_output)
|
||||
if l != self.n_layers - 1: raw_output = drop(raw_output)
|
||||
outputs.append(raw_output)
|
||||
|
||||
self.hidden = repackage_var(new_hidden)
|
||||
return raw_outputs, outputs
|
||||
|
||||
def one_hidden(self, l):
|
||||
nh = (self.n_hid if l != self.n_layers - 1 else self.emb_sz)//self.ndir
|
||||
if IS_TORCH_04: return Variable(self.weights.new(self.ndir, self.bs, nh).zero_())
|
||||
else: return Variable(self.weights.new(self.ndir, self.bs, nh).zero_(), volatile=not self.training)
|
||||
|
||||
def reset(self):
|
||||
if self.qrnn: [r.reset() for r in self.rnns]
|
||||
self.weights = next(self.parameters()).data
|
||||
if self.qrnn: self.hidden = [self.one_hidden(l) for l in range(self.n_layers)]
|
||||
else: self.hidden = [(self.one_hidden(l), self.one_hidden(l)) for l in range(self.n_layers)]
|
||||
|
||||
|
||||
class MultiBatchRNN(RNN_Encoder):
|
||||
def __init__(self, bptt, max_seq, *args, **kwargs):
|
||||
self.max_seq,self.bptt = max_seq,bptt
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def concat(self, arrs):
|
||||
return [torch.cat([l[si] for l in arrs]) for si in range(len(arrs[0]))]
|
||||
|
||||
def forward(self, input):
|
||||
sl,bs = input.size()
|
||||
for l in self.hidden:
|
||||
for h in l: h.data.zero_()
|
||||
raw_outputs, outputs = [],[]
|
||||
for i in range(0, sl, self.bptt):
|
||||
r, o = super().forward(input[i: min(i+self.bptt, sl)])
|
||||
if i>(sl-self.max_seq):
|
||||
raw_outputs.append(r)
|
||||
outputs.append(o)
|
||||
return self.concat(raw_outputs), self.concat(outputs)
|
||||
|
||||
class LinearDecoder(nn.Module):
|
||||
initrange=0.1
|
||||
def __init__(self, n_out, n_hid, dropout, tie_encoder=None, bias=False):
|
||||
super().__init__()
|
||||
self.decoder = nn.Linear(n_hid, n_out, bias=bias)
|
||||
self.decoder.weight.data.uniform_(-self.initrange, self.initrange)
|
||||
self.dropout = LockedDropout(dropout)
|
||||
if bias: self.decoder.bias.data.zero_()
|
||||
if tie_encoder: self.decoder.weight = tie_encoder.weight
|
||||
|
||||
def forward(self, input):
|
||||
raw_outputs, outputs = input
|
||||
output = self.dropout(outputs[-1])
|
||||
decoded = self.decoder(output.view(output.size(0)*output.size(1), output.size(2)))
|
||||
result = decoded.view(-1, decoded.size(1))
|
||||
return result, raw_outputs, outputs
|
||||
|
||||
|
||||
class LinearBlock(nn.Module):
|
||||
def __init__(self, ni, nf, drop):
|
||||
super().__init__()
|
||||
self.lin = nn.Linear(ni, nf)
|
||||
self.drop = nn.Dropout(drop)
|
||||
self.bn = nn.BatchNorm1d(ni)
|
||||
|
||||
def forward(self, x): return self.lin(self.drop(self.bn(x)))
|
||||
|
||||
|
||||
class PoolingLinearClassifier(nn.Module):
|
||||
def __init__(self, layers, drops):
|
||||
super().__init__()
|
||||
self.layers = nn.ModuleList([
|
||||
LinearBlock(layers[i], layers[i + 1], drops[i]) for i in range(len(layers) - 1)])
|
||||
|
||||
def pool(self, x, bs, is_max):
|
||||
f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d
|
||||
return f(x.permute(1,2,0), (1,)).view(bs,-1)
|
||||
|
||||
def forward(self, input):
|
||||
raw_outputs, outputs = input
|
||||
output = outputs[-1]
|
||||
sl,bs,_ = output.size()
|
||||
avgpool = self.pool(output, bs, False)
|
||||
mxpool = self.pool(output, bs, True)
|
||||
x = torch.cat([output[-1], mxpool, avgpool], 1)
|
||||
for l in self.layers:
|
||||
l_x = l(x)
|
||||
x = F.relu(l_x)
|
||||
return l_x, raw_outputs, outputs
|
||||
|
||||
|
||||
class SequentialRNN(nn.Sequential):
|
||||
def reset(self):
|
||||
for c in self.children():
|
||||
if hasattr(c, 'reset'): c.reset()
|
||||
|
||||
|
||||
def get_language_model(n_tok, emb_sz, n_hid, n_layers, pad_token,
|
||||
dropout=0.4, dropouth=0.3, dropouti=0.5, dropoute=0.1, wdrop=0.5, tie_weights=True, qrnn=False, bias=False):
|
||||
"""Returns a SequentialRNN model.
|
||||
|
||||
A RNN_Encoder layer is instantiated using the parameters provided.
|
||||
|
||||
This is followed by the creation of a LinearDecoder layer.
|
||||
|
||||
Also by default (i.e. tie_weights = True), the embedding matrix used in the RNN_Encoder
|
||||
is used to instantiate the weights for the LinearDecoder layer.
|
||||
|
||||
The SequentialRNN layer is the native torch's Sequential wrapper that puts the RNN_Encoder and
|
||||
LinearDecoder layers sequentially in the model.
|
||||
|
||||
Args:
|
||||
n_tok (int): number of unique vocabulary words (or tokens) in the source dataset
|
||||
emb_sz (int): the embedding size to use to encode each token
|
||||
n_hid (int): number of hidden activation per LSTM layer
|
||||
n_layers (int): number of LSTM layers to use in the architecture
|
||||
pad_token (int): the int value used for padding text.
|
||||
dropouth (float): dropout to apply to the activations going from one LSTM layer to another
|
||||
dropouti (float): dropout to apply to the input layer.
|
||||
dropoute (float): dropout to apply to the embedding layer.
|
||||
wdrop (float): dropout used for a LSTM's internal (or hidden) recurrent weights.
|
||||
tie_weights (bool): decide if the weights of the embedding matrix in the RNN encoder should be tied to the
|
||||
weights of the LinearDecoder layer.
|
||||
qrnn (bool): decide if the model is composed of LSTMS (False) or QRNNs (True).
|
||||
bias (bool): decide if the decoder should have a bias layer or not.
|
||||
Returns:
|
||||
A SequentialRNN model
|
||||
"""
|
||||
rnn_enc = RNN_Encoder(n_tok, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token,
|
||||
dropouth=dropouth, dropouti=dropouti, dropoute=dropoute, wdrop=wdrop, qrnn=qrnn)
|
||||
enc = rnn_enc.encoder if tie_weights else None
|
||||
return SequentialRNN(rnn_enc, LinearDecoder(n_tok, emb_sz, dropout, tie_encoder=enc, bias=bias))
|
||||
|
||||
|
||||
def get_rnn_classifier(bptt, max_seq, n_class, n_tok, emb_sz, n_hid, n_layers, pad_token, layers, drops, bidir=False,
|
||||
dropouth=0.3, dropouti=0.5, dropoute=0.1, wdrop=0.5, qrnn=False):
|
||||
rnn_enc = MultiBatchRNN(bptt, max_seq, n_tok, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir,
|
||||
dropouth=dropouth, dropouti=dropouti, dropoute=dropoute, wdrop=wdrop, qrnn=qrnn)
|
||||
return SequentialRNN(rnn_enc, PoolingLinearClassifier(layers, drops))
|
||||
|
||||
get_rnn_classifer=get_rnn_classifier
|
||||
@@ -0,0 +1,12 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
|
||||
def fbeta_torch(y_true, y_pred, beta, threshold, eps=1e-9):
|
||||
y_pred = (y_pred.float() > threshold).float()
|
||||
y_true = y_true.float()
|
||||
tp = (y_pred * y_true).sum(dim=1)
|
||||
precision = tp / (y_pred.sum(dim=1)+eps)
|
||||
recall = tp / (y_true.sum(dim=1)+eps)
|
||||
return torch.mean(
|
||||
precision*recall / (precision*(beta**2)+recall+eps) * (1+beta**2))
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
From https://github.com/ducha-aiki/LSUV-pytorch
|
||||
|
||||
Copyright (C) 2017, Dmytro Mishkin
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.init
|
||||
import torch.nn as nn
|
||||
|
||||
gg = {}
|
||||
gg['hook_position'] = 0
|
||||
gg['total_fc_conv_layers'] = 0
|
||||
gg['done_counter'] = -1
|
||||
gg['hook'] = None
|
||||
gg['act_dict'] = {}
|
||||
gg['counter_to_apply_correction'] = 0
|
||||
gg['correction_needed'] = False
|
||||
gg['current_coef'] = 1.0
|
||||
|
||||
# Orthonorm init code is taked from Lasagne
|
||||
# https://github.com/Lasagne/Lasagne/blob/master/lasagne/init.py
|
||||
def svd_orthonormal(w):
|
||||
shape = w.shape
|
||||
if len(shape) < 2:
|
||||
raise RuntimeError("Only shapes of length 2 or more are supported.")
|
||||
flat_shape = (shape[0], np.prod(shape[1:]))
|
||||
a = np.random.normal(0.0, 1.0, flat_shape)#w;
|
||||
u, _, v = np.linalg.svd(a, full_matrices=False)
|
||||
q = u if u.shape == flat_shape else v
|
||||
q = q.reshape(shape)
|
||||
return q.astype(np.float32)
|
||||
|
||||
def store_activations(self, input, output):
|
||||
gg['act_dict'] = output.data.cpu().numpy();
|
||||
return
|
||||
|
||||
def add_current_hook(m):
|
||||
if gg['hook'] is not None:
|
||||
return
|
||||
if (isinstance(m, nn.Conv2d)) or (isinstance(m, nn.Linear)):
|
||||
if gg['hook_position'] > gg['done_counter']:
|
||||
gg['hook'] = m.register_forward_hook(store_activations)
|
||||
else:
|
||||
gg['hook_position'] += 1
|
||||
return
|
||||
|
||||
def count_conv_fc_layers(m):
|
||||
if (isinstance(m, nn.Conv2d)) or (isinstance(m, nn.Linear)):
|
||||
gg['total_fc_conv_layers'] +=1
|
||||
return
|
||||
|
||||
def remove_hooks(hooks):
|
||||
for h in hooks:
|
||||
h.remove()
|
||||
return
|
||||
|
||||
def orthogonal_weights_init(m):
|
||||
if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
|
||||
if hasattr(m, 'weight_v'):
|
||||
w_ortho = svd_orthonormal(m.weight_v.data.cpu().numpy())
|
||||
m.weight_v.data = torch.from_numpy(w_ortho)
|
||||
try:
|
||||
nn.init.constant(m.bias, 0)
|
||||
except:
|
||||
pass
|
||||
else:
|
||||
w_ortho = svd_orthonormal(m.weight.data.cpu().numpy())
|
||||
m.weight.data = torch.from_numpy(w_ortho)
|
||||
try:
|
||||
nn.init.constant(m.bias, 0)
|
||||
except:
|
||||
pass
|
||||
return
|
||||
|
||||
def apply_weights_correction(m):
|
||||
if gg['hook'] is None:
|
||||
return
|
||||
if not gg['correction_needed']:
|
||||
return
|
||||
if (isinstance(m, nn.Conv2d)) or (isinstance(m, nn.Linear)):
|
||||
if gg['counter_to_apply_correction'] < gg['hook_position']:
|
||||
gg['counter_to_apply_correction'] += 1
|
||||
else:
|
||||
if hasattr(m, 'weight_g'):
|
||||
m.weight_g.data *= float(gg['current_coef'])
|
||||
gg['correction_needed'] = False
|
||||
else:
|
||||
m.weight.data *= gg['current_coef']
|
||||
gg['correction_needed'] = False
|
||||
return
|
||||
return
|
||||
|
||||
def apply_lsuv_init(model, data, needed_std=1.0, std_tol=0.1, max_attempts=10, do_orthonorm=True, cuda=True):
|
||||
model.eval();
|
||||
if cuda:
|
||||
model=model.cuda()
|
||||
data=data.cuda()
|
||||
else:
|
||||
model=model.cpu()
|
||||
data=data.cpu()
|
||||
|
||||
model.apply(count_conv_fc_layers)
|
||||
if do_orthonorm:
|
||||
model.apply(orthogonal_weights_init)
|
||||
if cuda:
|
||||
model=model.cuda()
|
||||
for layer_idx in range(gg['total_fc_conv_layers']):
|
||||
model.apply(add_current_hook)
|
||||
out = model(data)
|
||||
current_std = gg['act_dict'].std()
|
||||
attempts = 0
|
||||
while (np.abs(current_std - needed_std) > std_tol):
|
||||
gg['current_coef'] = needed_std / (current_std + 1e-8);
|
||||
gg['correction_needed'] = True
|
||||
model.apply(apply_weights_correction)
|
||||
if cuda:
|
||||
model=model.cuda()
|
||||
out = model(data)
|
||||
current_std = gg['act_dict'].std()
|
||||
attempts+=1
|
||||
if attempts > max_attempts:
|
||||
print(f'Cannot converge in {max_attempts} iterations')
|
||||
break
|
||||
if gg['hook'] is not None:
|
||||
gg['hook'].remove()
|
||||
gg['done_counter']+=1
|
||||
gg['counter_to_apply_correction'] = 0
|
||||
gg['hook_position'] = 0
|
||||
gg['hook'] = None
|
||||
if not cuda:
|
||||
model=model.cpu()
|
||||
return model
|
||||
@@ -0,0 +1,45 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
|
||||
def accuracy_np(preds, targs):
|
||||
preds = np.argmax(preds, 1)
|
||||
return (preds==targs).mean()
|
||||
|
||||
def accuracy(preds, targs):
|
||||
preds = torch.max(preds, dim=1)[1]
|
||||
return (preds==targs).float().mean()
|
||||
|
||||
def accuracy_thresh(thresh):
|
||||
return lambda preds,targs: accuracy_multi(preds, targs, thresh)
|
||||
|
||||
def accuracy_multi(preds, targs, thresh):
|
||||
return ((preds>thresh).float()==targs).float().mean()
|
||||
|
||||
def accuracy_multi_np(preds, targs, thresh):
|
||||
return ((preds>thresh)==targs).mean()
|
||||
|
||||
def recall(preds, targs, thresh=0.5):
|
||||
pred_pos = preds > thresh
|
||||
tpos = torch.mul((targs.byte() == pred_pos), targs.byte())
|
||||
return tpos.sum()/targs.sum()
|
||||
|
||||
def precision(preds, targs, thresh=0.5):
|
||||
pred_pos = preds > thresh
|
||||
tpos = torch.mul((targs.byte() == pred_pos), targs.byte())
|
||||
return tpos.sum()/pred_pos.sum()
|
||||
|
||||
def fbeta(preds, targs, beta, thresh=0.5):
|
||||
"""Calculates the F-beta score (the weighted harmonic mean of precision and recall).
|
||||
This is the micro averaged version where the true positives, false negatives and
|
||||
false positives are calculated globally (as opposed to on a per label basis).
|
||||
|
||||
beta == 1 places equal weight on precision and recall, b < 1 emphasizes precision and
|
||||
beta > 1 favors recall.
|
||||
"""
|
||||
assert beta > 0, 'beta needs to be greater than 0'
|
||||
beta2 = beta ** 2
|
||||
rec = recall(preds, targs, thresh)
|
||||
prec = precision(preds, targs, thresh)
|
||||
return (1 + beta2) * prec * rec / (beta2 * prec + rec)
|
||||
|
||||
def f1(preds, targs, thresh=0.5): return fbeta(preds, targs, 1, thresh)
|
||||
+302
@@ -0,0 +1,302 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
from .core import *
|
||||
from .layer_optimizer import *
|
||||
from .swa import *
|
||||
from .fp16 import *
|
||||
|
||||
IS_TORCH_04 = LooseVersion(torch.__version__) >= LooseVersion('0.4')
|
||||
|
||||
def cut_model(m, cut):
|
||||
return list(m.children())[:cut] if cut else [m]
|
||||
|
||||
def predict_to_bcolz(m, gen, arr, workers=4):
|
||||
arr.trim(len(arr))
|
||||
lock=threading.Lock()
|
||||
m.eval()
|
||||
for x,*_ in tqdm(gen):
|
||||
y = to_np(m(VV(x)).data)
|
||||
with lock:
|
||||
arr.append(y)
|
||||
arr.flush()
|
||||
|
||||
def num_features(m):
|
||||
c=children(m)
|
||||
if len(c)==0: return None
|
||||
for l in reversed(c):
|
||||
if hasattr(l, 'num_features'): return l.num_features
|
||||
res = num_features(l)
|
||||
if res is not None: return res
|
||||
|
||||
def torch_item(x): return x.item() if hasattr(x,'item') else x[0]
|
||||
|
||||
class Stepper():
|
||||
def __init__(self, m, opt, crit, clip=0, reg_fn=None, fp16=False, loss_scale=1):
|
||||
self.m,self.opt,self.crit,self.clip,self.reg_fn = m,opt,crit,clip,reg_fn
|
||||
self.fp16 = fp16
|
||||
self.reset(True)
|
||||
if self.fp16: self.fp32_params = copy_model_to_fp32(m, opt)
|
||||
self.loss_scale = loss_scale
|
||||
|
||||
def reset(self, train=True):
|
||||
if train: apply_leaf(self.m, set_train_mode)
|
||||
else: self.m.eval()
|
||||
if hasattr(self.m, 'reset'):
|
||||
self.m.reset()
|
||||
if self.fp16: self.fp32_params = copy_model_to_fp32(self.m, self.opt)
|
||||
|
||||
def step(self, xs, y, epoch):
|
||||
xtra = []
|
||||
output = self.m(*xs)
|
||||
if isinstance(output,tuple): output,*xtra = output
|
||||
if self.fp16: self.m.zero_grad()
|
||||
else: self.opt.zero_grad()
|
||||
loss = raw_loss = self.crit(output, y)
|
||||
if self.loss_scale != 1: assert(self.fp16); loss = loss*self.loss_scale
|
||||
if self.reg_fn: loss = self.reg_fn(output, xtra, raw_loss)
|
||||
loss.backward()
|
||||
if self.fp16: update_fp32_grads(self.fp32_params, self.m)
|
||||
if self.loss_scale != 1:
|
||||
for param in self.fp32_params: param.grad.data.div_(self.loss_scale)
|
||||
if self.clip: # Gradient clipping
|
||||
if IS_TORCH_04: nn.utils.clip_grad_norm_(trainable_params_(self.m), self.clip)
|
||||
else: nn.utils.clip_grad_norm(trainable_params_(self.m), self.clip)
|
||||
if 'wd' in self.opt.param_groups[0] and self.opt.param_groups[0]['wd'] != 0:
|
||||
#Weight decay out of the loss. After the gradient computation but before the step.
|
||||
for group in self.opt.param_groups:
|
||||
lr, wd = group['lr'], group['wd']
|
||||
for p in group['params']:
|
||||
if p.grad is not None: p.data = p.data.add(-wd * lr, p.data)
|
||||
self.opt.step()
|
||||
if self.fp16:
|
||||
copy_fp32_to_model(self.m, self.fp32_params)
|
||||
torch.cuda.synchronize()
|
||||
return torch_item(raw_loss.data)
|
||||
|
||||
def evaluate(self, xs, y):
|
||||
preds = self.m(*xs)
|
||||
if isinstance(preds,tuple): preds=preds[0]
|
||||
return preds, self.crit(preds, y)
|
||||
|
||||
def set_train_mode(m):
|
||||
if (hasattr(m, 'running_mean') and (getattr(m,'bn_freeze',False)
|
||||
or not getattr(m,'trainable',False))): m.eval()
|
||||
elif (getattr(m,'drop_freeze',False) and hasattr(m, 'p')
|
||||
and ('drop' in type(m).__name__.lower())): m.eval()
|
||||
else: m.train()
|
||||
|
||||
def fit(model, data, n_epochs, opt, crit, metrics=None, callbacks=None, stepper=Stepper,
|
||||
swa_model=None, swa_start=None, swa_eval_freq=None, visualize=False, **kwargs):
|
||||
""" Fits a model
|
||||
|
||||
Arguments:
|
||||
model (model): any pytorch module
|
||||
net = to_gpu(net)
|
||||
data (ModelData): see ModelData class and subclasses (can be a list)
|
||||
opts: an optimizer. Example: optim.Adam.
|
||||
If n_epochs is a list, it needs to be the layer_optimizer to get the optimizer as it changes.
|
||||
n_epochs(int or list): number of epochs (or list of number of epochs)
|
||||
crit: loss function to optimize. Example: F.cross_entropy
|
||||
"""
|
||||
|
||||
seq_first = kwargs.pop('seq_first', False)
|
||||
all_val = kwargs.pop('all_val', False)
|
||||
get_ep_vals = kwargs.pop('get_ep_vals', False)
|
||||
metrics = metrics or []
|
||||
callbacks = callbacks or []
|
||||
avg_mom=0.98
|
||||
batch_num,avg_loss=0,0.
|
||||
for cb in callbacks: cb.on_train_begin()
|
||||
names = ["epoch", "trn_loss", "val_loss"] + [f.__name__ for f in metrics]
|
||||
if swa_model is not None:
|
||||
swa_names = ['swa_loss'] + [f'swa_{f.__name__}' for f in metrics]
|
||||
names += swa_names
|
||||
# will use this to call evaluate later
|
||||
swa_stepper = stepper(swa_model, None, crit, **kwargs)
|
||||
|
||||
layout = "{!s:10} " * len(names)
|
||||
if not isinstance(n_epochs, Iterable): n_epochs=[n_epochs]
|
||||
if not isinstance(data, Iterable): data = [data]
|
||||
if len(data) == 1: data = data * len(n_epochs)
|
||||
for cb in callbacks: cb.on_phase_begin()
|
||||
model_stepper = stepper(model, opt.opt if hasattr(opt,'opt') else opt, crit, **kwargs)
|
||||
ep_vals = collections.OrderedDict()
|
||||
tot_epochs = int(np.ceil(np.array(n_epochs).sum()))
|
||||
cnt_phases = np.array([ep * len(dat.trn_dl) for (ep,dat) in zip(n_epochs,data)]).cumsum()
|
||||
phase = 0
|
||||
for epoch in tnrange(tot_epochs, desc='Epoch'):
|
||||
if phase >= len(n_epochs): break #Sometimes cumulated errors make this append.
|
||||
model_stepper.reset(True)
|
||||
cur_data = data[phase]
|
||||
if hasattr(cur_data, 'trn_sampler'): cur_data.trn_sampler.set_epoch(epoch)
|
||||
if hasattr(cur_data, 'val_sampler'): cur_data.val_sampler.set_epoch(epoch)
|
||||
num_batch = len(cur_data.trn_dl)
|
||||
t = tqdm(iter(cur_data.trn_dl), leave=False, total=num_batch, miniters=0)
|
||||
if all_val: val_iter = IterBatch(cur_data.val_dl)
|
||||
|
||||
for (*x,y) in t:
|
||||
batch_num += 1
|
||||
for cb in callbacks: cb.on_batch_begin()
|
||||
loss = model_stepper.step(V(x),V(y), epoch)
|
||||
avg_loss = avg_loss * avg_mom + loss * (1-avg_mom)
|
||||
debias_loss = avg_loss / (1 - avg_mom**batch_num)
|
||||
t.set_postfix(loss=debias_loss, refresh=False)
|
||||
stop=False
|
||||
los = debias_loss if not all_val else [debias_loss] + validate_next(model_stepper,metrics, val_iter)
|
||||
for cb in callbacks: stop = stop or cb.on_batch_end(los)
|
||||
if stop: return
|
||||
if batch_num >= cnt_phases[phase]:
|
||||
for cb in callbacks: cb.on_phase_end()
|
||||
phase += 1
|
||||
if phase >= len(n_epochs):
|
||||
t.close()
|
||||
break
|
||||
for cb in callbacks: cb.on_phase_begin()
|
||||
if isinstance(opt, LayerOptimizer): model_stepper.opt = opt.opt
|
||||
if cur_data != data[phase]:
|
||||
t.close()
|
||||
break
|
||||
|
||||
if not all_val:
|
||||
vals = validate(model_stepper, cur_data.val_dl, metrics, seq_first=seq_first)
|
||||
stop=False
|
||||
for cb in callbacks: stop = stop or cb.on_epoch_end(vals)
|
||||
if swa_model is not None:
|
||||
if (epoch + 1) >= swa_start and ((epoch + 1 - swa_start) % swa_eval_freq == 0 or epoch == tot_epochs - 1):
|
||||
fix_batchnorm(swa_model, cur_data.trn_dl)
|
||||
swa_vals = validate(swa_stepper, cur_data.val_dl, metrics)
|
||||
vals += swa_vals
|
||||
|
||||
if epoch > 0:
|
||||
print_stats(epoch, [debias_loss] + vals, visualize, prev_val)
|
||||
else:
|
||||
print(layout.format(*names))
|
||||
print_stats(epoch, [debias_loss] + vals, visualize)
|
||||
prev_val = [debias_loss] + vals
|
||||
ep_vals = append_stats(ep_vals, epoch, [debias_loss] + vals)
|
||||
if stop: break
|
||||
for cb in callbacks: cb.on_train_end()
|
||||
if get_ep_vals: return vals, ep_vals
|
||||
else: return vals
|
||||
|
||||
def append_stats(ep_vals, epoch, values, decimals=6):
|
||||
ep_vals[epoch]=list(np.round(values, decimals))
|
||||
return ep_vals
|
||||
|
||||
def print_stats(epoch, values, visualize, prev_val=[], decimals=6):
|
||||
layout = "{!s:^10}" + " {!s:10}" * len(values)
|
||||
values = [epoch] + list(np.round(values, decimals))
|
||||
sym = ""
|
||||
if visualize:
|
||||
if epoch == 0: pass
|
||||
elif values[1] > prev_val[0] and values[2] > prev_val[1]: sym = " △ △"
|
||||
elif values[1] > prev_val[0] and values[2] < prev_val[1]: sym = " △ ▼"
|
||||
elif values[1] < prev_val[0] and values[2] > prev_val[1]: sym = " ▼ △"
|
||||
elif values[1] < prev_val[0] and values[2] < prev_val[1]: sym = " ▼ ▼"
|
||||
print(layout.format(*values) + sym)
|
||||
|
||||
class IterBatch():
|
||||
def __init__(self, dl):
|
||||
self.idx = 0
|
||||
self.dl = dl
|
||||
self.iter = iter(dl)
|
||||
|
||||
def __iter__(self): return self
|
||||
|
||||
def next(self):
|
||||
res = next(self.iter)
|
||||
self.idx += 1
|
||||
if self.idx == len(self.dl):
|
||||
self.iter = iter(self.dl)
|
||||
self.idx=0
|
||||
return res
|
||||
|
||||
def validate_next(stepper, metrics, val_iter):
|
||||
"""Computes the loss on the next minibatch of the validation set."""
|
||||
stepper.reset(False)
|
||||
with no_grad_context():
|
||||
(*x,y) = val_iter.next()
|
||||
preds,l = stepper.evaluate(VV(x), VV(y))
|
||||
res = [delistify(to_np(l))]
|
||||
res += [f(preds.data,y) for f in metrics]
|
||||
stepper.reset(True)
|
||||
return res
|
||||
|
||||
def batch_sz(x, seq_first=False):
|
||||
if is_listy(x): x = x[0]
|
||||
return x.shape[1 if seq_first else 0]
|
||||
|
||||
def validate(stepper, dl, metrics, seq_first=False):
|
||||
batch_cnts,loss,res = [],[],[]
|
||||
stepper.reset(False)
|
||||
with no_grad_context():
|
||||
for (*x,y) in iter(dl):
|
||||
preds, l = stepper.evaluate(VV(x), VV(y))
|
||||
batch_cnts.append(batch_sz(x, seq_first=seq_first))
|
||||
loss.append(to_np(l))
|
||||
res.append([f(preds.data, y) for f in metrics])
|
||||
return [np.average(loss, 0, weights=batch_cnts)] + list(np.average(np.stack(res), 0, weights=batch_cnts))
|
||||
|
||||
def get_prediction(x):
|
||||
if is_listy(x): x=x[0]
|
||||
return x.data
|
||||
|
||||
def predict(m, dl):
|
||||
preda,_ = predict_with_targs_(m, dl)
|
||||
return np.concatenate(preda)
|
||||
|
||||
def predict_batch(m, x):
|
||||
m.eval()
|
||||
if hasattr(m, 'reset'): m.reset()
|
||||
return m(VV(x))
|
||||
|
||||
def predict_with_targs_(m, dl):
|
||||
m.eval()
|
||||
if hasattr(m, 'reset'): m.reset()
|
||||
res = []
|
||||
for *x,y in iter(dl): res.append([get_prediction(to_np(m(*VV(x)))),to_np(y)])
|
||||
return zip(*res)
|
||||
|
||||
def predict_with_targs(m, dl):
|
||||
preda,targa = predict_with_targs_(m, dl)
|
||||
return np.concatenate(preda), np.concatenate(targa)
|
||||
|
||||
# From https://github.com/ncullen93/torchsample
|
||||
def model_summary(m, inputs):
|
||||
def register_hook(module):
|
||||
def hook(module, input, output):
|
||||
class_name = str(module.__class__).split('.')[-1].split("'")[0]
|
||||
module_idx = len(summary)
|
||||
|
||||
m_key = '%s-%i' % (class_name, module_idx+1)
|
||||
summary[m_key] = OrderedDict()
|
||||
summary[m_key]['input_shape'] = list(input[0].size())
|
||||
summary[m_key]['input_shape'][0] = -1
|
||||
if is_listy(output):
|
||||
summary[m_key]['output_shape'] = [[-1] + list(o.size())[1:] for o in output]
|
||||
else:
|
||||
summary[m_key]['output_shape'] = list(output.size())
|
||||
summary[m_key]['output_shape'][0] = -1
|
||||
|
||||
params = 0
|
||||
if hasattr(module, 'weight'):
|
||||
params += torch.prod(torch.LongTensor(list(module.weight.size())))
|
||||
summary[m_key]['trainable'] = module.weight.requires_grad
|
||||
if hasattr(module, 'bias') and module.bias is not None:
|
||||
params += torch.prod(torch.LongTensor(list(module.bias.size())))
|
||||
summary[m_key]['nb_params'] = params
|
||||
|
||||
if (not isinstance(module, nn.Sequential) and
|
||||
not isinstance(module, nn.ModuleList) and
|
||||
not (module == m)):
|
||||
hooks.append(module.register_forward_hook(hook))
|
||||
|
||||
summary = OrderedDict()
|
||||
hooks = []
|
||||
m.apply(register_hook)
|
||||
xs = [to_gpu(Variable(x)) for x in inputs]
|
||||
m(*xs)
|
||||
|
||||
for h in hooks: h.remove()
|
||||
return summary
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
*.png
|
||||
*.tar
|
||||
checkpoint*
|
||||
log*
|
||||
wgts/
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
python main.py --lr=0.1
|
||||
python main.py --resume --lr=0.01
|
||||
python main.py --resume --lr=0.001
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
from __future__ import division
|
||||
|
||||
from senet import *
|
||||
import os, sys, shutil, time, random
|
||||
import argparse
|
||||
import torch
|
||||
import torch.backends.cudnn as cudnn
|
||||
import torchvision.datasets as dset
|
||||
import torchvision.transforms as transforms
|
||||
from utils import AverageMeter, RecorderMeter, time_string, convert_secs2time
|
||||
|
||||
parser = argparse.ArgumentParser(description='Trains ResNeXt on CIFAR or ImageNet', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--data_path', default='./data', type=str, help='Path to dataset')
|
||||
parser.add_argument('--dataset', default='cifar10', type=str, choices=['cifar10', 'cifar100', 'imagenet', 'svhn', 'stl10'], help='Choose between Cifar10/100 and ImageNet.')
|
||||
# Optimization options
|
||||
parser.add_argument('--epochs', type=int, default=300, help='Number of epochs to train.')
|
||||
parser.add_argument('--batch_size', type=int, default=64, help='Batch size.')
|
||||
parser.add_argument('--learning_rate', type=float, default=0.05, help='The Learning Rate.')
|
||||
parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.')
|
||||
parser.add_argument('--decay', type=float, default=0.0005, help='Weight decay (L2 penalty).')
|
||||
parser.add_argument('--schedule', type=int, nargs='+', default=[150, 225], help='Decrease learning rate at these epochs.')
|
||||
parser.add_argument('--gammas', type=float, nargs='+', default=[0.1, 0.1], help='LR is multiplied by gamma on schedule, number of gammas should be equal to schedule')
|
||||
# Checkpoints
|
||||
parser.add_argument('--print_freq', default=200, type=int, metavar='N', help='print frequency (default: 200)')
|
||||
parser.add_argument('--save_path', type=str, default='./', help='Folder to save checkpoints and log.')
|
||||
parser.add_argument('--resume', default='', type=str, metavar='PATH', help='path to latest checkpoint (default: none)')
|
||||
parser.add_argument('--start_epoch', default=0, type=int, metavar='N', help='manual epoch number (useful on restarts)')
|
||||
parser.add_argument('--evaluate', dest='evaluate', action='store_true', help='evaluate model on validation set')
|
||||
# Acceleration
|
||||
parser.add_argument('--workers', type=int, default=2, help='number of data loading workers (default: 2)')
|
||||
# random seed
|
||||
parser.add_argument('--manualSeed', type=int, help='manual seed')
|
||||
args = parser.parse_args()
|
||||
args.use_cuda = torch.cuda.is_available()
|
||||
torch.cuda.set_device(0)
|
||||
|
||||
if args.manualSeed is None: args.manualSeed = random.randint(1, 10000)
|
||||
random.seed(args.manualSeed)
|
||||
torch.manual_seed(args.manualSeed)
|
||||
if args.use_cuda: torch.cuda.manual_seed_all(args.manualSeed)
|
||||
cudnn.benchmark = True
|
||||
|
||||
def main():
|
||||
if not os.path.isdir(args.save_path): os.makedirs(args.save_path)
|
||||
log = open(os.path.join(args.save_path, 'log_seed_{}.txt'.format(args.manualSeed)), 'w')
|
||||
print_log('save path : {}'.format(args.save_path), log)
|
||||
state = {k: v for k, v in args._get_kwargs()}
|
||||
print_log(state, log)
|
||||
print_log("Random Seed: {}".format(args.manualSeed), log)
|
||||
print_log("python version : {}".format(sys.version.replace('\n', ' ')), log)
|
||||
print_log("torch version : {}".format(torch.__version__), log)
|
||||
print_log("cudnn version : {}".format(torch.backends.cudnn.version()), log)
|
||||
|
||||
# Init dataset
|
||||
if not os.path.isdir(args.data_path):
|
||||
os.makedirs(args.data_path)
|
||||
|
||||
if args.dataset == 'cifar10':
|
||||
mean = [x / 255 for x in [125.3, 123.0, 113.9]]
|
||||
std = [x / 255 for x in [63.0, 62.1, 66.7]]
|
||||
elif args.dataset == 'cifar100':
|
||||
mean = [x / 255 for x in [129.3, 124.1, 112.4]]
|
||||
std = [x / 255 for x in [68.2, 65.4, 70.4]]
|
||||
else:
|
||||
assert False, "Unknow dataset : {}".format(args.dataset)
|
||||
|
||||
train_transform = transforms.Compose(
|
||||
[transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(),
|
||||
transforms.Normalize(mean, std)])
|
||||
test_transform = transforms.Compose(
|
||||
[transforms.ToTensor(), transforms.Normalize(mean, std)])
|
||||
|
||||
if args.dataset == 'cifar10':
|
||||
train_data = dset.CIFAR10(args.data_path, train=True, transform=train_transform, download=True)
|
||||
test_data = dset.CIFAR10(args.data_path, train=False, transform=test_transform, download=True)
|
||||
num_classes = 10
|
||||
elif args.dataset == 'cifar100':
|
||||
train_data = dset.CIFAR100(args.data_path, train=True, transform=train_transform, download=True)
|
||||
test_data = dset.CIFAR100(args.data_path, train=False, transform=test_transform, download=True)
|
||||
num_classes = 100
|
||||
elif args.dataset == 'svhn':
|
||||
train_data = dset.SVHN(args.data_path, split='train', transform=train_transform, download=True)
|
||||
test_data = dset.SVHN(args.data_path, split='test', transform=test_transform, download=True)
|
||||
num_classes = 10
|
||||
elif args.dataset == 'stl10':
|
||||
train_data = dset.STL10(args.data_path, split='train', transform=train_transform, download=True)
|
||||
test_data = dset.STL10(args.data_path, split='test', transform=test_transform, download=True)
|
||||
num_classes = 10
|
||||
elif args.dataset == 'imagenet':
|
||||
assert False, 'Do not finish imagenet code'
|
||||
else:
|
||||
assert False, 'Do not support dataset : {}'.format(args.dataset)
|
||||
|
||||
train_loader = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True,
|
||||
num_workers=args.workers, pin_memory=True)
|
||||
test_loader = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False,
|
||||
num_workers=args.workers, pin_memory=True)
|
||||
|
||||
# Init model, criterion, and optimizer
|
||||
#net = models.__dict__[args.arch](num_classes).cuda()
|
||||
net = SENet34()
|
||||
|
||||
# define loss function (criterion) and optimizer
|
||||
criterion = F.nll_loss
|
||||
optimizer = torch.optim.SGD(net.parameters(), state['learning_rate'], momentum=state['momentum'],
|
||||
weight_decay=state['decay'], nesterov=True)
|
||||
|
||||
if args.use_cuda: net.cuda()
|
||||
|
||||
recorder = RecorderMeter(args.epochs)
|
||||
# optionally resume from a checkpoint
|
||||
if args.resume:
|
||||
if os.path.isfile(args.resume):
|
||||
print_log("=> loading checkpoint '{}'".format(args.resume), log)
|
||||
checkpoint = torch.load(args.resume)
|
||||
recorder = checkpoint['recorder']
|
||||
args.start_epoch = checkpoint['epoch']
|
||||
net.load_state_dict(checkpoint['state_dict'])
|
||||
optimizer.load_state_dict(checkpoint['optimizer'])
|
||||
print_log("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch']), log)
|
||||
else:
|
||||
print_log("=> no checkpoint found at '{}'".format(args.resume), log)
|
||||
else:
|
||||
print_log("=> do not use any checkpoint for model", log)
|
||||
|
||||
if args.evaluate:
|
||||
validate(test_loader, net, criterion, log)
|
||||
return
|
||||
|
||||
# Main loop
|
||||
start_time = time.time()
|
||||
epoch_time = AverageMeter()
|
||||
for epoch in range(args.start_epoch, args.epochs):
|
||||
current_learning_rate = adjust_learning_rate(optimizer, epoch, args.gammas, args.schedule)
|
||||
|
||||
need_hour, need_mins, need_secs = convert_secs2time(epoch_time.avg * (args.epochs-epoch))
|
||||
need_time = '[Need: {:02d}:{:02d}:{:02d}]'.format(need_hour, need_mins, need_secs)
|
||||
|
||||
print_log('\n==>>{:s} [Epoch={:03d}/{:03d}] {:s} [learning_rate={:6.4f}]'.format(time_string(), epoch, args.epochs, need_time, current_learning_rate) \
|
||||
+ ' [Best : Accuracy={:.2f}, Error={:.2f}]'.format(recorder.max_accuracy(False), 100-recorder.max_accuracy(False)), log)
|
||||
|
||||
# train for one epoch
|
||||
train_acc, train_los = train(train_loader, net, criterion, optimizer, epoch, log)
|
||||
|
||||
# evaluate on validation set
|
||||
val_acc, val_los = validate(test_loader, net, criterion, log)
|
||||
is_best = recorder.update(epoch, train_los, train_acc, val_los, val_acc)
|
||||
|
||||
save_checkpoint({
|
||||
'epoch': epoch + 1,
|
||||
'state_dict': net.state_dict(),
|
||||
'recorder': recorder,
|
||||
'optimizer' : optimizer.state_dict(),
|
||||
}, is_best, args.save_path, 'checkpoint.pth.tar')
|
||||
|
||||
# measure elapsed time
|
||||
epoch_time.update(time.time() - start_time)
|
||||
start_time = time.time()
|
||||
recorder.plot_curve( os.path.join(args.save_path, 'curve.png') )
|
||||
|
||||
log.close()
|
||||
|
||||
# train function (forward, backward, update)
|
||||
def train(train_loader, model, criterion, optimizer, epoch, log):
|
||||
batch_time = AverageMeter()
|
||||
data_time = AverageMeter()
|
||||
losses = AverageMeter()
|
||||
top1 = AverageMeter()
|
||||
top5 = AverageMeter()
|
||||
# switch to train mode
|
||||
model.train()
|
||||
|
||||
end = time.time()
|
||||
for i, (input, target) in enumerate(train_loader):
|
||||
# measure data loading time
|
||||
data_time.update(time.time() - end)
|
||||
|
||||
if args.use_cuda:
|
||||
target = target.cuda(async=True)
|
||||
input = input.cuda()
|
||||
input_var = torch.autograd.Variable(input)
|
||||
target_var = torch.autograd.Variable(target)
|
||||
|
||||
# compute output
|
||||
output = model(input_var)
|
||||
loss = criterion(output, target_var)
|
||||
|
||||
# measure accuracy and record loss
|
||||
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
|
||||
losses.update(loss.data[0], input.size(0))
|
||||
top1.update(prec1[0], input.size(0))
|
||||
top5.update(prec5[0], input.size(0))
|
||||
|
||||
# compute gradient and do SGD step
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
# measure elapsed time
|
||||
batch_time.update(time.time() - end)
|
||||
end = time.time()
|
||||
|
||||
print_log(' Epoch: [{:03d}][{:03d}/{:03d}] '
|
||||
'Time {batch_time.val:.3f} ({batch_time.avg:.3f}) '
|
||||
'Data {data_time.val:.3f} ({data_time.avg:.3f}) '
|
||||
'Loss {loss.val:.4f} ({loss.avg:.4f}) '
|
||||
'Prec@1 {top1.val:.3f} ({top1.avg:.3f}) '
|
||||
'Prec@5 {top5.val:.3f} ({top5.avg:.3f}) '.format(
|
||||
epoch, i, len(train_loader), batch_time=batch_time,
|
||||
data_time=data_time, loss=losses, top1=top1, top5=top5) + time_string(), log)
|
||||
return top1.avg, losses.avg
|
||||
|
||||
def validate(val_loader, model, criterion, log):
|
||||
losses = AverageMeter()
|
||||
top1 = AverageMeter()
|
||||
top5 = AverageMeter()
|
||||
|
||||
# switch to evaluate mode
|
||||
model.eval()
|
||||
|
||||
for i, (input, target) in enumerate(val_loader):
|
||||
if args.use_cuda:
|
||||
target = target.cuda(async=True)
|
||||
input = input.cuda()
|
||||
input_var = torch.autograd.Variable(input, volatile=True)
|
||||
target_var = torch.autograd.Variable(target, volatile=True)
|
||||
|
||||
# compute output
|
||||
output = model(input_var)
|
||||
loss = criterion(output, target_var)
|
||||
|
||||
# measure accuracy and record loss
|
||||
prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
|
||||
losses.update(loss.data[0], input.size(0))
|
||||
top1.update(prec1[0], input.size(0))
|
||||
top5.update(prec5[0], input.size(0))
|
||||
|
||||
print_log(' **Test** Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f} Error@1 {error1:.3f}'.format(top1=top1, top5=top5, error1=100-top1.avg), log)
|
||||
|
||||
return top1.avg, losses.avg
|
||||
|
||||
def print_log(print_string, log):
|
||||
print("{}".format(print_string))
|
||||
log.write('{}\n'.format(print_string))
|
||||
log.flush()
|
||||
|
||||
def save_checkpoint(state, is_best, save_path, filename):
|
||||
filename = os.path.join(save_path, filename)
|
||||
torch.save(state, filename)
|
||||
if is_best:
|
||||
bestname = os.path.join(save_path, 'model_best.pth.tar')
|
||||
shutil.copyfile(filename, bestname)
|
||||
|
||||
def adjust_learning_rate(optimizer, epoch, gammas, schedule):
|
||||
"""Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
|
||||
lr = args.learning_rate
|
||||
assert len(gammas) == len(schedule), "length of gammas and schedule should be equal"
|
||||
for (gamma, step) in zip(gammas, schedule):
|
||||
if (epoch >= step):
|
||||
lr = lr * gamma
|
||||
else:
|
||||
break
|
||||
for param_group in optimizer.param_groups:
|
||||
param_group['lr'] = lr
|
||||
return lr
|
||||
|
||||
def accuracy(output, target, topk=(1,)):
|
||||
"""Computes the precision@k for the specified values of k"""
|
||||
maxk = max(topk)
|
||||
batch_size = target.size(0)
|
||||
|
||||
_, pred = output.topk(maxk, 1, True, True)
|
||||
pred = pred.t()
|
||||
correct = pred.eq(target.view(1, -1).expand_as(pred))
|
||||
|
||||
res = []
|
||||
for k in topk:
|
||||
correct_k = correct[:k].view(-1).float().sum(0)
|
||||
res.append(correct_k.mul_(100.0 / batch_size))
|
||||
return res
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,145 @@
|
||||
'''Train CIFAR10 with PyTorch.'''
|
||||
from __future__ import print_function
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim as optim
|
||||
import torch.nn.functional as F
|
||||
import torch.backends.cudnn as cudnn
|
||||
|
||||
import torchvision
|
||||
import torchvision.transforms as transforms
|
||||
|
||||
import os
|
||||
import argparse
|
||||
|
||||
from senet import *
|
||||
from utils import progress_bar
|
||||
from torch.autograd import Variable
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description='PyTorch CIFAR10 Training')
|
||||
parser.add_argument('--lr', default=0.1, type=float, help='learning rate')
|
||||
parser.add_argument('--resume', '-r', action='store_true', help='resume from checkpoint')
|
||||
args = parser.parse_args()
|
||||
|
||||
use_cuda = torch.cuda.is_available()
|
||||
torch.cuda.set_device(3)
|
||||
best_acc = 0 # best test accuracy
|
||||
start_epoch = 0 # start from epoch 0 or last checkpoint epoch
|
||||
|
||||
# Data
|
||||
print('==> Preparing data..')
|
||||
transform_train = transforms.Compose([
|
||||
transforms.RandomCrop(32, padding=4),
|
||||
transforms.RandomHorizontalFlip(),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
|
||||
])
|
||||
|
||||
transform_test = transforms.Compose([
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
|
||||
])
|
||||
|
||||
trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)
|
||||
trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=4)
|
||||
|
||||
testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)
|
||||
testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=4)
|
||||
|
||||
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
|
||||
|
||||
# Model
|
||||
if args.resume:
|
||||
# Load checkpoint.
|
||||
print('==> Resuming from checkpoint..')
|
||||
assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
|
||||
checkpoint = torch.load('./checkpoint/ckpt.t7')
|
||||
net = checkpoint['net']
|
||||
best_acc = checkpoint['acc']
|
||||
start_epoch = checkpoint['epoch']
|
||||
else:
|
||||
print('==> Building model..')
|
||||
# net = VGG('VGG19')
|
||||
# net = ResNet18()
|
||||
# net = PreActResNet18()
|
||||
# net = GoogLeNet()
|
||||
# net = DenseNet121()
|
||||
# net = ResNeXt29_2x64d()
|
||||
# net = MobileNet()
|
||||
# net = DPN92()
|
||||
# net = ShuffleNetG2()
|
||||
net = SENet18()
|
||||
|
||||
if use_cuda:
|
||||
net.cuda()
|
||||
#net = torch.nn.DataParallel(net, device_ids=(0,3))
|
||||
#net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
|
||||
cudnn.benchmark = True
|
||||
|
||||
criterion = F.nll_loss
|
||||
optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4)
|
||||
|
||||
# Training
|
||||
def train(epoch):
|
||||
print('\nEpoch: %d' % epoch)
|
||||
net.train()
|
||||
train_loss = 0
|
||||
correct = 0
|
||||
total = 0
|
||||
for batch_idx, (inputs, targets) in enumerate(trainloader):
|
||||
if use_cuda: inputs, targets = inputs.cuda(), targets.cuda()
|
||||
optimizer.zero_grad()
|
||||
inputs, targets = Variable(inputs), Variable(targets)
|
||||
outputs = net(inputs)
|
||||
loss = criterion(outputs, targets)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
train_loss += loss.data[0]
|
||||
_, predicted = torch.max(outputs.data, 1)
|
||||
total += targets.size(0)
|
||||
correct += predicted.eq(targets.data).cpu().sum()
|
||||
|
||||
progress_bar(batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
|
||||
% (train_loss/(batch_idx+1), 100.*correct/total, correct, total))
|
||||
|
||||
def test(epoch):
|
||||
global best_acc
|
||||
net.eval()
|
||||
test_loss = 0
|
||||
correct = 0
|
||||
total = 0
|
||||
for batch_idx, (inputs, targets) in enumerate(testloader):
|
||||
if use_cuda: inputs, targets = inputs.cuda(), targets.cuda()
|
||||
inputs, targets = Variable(inputs, volatile=True), Variable(targets)
|
||||
outputs = net(inputs)
|
||||
loss = criterion(outputs, targets)
|
||||
|
||||
test_loss += loss.data[0]
|
||||
_, predicted = torch.max(outputs.data, 1)
|
||||
total += targets.size(0)
|
||||
correct += predicted.eq(targets.data).cpu().sum()
|
||||
|
||||
progress_bar(batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
|
||||
% (test_loss/(batch_idx+1), 100.*correct/total, correct, total))
|
||||
|
||||
# Save checkpoint.
|
||||
acc = 100.*correct/total
|
||||
if acc > best_acc:
|
||||
print('Saving..')
|
||||
state = {
|
||||
'net': net,
|
||||
'acc': acc,
|
||||
'epoch': epoch,
|
||||
}
|
||||
if not os.path.isdir('checkpoint'):
|
||||
os.mkdir('checkpoint')
|
||||
torch.save(state, './checkpoint/ckpt.t7')
|
||||
best_acc = acc
|
||||
|
||||
|
||||
for epoch in range(start_epoch, start_epoch+100):
|
||||
train(epoch)
|
||||
test(epoch)
|
||||
@@ -0,0 +1,102 @@
|
||||
'''Pre-activation ResNet in PyTorch.
|
||||
|
||||
Reference:
|
||||
[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
|
||||
Identity Mappings in Deep Residual Networks. arXiv:1603.05027
|
||||
'''
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from torch.autograd import Variable
|
||||
|
||||
|
||||
class PreActBlock(nn.Module):
|
||||
'''Pre-activation version of the BasicBlock.'''
|
||||
expansion = 1
|
||||
|
||||
def __init__(self, in_planes, planes, stride=1):
|
||||
super(PreActBlock, self).__init__()
|
||||
self.bn1 = nn.BatchNorm2d(in_planes)
|
||||
self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
|
||||
self.bn2 = nn.BatchNorm2d(planes)
|
||||
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
|
||||
|
||||
if stride != 1 or in_planes != self.expansion*planes:
|
||||
self.shortcut = nn.Sequential(
|
||||
nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
out = F.relu(self.bn1(x))
|
||||
shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x
|
||||
out = self.conv1(out)
|
||||
out = self.conv2(F.relu(self.bn2(out)))
|
||||
out += shortcut
|
||||
return out
|
||||
|
||||
|
||||
class PreActBottleneck(nn.Module):
|
||||
'''Pre-activation version of the original Bottleneck module.'''
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, in_planes, planes, stride=1):
|
||||
super(PreActBottleneck, self).__init__()
|
||||
self.bn1 = nn.BatchNorm2d(in_planes)
|
||||
self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)
|
||||
self.bn2 = nn.BatchNorm2d(planes)
|
||||
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
|
||||
self.bn3 = nn.BatchNorm2d(planes)
|
||||
self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False)
|
||||
|
||||
if stride != 1 or in_planes != self.expansion*planes:
|
||||
self.shortcut = nn.Sequential(
|
||||
nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
out = F.relu(self.bn1(x))
|
||||
shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x
|
||||
out = self.conv1(out)
|
||||
out = self.conv2(F.relu(self.bn2(out)))
|
||||
out = self.conv3(F.relu(self.bn3(out)))
|
||||
out += shortcut
|
||||
return out
|
||||
|
||||
|
||||
class PreActResNet(nn.Module):
|
||||
def __init__(self, block, num_blocks, num_classes=10):
|
||||
super(PreActResNet, self).__init__()
|
||||
self.in_planes = 64
|
||||
|
||||
self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)
|
||||
self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
|
||||
self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
|
||||
self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
|
||||
self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
|
||||
self.linear = nn.Linear(512*block.expansion, num_classes)
|
||||
|
||||
def _make_layer(self, block, planes, num_blocks, stride):
|
||||
strides = [stride] + [1]*(num_blocks-1)
|
||||
layers = []
|
||||
for stride in strides:
|
||||
layers.append(block(self.in_planes, planes, stride))
|
||||
self.in_planes = planes * block.expansion
|
||||
return nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x):
|
||||
out = self.conv1(x)
|
||||
out = self.layer1(out)
|
||||
out = self.layer2(out)
|
||||
out = self.layer3(out)
|
||||
out = self.layer4(out)
|
||||
out = F.adaptive_max_pool2d(out, 1)
|
||||
out = out.view(out.size(0), -1)
|
||||
return F.log_softmax(self.linear(out))
|
||||
|
||||
def PreActResNet18(): return PreActResNet(PreActBlock, [2,2,2,2])
|
||||
def PreActResNet34(): return PreActResNet(PreActBlock, [3,4,6,3])
|
||||
def PreActResNet50(): return PreActResNet(PreActBottleneck, [3,4,6,3])
|
||||
def PreActResNet101(): return PreActResNet(PreActBottleneck, [3,4,23,3])
|
||||
def PreActResNet152(): return PreActResNet(PreActBottleneck, [3,8,36,3])
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.nn import init
|
||||
import math
|
||||
|
||||
class ResNeXtBottleneck(nn.Module):
|
||||
expansion = 4
|
||||
"""
|
||||
RexNeXt bottleneck type C (https://github.com/facebookresearch/ResNeXt/blob/master/models/resnext.lua)
|
||||
"""
|
||||
def __init__(self, inplanes, planes, cardinality, base_width, stride=1, downsample=None):
|
||||
super(ResNeXtBottleneck, self).__init__()
|
||||
self.downsample = downsample
|
||||
|
||||
D = int(math.floor(planes * (base_width/64.0)))
|
||||
C = cardinality
|
||||
self.conv_reduce = nn.Conv2d(inplanes, D*C, kernel_size=1, stride=1, padding=0, bias=False)
|
||||
self.bn_reduce = nn.BatchNorm2d(D*C)
|
||||
|
||||
self.conv_conv = nn.Conv2d(D*C, D*C, kernel_size=3, stride=stride, padding=1, groups=cardinality, bias=False)
|
||||
self.bn = nn.BatchNorm2d(D*C)
|
||||
self.conv_expand = nn.Conv2d(D*C, planes*4, kernel_size=1, stride=1, padding=0, bias=False)
|
||||
self.bn_expand = nn.BatchNorm2d(planes*4)
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
|
||||
bottleneck = self.conv_reduce(x)
|
||||
bottleneck = F.relu(self.bn_reduce(bottleneck), inplace=True)
|
||||
|
||||
bottleneck = self.conv_conv(bottleneck)
|
||||
bottleneck = F.relu(self.bn(bottleneck), inplace=True)
|
||||
|
||||
bottleneck = self.conv_expand(bottleneck)
|
||||
bottleneck = self.bn_expand(bottleneck)
|
||||
|
||||
if self.downsample is not None: residual = self.downsample(x)
|
||||
return F.relu(residual + bottleneck, inplace=True)
|
||||
|
||||
|
||||
class CifarResNeXt(nn.Module):
|
||||
"""
|
||||
ResNext optimized for the Cifar dataset, as specified in
|
||||
https://arxiv.org/pdf/1611.05431.pdf
|
||||
"""
|
||||
def __init__(self, block, depth, cardinality, base_width, num_classes):
|
||||
super(CifarResNeXt, self).__init__()
|
||||
|
||||
# Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
|
||||
assert (depth - 2) % 9 == 0, 'depth should be one of 29, 38, 47, 56, 101'
|
||||
self.layer_blocks = (depth - 2) // 9
|
||||
|
||||
self.cardinality,self.base_width,self.num_classes,self.block = cardinality,base_width,num_classes,block
|
||||
|
||||
self.conv_1_3x3 = nn.Conv2d(3, 64, 3, 1, 1, bias=False)
|
||||
self.bn_1 = nn.BatchNorm2d(64)
|
||||
|
||||
self.inplanes = 64
|
||||
self.stage_1 = self._make_layer(64 , 1)
|
||||
self.stage_2 = self._make_layer(128, 2)
|
||||
self.stage_3 = self._make_layer(256, 2)
|
||||
self.avgpool = nn.AdaptiveAvgPool2d((1,1))
|
||||
self.classifier = nn.Linear(256*block.expansion, num_classes)
|
||||
|
||||
for m in self.modules():
|
||||
if isinstance(m, nn.Conv2d):
|
||||
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
|
||||
m.weight.data.normal_(0, math.sqrt(2. / n))
|
||||
elif isinstance(m, nn.BatchNorm2d):
|
||||
m.weight.data.fill_(1)
|
||||
m.bias.data.zero_()
|
||||
elif isinstance(m, nn.Linear):
|
||||
init.kaiming_normal(m.weight)
|
||||
m.bias.data.zero_()
|
||||
|
||||
def _make_layer(self, planes, stride=1):
|
||||
downsample = None
|
||||
exp_planes = planes * self.block.expansion
|
||||
if stride != 1 or self.inplanes != exp_planes:
|
||||
downsample = nn.Sequential(
|
||||
nn.Conv2d(self.inplanes, exp_planes, kernel_size=1, stride=stride, bias=False),
|
||||
nn.BatchNorm2d(exp_planes),
|
||||
)
|
||||
|
||||
layers = []
|
||||
layers.append(self.block(self.inplanes, planes, self.cardinality, self.base_width, stride, downsample))
|
||||
self.inplanes = exp_planes
|
||||
for i in range(1, self.layer_blocks):
|
||||
layers.append(self.block(self.inplanes, planes, self.cardinality, self.base_width))
|
||||
|
||||
return nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.conv_1_3x3(x)
|
||||
x = F.relu(self.bn_1(x), inplace=True)
|
||||
x = self.stage_1(x)
|
||||
x = self.stage_2(x)
|
||||
x = self.stage_3(x)
|
||||
x = self.avgpool(x)
|
||||
x = x.view(x.size(0), -1)
|
||||
return F.log_softmax(self.classifier(x))
|
||||
|
||||
def resnext29_16_64(num_classes=10):
|
||||
"""Constructs a ResNeXt-29, 16*64d model for CIFAR-10 (by default)
|
||||
|
||||
Args:
|
||||
num_classes (uint): number of classes
|
||||
"""
|
||||
model = CifarResNeXt(ResNeXtBottleneck, 29, 16, 64, num_classes)
|
||||
return model
|
||||
|
||||
def resnext29_8_64(num_classes=10):
|
||||
"""Constructs a ResNeXt-29, 8*64d model for CIFAR-10 (by default)
|
||||
|
||||
Args:
|
||||
num_classes (uint): number of classes
|
||||
"""
|
||||
model = CifarResNeXt(ResNeXtBottleneck, 29, 8, 64, num_classes)
|
||||
return model
|
||||
@@ -0,0 +1,116 @@
|
||||
'''SENet in PyTorch.
|
||||
|
||||
SENet is the winner of ImageNet-2017 (https://arxiv.org/abs/1709.01507).
|
||||
'''
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from torch.autograd import Variable
|
||||
|
||||
|
||||
class BasicBlock(nn.Module):
|
||||
def __init__(self, in_planes, planes, stride=1):
|
||||
super(BasicBlock, self).__init__()
|
||||
self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
|
||||
self.bn1 = nn.BatchNorm2d(planes)
|
||||
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
|
||||
self.bn2 = nn.BatchNorm2d(planes)
|
||||
|
||||
self.shortcut = nn.Sequential()
|
||||
if stride != 1 or in_planes != planes:
|
||||
self.shortcut = nn.Sequential(
|
||||
nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=False),
|
||||
nn.BatchNorm2d(planes)
|
||||
)
|
||||
|
||||
# SE layers
|
||||
self.fc1 = nn.Conv2d(planes, planes//16, kernel_size=1) # Use nn.Conv2d instead of nn.Linear
|
||||
self.fc2 = nn.Conv2d(planes//16, planes, kernel_size=1)
|
||||
|
||||
def forward(self, x):
|
||||
out = F.relu(self.bn1(self.conv1(x)))
|
||||
out = self.bn2(self.conv2(out))
|
||||
|
||||
# Squeeze
|
||||
w = F.avg_pool2d(out, out.size(2))
|
||||
w = F.relu(self.fc1(w))
|
||||
w = F.sigmoid(self.fc2(w))
|
||||
# Excitation
|
||||
out = out * w # New broadcasting feature from v0.2!
|
||||
|
||||
out += self.shortcut(x)
|
||||
out = F.relu(out)
|
||||
return out
|
||||
|
||||
|
||||
class PreActBlock(nn.Module):
|
||||
def __init__(self, in_planes, planes, stride=1):
|
||||
super(PreActBlock, self).__init__()
|
||||
self.bn1 = nn.BatchNorm2d(in_planes)
|
||||
self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
|
||||
self.bn2 = nn.BatchNorm2d(planes)
|
||||
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
|
||||
|
||||
if stride != 1 or in_planes != planes:
|
||||
self.shortcut = nn.Sequential(
|
||||
nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=False)
|
||||
)
|
||||
|
||||
# SE layers
|
||||
self.fc1 = nn.Conv2d(planes, planes//16, kernel_size=1)
|
||||
self.fc2 = nn.Conv2d(planes//16, planes, kernel_size=1)
|
||||
|
||||
def forward(self, x):
|
||||
out = F.relu(self.bn1(x))
|
||||
shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x
|
||||
out = self.conv1(out)
|
||||
out = self.conv2(F.relu(self.bn2(out)))
|
||||
|
||||
# Squeeze
|
||||
w = F.avg_pool2d(out, out.size(2))
|
||||
w = F.relu(self.fc1(w))
|
||||
w = F.sigmoid(self.fc2(w))
|
||||
# Excitation
|
||||
out = out * w
|
||||
|
||||
out += shortcut
|
||||
return out
|
||||
|
||||
|
||||
class SENet(nn.Module):
|
||||
def __init__(self, block, num_blocks, num_classes=10):
|
||||
super(SENet, self).__init__()
|
||||
self.in_planes = 64
|
||||
|
||||
self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)
|
||||
self.bn1 = nn.BatchNorm2d(64)
|
||||
self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
|
||||
self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
|
||||
self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
|
||||
self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
|
||||
self.linear = nn.Linear(512, num_classes)
|
||||
|
||||
def _make_layer(self, block, planes, num_blocks, stride):
|
||||
strides = [stride] + [1]*(num_blocks-1)
|
||||
layers = []
|
||||
for stride in strides:
|
||||
layers.append(block(self.in_planes, planes, stride))
|
||||
self.in_planes = planes
|
||||
return nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x):
|
||||
out = F.relu(self.bn1(self.conv1(x)))
|
||||
out = self.layer1(out)
|
||||
out = self.layer2(out)
|
||||
out = self.layer3(out)
|
||||
out = self.layer4(out)
|
||||
out = F.adaptive_max_pool2d(out, 1)
|
||||
out = out.view(out.size(0), -1)
|
||||
out = F.log_softmax(self.linear(out))
|
||||
return out
|
||||
|
||||
|
||||
def SENet18(): return SENet(PreActBlock, [2,2,2,2])
|
||||
def SENet34(): return SENet(PreActBlock, [3,4,6,3])
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import os, sys, time
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use('agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
class AverageMeter(object):
|
||||
"""Computes and stores the average and current value"""
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.val = 0
|
||||
self.avg = 0
|
||||
self.sum = 0
|
||||
self.count = 0
|
||||
|
||||
def update(self, val, n=1):
|
||||
self.val = val
|
||||
self.sum += val * n
|
||||
self.count += n
|
||||
self.avg = self.sum / self.count
|
||||
|
||||
|
||||
class RecorderMeter(object):
|
||||
"""Computes and stores the minimum loss value and its epoch index"""
|
||||
def __init__(self, total_epoch):
|
||||
self.reset(total_epoch)
|
||||
|
||||
def reset(self, total_epoch):
|
||||
assert total_epoch > 0
|
||||
self.total_epoch = total_epoch
|
||||
self.current_epoch = 0
|
||||
self.epoch_losses = np.zeros((self.total_epoch, 2), dtype=np.float32) # [epoch, train/val]
|
||||
self.epoch_losses = self.epoch_losses - 1
|
||||
|
||||
self.epoch_accuracy= np.zeros((self.total_epoch, 2), dtype=np.float32) # [epoch, train/val]
|
||||
self.epoch_accuracy= self.epoch_accuracy
|
||||
|
||||
def update(self, idx, train_loss, train_acc, val_loss, val_acc):
|
||||
assert idx >= 0 and idx < self.total_epoch, 'total_epoch : {} , but update with the {} index'.format(self.total_epoch, idx)
|
||||
self.epoch_losses [idx, 0] = train_loss
|
||||
self.epoch_losses [idx, 1] = val_loss
|
||||
self.epoch_accuracy[idx, 0] = train_acc
|
||||
self.epoch_accuracy[idx, 1] = val_acc
|
||||
self.current_epoch = idx + 1
|
||||
return self.max_accuracy(False) == val_acc
|
||||
|
||||
def max_accuracy(self, istrain):
|
||||
if self.current_epoch <= 0: return 0
|
||||
if istrain: return self.epoch_accuracy[:self.current_epoch, 0].max()
|
||||
else: return self.epoch_accuracy[:self.current_epoch, 1].max()
|
||||
|
||||
def plot_curve(self, save_path):
|
||||
title = 'the accuracy/loss curve of train/val'
|
||||
dpi = 80
|
||||
width, height = 1200, 800
|
||||
legend_fontsize = 10
|
||||
scale_distance = 48.8
|
||||
figsize = width / float(dpi), height / float(dpi)
|
||||
|
||||
fig = plt.figure(figsize=figsize)
|
||||
x_axis = np.array([i for i in range(self.total_epoch)]) # epochs
|
||||
y_axis = np.zeros(self.total_epoch)
|
||||
|
||||
plt.xlim(0, self.total_epoch)
|
||||
plt.ylim(0, 100)
|
||||
interval_y = 5
|
||||
interval_x = 5
|
||||
plt.xticks(np.arange(0, self.total_epoch + interval_x, interval_x))
|
||||
plt.yticks(np.arange(0, 100 + interval_y, interval_y))
|
||||
plt.grid()
|
||||
plt.title(title, fontsize=20)
|
||||
plt.xlabel('the training epoch', fontsize=16)
|
||||
plt.ylabel('accuracy', fontsize=16)
|
||||
|
||||
y_axis[:] = self.epoch_accuracy[:, 0]
|
||||
plt.plot(x_axis, y_axis, color='g', linestyle='-', label='train-accuracy', lw=2)
|
||||
plt.legend(loc=4, fontsize=legend_fontsize)
|
||||
|
||||
y_axis[:] = self.epoch_accuracy[:, 1]
|
||||
plt.plot(x_axis, y_axis, color='y', linestyle='-', label='valid-accuracy', lw=2)
|
||||
plt.legend(loc=4, fontsize=legend_fontsize)
|
||||
|
||||
|
||||
y_axis[:] = self.epoch_losses[:, 0]
|
||||
plt.plot(x_axis, y_axis*50, color='g', linestyle=':', label='train-loss-x50', lw=2)
|
||||
plt.legend(loc=4, fontsize=legend_fontsize)
|
||||
|
||||
y_axis[:] = self.epoch_losses[:, 1]
|
||||
plt.plot(x_axis, y_axis*50, color='y', linestyle=':', label='valid-loss-x50', lw=2)
|
||||
plt.legend(loc=4, fontsize=legend_fontsize)
|
||||
|
||||
if save_path is not None:
|
||||
fig.savefig(save_path, dpi=dpi, bbox_inches='tight')
|
||||
print ('---- save figure {} into {}'.format(title, save_path))
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def time_string():
|
||||
ISOTIMEFORMAT='%Y-%m-%d %X'
|
||||
string = '[{}]'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) ))
|
||||
return string
|
||||
|
||||
def convert_secs2time(epoch_time):
|
||||
need_hour = int(epoch_time / 3600)
|
||||
need_mins = int((epoch_time - 3600*need_hour) / 60)
|
||||
need_secs = int(epoch_time - 3600*need_hour - 60*need_mins)
|
||||
return need_hour, need_mins, need_secs
|
||||
|
||||
def time_file_str():
|
||||
ISOTIMEFORMAT='%Y-%m-%d'
|
||||
string = '{}'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) ))
|
||||
return string + '-{}'.format(random.randint(1, 10000))
|
||||
@@ -0,0 +1,124 @@
|
||||
'''Some helper functions for PyTorch, including:
|
||||
- get_mean_and_std: calculate the mean and std value of dataset.
|
||||
- msr_init: net parameter initialization.
|
||||
- progress_bar: progress bar mimic xlua.progress.
|
||||
'''
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import math
|
||||
|
||||
import torch.nn as nn
|
||||
import torch.nn.init as init
|
||||
|
||||
|
||||
def get_mean_and_std(dataset):
|
||||
'''Compute the mean and std value of dataset.'''
|
||||
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=2)
|
||||
mean = torch.zeros(3)
|
||||
std = torch.zeros(3)
|
||||
print('==> Computing mean and std..')
|
||||
for inputs, targets in dataloader:
|
||||
for i in range(3):
|
||||
mean[i] += inputs[:,i,:,:].mean()
|
||||
std[i] += inputs[:,i,:,:].std()
|
||||
mean.div_(len(dataset))
|
||||
std.div_(len(dataset))
|
||||
return mean, std
|
||||
|
||||
def init_params(net):
|
||||
'''Init layer parameters.'''
|
||||
for m in net.modules():
|
||||
if isinstance(m, nn.Conv2d):
|
||||
init.kaiming_normal(m.weight, mode='fan_out')
|
||||
if m.bias:
|
||||
init.constant(m.bias, 0)
|
||||
elif isinstance(m, nn.BatchNorm2d):
|
||||
init.constant(m.weight, 1)
|
||||
init.constant(m.bias, 0)
|
||||
elif isinstance(m, nn.Linear):
|
||||
init.normal(m.weight, std=1e-3)
|
||||
if m.bias:
|
||||
init.constant(m.bias, 0)
|
||||
|
||||
|
||||
_, term_width = os.popen('stty size', 'r').read().split()
|
||||
term_width = int(term_width)
|
||||
|
||||
TOTAL_BAR_LENGTH = 65.
|
||||
last_time = time.time()
|
||||
begin_time = last_time
|
||||
def progress_bar(current, total, msg=None):
|
||||
global last_time, begin_time
|
||||
if current == 0:
|
||||
begin_time = time.time() # Reset for new bar.
|
||||
|
||||
cur_len = int(TOTAL_BAR_LENGTH*current/total)
|
||||
rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1
|
||||
|
||||
sys.stdout.write(' [')
|
||||
for i in range(cur_len):
|
||||
sys.stdout.write('=')
|
||||
sys.stdout.write('>')
|
||||
for i in range(rest_len):
|
||||
sys.stdout.write('.')
|
||||
sys.stdout.write(']')
|
||||
|
||||
cur_time = time.time()
|
||||
step_time = cur_time - last_time
|
||||
last_time = cur_time
|
||||
tot_time = cur_time - begin_time
|
||||
|
||||
L = []
|
||||
L.append(' Step: %s' % format_time(step_time))
|
||||
L.append(' | Tot: %s' % format_time(tot_time))
|
||||
if msg:
|
||||
L.append(' | ' + msg)
|
||||
|
||||
msg = ''.join(L)
|
||||
sys.stdout.write(msg)
|
||||
for i in range(term_width-int(TOTAL_BAR_LENGTH)-len(msg)-3):
|
||||
sys.stdout.write(' ')
|
||||
|
||||
# Go back to the center of the bar.
|
||||
for i in range(term_width-int(TOTAL_BAR_LENGTH/2)+2):
|
||||
sys.stdout.write('\b')
|
||||
sys.stdout.write(' %d/%d ' % (current+1, total))
|
||||
|
||||
if current < total-1:
|
||||
sys.stdout.write('\r')
|
||||
else:
|
||||
sys.stdout.write('\n')
|
||||
sys.stdout.flush()
|
||||
|
||||
def format_time(seconds):
|
||||
days = int(seconds / 3600/24)
|
||||
seconds = seconds - days*3600*24
|
||||
hours = int(seconds / 3600)
|
||||
seconds = seconds - hours*3600
|
||||
minutes = int(seconds / 60)
|
||||
seconds = seconds - minutes*60
|
||||
secondsf = int(seconds)
|
||||
seconds = seconds - secondsf
|
||||
millis = int(seconds*1000)
|
||||
|
||||
f = ''
|
||||
i = 1
|
||||
if days > 0:
|
||||
f += str(days) + 'D'
|
||||
i += 1
|
||||
if hours > 0 and i <= 2:
|
||||
f += str(hours) + 'h'
|
||||
i += 1
|
||||
if minutes > 0 and i <= 2:
|
||||
f += str(minutes) + 'm'
|
||||
i += 1
|
||||
if secondsf > 0 and i <= 2:
|
||||
f += str(secondsf) + 's'
|
||||
i += 1
|
||||
if millis > 0 and i <= 2:
|
||||
f += str(millis) + 'ms'
|
||||
i += 1
|
||||
if f == '':
|
||||
f = '0ms'
|
||||
return f
|
||||
@@ -0,0 +1,69 @@
|
||||
# Cifar10 Wideresnet for Dawn Submission
|
||||
|
||||
import math
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from ...layers import *
|
||||
|
||||
def conv_2d(ni, nf, ks, stride): return nn.Conv2d(ni, nf, kernel_size=ks, stride=stride, padding=ks//2, bias=False)
|
||||
|
||||
def bn(ni, init_zero=False):
|
||||
m = nn.BatchNorm2d(ni)
|
||||
m.weight.data.fill_(0 if init_zero else 1)
|
||||
m.bias.data.zero_()
|
||||
return m
|
||||
|
||||
def bn_relu_conv(ni, nf, ks, stride, init_zero=False):
|
||||
bn_initzero = bn(ni, init_zero=init_zero)
|
||||
return nn.Sequential(bn_initzero, nn.ReLU(inplace=True), conv_2d(ni, nf, ks, stride))
|
||||
|
||||
def noop(x): return x
|
||||
|
||||
class BasicBlock(nn.Module):
|
||||
def __init__(self, ni, nf, stride, drop_p=0.0):
|
||||
super().__init__()
|
||||
self.bn = nn.BatchNorm2d(ni)
|
||||
self.conv1 = conv_2d(ni, nf, 3, stride)
|
||||
self.conv2 = bn_relu_conv(nf, nf, 3, 1)
|
||||
self.drop = nn.Dropout(drop_p, inplace=True) if drop_p else None
|
||||
self.shortcut = conv_2d(ni, nf, 1, stride) if ni != nf else noop
|
||||
|
||||
def forward(self, x):
|
||||
x2 = F.relu(self.bn(x), inplace=True)
|
||||
r = self.shortcut(x2)
|
||||
x = self.conv1(x2)
|
||||
if self.drop: x = self.drop(x)
|
||||
x = self.conv2(x) * 0.2
|
||||
return x.add_(r)
|
||||
|
||||
|
||||
def _make_group(N, ni, nf, block, stride, drop_p):
|
||||
return [block(ni if i == 0 else nf, nf, stride if i == 0 else 1, drop_p) for i in range(N)]
|
||||
|
||||
class WideResNet(nn.Module):
|
||||
def __init__(self, num_groups, N, num_classes, k=1, drop_p=0.0, start_nf=16):
|
||||
super().__init__()
|
||||
n_channels = [start_nf]
|
||||
for i in range(num_groups): n_channels.append(start_nf*(2**i)*k)
|
||||
|
||||
layers = [conv_2d(3, n_channels[0], 3, 1)] # conv1
|
||||
for i in range(num_groups):
|
||||
layers += _make_group(N, n_channels[i], n_channels[i+1], BasicBlock, (1 if i==0 else 2), drop_p)
|
||||
|
||||
layers += [nn.BatchNorm2d(n_channels[3]), nn.ReLU(inplace=True), nn.AdaptiveAvgPool2d(1),
|
||||
Flatten(), nn.Linear(n_channels[3], num_classes)]
|
||||
self.features = nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x): return self.features(x)
|
||||
|
||||
|
||||
def wrn_22(): return WideResNet(num_groups=3, N=3, num_classes=10, k=6, drop_p=0.)
|
||||
def wrn_22_k8(): return WideResNet(num_groups=3, N=3, num_classes=10, k=8, drop_p=0.)
|
||||
def wrn_22_k10(): return WideResNet(num_groups=3, N=3, num_classes=10, k=10, drop_p=0.)
|
||||
def wrn_22_k8_p2(): return WideResNet(num_groups=3, N=3, num_classes=10, k=8, drop_p=0.2)
|
||||
def wrn_28(): return WideResNet(num_groups=3, N=4, num_classes=10, k=6, drop_p=0.)
|
||||
def wrn_28_k8(): return WideResNet(num_groups=3, N=4, num_classes=10, k=8, drop_p=0.)
|
||||
def wrn_28_k8_p2(): return WideResNet(num_groups=3, N=4, num_classes=10, k=8, drop_p=0.2)
|
||||
def wrn_28_p2(): return WideResNet(num_groups=3, N=4, num_classes=10, k=6, drop_p=0.2)
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
from __future__ import print_function
|
||||
import argparse
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torch.optim as optim
|
||||
from torch.autograd import Variable
|
||||
from torch.utils.serialization import load_lua
|
||||
|
||||
import numpy as np
|
||||
import os
|
||||
import math
|
||||
from functools import reduce
|
||||
|
||||
class LambdaBase(nn.Sequential):
|
||||
def __init__(self, fn, *args):
|
||||
super(LambdaBase, self).__init__(*args)
|
||||
self.lambda_func = fn
|
||||
|
||||
def forward_prepare(self, input):
|
||||
output = []
|
||||
for module in self._modules.values():
|
||||
output.append(module(input))
|
||||
return output if output else input
|
||||
|
||||
class Lambda(LambdaBase):
|
||||
def forward(self, input):
|
||||
return self.lambda_func(self.forward_prepare(input))
|
||||
|
||||
class LambdaMap(LambdaBase):
|
||||
def forward(self, input):
|
||||
# result is Variables list [Variable1, Variable2, ...]
|
||||
return list(map(self.lambda_func,self.forward_prepare(input)))
|
||||
|
||||
class LambdaReduce(LambdaBase):
|
||||
def forward(self, input):
|
||||
# result is a Variable
|
||||
return reduce(self.lambda_func,self.forward_prepare(input))
|
||||
|
||||
|
||||
def copy_param(m,n):
|
||||
if m.weight is not None: n.weight.data.copy_(m.weight)
|
||||
if m.bias is not None: n.bias.data.copy_(m.bias)
|
||||
if hasattr(n,'running_mean'): n.running_mean.copy_(m.running_mean)
|
||||
if hasattr(n,'running_var'): n.running_var.copy_(m.running_var)
|
||||
|
||||
def add_submodule(seq, *args):
|
||||
for n in args:
|
||||
seq.add_module(str(len(seq._modules)),n)
|
||||
|
||||
def lua_recursive_model(module,seq):
|
||||
for m in module.modules:
|
||||
name = type(m).__name__
|
||||
real = m
|
||||
if name == 'TorchObject':
|
||||
name = m._typename.replace('cudnn.','')
|
||||
m = m._obj
|
||||
|
||||
if name == 'SpatialConvolution':
|
||||
if not hasattr(m,'groups'): m.groups=1
|
||||
n = nn.Conv2d(m.nInputPlane,m.nOutputPlane,(m.kW,m.kH),(m.dW,m.dH),(m.padW,m.padH),1,m.groups,bias=(m.bias is not None))
|
||||
copy_param(m,n)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'SpatialBatchNormalization':
|
||||
n = nn.BatchNorm2d(m.running_mean.size(0), m.eps, m.momentum, m.affine)
|
||||
copy_param(m,n)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'ReLU':
|
||||
n = nn.ReLU()
|
||||
add_submodule(seq,n)
|
||||
elif name == 'SpatialMaxPooling':
|
||||
n = nn.MaxPool2d((m.kW,m.kH),(m.dW,m.dH),(m.padW,m.padH),ceil_mode=m.ceil_mode)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'SpatialAveragePooling':
|
||||
n = nn.AvgPool2d((m.kW,m.kH),(m.dW,m.dH),(m.padW,m.padH),ceil_mode=m.ceil_mode)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'SpatialUpSamplingNearest':
|
||||
n = nn.UpsamplingNearest2d(scale_factor=m.scale_factor)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'View':
|
||||
n = Lambda(lambda x: x.view(x.size(0),-1))
|
||||
add_submodule(seq,n)
|
||||
elif name == 'Linear':
|
||||
# Linear in pytorch only accept 2D input
|
||||
n1 = Lambda(lambda x: x.view(1,-1) if 1==len(x.size()) else x )
|
||||
n2 = nn.Linear(m.weight.size(1),m.weight.size(0),bias=(m.bias is not None))
|
||||
copy_param(m,n2)
|
||||
n = nn.Sequential(n1,n2)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'Dropout':
|
||||
m.inplace = False
|
||||
n = nn.Dropout(m.p)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'SoftMax':
|
||||
n = nn.Softmax()
|
||||
add_submodule(seq,n)
|
||||
elif name == 'Identity':
|
||||
n = Lambda(lambda x: x) # do nothing
|
||||
add_submodule(seq,n)
|
||||
elif name == 'SpatialFullConvolution':
|
||||
n = nn.ConvTranspose2d(m.nInputPlane,m.nOutputPlane,(m.kW,m.kH),(m.dW,m.dH),(m.padW,m.padH))
|
||||
add_submodule(seq,n)
|
||||
elif name == 'SpatialReplicationPadding':
|
||||
n = nn.ReplicationPad2d((m.pad_l,m.pad_r,m.pad_t,m.pad_b))
|
||||
add_submodule(seq,n)
|
||||
elif name == 'SpatialReflectionPadding':
|
||||
n = nn.ReflectionPad2d((m.pad_l,m.pad_r,m.pad_t,m.pad_b))
|
||||
add_submodule(seq,n)
|
||||
elif name == 'Copy':
|
||||
n = Lambda(lambda x: x) # do nothing
|
||||
add_submodule(seq,n)
|
||||
elif name == 'Narrow':
|
||||
n = Lambda(lambda x,a=(m.dimension,m.index,m.length): x.narrow(*a))
|
||||
add_submodule(seq,n)
|
||||
elif name == 'SpatialCrossMapLRN':
|
||||
lrn = torch.legacy.nn.SpatialCrossMapLRN(m.size,m.alpha,m.beta,m.k)
|
||||
n = Lambda(lambda x,lrn=lrn: Variable(lrn.forward(x.data)))
|
||||
add_submodule(seq,n)
|
||||
elif name == 'Sequential':
|
||||
n = nn.Sequential()
|
||||
lua_recursive_model(m,n)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'ConcatTable': # output is list
|
||||
n = LambdaMap(lambda x: x)
|
||||
lua_recursive_model(m,n)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'CAddTable': # input is list
|
||||
n = LambdaReduce(lambda x,y: x+y)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'Concat':
|
||||
dim = m.dimension
|
||||
n = LambdaReduce(lambda x,y,dim=dim: torch.cat((x,y),dim))
|
||||
lua_recursive_model(m,n)
|
||||
add_submodule(seq,n)
|
||||
elif name == 'TorchObject':
|
||||
print('Not Implement',name,real._typename)
|
||||
else:
|
||||
print('Not Implement',name)
|
||||
|
||||
|
||||
def lua_recursive_source(module):
|
||||
s = []
|
||||
for m in module.modules:
|
||||
name = type(m).__name__
|
||||
real = m
|
||||
if name == 'TorchObject':
|
||||
name = m._typename.replace('cudnn.','')
|
||||
m = m._obj
|
||||
|
||||
if name == 'SpatialConvolution':
|
||||
if not hasattr(m,'groups'): m.groups=1
|
||||
s += ['nn.Conv2d({},{},{},{},{},{},{},bias={}),#Conv2d'.format(m.nInputPlane,
|
||||
m.nOutputPlane,(m.kW,m.kH),(m.dW,m.dH),(m.padW,m.padH),1,m.groups,m.bias is not None)]
|
||||
elif name == 'SpatialBatchNormalization':
|
||||
s += ['nn.BatchNorm2d({},{},{},{}),#BatchNorm2d'.format(m.running_mean.size(0), m.eps, m.momentum, m.affine)]
|
||||
elif name == 'ReLU':
|
||||
s += ['nn.ReLU()']
|
||||
elif name == 'SpatialMaxPooling':
|
||||
s += ['nn.MaxPool2d({},{},{},ceil_mode={}),#MaxPool2d'.format((m.kW,m.kH),(m.dW,m.dH),(m.padW,m.padH),m.ceil_mode)]
|
||||
elif name == 'SpatialAveragePooling':
|
||||
s += ['nn.AvgPool2d({},{},{},ceil_mode={}),#AvgPool2d'.format((m.kW,m.kH),(m.dW,m.dH),(m.padW,m.padH),m.ceil_mode)]
|
||||
elif name == 'SpatialUpSamplingNearest':
|
||||
s += ['nn.UpsamplingNearest2d(scale_factor={})'.format(m.scale_factor)]
|
||||
elif name == 'View':
|
||||
s += ['Lambda(lambda x: x.view(x.size(0),-1)), # View']
|
||||
elif name == 'Linear':
|
||||
s1 = 'Lambda(lambda x: x.view(1,-1) if 1==len(x.size()) else x )'
|
||||
s2 = 'nn.Linear({},{},bias={})'.format(m.weight.size(1),m.weight.size(0),(m.bias is not None))
|
||||
s += ['nn.Sequential({},{}),#Linear'.format(s1,s2)]
|
||||
elif name == 'Dropout':
|
||||
s += ['nn.Dropout({})'.format(m.p)]
|
||||
elif name == 'SoftMax':
|
||||
s += ['nn.Softmax()']
|
||||
elif name == 'Identity':
|
||||
s += ['Lambda(lambda x: x), # Identity']
|
||||
elif name == 'SpatialFullConvolution':
|
||||
s += ['nn.ConvTranspose2d({},{},{},{},{})'.format(m.nInputPlane,
|
||||
m.nOutputPlane,(m.kW,m.kH),(m.dW,m.dH),(m.padW,m.padH))]
|
||||
elif name == 'SpatialReplicationPadding':
|
||||
s += ['nn.ReplicationPad2d({})'.format((m.pad_l,m.pad_r,m.pad_t,m.pad_b))]
|
||||
elif name == 'SpatialReflectionPadding':
|
||||
s += ['nn.ReflectionPad2d({})'.format((m.pad_l,m.pad_r,m.pad_t,m.pad_b))]
|
||||
elif name == 'Copy':
|
||||
s += ['Lambda(lambda x: x), # Copy']
|
||||
elif name == 'Narrow':
|
||||
s += ['Lambda(lambda x,a={}: x.narrow(*a))'.format((m.dimension,m.index,m.length))]
|
||||
elif name == 'SpatialCrossMapLRN':
|
||||
lrn = 'torch.legacy.nn.SpatialCrossMapLRN(*{})'.format((m.size,m.alpha,m.beta,m.k))
|
||||
s += ['Lambda(lambda x,lrn={}: Variable(lrn.forward(x.data)))'.format(lrn)]
|
||||
|
||||
elif name == 'Sequential':
|
||||
s += ['nn.Sequential( # Sequential']
|
||||
s += lua_recursive_source(m)
|
||||
s += [')']
|
||||
elif name == 'ConcatTable':
|
||||
s += ['LambdaMap(lambda x: x, # ConcatTable']
|
||||
s += lua_recursive_source(m)
|
||||
s += [')']
|
||||
elif name == 'CAddTable':
|
||||
s += ['LambdaReduce(lambda x,y: x+y), # CAddTable']
|
||||
elif name == 'Concat':
|
||||
dim = m.dimension
|
||||
s += ['LambdaReduce(lambda x,y,dim={}: torch.cat((x,y),dim), # Concat'.format(m.dimension)]
|
||||
s += lua_recursive_source(m)
|
||||
s += [')']
|
||||
else:
|
||||
s += '# ' + name + ' Not Implement,\n'
|
||||
s = map(lambda x: '\t{}'.format(x),s)
|
||||
return s
|
||||
|
||||
def simplify_source(s):
|
||||
s = map(lambda x: x.replace(',(1, 1),(0, 0),1,1,bias=True),#Conv2d',')'),s)
|
||||
s = map(lambda x: x.replace(',(0, 0),1,1,bias=True),#Conv2d',')'),s)
|
||||
s = map(lambda x: x.replace(',1,1,bias=True),#Conv2d',')'),s)
|
||||
s = map(lambda x: x.replace(',bias=True),#Conv2d',')'),s)
|
||||
s = map(lambda x: x.replace('),#Conv2d',')'),s)
|
||||
s = map(lambda x: x.replace(',1e-05,0.1,True),#BatchNorm2d',')'),s)
|
||||
s = map(lambda x: x.replace('),#BatchNorm2d',')'),s)
|
||||
s = map(lambda x: x.replace(',(0, 0),ceil_mode=False),#MaxPool2d',')'),s)
|
||||
s = map(lambda x: x.replace(',ceil_mode=False),#MaxPool2d',')'),s)
|
||||
s = map(lambda x: x.replace('),#MaxPool2d',')'),s)
|
||||
s = map(lambda x: x.replace(',(0, 0),ceil_mode=False),#AvgPool2d',')'),s)
|
||||
s = map(lambda x: x.replace(',ceil_mode=False),#AvgPool2d',')'),s)
|
||||
s = map(lambda x: x.replace(',bias=True)),#Linear',')), # Linear'),s)
|
||||
s = map(lambda x: x.replace(')),#Linear',')), # Linear'),s)
|
||||
|
||||
s = map(lambda x: '{},\n'.format(x),s)
|
||||
s = map(lambda x: x[1:],s)
|
||||
s = reduce(lambda x,y: x+y, s)
|
||||
return s
|
||||
|
||||
def torch_to_pytorch(t7_filename,outputname=None):
|
||||
model = load_lua(t7_filename,unknown_classes=True)
|
||||
if type(model).__name__=='hashable_uniq_dict': model=model.model
|
||||
model.gradInput = None
|
||||
slist = lua_recursive_source(torch.legacy.nn.Sequential().add(model))
|
||||
s = simplify_source(slist)
|
||||
header = '''
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.autograd import Variable
|
||||
from functools import reduce
|
||||
|
||||
class LambdaBase(nn.Sequential):
|
||||
def __init__(self, fn, *args):
|
||||
super(LambdaBase, self).__init__(*args)
|
||||
self.lambda_func = fn
|
||||
|
||||
def forward_prepare(self, input):
|
||||
output = []
|
||||
for module in self._modules.values():
|
||||
output.append(module(input))
|
||||
return output if output else input
|
||||
|
||||
class Lambda(LambdaBase):
|
||||
def forward(self, input):
|
||||
return self.lambda_func(self.forward_prepare(input))
|
||||
|
||||
class LambdaMap(LambdaBase):
|
||||
def forward(self, input):
|
||||
return list(map(self.lambda_func,self.forward_prepare(input)))
|
||||
|
||||
class LambdaReduce(LambdaBase):
|
||||
def forward(self, input):
|
||||
return reduce(self.lambda_func,self.forward_prepare(input))
|
||||
'''
|
||||
varname = t7_filename.replace('.t7','').replace('.','_').replace('-','_')
|
||||
s = '{}\n\n{} = {}'.format(header,varname,s[:-2])
|
||||
|
||||
if outputname is None: outputname=varname
|
||||
with open(outputname+'.py', "w") as pyfile:
|
||||
pyfile.write(s)
|
||||
|
||||
n = nn.Sequential()
|
||||
lua_recursive_model(model,n)
|
||||
torch.save(n.state_dict(),outputname+'.pth')
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description='Convert torch t7 model to pytorch')
|
||||
parser.add_argument('--model','-m', type=str, required=True,
|
||||
help='torch model file in t7 format')
|
||||
parser.add_argument('--output', '-o', type=str, default=None,
|
||||
help='output file name prefix, xxx.py xxx.pth')
|
||||
args = parser.parse_args()
|
||||
|
||||
torch_to_pytorch(args.model,args.output)
|
||||
@@ -0,0 +1,51 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from .layers import *
|
||||
|
||||
class ConvBN(nn.Module):
|
||||
"convolutional layer then batchnorm"
|
||||
|
||||
def __init__(self, ch_in, ch_out, kernel_size = 3, stride=1, padding=0):
|
||||
super().__init__()
|
||||
self.conv = nn.Conv2d(ch_in, ch_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)
|
||||
self.bn = nn.BatchNorm2d(ch_out, momentum=0.01)
|
||||
self.relu = nn.LeakyReLU(0.1, inplace=True)
|
||||
|
||||
def forward(self, x): return self.relu(self.bn(self.conv(x)))
|
||||
|
||||
class DarknetBlock(nn.Module):
|
||||
def __init__(self, ch_in):
|
||||
super().__init__()
|
||||
ch_hid = ch_in//2
|
||||
self.conv1 = ConvBN(ch_in, ch_hid, kernel_size=1, stride=1, padding=0)
|
||||
self.conv2 = ConvBN(ch_hid, ch_in, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
def forward(self, x): return self.conv2(self.conv1(x)) + x
|
||||
|
||||
class Darknet(nn.Module):
|
||||
"Replicates the darknet classifier from the YOLOv3 paper (table 1)"
|
||||
|
||||
def make_group_layer(self, ch_in, num_blocks, stride=1):
|
||||
layers = [ConvBN(ch_in,ch_in*2,stride=stride)]
|
||||
for i in range(num_blocks): layers.append(DarknetBlock(ch_in*2))
|
||||
return layers
|
||||
|
||||
def __init__(self, num_blocks, num_classes=1000, start_nf=32):
|
||||
super().__init__()
|
||||
nf = start_nf
|
||||
layers = [ConvBN(3, nf, kernel_size=3, stride=1, padding=1)]
|
||||
for i,nb in enumerate(num_blocks):
|
||||
layers += self.make_group_layer(nf, nb, stride=(1 if i==1 else 2))
|
||||
nf *= 2
|
||||
layers += [nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(nf, num_classes)]
|
||||
self.layers = nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x): return self.layers(x)
|
||||
|
||||
def darknet_53(num_classes=1000): return Darknet([1,2,8,8,4], num_classes)
|
||||
def darknet_small(num_classes=1000): return Darknet([1,2,4,8,4], num_classes)
|
||||
def darknet_mini(num_classes=1000): return Darknet([1,2,4,4,2], num_classes, start_nf=24)
|
||||
def darknet_mini2(num_classes=1000): return Darknet([1,2,8,8,4], num_classes, start_nf=16)
|
||||
def darknet_mini3(num_classes=1000): return Darknet([1,2,4,4], num_classes)
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
import torch.nn as nn
|
||||
import math
|
||||
import torch.utils.model_zoo as model_zoo
|
||||
from ..layers import *
|
||||
|
||||
|
||||
model_urls = {
|
||||
'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
|
||||
'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
|
||||
'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
|
||||
'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
|
||||
'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
|
||||
}
|
||||
|
||||
|
||||
def conv3x3(in_planes, out_planes, stride=1):
|
||||
"3x3 convolution with padding"
|
||||
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
|
||||
padding=1, bias=False)
|
||||
|
||||
def bn1(planes):
|
||||
m = nn.BatchNorm1d(planes)
|
||||
m.weight.data.fill_(1)
|
||||
m.bias.data.zero_()
|
||||
return m
|
||||
|
||||
def bn(planes, init_zero=False):
|
||||
m = nn.BatchNorm2d(planes)
|
||||
m.weight.data.fill_(0 if init_zero else 1)
|
||||
m.bias.data.zero_()
|
||||
return m
|
||||
|
||||
class BasicBlock(nn.Module):
|
||||
expansion = 1
|
||||
|
||||
def __init__(self, inplanes, planes, stride=1, downsample=None):
|
||||
super().__init__()
|
||||
self.conv1 = conv3x3(inplanes, planes, stride)
|
||||
self.bn1 = bn(planes)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.conv2 = conv3x3(planes, planes)
|
||||
self.bn2 = bn(planes)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
if self.downsample is not None: residual = self.downsample(x)
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.relu(out)
|
||||
out = self.bn1(out)
|
||||
|
||||
out = self.conv2(out)
|
||||
|
||||
out += residual
|
||||
out = self.relu(out)
|
||||
out = self.bn2(out)
|
||||
|
||||
return out
|
||||
|
||||
class BottleneckFinal(nn.Module):
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, inplanes, planes, stride=1, downsample=None):
|
||||
super().__init__()
|
||||
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
|
||||
self.bn1 = bn(planes)
|
||||
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
|
||||
padding=1, bias=False)
|
||||
self.bn2 = bn(planes)
|
||||
self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
|
||||
self.bn3 = bn(planes * 4)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
if self.downsample is not None: residual = self.downsample(x)
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.bn1(out)
|
||||
out = self.relu(out)
|
||||
out = self.conv2(out)
|
||||
out = self.bn2(out)
|
||||
out = self.relu(out)
|
||||
out = self.conv3(out)
|
||||
|
||||
out += residual
|
||||
out = self.bn3(out)
|
||||
out = self.relu(out)
|
||||
|
||||
return out
|
||||
|
||||
class BottleneckZero(nn.Module):
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, inplanes, planes, stride=1, downsample=None):
|
||||
super().__init__()
|
||||
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
|
||||
self.bn1 = bn(planes)
|
||||
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
|
||||
padding=1, bias=False)
|
||||
self.bn2 = bn(planes)
|
||||
self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
|
||||
self.bn3 = bn(planes * 4, init_zero=True)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
if self.downsample is not None: residual = self.downsample(x)
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.bn1(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv2(out)
|
||||
out = self.bn2(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv3(out)
|
||||
out = self.bn3(out)
|
||||
|
||||
out += residual
|
||||
out = self.relu(out)
|
||||
|
||||
return out
|
||||
|
||||
class Bottleneck(nn.Module):
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, inplanes, planes, stride=1, downsample=None):
|
||||
super().__init__()
|
||||
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
|
||||
self.bn1 = bn(planes)
|
||||
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
|
||||
padding=1, bias=False)
|
||||
self.bn2 = bn(planes)
|
||||
self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
|
||||
self.bn3 = bn(planes * 4)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
if self.downsample is not None: residual = self.downsample(x)
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.bn1(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv2(out)
|
||||
out = self.bn2(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv3(out)
|
||||
out = self.bn3(out)
|
||||
|
||||
out += residual
|
||||
out = self.relu(out)
|
||||
|
||||
return out
|
||||
|
||||
class ResNet(nn.Module):
|
||||
def __init__(self, block, layers, num_classes=1000, k=1, vgg_head=False):
|
||||
super().__init__()
|
||||
self.inplanes = 64
|
||||
|
||||
features = [nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)
|
||||
, bn(64) , nn.ReLU(inplace=True) , nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
|
||||
, self._make_layer(block, int(64*k), layers[0])
|
||||
, self._make_layer(block, int(128*k), layers[1], stride=2)
|
||||
, self._make_layer(block, int(256*k), layers[2], stride=2)
|
||||
, self._make_layer(block, int(512*k), layers[3], stride=2)]
|
||||
out_sz = int(512*k) * block.expansion
|
||||
|
||||
if vgg_head:
|
||||
features += [nn.AdaptiveAvgPool2d(3), Flatten()
|
||||
, nn.Linear(out_sz*3*3, 4096), nn.ReLU(inplace=True), bn1(4096), nn.Dropout(0.25)
|
||||
, nn.Linear(4096, 4096), nn.ReLU(inplace=True), bn1(4096), nn.Dropout(0.25)
|
||||
, nn.Linear(4096, num_classes)]
|
||||
else: features += [nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(out_sz, num_classes)]
|
||||
|
||||
self.features = nn.Sequential(*features)
|
||||
|
||||
for m in self.modules():
|
||||
if isinstance(m, nn.Conv2d):
|
||||
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
|
||||
m.weight.data.normal_(0, math.sqrt(2. / n))
|
||||
|
||||
def _make_layer(self, block, planes, blocks, stride=1):
|
||||
downsample = None
|
||||
if stride != 1 or self.inplanes != planes * block.expansion:
|
||||
downsample = nn.Sequential(
|
||||
nn.Conv2d(self.inplanes, planes * block.expansion,
|
||||
kernel_size=1, stride=stride, bias=False),
|
||||
bn(planes * block.expansion),
|
||||
)
|
||||
|
||||
layers = []
|
||||
layers.append(block(self.inplanes, planes, stride, downsample))
|
||||
self.inplanes = planes * block.expansion
|
||||
for i in range(1, blocks): layers.append(block(self.inplanes, planes))
|
||||
return nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x): return self.features(x)
|
||||
|
||||
def load(model, pre, name):
|
||||
if pretrained: model.load_state_dict(model_zoo.load_url(model_urls[name]))
|
||||
return model
|
||||
|
||||
def fa_resnet18(pretrained=False, **kwargs): return load(ResNet(BasicBlock, [2, 2, 2, 2], **kwargs), pretrained, 'resnet18')
|
||||
def fa_resnet34(pretrained=False, **kwargs): return load(ResNet(BasicBlock, [3, 4, 6, 3], **kwargs), pretrained, 'resnet34')
|
||||
def fa_resnet50(pretrained=False, **kwargs): return load(ResNet(Bottleneck, [3, 4, 6, 3], **kwargs), pretrained, 'resnet50')
|
||||
def fa_resnet101(pretrained=False, **kwargs): return load(ResNet(Bottleneck, [3, 4, 23, 3], **kwargs), pretrained, 'resnet101')
|
||||
def fa_resnet152(pretrained=False, **kwargs): return load(ResNet(Bottleneck, [3, 8, 36, 3], **kwargs), pretrained, 'resnet152')
|
||||
def bnf_resnet50 (): return ResNet(BottleneckFinal, [3, 4, 6, 3])
|
||||
def bnz_resnet50 (): return ResNet(BottleneckZero, [3, 4, 6, 3])
|
||||
def w5_resnet50 (): return ResNet(Bottleneck, [2, 3, 3, 2], k=1.5)
|
||||
def w25_resnet50(): return ResNet(Bottleneck, [3, 4, 4, 3], k=1.25)
|
||||
def w125_resnet50(): return ResNet(Bottleneck,[3, 4, 6, 3], k=1.125)
|
||||
def vgg_resnet50(): return ResNet(Bottleneck, [3, 4, 6, 3], vgg_head=True)
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.utils.model_zoo as model_zoo
|
||||
import os
|
||||
import sys
|
||||
|
||||
model_urls = {
|
||||
'imagenet': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionresnetv2-520b38e4.pth'
|
||||
}
|
||||
|
||||
|
||||
class BasicConv2d(nn.Module):
|
||||
|
||||
def __init__(self, in_planes, out_planes, kernel_size, stride, padding=0):
|
||||
super(BasicConv2d, self).__init__()
|
||||
self.conv = nn.Conv2d(in_planes, out_planes,
|
||||
kernel_size=kernel_size, stride=stride,
|
||||
padding=padding, bias=False) # verify bias false
|
||||
self.bn = nn.BatchNorm2d(out_planes,
|
||||
eps=0.001, # value found in tensorflow
|
||||
momentum=0.1, # default pytorch value
|
||||
affine=True)
|
||||
self.relu = nn.ReLU(inplace=False)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.conv(x)
|
||||
x = self.bn(x)
|
||||
x = self.relu(x)
|
||||
return x
|
||||
|
||||
|
||||
class Mixed_5b(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Mixed_5b, self).__init__()
|
||||
|
||||
self.branch0 = BasicConv2d(192, 96, kernel_size=1, stride=1)
|
||||
|
||||
self.branch1 = nn.Sequential(
|
||||
BasicConv2d(192, 48, kernel_size=1, stride=1),
|
||||
BasicConv2d(48, 64, kernel_size=5, stride=1, padding=2)
|
||||
)
|
||||
|
||||
self.branch2 = nn.Sequential(
|
||||
BasicConv2d(192, 64, kernel_size=1, stride=1),
|
||||
BasicConv2d(64, 96, kernel_size=3, stride=1, padding=1),
|
||||
BasicConv2d(96, 96, kernel_size=3, stride=1, padding=1)
|
||||
)
|
||||
|
||||
self.branch3 = nn.Sequential(
|
||||
nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),
|
||||
BasicConv2d(192, 64, kernel_size=1, stride=1)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.branch0(x)
|
||||
x1 = self.branch1(x)
|
||||
x2 = self.branch2(x)
|
||||
x3 = self.branch3(x)
|
||||
out = torch.cat((x0, x1, x2, x3), 1)
|
||||
return out
|
||||
|
||||
|
||||
class Block35(nn.Module):
|
||||
|
||||
def __init__(self, scale=1.0):
|
||||
super(Block35, self).__init__()
|
||||
|
||||
self.scale = scale
|
||||
|
||||
self.branch0 = BasicConv2d(320, 32, kernel_size=1, stride=1)
|
||||
|
||||
self.branch1 = nn.Sequential(
|
||||
BasicConv2d(320, 32, kernel_size=1, stride=1),
|
||||
BasicConv2d(32, 32, kernel_size=3, stride=1, padding=1)
|
||||
)
|
||||
|
||||
self.branch2 = nn.Sequential(
|
||||
BasicConv2d(320, 32, kernel_size=1, stride=1),
|
||||
BasicConv2d(32, 48, kernel_size=3, stride=1, padding=1),
|
||||
BasicConv2d(48, 64, kernel_size=3, stride=1, padding=1)
|
||||
)
|
||||
|
||||
self.conv2d = nn.Conv2d(128, 320, kernel_size=1, stride=1)
|
||||
self.relu = nn.ReLU(inplace=False)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.branch0(x)
|
||||
x1 = self.branch1(x)
|
||||
x2 = self.branch2(x)
|
||||
out = torch.cat((x0, x1, x2), 1)
|
||||
out = self.conv2d(out)
|
||||
out = out * self.scale + x
|
||||
out = self.relu(out)
|
||||
return out
|
||||
|
||||
|
||||
class Mixed_6a(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Mixed_6a, self).__init__()
|
||||
|
||||
self.branch0 = BasicConv2d(320, 384, kernel_size=3, stride=2)
|
||||
|
||||
self.branch1 = nn.Sequential(
|
||||
BasicConv2d(320, 256, kernel_size=1, stride=1),
|
||||
BasicConv2d(256, 256, kernel_size=3, stride=1, padding=1),
|
||||
BasicConv2d(256, 384, kernel_size=3, stride=2)
|
||||
)
|
||||
|
||||
self.branch2 = nn.MaxPool2d(3, stride=2)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.branch0(x)
|
||||
x1 = self.branch1(x)
|
||||
x2 = self.branch2(x)
|
||||
out = torch.cat((x0, x1, x2), 1)
|
||||
return out
|
||||
|
||||
|
||||
class Block17(nn.Module):
|
||||
|
||||
def __init__(self, scale=1.0):
|
||||
super(Block17, self).__init__()
|
||||
|
||||
self.scale = scale
|
||||
|
||||
self.branch0 = BasicConv2d(1088, 192, kernel_size=1, stride=1)
|
||||
|
||||
self.branch1 = nn.Sequential(
|
||||
BasicConv2d(1088, 128, kernel_size=1, stride=1),
|
||||
BasicConv2d(128, 160, kernel_size=(1,7), stride=1, padding=(0,3)),
|
||||
BasicConv2d(160, 192, kernel_size=(7,1), stride=1, padding=(3,0))
|
||||
)
|
||||
|
||||
self.conv2d = nn.Conv2d(384, 1088, kernel_size=1, stride=1)
|
||||
self.relu = nn.ReLU(inplace=False)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.branch0(x)
|
||||
x1 = self.branch1(x)
|
||||
out = torch.cat((x0, x1), 1)
|
||||
out = self.conv2d(out)
|
||||
out = out * self.scale + x
|
||||
out = self.relu(out)
|
||||
return out
|
||||
|
||||
|
||||
class Mixed_7a(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Mixed_7a, self).__init__()
|
||||
|
||||
self.branch0 = nn.Sequential(
|
||||
BasicConv2d(1088, 256, kernel_size=1, stride=1),
|
||||
BasicConv2d(256, 384, kernel_size=3, stride=2)
|
||||
)
|
||||
|
||||
self.branch1 = nn.Sequential(
|
||||
BasicConv2d(1088, 256, kernel_size=1, stride=1),
|
||||
BasicConv2d(256, 288, kernel_size=3, stride=2)
|
||||
)
|
||||
|
||||
self.branch2 = nn.Sequential(
|
||||
BasicConv2d(1088, 256, kernel_size=1, stride=1),
|
||||
BasicConv2d(256, 288, kernel_size=3, stride=1, padding=1),
|
||||
BasicConv2d(288, 320, kernel_size=3, stride=2)
|
||||
)
|
||||
|
||||
self.branch3 = nn.MaxPool2d(3, stride=2)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.branch0(x)
|
||||
x1 = self.branch1(x)
|
||||
x2 = self.branch2(x)
|
||||
x3 = self.branch3(x)
|
||||
out = torch.cat((x0, x1, x2, x3), 1)
|
||||
return out
|
||||
|
||||
|
||||
class Block8(nn.Module):
|
||||
|
||||
def __init__(self, scale=1.0, noReLU=False):
|
||||
super(Block8, self).__init__()
|
||||
|
||||
self.scale = scale
|
||||
self.noReLU = noReLU
|
||||
|
||||
self.branch0 = BasicConv2d(2080, 192, kernel_size=1, stride=1)
|
||||
|
||||
self.branch1 = nn.Sequential(
|
||||
BasicConv2d(2080, 192, kernel_size=1, stride=1),
|
||||
BasicConv2d(192, 224, kernel_size=(1,3), stride=1, padding=(0,1)),
|
||||
BasicConv2d(224, 256, kernel_size=(3,1), stride=1, padding=(1,0))
|
||||
)
|
||||
|
||||
self.conv2d = nn.Conv2d(448, 2080, kernel_size=1, stride=1)
|
||||
if not self.noReLU:
|
||||
self.relu = nn.ReLU(inplace=False)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.branch0(x)
|
||||
x1 = self.branch1(x)
|
||||
out = torch.cat((x0, x1), 1)
|
||||
out = self.conv2d(out)
|
||||
out = out * self.scale + x
|
||||
if not self.noReLU:
|
||||
out = self.relu(out)
|
||||
return out
|
||||
|
||||
|
||||
class InceptionResnetV2(nn.Module):
|
||||
|
||||
def __init__(self, num_classes=1001):
|
||||
super(InceptionResnetV2, self).__init__()
|
||||
# Special attributs
|
||||
self.input_space = None
|
||||
self.input_size = (299, 299, 3)
|
||||
self.mean = None
|
||||
self.std = None
|
||||
# Modules
|
||||
self.conv2d_1a = BasicConv2d(3, 32, kernel_size=3, stride=2)
|
||||
self.conv2d_2a = BasicConv2d(32, 32, kernel_size=3, stride=1)
|
||||
self.conv2d_2b = BasicConv2d(32, 64, kernel_size=3, stride=1, padding=1)
|
||||
self.maxpool_3a = nn.MaxPool2d(3, stride=2)
|
||||
self.conv2d_3b = BasicConv2d(64, 80, kernel_size=1, stride=1)
|
||||
self.conv2d_4a = BasicConv2d(80, 192, kernel_size=3, stride=1)
|
||||
self.maxpool_5a = nn.MaxPool2d(3, stride=2)
|
||||
self.mixed_5b = Mixed_5b()
|
||||
self.repeat = nn.Sequential(
|
||||
Block35(scale=0.17),
|
||||
Block35(scale=0.17),
|
||||
Block35(scale=0.17),
|
||||
Block35(scale=0.17),
|
||||
Block35(scale=0.17),
|
||||
Block35(scale=0.17),
|
||||
Block35(scale=0.17),
|
||||
Block35(scale=0.17),
|
||||
Block35(scale=0.17),
|
||||
Block35(scale=0.17)
|
||||
)
|
||||
self.mixed_6a = Mixed_6a()
|
||||
self.repeat_1 = nn.Sequential(
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10),
|
||||
Block17(scale=0.10)
|
||||
)
|
||||
self.mixed_7a = Mixed_7a()
|
||||
self.repeat_2 = nn.Sequential(
|
||||
Block8(scale=0.20),
|
||||
Block8(scale=0.20),
|
||||
Block8(scale=0.20),
|
||||
Block8(scale=0.20),
|
||||
Block8(scale=0.20),
|
||||
Block8(scale=0.20),
|
||||
Block8(scale=0.20),
|
||||
Block8(scale=0.20),
|
||||
Block8(scale=0.20)
|
||||
)
|
||||
self.block8 = Block8(noReLU=True)
|
||||
self.conv2d_7b = BasicConv2d(2080, 1536, kernel_size=1, stride=1)
|
||||
self.avgpool_1a = nn.AvgPool2d(8, count_include_pad=False)
|
||||
self.last_linear = nn.Linear(1536, num_classes)
|
||||
|
||||
def features(self, input):
|
||||
x = self.conv2d_1a(input)
|
||||
x = self.conv2d_2a(x)
|
||||
x = self.conv2d_2b(x)
|
||||
x = self.maxpool_3a(x)
|
||||
x = self.conv2d_3b(x)
|
||||
x = self.conv2d_4a(x)
|
||||
x = self.maxpool_5a(x)
|
||||
x = self.mixed_5b(x)
|
||||
x = self.repeat(x)
|
||||
x = self.mixed_6a(x)
|
||||
x = self.repeat_1(x)
|
||||
x = self.mixed_7a(x)
|
||||
x = self.repeat_2(x)
|
||||
x = self.block8(x)
|
||||
x = self.conv2d_7b(x)
|
||||
return x
|
||||
|
||||
def logits(self, features):
|
||||
x = self.avgpool_1a(features)
|
||||
x = x.view(x.size(0), -1)
|
||||
x = self.last_linear(x)
|
||||
return x
|
||||
|
||||
def forward(self, input):
|
||||
x = self.features(input)
|
||||
x = self.logits(x)
|
||||
return x
|
||||
|
||||
def inceptionresnetv2(num_classes=1000, pretrained='imagenet'):
|
||||
r"""InceptionResNetV2 model architecture from the
|
||||
`"InceptionV4, Inception-ResNet..." <https://arxiv.org/abs/1602.07261>`_ paper.
|
||||
"""
|
||||
if pretrained:
|
||||
settings = pretrained_settings['inceptionresnetv2'][pretrained]
|
||||
assert num_classes == settings['num_classes'], \
|
||||
"num_classes should be {}, but is {}".format(settings['num_classes'], num_classes)
|
||||
|
||||
# both 'imagenet'&'imagenet+background' are loaded from same parameters
|
||||
model = InceptionResNetV2(num_classes=1001)
|
||||
model.load_state_dict(model_zoo.load_url(settings['url']))
|
||||
|
||||
if pretrained == 'imagenet':
|
||||
new_last_linear = nn.Linear(1536, 1000)
|
||||
new_last_linear.weight.data = model.last_linear.weight.data[1:]
|
||||
new_last_linear.bias.data = model.last_linear.bias.data[1:]
|
||||
model.last_linear = new_last_linear
|
||||
|
||||
model.input_space = settings['input_space']
|
||||
model.input_size = settings['input_size']
|
||||
model.input_range = settings['input_range']
|
||||
|
||||
model.mean = settings['mean']
|
||||
model.std = settings['std']
|
||||
else:
|
||||
model = InceptionResNetV2(num_classes=num_classes)
|
||||
return model
|
||||
|
||||
'''
|
||||
TEST
|
||||
Run this code with:
|
||||
```
|
||||
cd $HOME/pretrained-models.pytorch
|
||||
python -m pretrainedmodels.inceptionresnetv2
|
||||
```
|
||||
'''
|
||||
if __name__ == '__main__':
|
||||
|
||||
assert inceptionresnetv2(num_classes=10, pretrained=None)
|
||||
print('success')
|
||||
assert inceptionresnetv2(num_classes=1000, pretrained='imagenet')
|
||||
print('success')
|
||||
assert inceptionresnetv2(num_classes=1001, pretrained='imagenet+background')
|
||||
print('success')
|
||||
|
||||
# fail
|
||||
assert inceptionresnetv2(num_classes=1001, pretrained='imagenet')
|
||||
@@ -0,0 +1,442 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.utils.model_zoo as model_zoo
|
||||
import os
|
||||
import sys
|
||||
|
||||
model_urls = {
|
||||
'imagenet': 'https://s3.amazonaws.com/pytorch/models/inceptionv4-58153ba9.pth'
|
||||
}
|
||||
|
||||
class BasicConv2d(nn.Module):
|
||||
|
||||
def __init__(self, in_planes, out_planes, kernel_size, stride, padding=0):
|
||||
super(BasicConv2d, self).__init__()
|
||||
self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, bias=False) # verify bias false
|
||||
self.bn = nn.BatchNorm2d(out_planes)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.conv(x)
|
||||
x = self.bn(x)
|
||||
x = self.relu(x)
|
||||
return x
|
||||
|
||||
class Mixed_3a(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Mixed_3a, self).__init__()
|
||||
self.maxpool = nn.MaxPool2d(3, stride=2)
|
||||
self.conv = BasicConv2d(64, 96, kernel_size=3, stride=2)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.maxpool(x)
|
||||
x1 = self.conv(x)
|
||||
out = torch.cat((x0, x1), 1)
|
||||
return out
|
||||
|
||||
class Mixed_4a(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Mixed_4a, self).__init__()
|
||||
|
||||
self.block0 = nn.Sequential(
|
||||
BasicConv2d(160, 64, kernel_size=1, stride=1),
|
||||
BasicConv2d(64, 96, kernel_size=3, stride=1)
|
||||
)
|
||||
|
||||
self.block1 = nn.Sequential(
|
||||
BasicConv2d(160, 64, kernel_size=1, stride=1),
|
||||
BasicConv2d(64, 64, kernel_size=(1,7), stride=1, padding=(0,3)),
|
||||
BasicConv2d(64, 64, kernel_size=(7,1), stride=1, padding=(3,0)),
|
||||
BasicConv2d(64, 96, kernel_size=(3,3), stride=1)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.block0(x)
|
||||
x1 = self.block1(x)
|
||||
out = torch.cat((x0, x1), 1)
|
||||
return out
|
||||
|
||||
class Mixed_5a(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Mixed_5a, self).__init__()
|
||||
self.conv = BasicConv2d(192, 192, kernel_size=3, stride=2)
|
||||
self.maxpool = nn.MaxPool2d(3, stride=2)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.conv(x)
|
||||
x1 = self.maxpool(x)
|
||||
out = torch.cat((x0, x1), 1)
|
||||
return out
|
||||
|
||||
class Inception_A(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Inception_A, self).__init__()
|
||||
self.block0 = BasicConv2d(384, 96, kernel_size=1, stride=1)
|
||||
|
||||
self.block1 = nn.Sequential(
|
||||
BasicConv2d(384, 64, kernel_size=1, stride=1),
|
||||
BasicConv2d(64, 96, kernel_size=3, stride=1, padding=1)
|
||||
)
|
||||
|
||||
self.block2 = nn.Sequential(
|
||||
BasicConv2d(384, 64, kernel_size=1, stride=1),
|
||||
BasicConv2d(64, 96, kernel_size=3, stride=1, padding=1),
|
||||
BasicConv2d(96, 96, kernel_size=3, stride=1, padding=1)
|
||||
)
|
||||
|
||||
self.block3 = nn.Sequential(
|
||||
nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),
|
||||
BasicConv2d(384, 96, kernel_size=1, stride=1)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.block0(x)
|
||||
x1 = self.block1(x)
|
||||
x2 = self.block2(x)
|
||||
x3 = self.block3(x)
|
||||
out = torch.cat((x0, x1, x2, x3), 1)
|
||||
return out
|
||||
|
||||
class Reduction_A(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Reduction_A, self).__init__()
|
||||
self.block0 = BasicConv2d(384, 384, kernel_size=3, stride=2)
|
||||
|
||||
self.block1 = nn.Sequential(
|
||||
BasicConv2d(384, 192, kernel_size=1, stride=1),
|
||||
BasicConv2d(192, 224, kernel_size=3, stride=1, padding=1),
|
||||
BasicConv2d(224, 256, kernel_size=3, stride=2)
|
||||
)
|
||||
|
||||
self.block2 = nn.MaxPool2d(3, stride=2)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.block0(x)
|
||||
x1 = self.block1(x)
|
||||
x2 = self.block2(x)
|
||||
out = torch.cat((x0, x1, x2), 1)
|
||||
return out
|
||||
|
||||
class Inception_B(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Inception_B, self).__init__()
|
||||
self.block0 = BasicConv2d(1024, 384, kernel_size=1, stride=1)
|
||||
|
||||
self.block1 = nn.Sequential(
|
||||
BasicConv2d(1024, 192, kernel_size=1, stride=1),
|
||||
BasicConv2d(192, 224, kernel_size=(1,7), stride=1, padding=(0,3)),
|
||||
BasicConv2d(224, 256, kernel_size=(7,1), stride=1, padding=(3,0))
|
||||
)
|
||||
|
||||
self.block2 = nn.Sequential(
|
||||
BasicConv2d(1024, 192, kernel_size=1, stride=1),
|
||||
BasicConv2d(192, 192, kernel_size=(7,1), stride=1, padding=(3,0)),
|
||||
BasicConv2d(192, 224, kernel_size=(1,7), stride=1, padding=(0,3)),
|
||||
BasicConv2d(224, 224, kernel_size=(7,1), stride=1, padding=(3,0)),
|
||||
BasicConv2d(224, 256, kernel_size=(1,7), stride=1, padding=(0,3))
|
||||
)
|
||||
|
||||
self.block3 = nn.Sequential(
|
||||
nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),
|
||||
BasicConv2d(1024, 128, kernel_size=1, stride=1)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.block0(x)
|
||||
x1 = self.block1(x)
|
||||
x2 = self.block2(x)
|
||||
x3 = self.block3(x)
|
||||
out = torch.cat((x0, x1, x2, x3), 1)
|
||||
return out
|
||||
|
||||
class Reduction_B(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Reduction_B, self).__init__()
|
||||
|
||||
self.block0 = nn.Sequential(
|
||||
BasicConv2d(1024, 192, kernel_size=1, stride=1),
|
||||
BasicConv2d(192, 192, kernel_size=3, stride=2)
|
||||
)
|
||||
|
||||
self.block1 = nn.Sequential(
|
||||
BasicConv2d(1024, 256, kernel_size=1, stride=1),
|
||||
BasicConv2d(256, 256, kernel_size=(1,7), stride=1, padding=(0,3)),
|
||||
BasicConv2d(256, 320, kernel_size=(7,1), stride=1, padding=(3,0)),
|
||||
BasicConv2d(320, 320, kernel_size=3, stride=2)
|
||||
)
|
||||
|
||||
self.block2 = nn.MaxPool2d(3, stride=2)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.block0(x)
|
||||
x1 = self.block1(x)
|
||||
x2 = self.block2(x)
|
||||
out = torch.cat((x0, x1, x2), 1)
|
||||
return out
|
||||
|
||||
class Inception_C(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(Inception_C, self).__init__()
|
||||
self.block0 = BasicConv2d(1536, 256, kernel_size=1, stride=1)
|
||||
|
||||
self.block1_0 = BasicConv2d(1536, 384, kernel_size=1, stride=1)
|
||||
self.block1_1a = BasicConv2d(384, 256, kernel_size=(1,3), stride=1, padding=(0,1))
|
||||
self.block1_1b = BasicConv2d(384, 256, kernel_size=(3,1), stride=1, padding=(1,0))
|
||||
|
||||
self.block2_0 = BasicConv2d(1536, 384, kernel_size=1, stride=1)
|
||||
self.block2_1 = BasicConv2d(384, 448, kernel_size=(3,1), stride=1, padding=(1,0))
|
||||
self.block2_2 = BasicConv2d(448, 512, kernel_size=(1,3), stride=1, padding=(0,1))
|
||||
self.block2_3a = BasicConv2d(512, 256, kernel_size=(1,3), stride=1, padding=(0,1))
|
||||
self.block2_3b = BasicConv2d(512, 256, kernel_size=(3,1), stride=1, padding=(1,0))
|
||||
|
||||
self.block3 = nn.Sequential(
|
||||
nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),
|
||||
BasicConv2d(1536, 256, kernel_size=1, stride=1)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x0 = self.block0(x)
|
||||
|
||||
x1_0 = self.block1_0(x)
|
||||
x1_1a = self.block1_1a(x1_0)
|
||||
x1_1b = self.block1_1b(x1_0)
|
||||
x1 = torch.cat((x1_1a, x1_1b), 1)
|
||||
|
||||
x2_0 = self.block2_0(x)
|
||||
x2_1 = self.block2_1(x2_0)
|
||||
x2_2 = self.block2_2(x2_1)
|
||||
x2_3a = self.block2_3a(x2_2)
|
||||
x2_3b = self.block2_3b(x2_2)
|
||||
x2 = torch.cat((x2_3a, x2_3b), 1)
|
||||
|
||||
x3 = self.block3(x)
|
||||
|
||||
out = torch.cat((x0, x1, x2, x3), 1)
|
||||
return out
|
||||
|
||||
class InceptionV4(nn.Module):
|
||||
|
||||
def __init__(self, num_classes=1001):
|
||||
super(InceptionV4, self).__init__()
|
||||
self.features = nn.Sequential(
|
||||
BasicConv2d(3, 32, kernel_size=3, stride=2),
|
||||
BasicConv2d(32, 32, kernel_size=3, stride=1),
|
||||
BasicConv2d(32, 64, kernel_size=3, stride=1, padding=1),
|
||||
Mixed_3a(),
|
||||
Mixed_4a(),
|
||||
Mixed_5a(),
|
||||
Inception_A(),
|
||||
Inception_A(),
|
||||
Inception_A(),
|
||||
Inception_A(),
|
||||
Reduction_A(), # Mixed_6a
|
||||
Inception_B(),
|
||||
Inception_B(),
|
||||
Inception_B(),
|
||||
Inception_B(),
|
||||
Inception_B(),
|
||||
Inception_B(),
|
||||
Inception_B(),
|
||||
Reduction_B(), # Mixed_7a
|
||||
Inception_C(),
|
||||
Inception_C(),
|
||||
Inception_C(),
|
||||
nn.AdaptiveAvgPool2d((1,1))
|
||||
)
|
||||
self.classif = nn.Linear(1536, num_classes)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.features(x)
|
||||
x = x.view(x.size(0), -1)
|
||||
x = self.classif(x)
|
||||
return x
|
||||
|
||||
|
||||
def inceptionv4(pretrained=True):
|
||||
r"""InceptionV4 model architecture from the
|
||||
`"Inception-v4, Inception-ResNet..." <https://arxiv.org/abs/1602.07261>`_ paper.
|
||||
|
||||
Args:
|
||||
pretrained (bool): If True, returns a model pre-trained on ImageNet
|
||||
"""
|
||||
model = InceptionV4()
|
||||
if pretrained:
|
||||
model.load_state_dict(model_zoo.load_url(model_urls['imagenet']))
|
||||
return model
|
||||
|
||||
######################################################################
|
||||
## Load parameters from HDF5 to Dict
|
||||
######################################################################
|
||||
|
||||
def load_conv2d(state_dict, name_pth, name_tf):
|
||||
h5f = h5py.File('dump/InceptionV4/'+name_tf+'.h5', 'r')
|
||||
state_dict[name_pth+'.conv.weight'] = torch.from_numpy(h5f['weights'][()]).permute(3, 2, 0, 1)
|
||||
out_planes = state_dict[name_pth+'.conv.weight'].size(0)
|
||||
state_dict[name_pth+'.bn.weight'] = torch.ones(out_planes)
|
||||
state_dict[name_pth+'.bn.bias'] = torch.from_numpy(h5f['beta'][()])
|
||||
state_dict[name_pth+'.bn.running_mean'] = torch.from_numpy(h5f['mean'][()])
|
||||
state_dict[name_pth+'.bn.running_var'] = torch.from_numpy(h5f['var'][()])
|
||||
h5f.close()
|
||||
|
||||
def load_linear(state_dict, name_pth, name_tf):
|
||||
h5f = h5py.File('dump/InceptionV4/'+name_tf+'.h5', 'r')
|
||||
state_dict[name_pth+'.weight'] = torch.from_numpy(h5f['weights'][()]).t()
|
||||
state_dict[name_pth+'.bias'] = torch.from_numpy(h5f['biases'][()])
|
||||
h5f.close()
|
||||
|
||||
def load_mixed_4a_7a(state_dict, name_pth, name_tf):
|
||||
load_conv2d(state_dict, name_pth+'.branch0.0', name_tf+'/Branch_0/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch0.1', name_tf+'/Branch_0/Conv2d_1a_3x3')
|
||||
load_conv2d(state_dict, name_pth+'.branch1.0', name_tf+'/Branch_1/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch1.1', name_tf+'/Branch_1/Conv2d_0b_1x7')
|
||||
load_conv2d(state_dict, name_pth+'.branch1.2', name_tf+'/Branch_1/Conv2d_0c_7x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch1.3', name_tf+'/Branch_1/Conv2d_1a_3x3')
|
||||
|
||||
def load_mixed_5(state_dict, name_pth, name_tf):
|
||||
load_conv2d(state_dict, name_pth+'.branch0', name_tf+'/Branch_0/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch1.0', name_tf+'/Branch_1/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch1.1', name_tf+'/Branch_1/Conv2d_0b_3x3')
|
||||
load_conv2d(state_dict, name_pth+'.branch2.0', name_tf+'/Branch_2/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch2.1', name_tf+'/Branch_2/Conv2d_0b_3x3')
|
||||
load_conv2d(state_dict, name_pth+'.branch2.2', name_tf+'/Branch_2/Conv2d_0c_3x3')
|
||||
load_conv2d(state_dict, name_pth+'.branch3.1', name_tf+'/Branch_3/Conv2d_0b_1x1')
|
||||
|
||||
def load_mixed_6(state_dict, name_pth, name_tf):
|
||||
load_conv2d(state_dict, name_pth+'.branch0', name_tf+'/Branch_0/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch1.0', name_tf+'/Branch_1/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch1.1', name_tf+'/Branch_1/Conv2d_0b_1x7')
|
||||
load_conv2d(state_dict, name_pth+'.branch1.2', name_tf+'/Branch_1/Conv2d_0c_7x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch2.0', name_tf+'/Branch_2/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch2.1', name_tf+'/Branch_2/Conv2d_0b_7x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch2.2', name_tf+'/Branch_2/Conv2d_0c_1x7')
|
||||
load_conv2d(state_dict, name_pth+'.branch2.3', name_tf+'/Branch_2/Conv2d_0d_7x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch2.4', name_tf+'/Branch_2/Conv2d_0e_1x7')
|
||||
load_conv2d(state_dict, name_pth+'.branch3.1', name_tf+'/Branch_3/Conv2d_0b_1x1')
|
||||
|
||||
def load_mixed_7(state_dict, name_pth, name_tf):
|
||||
load_conv2d(state_dict, name_pth+'.branch0', name_tf+'/Branch_0/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch1_0', name_tf+'/Branch_1/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch1_1a', name_tf+'/Branch_1/Conv2d_0b_1x3')
|
||||
load_conv2d(state_dict, name_pth+'.branch1_1b', name_tf+'/Branch_1/Conv2d_0c_3x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch2_0', name_tf+'/Branch_2/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch2_1', name_tf+'/Branch_2/Conv2d_0b_3x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch2_2', name_tf+'/Branch_2/Conv2d_0c_1x3')
|
||||
load_conv2d(state_dict, name_pth+'.branch2_3a', name_tf+'/Branch_2/Conv2d_0d_1x3')
|
||||
load_conv2d(state_dict, name_pth+'.branch2_3b', name_tf+'/Branch_2/Conv2d_0e_3x1')
|
||||
load_conv2d(state_dict, name_pth+'.branch3.1', name_tf+'/Branch_3/Conv2d_0b_1x1')
|
||||
|
||||
|
||||
def load():
|
||||
state_dict={}
|
||||
|
||||
load_conv2d(state_dict, name_pth='features.0', name_tf='Conv2d_1a_3x3')
|
||||
load_conv2d(state_dict, name_pth='features.1', name_tf='Conv2d_2a_3x3')
|
||||
load_conv2d(state_dict, name_pth='features.2', name_tf='Conv2d_2b_3x3')
|
||||
|
||||
load_conv2d(state_dict, name_pth='features.3.conv', name_tf='Mixed_3a/Branch_1/Conv2d_0a_3x3')
|
||||
|
||||
load_mixed_4a_7a(state_dict, name_pth='features.4', name_tf='Mixed_4a')
|
||||
|
||||
load_conv2d(state_dict, name_pth='features.5.conv', name_tf='Mixed_5a/Branch_0/Conv2d_1a_3x3')
|
||||
|
||||
load_mixed_5(state_dict, name_pth='features.6', name_tf='Mixed_5b')
|
||||
load_mixed_5(state_dict, name_pth='features.7', name_tf='Mixed_5c')
|
||||
load_mixed_5(state_dict, name_pth='features.8', name_tf='Mixed_5d')
|
||||
load_mixed_5(state_dict, name_pth='features.9', name_tf='Mixed_5e')
|
||||
|
||||
load_conv2d(state_dict, name_pth='features.10.branch0', name_tf='Mixed_6a/Branch_0/Conv2d_1a_3x3')
|
||||
load_conv2d(state_dict, name_pth='features.10.branch1.0', name_tf='Mixed_6a/Branch_1/Conv2d_0a_1x1')
|
||||
load_conv2d(state_dict, name_pth='features.10.branch1.1', name_tf='Mixed_6a/Branch_1/Conv2d_0b_3x3')
|
||||
load_conv2d(state_dict, name_pth='features.10.branch1.2', name_tf='Mixed_6a/Branch_1/Conv2d_1a_3x3')
|
||||
|
||||
load_mixed_6(state_dict, name_pth='features.11', name_tf='Mixed_6b')
|
||||
load_mixed_6(state_dict, name_pth='features.12', name_tf='Mixed_6c')
|
||||
load_mixed_6(state_dict, name_pth='features.13', name_tf='Mixed_6d')
|
||||
load_mixed_6(state_dict, name_pth='features.14', name_tf='Mixed_6e')
|
||||
load_mixed_6(state_dict, name_pth='features.15', name_tf='Mixed_6f')
|
||||
load_mixed_6(state_dict, name_pth='features.16', name_tf='Mixed_6g')
|
||||
load_mixed_6(state_dict, name_pth='features.17', name_tf='Mixed_6h')
|
||||
|
||||
load_mixed_4a_7a(state_dict, name_pth='features.18', name_tf='Mixed_7a')
|
||||
|
||||
load_mixed_7(state_dict, name_pth='features.19', name_tf='Mixed_7b')
|
||||
load_mixed_7(state_dict, name_pth='features.20', name_tf='Mixed_7c')
|
||||
load_mixed_7(state_dict, name_pth='features.21', name_tf='Mixed_7d')
|
||||
|
||||
load_linear(state_dict, name_pth='classif', name_tf='Logits')
|
||||
|
||||
return state_dict
|
||||
|
||||
######################################################################
|
||||
## Test
|
||||
######################################################################
|
||||
|
||||
def test(model):
|
||||
model.eval()
|
||||
from scipy import misc
|
||||
img = misc.imread('lena_299.png')
|
||||
inputs = torch.zeros(1,299,299,3)
|
||||
inputs[0] = torch.from_numpy(img)
|
||||
inputs.transpose_(1,3)
|
||||
inputs.transpose_(2,3)
|
||||
# 1, 3, 299, 299
|
||||
outputs = model.forward(torch.autograd.Variable(inputs))
|
||||
h5f = h5py.File('dump/InceptionV4/Logits.h5', 'r')
|
||||
outputs_tf = torch.from_numpy(h5f['out'][()])
|
||||
h5f.close()
|
||||
outputs = torch.nn.functional.softmax(outputs)
|
||||
print(torch.dist(outputs.data, outputs_tf))
|
||||
return outputs
|
||||
|
||||
def test_conv2d(module, name):
|
||||
#global output_tf
|
||||
h5f = h5py.File('dump/InceptionV4/'+name+'.h5', 'r')
|
||||
output_tf = torch.from_numpy(h5f['relu_out'][()])
|
||||
output_tf.transpose_(1,3)
|
||||
output_tf.transpose_(2,3)
|
||||
h5f.close()
|
||||
def test_dist(self, input, output):
|
||||
print(name, torch.dist(output.data, output_tf))
|
||||
module.register_forward_hook(test_dist)
|
||||
|
||||
def test_mixed_4a_7a(module, name):
|
||||
test_conv2d(module.branch0[0], name+'/Branch_0/Conv2d_0a_1x1')
|
||||
test_conv2d(module.branch0[1], name+'/Branch_0/Conv2d_1a_3x3')
|
||||
test_conv2d(module.branch1[0], name+'/Branch_1/Conv2d_0a_1x1')
|
||||
test_conv2d(module.branch1[1], name+'/Branch_1/Conv2d_0b_1x7')
|
||||
test_conv2d(module.branch1[2], name+'/Branch_1/Conv2d_0c_7x1')
|
||||
test_conv2d(module.branch1[3], name+'/Branch_1/Conv2d_1a_3x3')
|
||||
|
||||
######################################################################
|
||||
## Main
|
||||
######################################################################
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
import h5py
|
||||
|
||||
model = InceptionV4()
|
||||
state_dict = load()
|
||||
model.load_state_dict(state_dict)
|
||||
|
||||
# test_conv2d(model.features[0], 'Conv2d_1a_3x3')
|
||||
# test_conv2d(model.features[1], 'Conv2d_2a_3x3')
|
||||
# test_conv2d(model.features[2], 'Conv2d_2b_3x3')
|
||||
# test_conv2d(model.features[3].conv, 'Mixed_3a/Branch_1/Conv2d_0a_3x3')
|
||||
# test_mixed_4a_7a(model.features[4], 'Mixed_4a')
|
||||
|
||||
os.system('mkdir -p save')
|
||||
torch.save(model, 'save/inceptionv4.pth')
|
||||
torch.save(state_dict, 'save/inceptionv4_state.pth')
|
||||
|
||||
outputs = test(model)
|
||||
|
||||
|
||||
@@ -0,0 +1,620 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torch.utils.model_zoo as model_zoo
|
||||
from torch.autograd import Variable
|
||||
|
||||
pretrained_settings = {
|
||||
'nasnetalarge': {
|
||||
'imagenet': {
|
||||
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/nasnetalarge-a1897284.pth',
|
||||
'input_space': 'RGB',
|
||||
'input_size': [3, 331, 331], # resize 354
|
||||
'input_range': [0, 1],
|
||||
'mean': [0.5, 0.5, 0.5],
|
||||
'std': [0.5, 0.5, 0.5],
|
||||
'num_classes': 1000
|
||||
},
|
||||
'imagenet+background': {
|
||||
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/nasnetalarge-a1897284.pth',
|
||||
'input_space': 'RGB',
|
||||
'input_size': [3, 331, 331], # resize 354
|
||||
'input_range': [0, 1],
|
||||
'mean': [0.5, 0.5, 0.5],
|
||||
'std': [0.5, 0.5, 0.5],
|
||||
'num_classes': 1001
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MaxPoolPad(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(MaxPoolPad, self).__init__()
|
||||
self.pad = nn.ZeroPad2d((1, 0, 1, 0))
|
||||
self.pool = nn.MaxPool2d(3, stride=2, padding=1)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.pad(x)
|
||||
x = self.pool(x)
|
||||
x = x[:, :, 1:, 1:]
|
||||
return x
|
||||
|
||||
|
||||
class AvgPoolPad(nn.Module):
|
||||
|
||||
def __init__(self, stride=2, padding=1):
|
||||
super(AvgPoolPad, self).__init__()
|
||||
self.pad = nn.ZeroPad2d((1, 0, 1, 0))
|
||||
self.pool = nn.AvgPool2d(3, stride=stride, padding=padding, count_include_pad=False)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.pad(x)
|
||||
x = self.pool(x)
|
||||
x = x[:, :, 1:, 1:]
|
||||
return x
|
||||
|
||||
|
||||
class SeparableConv2d(nn.Module):
|
||||
|
||||
def __init__(self, in_channels, out_channels, dw_kernel, dw_stride, dw_padding, bias=False):
|
||||
super(SeparableConv2d, self).__init__()
|
||||
self.depthwise_conv2d = nn.Conv2d(in_channels, in_channels, dw_kernel,
|
||||
stride=dw_stride,
|
||||
padding=dw_padding,
|
||||
bias=bias,
|
||||
groups=in_channels)
|
||||
self.pointwise_conv2d = nn.Conv2d(in_channels, out_channels, 1, stride=1, bias=bias)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.depthwise_conv2d(x)
|
||||
x = self.pointwise_conv2d(x)
|
||||
return x
|
||||
|
||||
|
||||
class BranchSeparables(nn.Module):
|
||||
|
||||
def __init__(self, in_channels, out_channels, kernel_size, stride, padding, bias=False):
|
||||
super(BranchSeparables, self).__init__()
|
||||
self.relu = nn.ReLU()
|
||||
self.separable_1 = SeparableConv2d(in_channels, in_channels, kernel_size, stride, padding, bias=bias)
|
||||
self.bn_sep_1 = nn.BatchNorm2d(in_channels, eps=0.001, momentum=0.1, affine=True)
|
||||
self.relu1 = nn.ReLU()
|
||||
self.separable_2 = SeparableConv2d(in_channels, out_channels, kernel_size, 1, padding, bias=bias)
|
||||
self.bn_sep_2 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.1, affine=True)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.relu(x)
|
||||
x = self.separable_1(x)
|
||||
x = self.bn_sep_1(x)
|
||||
x = self.relu1(x)
|
||||
x = self.separable_2(x)
|
||||
x = self.bn_sep_2(x)
|
||||
return x
|
||||
|
||||
|
||||
class BranchSeparablesStem(nn.Module):
|
||||
|
||||
def __init__(self, in_channels, out_channels, kernel_size, stride, padding, bias=False):
|
||||
super(BranchSeparablesStem, self).__init__()
|
||||
self.relu = nn.ReLU()
|
||||
self.separable_1 = SeparableConv2d(in_channels, out_channels, kernel_size, stride, padding, bias=bias)
|
||||
self.bn_sep_1 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.1, affine=True)
|
||||
self.relu1 = nn.ReLU()
|
||||
self.separable_2 = SeparableConv2d(out_channels, out_channels, kernel_size, 1, padding, bias=bias)
|
||||
self.bn_sep_2 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.1, affine=True)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.relu(x)
|
||||
x = self.separable_1(x)
|
||||
x = self.bn_sep_1(x)
|
||||
x = self.relu1(x)
|
||||
x = self.separable_2(x)
|
||||
x = self.bn_sep_2(x)
|
||||
return x
|
||||
|
||||
|
||||
class BranchSeparablesReduction(BranchSeparables):
|
||||
|
||||
def __init__(self, in_channels, out_channels, kernel_size, stride, padding, z_padding=1, bias=False):
|
||||
BranchSeparables.__init__(self, in_channels, out_channels, kernel_size, stride, padding, bias)
|
||||
self.padding = nn.ZeroPad2d((z_padding, 0, z_padding, 0))
|
||||
|
||||
def forward(self, x):
|
||||
x = self.relu(x)
|
||||
x = self.padding(x)
|
||||
x = self.separable_1(x)
|
||||
x = x[:, :, 1:, 1:].contiguous()
|
||||
x = self.bn_sep_1(x)
|
||||
x = self.relu1(x)
|
||||
x = self.separable_2(x)
|
||||
x = self.bn_sep_2(x)
|
||||
return x
|
||||
|
||||
|
||||
class CellStem0(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(CellStem0, self).__init__()
|
||||
self.conv_1x1 = nn.Sequential()
|
||||
self.conv_1x1.add_module('relu', nn.ReLU())
|
||||
self.conv_1x1.add_module('conv', nn.Conv2d(96, 42, 1, stride=1, bias=False))
|
||||
self.conv_1x1.add_module('bn', nn.BatchNorm2d(42, eps=0.001, momentum=0.1, affine=True))
|
||||
|
||||
self.comb_iter_0_left = BranchSeparables(42, 42, 5, 2, 2)
|
||||
self.comb_iter_0_right = BranchSeparablesStem(96, 42, 7, 2, 3, bias=False)
|
||||
|
||||
self.comb_iter_1_left = nn.MaxPool2d(3, stride=2, padding=1)
|
||||
self.comb_iter_1_right = BranchSeparablesStem(96, 42, 7, 2, 3, bias=False)
|
||||
|
||||
self.comb_iter_2_left = nn.AvgPool2d(3, stride=2, padding=1, count_include_pad=False)
|
||||
self.comb_iter_2_right = BranchSeparablesStem(96, 42, 5, 2, 2, bias=False)
|
||||
|
||||
self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
|
||||
|
||||
self.comb_iter_4_left = BranchSeparables(42, 42, 3, 1, 1, bias=False)
|
||||
self.comb_iter_4_right = nn.MaxPool2d(3, stride=2, padding=1)
|
||||
|
||||
def forward(self, x):
|
||||
x1 = self.conv_1x1(x)
|
||||
|
||||
x_comb_iter_0_left = self.comb_iter_0_left(x1)
|
||||
x_comb_iter_0_right = self.comb_iter_0_right(x)
|
||||
x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
|
||||
|
||||
x_comb_iter_1_left = self.comb_iter_1_left(x1)
|
||||
x_comb_iter_1_right = self.comb_iter_1_right(x)
|
||||
x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
|
||||
|
||||
x_comb_iter_2_left = self.comb_iter_2_left(x1)
|
||||
x_comb_iter_2_right = self.comb_iter_2_right(x)
|
||||
x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right
|
||||
|
||||
x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)
|
||||
x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1
|
||||
|
||||
x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)
|
||||
x_comb_iter_4_right = self.comb_iter_4_right(x1)
|
||||
x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right
|
||||
|
||||
x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
|
||||
return x_out
|
||||
|
||||
|
||||
class CellStem1(nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super(CellStem1, self).__init__()
|
||||
self.conv_1x1 = nn.Sequential()
|
||||
self.conv_1x1.add_module('relu', nn.ReLU())
|
||||
self.conv_1x1.add_module('conv', nn.Conv2d(168, 84, 1, stride=1, bias=False))
|
||||
self.conv_1x1.add_module('bn', nn.BatchNorm2d(84, eps=0.001, momentum=0.1, affine=True))
|
||||
|
||||
self.relu = nn.ReLU()
|
||||
self.path_1 = nn.Sequential()
|
||||
self.path_1.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))
|
||||
self.path_1.add_module('conv', nn.Conv2d(96, 42, 1, stride=1, bias=False))
|
||||
self.path_2 = nn.ModuleList()
|
||||
self.path_2.add_module('pad', nn.ZeroPad2d((0, 1, 0, 1)))
|
||||
self.path_2.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))
|
||||
self.path_2.add_module('conv', nn.Conv2d(96, 42, 1, stride=1, bias=False))
|
||||
|
||||
self.final_path_bn = nn.BatchNorm2d(84, eps=0.001, momentum=0.1, affine=True)
|
||||
|
||||
self.comb_iter_0_left = BranchSeparables(84, 84, 5, 2, 2, bias=False)
|
||||
self.comb_iter_0_right = BranchSeparables(84, 84, 7, 2, 3, bias=False)
|
||||
|
||||
self.comb_iter_1_left = nn.MaxPool2d(3, stride=2, padding=1)
|
||||
self.comb_iter_1_right = BranchSeparables(84, 84, 7, 2, 3, bias=False)
|
||||
|
||||
self.comb_iter_2_left = nn.AvgPool2d(3, stride=2, padding=1, count_include_pad=False)
|
||||
self.comb_iter_2_right = BranchSeparables(84, 84, 5, 2, 2, bias=False)
|
||||
|
||||
self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
|
||||
|
||||
self.comb_iter_4_left = BranchSeparables(84, 84, 3, 1, 1, bias=False)
|
||||
self.comb_iter_4_right = nn.MaxPool2d(3, stride=2, padding=1)
|
||||
|
||||
def forward(self, x_conv0, x_stem_0):
|
||||
x_left = self.conv_1x1(x_stem_0)
|
||||
|
||||
x_relu = self.relu(x_conv0)
|
||||
# path 1
|
||||
x_path1 = self.path_1(x_relu)
|
||||
# path 2
|
||||
x_path2 = self.path_2.pad(x_relu)
|
||||
x_path2 = x_path2[:, :, 1:, 1:]
|
||||
x_path2 = self.path_2.avgpool(x_path2)
|
||||
x_path2 = self.path_2.conv(x_path2)
|
||||
# final path
|
||||
x_right = self.final_path_bn(torch.cat([x_path1, x_path2], 1))
|
||||
|
||||
x_comb_iter_0_left = self.comb_iter_0_left(x_left)
|
||||
x_comb_iter_0_right = self.comb_iter_0_right(x_right)
|
||||
x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
|
||||
|
||||
x_comb_iter_1_left = self.comb_iter_1_left(x_left)
|
||||
x_comb_iter_1_right = self.comb_iter_1_right(x_right)
|
||||
x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
|
||||
|
||||
x_comb_iter_2_left = self.comb_iter_2_left(x_left)
|
||||
x_comb_iter_2_right = self.comb_iter_2_right(x_right)
|
||||
x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right
|
||||
|
||||
x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)
|
||||
x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1
|
||||
|
||||
x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)
|
||||
x_comb_iter_4_right = self.comb_iter_4_right(x_left)
|
||||
x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right
|
||||
|
||||
x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
|
||||
return x_out
|
||||
|
||||
|
||||
class FirstCell(nn.Module):
|
||||
|
||||
def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):
|
||||
super(FirstCell, self).__init__()
|
||||
self.conv_1x1 = nn.Sequential()
|
||||
self.conv_1x1.add_module('relu', nn.ReLU())
|
||||
self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))
|
||||
self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))
|
||||
|
||||
self.relu = nn.ReLU()
|
||||
self.path_1 = nn.Sequential()
|
||||
self.path_1.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))
|
||||
self.path_1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))
|
||||
self.path_2 = nn.ModuleList()
|
||||
self.path_2.add_module('pad', nn.ZeroPad2d((0, 1, 0, 1)))
|
||||
self.path_2.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))
|
||||
self.path_2.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))
|
||||
|
||||
self.final_path_bn = nn.BatchNorm2d(out_channels_left * 2, eps=0.001, momentum=0.1, affine=True)
|
||||
|
||||
self.comb_iter_0_left = BranchSeparables(out_channels_right, out_channels_right, 5, 1, 2, bias=False)
|
||||
self.comb_iter_0_right = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
|
||||
|
||||
self.comb_iter_1_left = BranchSeparables(out_channels_right, out_channels_right, 5, 1, 2, bias=False)
|
||||
self.comb_iter_1_right = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
|
||||
|
||||
self.comb_iter_2_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
|
||||
|
||||
self.comb_iter_3_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
|
||||
self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
|
||||
|
||||
self.comb_iter_4_left = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
|
||||
|
||||
def forward(self, x, x_prev):
|
||||
x_relu = self.relu(x_prev)
|
||||
# path 1
|
||||
x_path1 = self.path_1(x_relu)
|
||||
# path 2
|
||||
x_path2 = self.path_2.pad(x_relu)
|
||||
x_path2 = x_path2[:, :, 1:, 1:]
|
||||
x_path2 = self.path_2.avgpool(x_path2)
|
||||
x_path2 = self.path_2.conv(x_path2)
|
||||
# final path
|
||||
x_left = self.final_path_bn(torch.cat([x_path1, x_path2], 1))
|
||||
|
||||
x_right = self.conv_1x1(x)
|
||||
|
||||
x_comb_iter_0_left = self.comb_iter_0_left(x_right)
|
||||
x_comb_iter_0_right = self.comb_iter_0_right(x_left)
|
||||
x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
|
||||
|
||||
x_comb_iter_1_left = self.comb_iter_1_left(x_left)
|
||||
x_comb_iter_1_right = self.comb_iter_1_right(x_left)
|
||||
x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
|
||||
|
||||
x_comb_iter_2_left = self.comb_iter_2_left(x_right)
|
||||
x_comb_iter_2 = x_comb_iter_2_left + x_left
|
||||
|
||||
x_comb_iter_3_left = self.comb_iter_3_left(x_left)
|
||||
x_comb_iter_3_right = self.comb_iter_3_right(x_left)
|
||||
x_comb_iter_3 = x_comb_iter_3_left + x_comb_iter_3_right
|
||||
|
||||
x_comb_iter_4_left = self.comb_iter_4_left(x_right)
|
||||
x_comb_iter_4 = x_comb_iter_4_left + x_right
|
||||
|
||||
x_out = torch.cat([x_left, x_comb_iter_0, x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
|
||||
return x_out
|
||||
|
||||
|
||||
class NormalCell(nn.Module):
|
||||
|
||||
def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):
|
||||
super(NormalCell, self).__init__()
|
||||
self.conv_prev_1x1 = nn.Sequential()
|
||||
self.conv_prev_1x1.add_module('relu', nn.ReLU())
|
||||
self.conv_prev_1x1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))
|
||||
self.conv_prev_1x1.add_module('bn', nn.BatchNorm2d(out_channels_left, eps=0.001, momentum=0.1, affine=True))
|
||||
|
||||
self.conv_1x1 = nn.Sequential()
|
||||
self.conv_1x1.add_module('relu', nn.ReLU())
|
||||
self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))
|
||||
self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))
|
||||
|
||||
self.comb_iter_0_left = BranchSeparables(out_channels_right, out_channels_right, 5, 1, 2, bias=False)
|
||||
self.comb_iter_0_right = BranchSeparables(out_channels_left, out_channels_left, 3, 1, 1, bias=False)
|
||||
|
||||
self.comb_iter_1_left = BranchSeparables(out_channels_left, out_channels_left, 5, 1, 2, bias=False)
|
||||
self.comb_iter_1_right = BranchSeparables(out_channels_left, out_channels_left, 3, 1, 1, bias=False)
|
||||
|
||||
self.comb_iter_2_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
|
||||
|
||||
self.comb_iter_3_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
|
||||
self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
|
||||
|
||||
self.comb_iter_4_left = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
|
||||
|
||||
def forward(self, x, x_prev):
|
||||
x_left = self.conv_prev_1x1(x_prev)
|
||||
x_right = self.conv_1x1(x)
|
||||
|
||||
x_comb_iter_0_left = self.comb_iter_0_left(x_right)
|
||||
x_comb_iter_0_right = self.comb_iter_0_right(x_left)
|
||||
x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
|
||||
|
||||
x_comb_iter_1_left = self.comb_iter_1_left(x_left)
|
||||
x_comb_iter_1_right = self.comb_iter_1_right(x_left)
|
||||
x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
|
||||
|
||||
x_comb_iter_2_left = self.comb_iter_2_left(x_right)
|
||||
x_comb_iter_2 = x_comb_iter_2_left + x_left
|
||||
|
||||
x_comb_iter_3_left = self.comb_iter_3_left(x_left)
|
||||
x_comb_iter_3_right = self.comb_iter_3_right(x_left)
|
||||
x_comb_iter_3 = x_comb_iter_3_left + x_comb_iter_3_right
|
||||
|
||||
x_comb_iter_4_left = self.comb_iter_4_left(x_right)
|
||||
x_comb_iter_4 = x_comb_iter_4_left + x_right
|
||||
|
||||
x_out = torch.cat([x_left, x_comb_iter_0, x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
|
||||
return x_out
|
||||
|
||||
|
||||
class ReductionCell0(nn.Module):
|
||||
|
||||
def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):
|
||||
super(ReductionCell0, self).__init__()
|
||||
self.conv_prev_1x1 = nn.Sequential()
|
||||
self.conv_prev_1x1.add_module('relu', nn.ReLU())
|
||||
self.conv_prev_1x1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))
|
||||
self.conv_prev_1x1.add_module('bn', nn.BatchNorm2d(out_channels_left, eps=0.001, momentum=0.1, affine=True))
|
||||
|
||||
self.conv_1x1 = nn.Sequential()
|
||||
self.conv_1x1.add_module('relu', nn.ReLU())
|
||||
self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))
|
||||
self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))
|
||||
|
||||
self.comb_iter_0_left = BranchSeparablesReduction(out_channels_right, out_channels_right, 5, 2, 2, bias=False)
|
||||
self.comb_iter_0_right = BranchSeparablesReduction(out_channels_right, out_channels_right, 7, 2, 3, bias=False)
|
||||
|
||||
self.comb_iter_1_left = MaxPoolPad()
|
||||
self.comb_iter_1_right = BranchSeparablesReduction(out_channels_right, out_channels_right, 7, 2, 3, bias=False)
|
||||
|
||||
self.comb_iter_2_left = AvgPoolPad()
|
||||
self.comb_iter_2_right = BranchSeparablesReduction(out_channels_right, out_channels_right, 5, 2, 2, bias=False)
|
||||
|
||||
self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
|
||||
|
||||
self.comb_iter_4_left = BranchSeparablesReduction(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
|
||||
self.comb_iter_4_right = MaxPoolPad()
|
||||
|
||||
def forward(self, x, x_prev):
|
||||
x_left = self.conv_prev_1x1(x_prev)
|
||||
x_right = self.conv_1x1(x)
|
||||
|
||||
x_comb_iter_0_left = self.comb_iter_0_left(x_right)
|
||||
x_comb_iter_0_right = self.comb_iter_0_right(x_left)
|
||||
x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
|
||||
|
||||
x_comb_iter_1_left = self.comb_iter_1_left(x_right)
|
||||
x_comb_iter_1_right = self.comb_iter_1_right(x_left)
|
||||
x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
|
||||
|
||||
x_comb_iter_2_left = self.comb_iter_2_left(x_right)
|
||||
x_comb_iter_2_right = self.comb_iter_2_right(x_left)
|
||||
x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right
|
||||
|
||||
x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)
|
||||
x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1
|
||||
|
||||
x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)
|
||||
x_comb_iter_4_right = self.comb_iter_4_right(x_right)
|
||||
x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right
|
||||
|
||||
x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
|
||||
return x_out
|
||||
|
||||
|
||||
class ReductionCell1(nn.Module):
|
||||
|
||||
def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):
|
||||
super(ReductionCell1, self).__init__()
|
||||
self.conv_prev_1x1 = nn.Sequential()
|
||||
self.conv_prev_1x1.add_module('relu', nn.ReLU())
|
||||
self.conv_prev_1x1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))
|
||||
self.conv_prev_1x1.add_module('bn', nn.BatchNorm2d(out_channels_left, eps=0.001, momentum=0.1, affine=True))
|
||||
|
||||
self.conv_1x1 = nn.Sequential()
|
||||
self.conv_1x1.add_module('relu', nn.ReLU())
|
||||
self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))
|
||||
self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))
|
||||
|
||||
self.comb_iter_0_left = BranchSeparables(out_channels_right, out_channels_right, 5, 2, 2, bias=False)
|
||||
self.comb_iter_0_right = BranchSeparables(out_channels_right, out_channels_right, 7, 2, 3, bias=False)
|
||||
|
||||
self.comb_iter_1_left = nn.MaxPool2d(3, stride=2, padding=1)
|
||||
self.comb_iter_1_right = BranchSeparables(out_channels_right, out_channels_right, 7, 2, 3, bias=False)
|
||||
|
||||
self.comb_iter_2_left = nn.AvgPool2d(3, stride=2, padding=1, count_include_pad=False)
|
||||
self.comb_iter_2_right = BranchSeparables(out_channels_right, out_channels_right, 5, 2, 2, bias=False)
|
||||
|
||||
self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
|
||||
|
||||
self.comb_iter_4_left = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
|
||||
self.comb_iter_4_right = nn.MaxPool2d(3, stride=2, padding=1)
|
||||
|
||||
def forward(self, x, x_prev):
|
||||
x_left = self.conv_prev_1x1(x_prev)
|
||||
x_right = self.conv_1x1(x)
|
||||
|
||||
x_comb_iter_0_left = self.comb_iter_0_left(x_right)
|
||||
x_comb_iter_0_right = self.comb_iter_0_right(x_left)
|
||||
x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
|
||||
|
||||
x_comb_iter_1_left = self.comb_iter_1_left(x_right)
|
||||
x_comb_iter_1_right = self.comb_iter_1_right(x_left)
|
||||
x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
|
||||
|
||||
x_comb_iter_2_left = self.comb_iter_2_left(x_right)
|
||||
x_comb_iter_2_right = self.comb_iter_2_right(x_left)
|
||||
x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right
|
||||
|
||||
x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)
|
||||
x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1
|
||||
|
||||
x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)
|
||||
x_comb_iter_4_right = self.comb_iter_4_right(x_right)
|
||||
x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right
|
||||
|
||||
x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
|
||||
return x_out
|
||||
|
||||
|
||||
class NASNetALarge(nn.Module):
|
||||
|
||||
def __init__(self, use_classifier=False, num_classes=1001):
|
||||
super(NASNetALarge, self).__init__()
|
||||
self.use_classifier,self.num_classes = use_classifier,num_classes
|
||||
|
||||
self.conv0 = nn.Sequential()
|
||||
self.conv0.add_module('conv', nn.Conv2d(in_channels=3, out_channels=96, kernel_size=3, padding=0, stride=2,
|
||||
bias=False))
|
||||
self.conv0.add_module('bn', nn.BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True))
|
||||
|
||||
self.cell_stem_0 = CellStem0()
|
||||
self.cell_stem_1 = CellStem1()
|
||||
|
||||
self.cell_0 = FirstCell(in_channels_left=168, out_channels_left=84,
|
||||
in_channels_right=336, out_channels_right=168)
|
||||
self.cell_1 = NormalCell(in_channels_left=336, out_channels_left=168,
|
||||
in_channels_right=1008, out_channels_right=168)
|
||||
self.cell_2 = NormalCell(in_channels_left=1008, out_channels_left=168,
|
||||
in_channels_right=1008, out_channels_right=168)
|
||||
self.cell_3 = NormalCell(in_channels_left=1008, out_channels_left=168,
|
||||
in_channels_right=1008, out_channels_right=168)
|
||||
self.cell_4 = NormalCell(in_channels_left=1008, out_channels_left=168,
|
||||
in_channels_right=1008, out_channels_right=168)
|
||||
self.cell_5 = NormalCell(in_channels_left=1008, out_channels_left=168,
|
||||
in_channels_right=1008, out_channels_right=168)
|
||||
|
||||
self.reduction_cell_0 = ReductionCell0(in_channels_left=1008, out_channels_left=336,
|
||||
in_channels_right=1008, out_channels_right=336)
|
||||
|
||||
self.cell_6 = FirstCell(in_channels_left=1008, out_channels_left=168,
|
||||
in_channels_right=1344, out_channels_right=336)
|
||||
self.cell_7 = NormalCell(in_channels_left=1344, out_channels_left=336,
|
||||
in_channels_right=2016, out_channels_right=336)
|
||||
self.cell_8 = NormalCell(in_channels_left=2016, out_channels_left=336,
|
||||
in_channels_right=2016, out_channels_right=336)
|
||||
self.cell_9 = NormalCell(in_channels_left=2016, out_channels_left=336,
|
||||
in_channels_right=2016, out_channels_right=336)
|
||||
self.cell_10 = NormalCell(in_channels_left=2016, out_channels_left=336,
|
||||
in_channels_right=2016, out_channels_right=336)
|
||||
self.cell_11 = NormalCell(in_channels_left=2016, out_channels_left=336,
|
||||
in_channels_right=2016, out_channels_right=336)
|
||||
|
||||
self.reduction_cell_1 = ReductionCell1(in_channels_left=2016, out_channels_left=672,
|
||||
in_channels_right=2016, out_channels_right=672)
|
||||
|
||||
self.cell_12 = FirstCell(in_channels_left=2016, out_channels_left=336,
|
||||
in_channels_right=2688, out_channels_right=672)
|
||||
self.cell_13 = NormalCell(in_channels_left=2688, out_channels_left=672,
|
||||
in_channels_right=4032, out_channels_right=672)
|
||||
self.cell_14 = NormalCell(in_channels_left=4032, out_channels_left=672,
|
||||
in_channels_right=4032, out_channels_right=672)
|
||||
self.cell_15 = NormalCell(in_channels_left=4032, out_channels_left=672,
|
||||
in_channels_right=4032, out_channels_right=672)
|
||||
self.cell_16 = NormalCell(in_channels_left=4032, out_channels_left=672,
|
||||
in_channels_right=4032, out_channels_right=672)
|
||||
self.cell_17 = NormalCell(in_channels_left=4032, out_channels_left=672,
|
||||
in_channels_right=4032, out_channels_right=672)
|
||||
|
||||
self.relu = nn.ReLU()
|
||||
self.dropout = nn.Dropout()
|
||||
self.last_linear = nn.Linear(4032, self.num_classes)
|
||||
|
||||
def features(self, x):
|
||||
x_conv0 = self.conv0(x)
|
||||
x_stem_0 = self.cell_stem_0(x_conv0)
|
||||
x_stem_1 = self.cell_stem_1(x_conv0, x_stem_0)
|
||||
|
||||
x_cell_0 = self.cell_0(x_stem_1, x_stem_0)
|
||||
x_cell_1 = self.cell_1(x_cell_0, x_stem_1)
|
||||
x_cell_2 = self.cell_2(x_cell_1, x_cell_0)
|
||||
x_cell_3 = self.cell_3(x_cell_2, x_cell_1)
|
||||
x_cell_4 = self.cell_4(x_cell_3, x_cell_2)
|
||||
x_cell_5 = self.cell_5(x_cell_4, x_cell_3)
|
||||
|
||||
x_reduction_cell_0 = self.reduction_cell_0(x_cell_5, x_cell_4)
|
||||
|
||||
x_cell_6 = self.cell_6(x_reduction_cell_0, x_cell_4)
|
||||
x_cell_7 = self.cell_7(x_cell_6, x_reduction_cell_0)
|
||||
x_cell_8 = self.cell_8(x_cell_7, x_cell_6)
|
||||
x_cell_9 = self.cell_9(x_cell_8, x_cell_7)
|
||||
x_cell_10 = self.cell_10(x_cell_9, x_cell_8)
|
||||
x_cell_11 = self.cell_11(x_cell_10, x_cell_9)
|
||||
|
||||
x_reduction_cell_1 = self.reduction_cell_1(x_cell_11, x_cell_10)
|
||||
|
||||
x_cell_12 = self.cell_12(x_reduction_cell_1, x_cell_10)
|
||||
x_cell_13 = self.cell_13(x_cell_12, x_reduction_cell_1)
|
||||
x_cell_14 = self.cell_14(x_cell_13, x_cell_12)
|
||||
x_cell_15 = self.cell_15(x_cell_14, x_cell_13)
|
||||
x_cell_16 = self.cell_16(x_cell_15, x_cell_14)
|
||||
x_cell_17 = self.cell_17(x_cell_16, x_cell_15)
|
||||
return self.relu(x_cell_17)
|
||||
|
||||
def classifier(self, x):
|
||||
x = F.adaptive_max_pool2d(x, 1)
|
||||
x = x.view(x.size(0), -1)
|
||||
x = self.dropout(x)
|
||||
return F.log_softmax(self.linear(x))
|
||||
|
||||
def forward(self, x):
|
||||
x = self.features(x)
|
||||
if self.use_classifier: x = self.classifier(x)
|
||||
return x
|
||||
|
||||
|
||||
def nasnetalarge(num_classes=1000, pretrained='imagenet'):
|
||||
r"""NASNetALarge model architecture from the
|
||||
`"NASNet" <https://arxiv.org/abs/1707.07012>`_ paper.
|
||||
"""
|
||||
if pretrained:
|
||||
settings = pretrained_settings['nasnetalarge'][pretrained]
|
||||
assert num_classes == settings['num_classes'], \
|
||||
"num_classes should be {}, but is {}".format(settings['num_classes'], num_classes)
|
||||
|
||||
# both 'imagenet'&'imagenet+background' are loaded from same parameters
|
||||
model = NASNetALarge(num_classes=1001)
|
||||
model.load_state_dict(model_zoo.load_url(settings['url']))
|
||||
|
||||
if pretrained == 'imagenet':
|
||||
new_last_linear = nn.Linear(model.last_linear.in_features, 1000)
|
||||
new_last_linear.weight.data = model.last_linear.weight.data[1:]
|
||||
new_last_linear.bias.data = model.last_linear.bias.data[1:]
|
||||
model.last_linear = new_last_linear
|
||||
|
||||
model.input_space = settings['input_space']
|
||||
model.input_size = settings['input_size']
|
||||
model.input_range = settings['input_range']
|
||||
|
||||
model.mean = settings['mean']
|
||||
model.std = settings['std']
|
||||
else:
|
||||
model = NASNetALarge(num_classes=num_classes)
|
||||
return model
|
||||
@@ -0,0 +1,217 @@
|
||||
import torch.nn as nn
|
||||
import math
|
||||
import torch.utils.model_zoo as model_zoo
|
||||
from ..layers import *
|
||||
|
||||
__all__ = ['vgg_resnet50']
|
||||
|
||||
model_urls = {
|
||||
'vgg_resnet50': 'https://download.pytorch.org/models/vggresnet.pth',
|
||||
}
|
||||
|
||||
|
||||
def conv(ni, nf, ks=3, stride=1):
|
||||
return nn.Conv2d(ni, nf, kernel_size=ks, stride=stride, padding=ks//2, bias=False)
|
||||
|
||||
def bn1(planes):
|
||||
m = nn.BatchNorm1d(planes)
|
||||
m.weight.data.fill_(1)
|
||||
m.bias.data.zero_()
|
||||
return m
|
||||
|
||||
def bn(planes, init_zero=False):
|
||||
m = nn.BatchNorm2d(planes)
|
||||
m.weight.data.fill_(0 if init_zero else 1)
|
||||
m.bias.data.zero_()
|
||||
return m
|
||||
|
||||
|
||||
class BasicBlock(nn.Module):
|
||||
expansion = 1
|
||||
|
||||
def __init__(self, inplanes, planes, stride=1, downsample=None):
|
||||
super().__init__()
|
||||
self.conv1 = conv(inplanes, planes, stride=stride)
|
||||
self.bn1 = bn(planes)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.conv2 = conv(planes, planes)
|
||||
self.bn2 = bn(planes)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
if self.downsample is not None: residual = self.downsample(x)
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.relu(out)
|
||||
out = self.bn1(out)
|
||||
|
||||
out = self.conv2(out)
|
||||
|
||||
out = residual + out
|
||||
out = self.relu(out)
|
||||
out = self.bn2(out)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
class BottleneckFinal(nn.Module):
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, inplanes, planes, stride=1, downsample=None):
|
||||
super().__init__()
|
||||
self.conv1 = conv(inplanes, planes, ks=1)
|
||||
self.bn1 = bn(planes)
|
||||
self.conv2 = conv(planes, planes, stride=stride)
|
||||
self.bn2 = bn(planes)
|
||||
self.conv3 = conv(planes, planes*4, ks=1)
|
||||
self.bn3 = bn(planes * 4)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
if self.downsample is not None: residual = self.downsample(x)
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.bn1(out)
|
||||
out = self.relu(out)
|
||||
out = self.conv2(out)
|
||||
out = self.bn2(out)
|
||||
out = self.relu(out)
|
||||
out = self.conv3(out)
|
||||
|
||||
out = residual + out
|
||||
out = self.bn3(out)
|
||||
out = self.relu(out)
|
||||
|
||||
return out
|
||||
|
||||
class BottleneckZero(nn.Module):
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, inplanes, planes, stride=1, downsample=None):
|
||||
super().__init__()
|
||||
self.conv1 = conv(inplanes, planes, ks=1)
|
||||
self.bn1 = bn(planes)
|
||||
self.conv2 = conv(planes, planes, stride=stride)
|
||||
self.bn2 = bn(planes)
|
||||
self.conv3 = conv(planes, planes*4, ks=1)
|
||||
self.bn3 = bn(planes * 4, init_zero=True)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
if self.downsample is not None: residual = self.downsample(x)
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.bn1(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv2(out)
|
||||
out = self.bn2(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv3(out)
|
||||
out = self.bn3(out)
|
||||
|
||||
out = residual + out
|
||||
out = self.relu(out)
|
||||
|
||||
return out
|
||||
|
||||
class Bottleneck(nn.Module):
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, inplanes, planes, stride=1, downsample=None):
|
||||
super().__init__()
|
||||
self.conv1 = conv(inplanes, planes, ks=1)
|
||||
self.bn1 = bn(planes)
|
||||
self.conv2 = conv(planes, planes, stride=stride)
|
||||
self.bn2 = bn(planes)
|
||||
self.conv3 = conv(planes, planes*4, ks=1)
|
||||
self.bn3 = bn(planes * 4)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
if self.downsample is not None: residual = self.downsample(x)
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.bn1(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv2(out)
|
||||
out = self.bn2(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv3(out)
|
||||
out = self.bn3(out)
|
||||
|
||||
out = residual + out
|
||||
out = self.relu(out)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
class ResNet(nn.Module):
|
||||
def __init__(self, block, layers, num_classes=1000, k=1, vgg_head=False):
|
||||
super().__init__()
|
||||
self.inplanes = 64
|
||||
|
||||
features = [conv(3, 64, ks=7, stride=2)
|
||||
, bn(64) , nn.ReLU(inplace=True) , nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
|
||||
, self._make_layer(block, int(64*k), layers[0])
|
||||
, self._make_layer(block, int(128*k), layers[1], stride=2)
|
||||
, self._make_layer(block, int(256*k), layers[2], stride=2)
|
||||
, self._make_layer(block, int(512*k), layers[3], stride=2)]
|
||||
out_sz = int(512*k) * block.expansion
|
||||
|
||||
if vgg_head:
|
||||
features += [nn.AdaptiveAvgPool2d(3), Flatten()
|
||||
, nn.Linear(out_sz*3*3, 4096), nn.ReLU(inplace=True), bn1(4096), nn.Dropout(0.25)
|
||||
, nn.Linear(4096, 4096), nn.ReLU(inplace=True), bn1(4096), nn.Dropout(0.25)
|
||||
, nn.Linear(4096, num_classes)]
|
||||
else: features += [nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(out_sz, num_classes)]
|
||||
|
||||
self.features = nn.Sequential(*features)
|
||||
|
||||
for m in self.modules():
|
||||
if isinstance(m, nn.Conv2d):
|
||||
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
|
||||
m.weight.data.normal_(0, math.sqrt(2. / n))
|
||||
|
||||
def _make_layer(self, block, planes, blocks, stride=1):
|
||||
downsample = None
|
||||
if stride != 1 or self.inplanes != planes * block.expansion:
|
||||
downsample = nn.Sequential(
|
||||
conv(self.inplanes, planes*block.expansion, ks=1, stride=stride),
|
||||
bn(planes * block.expansion),
|
||||
)
|
||||
|
||||
layers = []
|
||||
layers.append(block(self.inplanes, planes, stride, downsample))
|
||||
self.inplanes = planes * block.expansion
|
||||
for i in range(1, blocks): layers.append(block(self.inplanes, planes))
|
||||
return nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x): return self.features(x)
|
||||
|
||||
|
||||
def bnf_resnet50 (): return ResNet(BottleneckFinal, [3, 4, 6, 3])
|
||||
def bnz_resnet50 (): return ResNet(BottleneckZero, [3, 4, 6, 3])
|
||||
def w5_resnet50 (): return ResNet(Bottleneck, [2, 3, 3, 2], k=1.5)
|
||||
def w25_resnet50(): return ResNet(Bottleneck, [3, 4, 4, 3], k=1.25)
|
||||
def w125_resnet50(): return ResNet(Bottleneck, [3, 4, 6, 3], k=1.125)
|
||||
def vgg_resnet34(): return ResNet(BasicBlock, [3, 4, 6, 3], vgg_head=True)
|
||||
def vgg_resnet50(pretrained=False):
|
||||
model = ResNet(Bottleneck, [3, 4, 6, 3], vgg_head=True)
|
||||
if pretrained: model.load_state_dict(torch.load('/home/jhoward/.torch/models/vgg_resnet50.pth'))
|
||||
return model
|
||||
|
||||
@@ -0,0 +1,686 @@
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.autograd import Variable
|
||||
from functools import reduce
|
||||
|
||||
class LambdaBase(nn.Sequential):
|
||||
def __init__(self, fn, *args):
|
||||
super(LambdaBase, self).__init__(*args)
|
||||
self.lambda_func = fn
|
||||
|
||||
def forward_prepare(self, input):
|
||||
output = []
|
||||
for module in self._modules.values():
|
||||
output.append(module(input))
|
||||
return output if output else input
|
||||
|
||||
class Lambda(LambdaBase):
|
||||
def forward(self, input):
|
||||
return self.lambda_func(self.forward_prepare(input))
|
||||
|
||||
class LambdaMap(LambdaBase):
|
||||
def forward(self, input):
|
||||
return list(map(self.lambda_func,self.forward_prepare(input)))
|
||||
|
||||
class LambdaReduce(LambdaBase):
|
||||
def forward(self, input):
|
||||
return reduce(self.lambda_func,self.forward_prepare(input))
|
||||
|
||||
|
||||
def resnext_101_32x4d(): return nn.Sequential( # Sequential,
|
||||
nn.Conv2d(3,64,(7, 7),(2, 2),(3, 3),1,1,bias=False),
|
||||
nn.BatchNorm2d(64),
|
||||
nn.ReLU(),
|
||||
nn.MaxPool2d((3, 3),(2, 2),(1, 1)),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(64,128,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,128,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(128,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(64,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,128,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,128,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(128,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,128,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,128,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(128,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(2, 2),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,512,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(2, 2),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,1024,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(2, 2),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,2048,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(2048,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(2048,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.AvgPool2d((7, 7),(1, 1)),
|
||||
Lambda(lambda x: x.view(x.size(0),-1)), # View,
|
||||
nn.Sequential(Lambda(lambda x: x.view(1,-1) if 1==len(x.size()) else x ),nn.Linear(2048,1000)), # Linear,
|
||||
)
|
||||
@@ -0,0 +1,686 @@
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.autograd import Variable
|
||||
from functools import reduce
|
||||
|
||||
class LambdaBase(nn.Sequential):
|
||||
def __init__(self, fn, *args):
|
||||
super(LambdaBase, self).__init__(*args)
|
||||
self.lambda_func = fn
|
||||
|
||||
def forward_prepare(self, input):
|
||||
output = []
|
||||
for module in self._modules.values():
|
||||
output.append(module(input))
|
||||
return output if output else input
|
||||
|
||||
class Lambda(LambdaBase):
|
||||
def forward(self, input):
|
||||
return self.lambda_func(self.forward_prepare(input))
|
||||
|
||||
class LambdaMap(LambdaBase):
|
||||
def forward(self, input):
|
||||
return list(map(self.lambda_func,self.forward_prepare(input)))
|
||||
|
||||
class LambdaReduce(LambdaBase):
|
||||
def forward(self, input):
|
||||
return reduce(self.lambda_func,self.forward_prepare(input))
|
||||
|
||||
|
||||
def resnext_101_64x4d(): return nn.Sequential( # Sequential,
|
||||
nn.Conv2d(3,64,(7, 7),(2, 2),(3, 3),1,1,bias=False),
|
||||
nn.BatchNorm2d(64),
|
||||
nn.ReLU(),
|
||||
nn.MaxPool2d((3, 3),(2, 2),(1, 1)),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(64,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(64,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(2, 2),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,512,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(2, 2),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,1024,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(2048,2048,(3, 3),(2, 2),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(2048,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,2048,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(2048,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(2048,2048,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(2048,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(2048,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(2048,2048,(3, 3),(1, 1),(1, 1),1,64,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(2048,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.AvgPool2d((7, 7),(1, 1)),
|
||||
Lambda(lambda x: x.view(x.size(0),-1)), # View,
|
||||
nn.Sequential(Lambda(lambda x: x.view(1,-1) if 1==len(x.size()) else x ),nn.Linear(2048,1000)), # Linear,
|
||||
)
|
||||
@@ -0,0 +1,363 @@
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.autograd import Variable
|
||||
from functools import reduce
|
||||
|
||||
class LambdaBase(nn.Sequential):
|
||||
def __init__(self, fn, *args):
|
||||
super(LambdaBase, self).__init__(*args)
|
||||
self.lambda_func = fn
|
||||
|
||||
def forward_prepare(self, input):
|
||||
output = []
|
||||
for module in self._modules.values():
|
||||
output.append(module(input))
|
||||
return output if output else input
|
||||
|
||||
class Lambda(LambdaBase):
|
||||
def forward(self, input):
|
||||
return self.lambda_func(self.forward_prepare(input))
|
||||
|
||||
class LambdaMap(LambdaBase):
|
||||
def forward(self, input):
|
||||
return list(map(self.lambda_func,self.forward_prepare(input)))
|
||||
|
||||
class LambdaReduce(LambdaBase):
|
||||
def forward(self, input):
|
||||
return reduce(self.lambda_func,self.forward_prepare(input))
|
||||
|
||||
|
||||
def resnext_50_32x4d(): return nn.Sequential( # Sequential,
|
||||
nn.Conv2d(3,64,(7, 7),(2, 2),(3, 3),1,1,bias=False),
|
||||
nn.BatchNorm2d(64),
|
||||
nn.ReLU(),
|
||||
nn.MaxPool2d((3, 3),(2, 2),(1, 1)),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(64,128,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,128,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(128,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(64,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,128,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,128,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(128,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,128,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,128,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(128,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(2, 2),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,512,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(2, 2),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,1024,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(2, 2),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,2048,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(2048,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(2048,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,32,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Conv2d(1024,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.AdaptiveAvgPool2d(1),
|
||||
Lambda(lambda x: x.view(x.size(0),-1)), # View,
|
||||
nn.Sequential(Lambda(lambda x: x.view(1,-1) if 1==len(x.size()) else x ),nn.Linear(2048,1000)), # Linear,
|
||||
)
|
||||
@@ -0,0 +1,445 @@
|
||||
'''
|
||||
File name: senet.py
|
||||
Squeeze-and-Excitation Networks (SeNet) implementation for fast.ai/pytorch with pretrained model
|
||||
Credit https://github.com/hujie-frank/SENet
|
||||
SENet is the winner of ImageNet-2017 (https://arxiv.org/pdf/1709.01507.pdf).
|
||||
'''
|
||||
|
||||
from collections import OrderedDict
|
||||
import math
|
||||
import torch.utils.model_zoo as model_zoo
|
||||
from ..layers import *
|
||||
import torch.nn as nn
|
||||
from torch.utils import model_zoo
|
||||
|
||||
__all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',
|
||||
'se_resnext50_32x4d', 'se_resnext101_32x4d']
|
||||
|
||||
pretrained_settings = {
|
||||
'senet154': {
|
||||
'imagenet': {
|
||||
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',
|
||||
'input_space': 'RGB',
|
||||
'input_size': [3, 224, 224],
|
||||
'input_range': [0, 1],
|
||||
'mean': [0.485, 0.456, 0.406],
|
||||
'std': [0.229, 0.224, 0.225],
|
||||
'num_classes': 1000
|
||||
}
|
||||
},
|
||||
'se_resnet50': {
|
||||
'imagenet': {
|
||||
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',
|
||||
'input_space': 'RGB',
|
||||
'input_size': [3, 224, 224],
|
||||
'input_range': [0, 1],
|
||||
'mean': [0.485, 0.456, 0.406],
|
||||
'std': [0.229, 0.224, 0.225],
|
||||
'num_classes': 1000
|
||||
}
|
||||
},
|
||||
'se_resnet101': {
|
||||
'imagenet': {
|
||||
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',
|
||||
'input_space': 'RGB',
|
||||
'input_size': [3, 224, 224],
|
||||
'input_range': [0, 1],
|
||||
'mean': [0.485, 0.456, 0.406],
|
||||
'std': [0.229, 0.224, 0.225],
|
||||
'num_classes': 1000
|
||||
}
|
||||
},
|
||||
'se_resnet152': {
|
||||
'imagenet': {
|
||||
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',
|
||||
'input_space': 'RGB',
|
||||
'input_size': [3, 224, 224],
|
||||
'input_range': [0, 1],
|
||||
'mean': [0.485, 0.456, 0.406],
|
||||
'std': [0.229, 0.224, 0.225],
|
||||
'num_classes': 1000
|
||||
}
|
||||
},
|
||||
'se_resnext50_32x4d': {
|
||||
'imagenet': {
|
||||
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',
|
||||
'input_space': 'RGB',
|
||||
'input_size': [3, 224, 224],
|
||||
'input_range': [0, 1],
|
||||
'mean': [0.485, 0.456, 0.406],
|
||||
'std': [0.229, 0.224, 0.225],
|
||||
'num_classes': 1000
|
||||
}
|
||||
},
|
||||
'se_resnext101_32x4d': {
|
||||
'imagenet': {
|
||||
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
|
||||
'input_space': 'RGB',
|
||||
'input_size': [3, 224, 224],
|
||||
'input_range': [0, 1],
|
||||
'mean': [0.485, 0.456, 0.406],
|
||||
'std': [0.229, 0.224, 0.225],
|
||||
'num_classes': 1000
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class SEModule(nn.Module):
|
||||
|
||||
def __init__(self, channels, reduction):
|
||||
super(SEModule, self).__init__()
|
||||
self.avg_pool = nn.AdaptiveAvgPool2d(1)
|
||||
self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
|
||||
padding=0)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
|
||||
padding=0)
|
||||
self.sigmoid = nn.Sigmoid()
|
||||
|
||||
def forward(self, x):
|
||||
module_input = x
|
||||
x = self.avg_pool(x)
|
||||
x = self.fc1(x)
|
||||
x = self.relu(x)
|
||||
x = self.fc2(x)
|
||||
x = self.sigmoid(x)
|
||||
return module_input * x
|
||||
|
||||
|
||||
class Bottleneck(nn.Module):
|
||||
"""
|
||||
Base class for bottlenecks that implements `forward()` method.
|
||||
"""
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.bn1(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv2(out)
|
||||
out = self.bn2(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv3(out)
|
||||
out = self.bn3(out)
|
||||
|
||||
if self.downsample is not None:
|
||||
residual = self.downsample(x)
|
||||
|
||||
out = self.se_module(out) + residual
|
||||
out = self.relu(out)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
class SEBottleneck(Bottleneck):
|
||||
"""
|
||||
Bottleneck for SENet154.
|
||||
"""
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, inplanes, planes, groups, reduction, stride=1,
|
||||
downsample=None):
|
||||
super(SEBottleneck, self).__init__()
|
||||
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
|
||||
self.bn1 = nn.BatchNorm2d(planes * 2)
|
||||
self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
|
||||
stride=stride, padding=1, groups=groups,
|
||||
bias=False)
|
||||
self.bn2 = nn.BatchNorm2d(planes * 4)
|
||||
self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
|
||||
bias=False)
|
||||
self.bn3 = nn.BatchNorm2d(planes * 4)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.se_module = SEModule(planes * 4, reduction=reduction)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
|
||||
class SEResNetBottleneck(Bottleneck):
|
||||
"""
|
||||
ResNet bottleneck with a Squeeze-and-Excitation module. It follows Caffe
|
||||
implementation and uses `stride=stride` in `conv1` and not in `conv2`
|
||||
(the latter is used in the torchvision implementation of ResNet).
|
||||
"""
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, inplanes, planes, groups, reduction, stride=1,
|
||||
downsample=None):
|
||||
super(SEResNetBottleneck, self).__init__()
|
||||
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
|
||||
stride=stride)
|
||||
self.bn1 = nn.BatchNorm2d(planes)
|
||||
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
|
||||
groups=groups, bias=False)
|
||||
self.bn2 = nn.BatchNorm2d(planes)
|
||||
self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
|
||||
self.bn3 = nn.BatchNorm2d(planes * 4)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.se_module = SEModule(planes * 4, reduction=reduction)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
|
||||
class SEResNeXtBottleneck(Bottleneck):
|
||||
"""
|
||||
ResNeXt bottleneck type C with a Squeeze-and-Excitation module.
|
||||
"""
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, inplanes, planes, groups, reduction, stride=1,
|
||||
downsample=None, base_width=4):
|
||||
super(SEResNeXtBottleneck, self).__init__()
|
||||
width = math.floor(planes * (base_width / 64)) * groups
|
||||
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
|
||||
stride=1)
|
||||
self.bn1 = nn.BatchNorm2d(width)
|
||||
self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
|
||||
padding=1, groups=groups, bias=False)
|
||||
self.bn2 = nn.BatchNorm2d(width)
|
||||
self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
|
||||
self.bn3 = nn.BatchNorm2d(planes * 4)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.se_module = SEModule(planes * 4, reduction=reduction)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
|
||||
class SENet(nn.Module):
|
||||
|
||||
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
|
||||
inplanes=128, input_3x3=True, downsample_kernel_size=3,
|
||||
downsample_padding=1, num_classes=1000):
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
block (nn.Module): Bottleneck class.
|
||||
- For SENet154: SEBottleneck
|
||||
- For SE-ResNet models: SEResNetBottleneck
|
||||
- For SE-ResNeXt models: SEResNeXtBottleneck
|
||||
layers (list of ints): Number of residual blocks for 4 layers of the
|
||||
network (layer1...layer4).
|
||||
groups (int): Number of groups for the 3x3 convolution in each
|
||||
bottleneck block.
|
||||
- For SENet154: 64
|
||||
- For SE-ResNet models: 1
|
||||
- For SE-ResNeXt models: 32
|
||||
reduction (int): Reduction ratio for Squeeze-and-Excitation modules.
|
||||
- For all models: 16
|
||||
dropout_p (float or None): Drop probability for the Dropout layer.
|
||||
If `None` the Dropout layer is not used.
|
||||
- For SENet154: 0.2
|
||||
- For SE-ResNet models: None
|
||||
- For SE-ResNeXt models: None
|
||||
inplanes (int): Number of input channels for layer1.
|
||||
- For SENet154: 128
|
||||
- For SE-ResNet models: 64
|
||||
- For SE-ResNeXt models: 64
|
||||
input_3x3 (bool): If `True`, use three 3x3 convolutions instead of
|
||||
a single 7x7 convolution in layer0.
|
||||
- For SENet154: True
|
||||
- For SE-ResNet models: False
|
||||
- For SE-ResNeXt models: False
|
||||
downsample_kernel_size (int): Kernel size for downsampling convolutions
|
||||
in layer2, layer3 and layer4.
|
||||
- For SENet154: 3
|
||||
- For SE-ResNet models: 1
|
||||
- For SE-ResNeXt models: 1
|
||||
downsample_padding (int): Padding for downsampling convolutions in
|
||||
layer2, layer3 and layer4.
|
||||
- For SENet154: 1
|
||||
- For SE-ResNet models: 0
|
||||
- For SE-ResNeXt models: 0
|
||||
num_classes (int): Number of outputs in `last_linear` layer.
|
||||
- For all models: 1000
|
||||
"""
|
||||
super(SENet, self).__init__()
|
||||
self.inplanes = inplanes
|
||||
if input_3x3:
|
||||
layer0_modules = [
|
||||
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
|
||||
bias=False)),
|
||||
('bn1', nn.BatchNorm2d(64)),
|
||||
('relu1', nn.ReLU(inplace=True)),
|
||||
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
|
||||
bias=False)),
|
||||
('bn2', nn.BatchNorm2d(64)),
|
||||
('relu2', nn.ReLU(inplace=True)),
|
||||
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
|
||||
bias=False)),
|
||||
('bn3', nn.BatchNorm2d(inplanes)),
|
||||
('relu3', nn.ReLU(inplace=True)),
|
||||
]
|
||||
else:
|
||||
layer0_modules = [
|
||||
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
|
||||
padding=3, bias=False)),
|
||||
('bn1', nn.BatchNorm2d(inplanes)),
|
||||
('relu1', nn.ReLU(inplace=True)),
|
||||
]
|
||||
# To preserve compatibility with Caffe weights `ceil_mode=True`
|
||||
# is used instead of `padding=1`.
|
||||
layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2,
|
||||
ceil_mode=True)))
|
||||
self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
|
||||
self.layer1 = self._make_layer(
|
||||
block,
|
||||
planes=64,
|
||||
blocks=layers[0],
|
||||
groups=groups,
|
||||
reduction=reduction,
|
||||
downsample_kernel_size=1,
|
||||
downsample_padding=0
|
||||
)
|
||||
self.layer2 = self._make_layer(
|
||||
block,
|
||||
planes=128,
|
||||
blocks=layers[1],
|
||||
stride=2,
|
||||
groups=groups,
|
||||
reduction=reduction,
|
||||
downsample_kernel_size=downsample_kernel_size,
|
||||
downsample_padding=downsample_padding
|
||||
)
|
||||
self.layer3 = self._make_layer(
|
||||
block,
|
||||
planes=256,
|
||||
blocks=layers[2],
|
||||
stride=2,
|
||||
groups=groups,
|
||||
reduction=reduction,
|
||||
downsample_kernel_size=downsample_kernel_size,
|
||||
downsample_padding=downsample_padding
|
||||
)
|
||||
self.layer4 = self._make_layer(
|
||||
block,
|
||||
planes=512,
|
||||
blocks=layers[3],
|
||||
stride=2,
|
||||
groups=groups,
|
||||
reduction=reduction,
|
||||
downsample_kernel_size=downsample_kernel_size,
|
||||
downsample_padding=downsample_padding
|
||||
)
|
||||
self.avg_pool = nn.AvgPool2d(7, stride=1)
|
||||
self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None
|
||||
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
|
||||
|
||||
def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
|
||||
downsample_kernel_size=1, downsample_padding=0):
|
||||
downsample = None
|
||||
if stride != 1 or self.inplanes != planes * block.expansion:
|
||||
downsample = nn.Sequential(
|
||||
nn.Conv2d(self.inplanes, planes * block.expansion,
|
||||
kernel_size=downsample_kernel_size, stride=stride,
|
||||
padding=downsample_padding, bias=False),
|
||||
nn.BatchNorm2d(planes * block.expansion),
|
||||
)
|
||||
|
||||
layers = []
|
||||
layers.append(block(self.inplanes, planes, groups, reduction, stride,
|
||||
downsample))
|
||||
self.inplanes = planes * block.expansion
|
||||
for i in range(1, blocks):
|
||||
layers.append(block(self.inplanes, planes, groups, reduction))
|
||||
|
||||
return nn.Sequential(*layers)
|
||||
|
||||
def features(self, x):
|
||||
x = self.layer0(x)
|
||||
x = self.layer1(x)
|
||||
x = self.layer2(x)
|
||||
x = self.layer3(x)
|
||||
x = self.layer4(x)
|
||||
return x
|
||||
|
||||
def logits(self, x):
|
||||
x = self.avg_pool(x)
|
||||
if self.dropout is not None:
|
||||
x = self.dropout(x)
|
||||
x = x.view(x.size(0), -1)
|
||||
x = self.last_linear(x)
|
||||
return x
|
||||
|
||||
def forward(self, x):
|
||||
x = self.features(x)
|
||||
x = self.logits(x)
|
||||
return x
|
||||
|
||||
|
||||
def initialize_pretrained_model(model, num_classes, settings):
|
||||
assert num_classes == settings['num_classes'], \
|
||||
'num_classes should be {}, but is {}'.format(
|
||||
settings['num_classes'], num_classes)
|
||||
model.load_state_dict(model_zoo.load_url(settings['url']))
|
||||
model.input_space = settings['input_space']
|
||||
model.input_size = settings['input_size']
|
||||
model.input_range = settings['input_range']
|
||||
model.mean = settings['mean']
|
||||
model.std = settings['std']
|
||||
|
||||
|
||||
def senet154(num_classes=1000, pretrained='imagenet'):
|
||||
model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,
|
||||
dropout_p=0.2, num_classes=num_classes)
|
||||
if pretrained is not None:
|
||||
settings = pretrained_settings['senet154'][pretrained]
|
||||
initialize_pretrained_model(model, num_classes, settings)
|
||||
return model
|
||||
|
||||
|
||||
def se_resnet50(num_classes=1000, pretrained='imagenet'):
|
||||
model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16,
|
||||
dropout_p=None, inplanes=64, input_3x3=False,
|
||||
downsample_kernel_size=1, downsample_padding=0,
|
||||
num_classes=num_classes)
|
||||
if pretrained is not None:
|
||||
settings = pretrained_settings['se_resnet50'][pretrained]
|
||||
initialize_pretrained_model(model, num_classes, settings)
|
||||
return model
|
||||
|
||||
|
||||
def se_resnet101(num_classes=1000, pretrained='imagenet'):
|
||||
model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16,
|
||||
dropout_p=None, inplanes=64, input_3x3=False,
|
||||
downsample_kernel_size=1, downsample_padding=0,
|
||||
num_classes=num_classes)
|
||||
if pretrained is not None:
|
||||
settings = pretrained_settings['se_resnet101'][pretrained]
|
||||
initialize_pretrained_model(model, num_classes, settings)
|
||||
return model
|
||||
|
||||
|
||||
def se_resnet152(num_classes=1000, pretrained='imagenet'):
|
||||
model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,
|
||||
dropout_p=None, inplanes=64, input_3x3=False,
|
||||
downsample_kernel_size=1, downsample_padding=0,
|
||||
num_classes=num_classes)
|
||||
if pretrained is not None:
|
||||
settings = pretrained_settings['se_resnet152'][pretrained]
|
||||
initialize_pretrained_model(model, num_classes, settings)
|
||||
return model
|
||||
|
||||
|
||||
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
|
||||
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
|
||||
dropout_p=None, inplanes=64, input_3x3=False,
|
||||
downsample_kernel_size=1, downsample_padding=0,
|
||||
num_classes=num_classes)
|
||||
if pretrained is not None:
|
||||
settings = pretrained_settings['se_resnext50_32x4d'][pretrained]
|
||||
initialize_pretrained_model(model, num_classes, settings)
|
||||
return model
|
||||
|
||||
|
||||
def se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'):
|
||||
model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
|
||||
dropout_p=None, inplanes=64, input_3x3=False,
|
||||
downsample_kernel_size=1, downsample_padding=0,
|
||||
num_classes=num_classes)
|
||||
if pretrained is not None:
|
||||
settings = pretrained_settings['se_resnext101_32x4d'][pretrained]
|
||||
initialize_pretrained_model(model, num_classes, settings)
|
||||
return model
|
||||
@@ -0,0 +1,128 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.autograd import Variable
|
||||
import numpy as np
|
||||
|
||||
def get_sfs_idxs(sfs, last=True):
|
||||
"""
|
||||
Return the saved feature indexes that will be concatenated
|
||||
Inputs:
|
||||
sfs (list): saved features by hook function, in other words intermediate activations
|
||||
last (bool): whether to concatenate only last different activation, or all from the encoder model
|
||||
"""
|
||||
if last:
|
||||
feature_szs = [sfs_feats.features.size()[-1] for sfs_feats in sfs]
|
||||
sfs_idxs = list(np.where(np.array(feature_szs[:-1]) != np.array(feature_szs[1:]))[0])
|
||||
if feature_szs[0] != feature_szs[1]: sfs_idxs = [0] + sfs_idxs
|
||||
else: sfs_idxs = list(range(len(sfs)))
|
||||
return sfs_idxs
|
||||
|
||||
|
||||
def conv_bn_relu(in_c, out_c, kernel_size, stride, padding):
|
||||
return [
|
||||
nn.Conv2d(in_c, out_c, kernel_size=kernel_size, stride=stride, padding=padding),
|
||||
nn.ReLU(),
|
||||
nn.BatchNorm2d(out_c)]
|
||||
|
||||
|
||||
class UnetBlock(nn.Module):
|
||||
#TODO: ADAPT KERNEL SIZE, STRIDE AND PADDING SO THAT ANY SIZE DECAY WILL BE SUPPORTED
|
||||
def __init__(self, up_in_c, x_in_c):
|
||||
super().__init__()
|
||||
self.upconv = nn.ConvTranspose2d(up_in_c, up_in_c // 2, 2, 2) # H, W -> 2H, 2W
|
||||
self.conv1 = nn.Conv2d(x_in_c + up_in_c // 2, (x_in_c + up_in_c // 2) // 2, 3, 1, 1)
|
||||
self.conv2 = nn.Conv2d((x_in_c + up_in_c // 2) // 2, (x_in_c + up_in_c // 2) // 2, 3, 1, 1)
|
||||
self.bn = nn.BatchNorm2d((x_in_c + up_in_c // 2) // 2)
|
||||
|
||||
def forward(self, up_in, x_in):
|
||||
up_out = self.upconv(up_in)
|
||||
cat_x = torch.cat([up_out, x_in], dim=1)
|
||||
x = F.relu(self.conv1(cat_x))
|
||||
x = F.relu(self.conv2(x))
|
||||
return self.bn(x)
|
||||
|
||||
class SaveFeatures():
|
||||
""" Extract pretrained activations"""
|
||||
features=None
|
||||
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
|
||||
def hook_fn(self, module, input, output): self.features = output
|
||||
def remove(self): self.hook.remove()
|
||||
|
||||
|
||||
class DynamicUnet(nn.Module):
|
||||
"""
|
||||
A dynamic implementation of Unet architecture, because calculating connections
|
||||
and channels suck!. When an encoder is passed, this network will
|
||||
automatically construct a decoder after the first single forward pass for any
|
||||
given encoder architecture.
|
||||
|
||||
Decoder part is heavily based on the original Unet paper:
|
||||
https://arxiv.org/abs/1505.04597.
|
||||
|
||||
Inputs:
|
||||
encoder(nn.Module): Preferably a pretrained model, such as VGG or ResNet
|
||||
last (bool): Whether to concat only last activation just before a size change
|
||||
n_classes (int): Number of classes to output in final step of decoder
|
||||
|
||||
Important Note: If architecture directly reduces the dimension of an image as soon as the
|
||||
first forward pass then output size will not be same as the input size, e.g. ResNet.
|
||||
In order to resolve this problem architecture will add an additional extra conv transpose
|
||||
layer. Also, currently Dynamic Unet expects size change to be H,W -> H/2, W/2. This is
|
||||
not a problem for state-of-the-art architectures as they follow this pattern but it should
|
||||
be changed for custom encoders that might have a different size decay.
|
||||
"""
|
||||
|
||||
def __init__(self, encoder, last=True, n_classes=3):
|
||||
super().__init__()
|
||||
self.encoder = encoder
|
||||
self.n_children = len(list(encoder.children()))
|
||||
self.sfs = [SaveFeatures(encoder[i]) for i in range(self.n_children)]
|
||||
self.last = last
|
||||
self.n_classes = n_classes
|
||||
|
||||
def forward(self, x):
|
||||
# get imsize
|
||||
imsize = x.size()[-2:]
|
||||
|
||||
# encoder output
|
||||
x = F.relu(self.encoder(x))
|
||||
|
||||
# initialize sfs_idxs, sfs_szs, middle_in_c and middle_conv only once
|
||||
if not hasattr(self, 'middle_conv'):
|
||||
self.sfs_szs = [sfs_feats.features.size() for sfs_feats in self.sfs]
|
||||
self.sfs_idxs = get_sfs_idxs(self.sfs, self.last)
|
||||
middle_in_c = self.sfs_szs[-1][1]
|
||||
middle_conv = nn.Sequential(*conv_bn_relu(middle_in_c, middle_in_c * 2, 3, 1, 1),
|
||||
*conv_bn_relu(middle_in_c * 2, middle_in_c, 3, 1, 1))
|
||||
self.middle_conv = middle_conv
|
||||
|
||||
# middle conv
|
||||
x = self.middle_conv(x)
|
||||
|
||||
# initialize upmodel, extra_block and 1x1 final conv
|
||||
if not hasattr(self, 'upmodel'):
|
||||
x_copy = Variable(x.data, requires_grad=False)
|
||||
upmodel = []
|
||||
for idx in self.sfs_idxs[::-1]:
|
||||
up_in_c, x_in_c = int(x_copy.size()[1]), int(self.sfs_szs[idx][1])
|
||||
unet_block = UnetBlock(up_in_c, x_in_c)
|
||||
upmodel.append(unet_block)
|
||||
x_copy = unet_block(x_copy, self.sfs[idx].features)
|
||||
self.upmodel = nn.Sequential(*upmodel)
|
||||
|
||||
if imsize != self.sfs_szs[0][-2:]:
|
||||
extra_in_c = self.upmodel[-1].conv2.out_channels
|
||||
self.extra_block = nn.ConvTranspose2d(extra_in_c, extra_in_c, 2, 2)
|
||||
|
||||
final_in_c = self.upmodel[-1].conv2.out_channels
|
||||
self.final_conv = nn.Conv2d(final_in_c, self.n_classes, 1)
|
||||
|
||||
# run upsample
|
||||
for block, idx in zip(self.upmodel, self.sfs_idxs[::-1]):
|
||||
x = block(x, self.sfs[idx].features)
|
||||
if hasattr(self, 'extra_block'):
|
||||
x = self.extra_block(x)
|
||||
|
||||
out = self.final_conv(x)
|
||||
return out
|
||||
@@ -0,0 +1,78 @@
|
||||
# https://github.com/uoguelph-mlrg/Cutout
|
||||
|
||||
import math
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class BasicBlock(nn.Module):
|
||||
def __init__(self, in_planes, out_planes, stride, dropRate=0.0):
|
||||
super().__init__()
|
||||
self.bn1 = nn.BatchNorm2d(in_planes)
|
||||
self.relu1 = nn.ReLU(inplace=True)
|
||||
self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
|
||||
padding=1, bias=False)
|
||||
self.bn2 = nn.BatchNorm2d(out_planes)
|
||||
self.relu2 = nn.ReLU(inplace=True)
|
||||
self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1,
|
||||
padding=1, bias=False)
|
||||
self.droprate = dropRate
|
||||
self.equalInOut = (in_planes == out_planes)
|
||||
self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride,
|
||||
padding=0, bias=False) or None
|
||||
def forward(self, x):
|
||||
if not self.equalInOut: x = self.relu1(self.bn1(x))
|
||||
else: out = self.relu1(self.bn1(x))
|
||||
out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x)))
|
||||
if self.droprate > 0:
|
||||
out = F.dropout(out, p=self.droprate, training=self.training)
|
||||
out = self.conv2(out)
|
||||
return torch.add(x if self.equalInOut else self.convShortcut(x), out)
|
||||
|
||||
class NetworkBlock(nn.Module):
|
||||
def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0):
|
||||
super().__init__()
|
||||
self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate)
|
||||
def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate):
|
||||
layers = []
|
||||
for i in range(nb_layers):
|
||||
layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate))
|
||||
return nn.Sequential(*layers)
|
||||
def forward(self, x): return self.layer(x)
|
||||
|
||||
class WideResNet(nn.Module):
|
||||
def __init__(self, depth, num_classes, widen_factor=1, dropRate=0.0):
|
||||
super().__init__()
|
||||
nChannels = [16, 16*widen_factor, 32*widen_factor, 64*widen_factor]
|
||||
assert((depth - 4) % 6 == 0)
|
||||
n = (depth - 4) // 6
|
||||
block = BasicBlock
|
||||
# 1st conv before any network block
|
||||
self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1,
|
||||
padding=1, bias=False)
|
||||
self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate)
|
||||
self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate)
|
||||
self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate)
|
||||
self.bn1 = nn.BatchNorm2d(nChannels[3])
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.fc = nn.Linear(nChannels[3], num_classes)
|
||||
self.nChannels = nChannels[3]
|
||||
|
||||
for m in self.modules():
|
||||
if isinstance(m, nn.Conv2d):
|
||||
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
|
||||
m.weight.data.normal_(0, math.sqrt(2. / n))
|
||||
elif isinstance(m, nn.BatchNorm2d):
|
||||
m.weight.data.fill_(1)
|
||||
m.bias.data.zero_()
|
||||
elif isinstance(m, nn.Linear): m.bias.data.zero_()
|
||||
def forward(self, x):
|
||||
out = self.conv1(x)
|
||||
out = self.block1(out)
|
||||
out = self.block2(out)
|
||||
out = self.block3(out)
|
||||
out = self.relu(self.bn1(out))
|
||||
out = F.adaptive_avg_pool2d(out, 1)
|
||||
out = out.view(-1, self.nChannels)
|
||||
return self.fc(out)
|
||||
@@ -0,0 +1,331 @@
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.autograd import Variable
|
||||
from functools import reduce
|
||||
|
||||
class LambdaBase(nn.Sequential):
|
||||
def __init__(self, fn, *args):
|
||||
super(LambdaBase, self).__init__(*args)
|
||||
self.lambda_func = fn
|
||||
|
||||
def forward_prepare(self, input):
|
||||
output = []
|
||||
for module in self._modules.values():
|
||||
output.append(module(input))
|
||||
return output if output else input
|
||||
|
||||
class Lambda(LambdaBase):
|
||||
def forward(self, input):
|
||||
return self.lambda_func(self.forward_prepare(input))
|
||||
|
||||
class LambdaMap(LambdaBase):
|
||||
def forward(self, input):
|
||||
return list(map(self.lambda_func,self.forward_prepare(input)))
|
||||
|
||||
class LambdaReduce(LambdaBase):
|
||||
def forward(self, input):
|
||||
return reduce(self.lambda_func,self.forward_prepare(input))
|
||||
|
||||
|
||||
def wrn_50_2f(): return nn.Sequential( # Sequential,
|
||||
nn.Conv2d(3,64,(7, 7),(2, 2),(3, 3),1,1,bias=False),
|
||||
nn.BatchNorm2d(64),
|
||||
nn.ReLU(),
|
||||
nn.MaxPool2d((3, 3),(2, 2),(1, 1)),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(64,128,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,128,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(64,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,128,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,128,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,128,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,128,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(128),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(128,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(2, 2),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(256,512,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,256,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,256,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(256),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(256,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(2, 2),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(512,1024,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,512,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,512,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(512),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(512,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(2, 2),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(1024,2048,(1, 1),(2, 2),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(2048,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
nn.Sequential( # Sequential,
|
||||
LambdaMap(lambda x: x, # ConcatTable,
|
||||
nn.Sequential( # Sequential,
|
||||
nn.Conv2d(2048,1024,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,1024,(3, 3),(1, 1),(1, 1),1,1,bias=False),
|
||||
nn.BatchNorm2d(1024),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(1024,2048,(1, 1),(1, 1),(0, 0),1,1,bias=False),
|
||||
nn.BatchNorm2d(2048),
|
||||
),
|
||||
Lambda(lambda x: x), # Identity,
|
||||
),
|
||||
LambdaReduce(lambda x,y: x+y), # CAddTable,
|
||||
nn.ReLU(),
|
||||
),
|
||||
),
|
||||
nn.AvgPool2d((7, 7),(1, 1)),
|
||||
Lambda(lambda x: x.view(x.size(0),-1)), # View,
|
||||
nn.Sequential(Lambda(lambda x: x.view(1,-1) if 1==len(x.size()) else x ),nn.Linear(2048,1000)), # Linear,
|
||||
)
|
||||
+368
@@ -0,0 +1,368 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
from .core import *
|
||||
from .model import *
|
||||
from .dataset import *
|
||||
from .learner import *
|
||||
from .text import *
|
||||
from .lm_rnn import *
|
||||
|
||||
from sklearn.feature_extraction.text import CountVectorizer
|
||||
from sklearn.model_selection import train_test_split
|
||||
from torchtext.datasets import language_modeling
|
||||
|
||||
class DotProdNB(nn.Module):
|
||||
def __init__(self, nf, ny, w_adj=0.4, r_adj=10):
|
||||
super().__init__()
|
||||
self.w_adj,self.r_adj = w_adj,r_adj
|
||||
self.w = nn.Embedding(nf+1, 1, padding_idx=0)
|
||||
self.w.weight.data.uniform_(-0.1,0.1)
|
||||
self.r = nn.Embedding(nf+1, ny)
|
||||
|
||||
def forward(self, feat_idx, feat_cnt, sz):
|
||||
w = self.w(feat_idx)
|
||||
r = self.r(feat_idx)
|
||||
x = ((w+self.w_adj)*r/self.r_adj).sum(1)
|
||||
return F.softmax(x)
|
||||
|
||||
class SimpleNB(nn.Module):
|
||||
def __init__(self, nf, ny):
|
||||
super().__init__()
|
||||
self.r = nn.Embedding(nf+1, ny, padding_idx=0)
|
||||
self.b = nn.Parameter(torch.zeros(ny,))
|
||||
|
||||
def forward(self, feat_idx, feat_cnt, sz):
|
||||
r = self.r(feat_idx)
|
||||
x = r.sum(1)+self.b
|
||||
return F.softmax(x)
|
||||
|
||||
class BOW_Learner(Learner):
|
||||
def __init__(self, data, models, **kwargs):
|
||||
super().__init__(data, models, **kwargs)
|
||||
|
||||
def _get_crit(self, data): return F.l1_loss
|
||||
|
||||
def calc_pr(y_i, x, y, b):
|
||||
idx = np.argwhere((y==y_i)==b)
|
||||
ct = x[idx[:,0]].sum(0)+1
|
||||
tot = ((y==y_i)==b).sum()+1
|
||||
return ct/tot
|
||||
|
||||
def calc_r(y_i, x, y):
|
||||
return np.log(calc_pr(y_i, x, y, True) / calc_pr(y_i, x, y, False))
|
||||
|
||||
class BOW_Dataset(Dataset):
|
||||
def __init__(self, bow, y, max_len):
|
||||
self.bow,self.max_len = bow,max_len
|
||||
self.c = int(y.max())+1
|
||||
self.n,self.vocab_size = bow.shape
|
||||
self.y = one_hot(y,self.c).astype(np.float32)
|
||||
x = self.bow.sign()
|
||||
self.r = np.stack([calc_r(i, x, y).A1 for i in range(self.c)]).T
|
||||
|
||||
def __getitem__(self, i):
|
||||
row = self.bow.getrow(i)
|
||||
|
||||
num_row_entries = row.indices.shape[0]
|
||||
indices = (row.indices + 1).astype(np.int64)
|
||||
data = (row.data).astype(np.int64)
|
||||
|
||||
if num_row_entries < self.max_len:
|
||||
# If short, pad
|
||||
indices = np.pad(indices, (self.max_len - num_row_entries, 0), mode='constant')
|
||||
data = np.pad(data, (self.max_len - num_row_entries, 0), mode='constant')
|
||||
else:
|
||||
# If long, truncate
|
||||
indices, data = indices[-self.max_len:], data[-self.max_len:]
|
||||
|
||||
return indices, data, min(self.max_len, num_row_entries), self.y[i]
|
||||
|
||||
def __len__(self): return len(self.bow.indptr)-1
|
||||
|
||||
|
||||
class TextClassifierData(ModelData):
|
||||
@property
|
||||
def c(self): return self.trn_ds.c
|
||||
|
||||
@property
|
||||
def r(self):
|
||||
return torch.Tensor(np.concatenate([np.zeros((1,self.c)), self.trn_ds.r]))
|
||||
|
||||
def get_model(self, f, **kwargs):
|
||||
m = to_gpu(f(self.trn_ds.vocab_size, self.c, **kwargs))
|
||||
m.r.weight.data = to_gpu(self.r)
|
||||
m.r.weight.requires_grad = False
|
||||
model = BasicModel(m)
|
||||
return BOW_Learner(self, model, metrics=[accuracy_thresh(0.5)], opt_fn=optim.Adam)
|
||||
|
||||
def dotprod_nb_learner(self, **kwargs): return self.get_model(DotProdNB, **kwargs)
|
||||
def nb_learner(self, **kwargs): return self.get_model(SimpleNB, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def from_bow(cls, trn_bow, trn_y, val_bow, val_y, sl):
|
||||
trn_ds = BOW_Dataset(trn_bow, trn_y, sl)
|
||||
val_ds = BOW_Dataset(val_bow, val_y, sl)
|
||||
trn_dl = DataLoader(trn_ds, 64, True)
|
||||
val_dl = DataLoader(val_ds, 64, False)
|
||||
return cls('.', trn_dl, val_dl)
|
||||
|
||||
|
||||
def flip_tensor(x, dim):
|
||||
xsize = x.size()
|
||||
dim = x.dim() + dim if dim < 0 else dim
|
||||
x = x.view(-1, *xsize[dim:])
|
||||
x = x.view(x.size(0), x.size(1), -1)[:, getattr(torch.arange(x.size(1)-1,
|
||||
-1, -1), ('cpu','cuda')[x.is_cuda])().long(), :]
|
||||
return x.view(xsize)
|
||||
|
||||
|
||||
class LanguageModelLoader():
|
||||
|
||||
def __init__(self, ds, bs, bptt, backwards=False):
|
||||
self.bs,self.bptt,self.backwards = bs,bptt,backwards
|
||||
text = sum([o.text for o in ds], [])
|
||||
fld = ds.fields['text']
|
||||
nums = fld.numericalize([text],device=None if torch.cuda.is_available() else -1)
|
||||
self.data = self.batchify(nums)
|
||||
self.i,self.iter = 0,0
|
||||
self.n = len(self.data)
|
||||
|
||||
def __iter__(self):
|
||||
self.i,self.iter = 0,0
|
||||
return self
|
||||
|
||||
def __len__(self): return self.n // self.bptt - 1
|
||||
|
||||
def __next__(self):
|
||||
if self.i >= self.n-1 or self.iter>=len(self): raise StopIteration
|
||||
bptt = self.bptt if np.random.random() < 0.95 else self.bptt / 2.
|
||||
seq_len = max(5, int(np.random.normal(bptt, 5)))
|
||||
res = self.get_batch(self.i, seq_len)
|
||||
self.i += seq_len
|
||||
self.iter += 1
|
||||
return res
|
||||
|
||||
def batchify(self, data):
|
||||
nb = data.size(0) // self.bs
|
||||
data = data[:nb*self.bs]
|
||||
data = data.view(self.bs, -1).t().contiguous()
|
||||
if self.backwards: data=flip_tensor(data, 0)
|
||||
return to_gpu(data)
|
||||
|
||||
def get_batch(self, i, seq_len):
|
||||
source = self.data
|
||||
seq_len = min(seq_len, len(source) - 1 - i)
|
||||
return source[i:i+seq_len], source[i+1:i+1+seq_len].view(-1)
|
||||
|
||||
|
||||
class RNN_Learner(Learner):
|
||||
def __init__(self, data, models, **kwargs):
|
||||
super().__init__(data, models, **kwargs)
|
||||
|
||||
def _get_crit(self, data): return F.cross_entropy
|
||||
|
||||
def save_encoder(self, name): save_model(self.model[0], self.get_model_path(name))
|
||||
|
||||
def load_encoder(self, name): load_model(self.model[0], self.get_model_path(name))
|
||||
|
||||
|
||||
class ConcatTextDataset(torchtext.data.Dataset):
|
||||
def __init__(self, path, text_field, newline_eos=True, encoding='utf-8', **kwargs):
|
||||
fields = [('text', text_field)]
|
||||
text = []
|
||||
if os.path.isdir(path): paths=glob(f'{path}/*.*')
|
||||
else: paths=[path]
|
||||
for p in paths:
|
||||
for line in open(p, encoding=encoding): text += text_field.preprocess(line)
|
||||
if newline_eos: text.append('<eos>')
|
||||
|
||||
examples = [torchtext.data.Example.fromlist([text], fields)]
|
||||
super().__init__(examples, fields, **kwargs)
|
||||
|
||||
|
||||
class ConcatTextDatasetFromDataFrames(torchtext.data.Dataset):
|
||||
def __init__(self, df, text_field, col, newline_eos=True, **kwargs):
|
||||
fields = [('text', text_field)]
|
||||
text = []
|
||||
|
||||
text += text_field.preprocess(df[col].str.cat(sep=' <eos> '))
|
||||
if (newline_eos): text.append('<eos>')
|
||||
|
||||
examples = [torchtext.data.Example.fromlist([text], fields)]
|
||||
|
||||
super().__init__(examples, fields, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def splits(cls, train_df=None, val_df=None, test_df=None, keep_nones=False, **kwargs):
|
||||
res = (
|
||||
cls(train_df, **kwargs),
|
||||
cls(val_df, **kwargs),
|
||||
map_none(test_df, partial(cls, **kwargs))) # not required
|
||||
return res if keep_nones else tuple(d for d in res if d is not None)
|
||||
|
||||
|
||||
class LanguageModelData():
|
||||
"""
|
||||
This class provides the entry point for dealing with supported NLP tasks.
|
||||
Usage:
|
||||
1. Use one of the factory constructors (from_dataframes, from_text_files) to
|
||||
obtain an instance of the class.
|
||||
2. Use the get_model method to return a RNN_Learner instance (a network suited
|
||||
for NLP tasks), then proceed with training.
|
||||
|
||||
Example:
|
||||
>> TEXT = data.Field(lower=True, tokenize=spacy_tok)
|
||||
>> FILES = dict(train=TRN_PATH, validation=VAL_PATH, test=VAL_PATH)
|
||||
>> md = LanguageModelData.from_text_files(PATH, TEXT, **FILES, bs=64, bptt=70, min_freq=10)
|
||||
|
||||
>> em_sz = 200 # size of each embedding vector
|
||||
>> nh = 500 # number of hidden activations per layer
|
||||
>> nl = 3 # number of layers
|
||||
|
||||
>> opt_fn = partial(optim.Adam, betas=(0.7, 0.99))
|
||||
>> learner = md.get_model(opt_fn, em_sz, nh, nl,
|
||||
dropouti=0.05, dropout=0.05, wdrop=0.1, dropoute=0.02, dropouth=0.05)
|
||||
>> learner.reg_fn = seq2seq_reg
|
||||
>> learner.clip=0.3
|
||||
|
||||
>> learner.fit(3e-3, 4, wds=1e-6, cycle_len=1, cycle_mult=2)
|
||||
|
||||
"""
|
||||
def __init__(self, path, field, trn_ds, val_ds, test_ds, bs, bptt, backwards=False, **kwargs):
|
||||
""" Constructor for the class. An important thing that happens here is
|
||||
that the field's "build_vocab" method is invoked, which builds the vocabulary
|
||||
for this NLP model.
|
||||
|
||||
Also, three instances of the LanguageModelLoader are constructed; one each
|
||||
for training data (self.trn_dl), validation data (self.val_dl), and the
|
||||
testing data (self.test_dl)
|
||||
|
||||
Args:
|
||||
path (str): testing path
|
||||
field (Field): torchtext field object
|
||||
trn_ds (Dataset): training dataset
|
||||
val_ds (Dataset): validation dataset
|
||||
test_ds (Dataset): testing dataset
|
||||
bs (int): batch size
|
||||
bptt (int): back propagation through time
|
||||
kwargs: other arguments
|
||||
"""
|
||||
self.bs = bs
|
||||
self.path = path
|
||||
self.trn_ds = trn_ds; self.val_ds = val_ds; self.test_ds = test_ds
|
||||
if not hasattr(field, 'vocab'): field.build_vocab(self.trn_ds, **kwargs)
|
||||
|
||||
self.pad_idx = field.vocab.stoi[field.pad_token]
|
||||
self.nt = len(field.vocab)
|
||||
|
||||
factory = lambda ds: LanguageModelLoader(ds, bs, bptt, backwards=backwards)
|
||||
self.trn_dl = factory(self.trn_ds)
|
||||
self.val_dl = factory(self.val_ds)
|
||||
self.test_dl = map_none(self.test_ds, factory) # not required
|
||||
|
||||
def get_model(self, opt_fn, emb_sz, n_hid, n_layers, **kwargs):
|
||||
""" Method returns a RNN_Learner object, that wraps an instance of the RNN_Encoder module.
|
||||
|
||||
Args:
|
||||
opt_fn (Optimizer): the torch optimizer function to use
|
||||
emb_sz (int): embedding size
|
||||
n_hid (int): number of hidden inputs
|
||||
n_layers (int): number of hidden layers
|
||||
kwargs: other arguments
|
||||
|
||||
Returns:
|
||||
An instance of the RNN_Learner class.
|
||||
|
||||
"""
|
||||
m = get_language_model(self.nt, emb_sz, n_hid, n_layers, self.pad_idx, **kwargs)
|
||||
model = SingleModel(to_gpu(m))
|
||||
return RNN_Learner(self, model, opt_fn=opt_fn)
|
||||
|
||||
@classmethod
|
||||
def from_dataframes(cls, path, field, col, train_df, val_df, test_df=None, bs=64, bptt=70, **kwargs):
|
||||
trn_ds, val_ds, test_ds = ConcatTextDatasetFromDataFrames.splits(
|
||||
text_field=field, col=col, train_df=train_df, val_df=val_df, test_df=test_df, keep_nones=True)
|
||||
return cls(path, field, trn_ds, val_ds, test_ds, bs, bptt, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def from_text_files(cls, path, field, train, validation, test=None, bs=64, bptt=70, **kwargs):
|
||||
""" Method used to instantiate a LanguageModelData object that can be used for a
|
||||
supported nlp task.
|
||||
|
||||
Args:
|
||||
path (str): the absolute path in which temporary model data will be saved
|
||||
field (Field): torchtext field
|
||||
train (str): file location of the training data
|
||||
validation (str): file location of the validation data
|
||||
test (str): file location of the testing data
|
||||
bs (int): batch size to use
|
||||
bptt (int): back propagation through time hyper-parameter
|
||||
kwargs: other arguments
|
||||
|
||||
Returns:
|
||||
a LanguageModelData instance, which most importantly, provides us the datasets for training,
|
||||
validation, and testing
|
||||
|
||||
Note:
|
||||
The train, validation, and test path can be pointed to any file (or folder) that contains a valid
|
||||
text corpus.
|
||||
|
||||
"""
|
||||
trn_ds, val_ds, test_ds = ConcatTextDataset.splits(
|
||||
path, text_field=field, train=train, validation=validation, test=test)
|
||||
return cls(path, field, trn_ds, val_ds, test_ds, bs, bptt, **kwargs)
|
||||
|
||||
|
||||
class TextDataLoader():
|
||||
def __init__(self, src, x_fld, y_fld):
|
||||
self.src,self.x_fld,self.y_fld = src,x_fld,y_fld
|
||||
|
||||
def __len__(self): return len(self.src)
|
||||
|
||||
def __iter__(self):
|
||||
it = iter(self.src)
|
||||
for i in range(len(self)):
|
||||
b = next(it)
|
||||
yield getattr(b, self.x_fld).data, getattr(b, self.y_fld).data
|
||||
|
||||
|
||||
class TextModel(BasicModel):
|
||||
def get_layer_groups(self):
|
||||
m = self.model[0]
|
||||
return [(m.encoder, m.dropouti), *zip(m.rnns, m.dropouths), (self.model[1])]
|
||||
|
||||
|
||||
class TextData(ModelData):
|
||||
def create_td(self, it): return TextDataLoader(it, self.text_fld, self.label_fld)
|
||||
|
||||
@classmethod
|
||||
def from_splits(cls, path, splits, bs, text_name='text', label_name='label'):
|
||||
text_fld = splits[0].fields[text_name]
|
||||
label_fld = splits[0].fields[label_name]
|
||||
if hasattr(label_fld, 'build_vocab'): label_fld.build_vocab(splits[0])
|
||||
iters = torchtext.data.BucketIterator.splits(splits, batch_size=bs)
|
||||
trn_iter,val_iter,test_iter = iters[0],iters[1],None
|
||||
test_dl = None
|
||||
if len(iters) == 3:
|
||||
test_iter = iters[2]
|
||||
test_dl = TextDataLoader(test_iter, text_name, label_name)
|
||||
trn_dl = TextDataLoader(trn_iter, text_name, label_name)
|
||||
val_dl = TextDataLoader(val_iter, text_name, label_name)
|
||||
obj = cls.from_dls(path, trn_dl, val_dl, test_dl)
|
||||
obj.bs = bs
|
||||
obj.pad_idx = text_fld.vocab.stoi[text_fld.pad_token]
|
||||
obj.nt = len(text_fld.vocab)
|
||||
obj.c = (len(label_fld.vocab) if hasattr(label_fld, 'vocab')
|
||||
else len(getattr(splits[0][0], label_name)))
|
||||
return obj
|
||||
|
||||
def to_model(self, m, opt_fn):
|
||||
model = TextModel(to_gpu(m))
|
||||
return RNN_Learner(self, model, opt_fn=opt_fn)
|
||||
|
||||
def get_model(self, opt_fn, max_sl, bptt, emb_sz, n_hid, n_layers, dropout, **kwargs):
|
||||
m = get_rnn_classifier(bptt, max_sl, self.c, self.nt,
|
||||
layers=[emb_sz*3, self.c], drops=[dropout],
|
||||
emb_sz=emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=self.pad_idx, **kwargs)
|
||||
return self.to_model(m, opt_fn)
|
||||
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
from .imports import *
|
||||
from .torch_imports import *
|
||||
from sklearn.metrics import confusion_matrix
|
||||
|
||||
def ceildiv(a, b):
|
||||
return -(-a // b)
|
||||
|
||||
def plots(ims, figsize=(12,6), rows=1, interp=False, titles=None, maintitle=None):
|
||||
if type(ims[0]) is np.ndarray:
|
||||
ims = np.array(ims)
|
||||
if (ims.shape[-1] != 3): ims = ims.transpose((0,2,3,1))
|
||||
f = plt.figure(figsize=figsize)
|
||||
if maintitle is not None:
|
||||
plt.suptitle(maintitle, fontsize=16)
|
||||
for i in range(len(ims)):
|
||||
sp = f.add_subplot(rows, ceildiv(len(ims), rows), i+1)
|
||||
sp.axis('Off')
|
||||
if titles is not None: sp.set_title(titles[i], fontsize=16)
|
||||
plt.imshow(ims[i], interpolation=None if interp else 'none')
|
||||
|
||||
|
||||
def plots_from_files(imspaths, figsize=(10,5), rows=1, titles=None, maintitle=None):
|
||||
"""Plots images given image files.
|
||||
|
||||
Arguments:
|
||||
im_paths (list): list of paths
|
||||
figsize (tuple): figure size
|
||||
rows (int): number of rows
|
||||
titles (list): list of titles
|
||||
maintitle (string): main title
|
||||
"""
|
||||
f = plt.figure(figsize=figsize)
|
||||
if maintitle is not None: plt.suptitle(maintitle, fontsize=16)
|
||||
for i in range(len(imspaths)):
|
||||
sp = f.add_subplot(rows, ceildiv(len(imspaths), rows), i+1)
|
||||
sp.axis('Off')
|
||||
if titles is not None: sp.set_title(titles[i], fontsize=16)
|
||||
img = plt.imread(imspaths[i])
|
||||
plt.imshow(img)
|
||||
|
||||
|
||||
def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues, figsize=None):
|
||||
"""
|
||||
This function prints and plots the confusion matrix.
|
||||
Normalization can be applied by setting `normalize=True`.
|
||||
(This function is copied from the scikit docs.)
|
||||
"""
|
||||
plt.figure(figsize=figsize)
|
||||
plt.imshow(cm, interpolation='nearest', cmap=cmap)
|
||||
plt.title(title)
|
||||
plt.colorbar()
|
||||
tick_marks = np.arange(len(classes))
|
||||
plt.xticks(tick_marks, classes, rotation=45)
|
||||
plt.yticks(tick_marks, classes)
|
||||
|
||||
if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
|
||||
print(cm)
|
||||
thresh = cm.max() / 2.
|
||||
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
|
||||
plt.text(j, i, cm[i, j], horizontalalignment="center", color="white" if cm[i, j] > thresh else "black")
|
||||
|
||||
plt.tight_layout()
|
||||
plt.ylabel('True label')
|
||||
plt.xlabel('Predicted label')
|
||||
|
||||
def plots_raw(ims, figsize=(12,6), rows=1, titles=None):
|
||||
f = plt.figure(figsize=figsize)
|
||||
for i in range(len(ims)):
|
||||
sp = f.add_subplot(rows, ceildiv(len(ims), rows), i+1)
|
||||
sp.axis('Off')
|
||||
if titles is not None: sp.set_title(titles[i], fontsize=16)
|
||||
plt.imshow(ims[i])
|
||||
|
||||
def load_img_id(ds, idx, path): return np.array(PIL.Image.open(os.path.join(path, ds.fnames[idx])))
|
||||
|
||||
|
||||
class ImageModelResults():
|
||||
""" Visualize the results of an image model
|
||||
|
||||
Arguments:
|
||||
ds (dataset): a dataset which contains the images
|
||||
log_preds (numpy.ndarray): predictions for the dataset in log scale
|
||||
|
||||
Returns:
|
||||
ImageModelResults
|
||||
"""
|
||||
def __init__(self, ds, log_preds):
|
||||
"""Initialize an ImageModelResults class instance"""
|
||||
self.ds = ds
|
||||
# returns the indices of the maximum value of predictions along axis 1, representing the predicted class
|
||||
# log_preds.shape = (number_of_samples, number_of_classes);
|
||||
# preds.shape = (number_of_samples,)
|
||||
self.preds = np.argmax(log_preds, axis=1)
|
||||
# computes the probabilities
|
||||
self.probs = np.exp(log_preds)
|
||||
# extracts the number of classes
|
||||
self.num_classes = log_preds.shape[1]
|
||||
|
||||
def plot_val_with_title(self, idxs, y):
|
||||
""" Displays the images and their probabilities of belonging to a certain class
|
||||
|
||||
Arguments:
|
||||
idxs (numpy.ndarray): indexes of the image samples from the dataset
|
||||
y (int): the selected class
|
||||
|
||||
Returns:
|
||||
Plots the images in n rows [rows = n]
|
||||
"""
|
||||
# if there are any samples to be displayed
|
||||
if len(idxs) > 0:
|
||||
imgs = np.stack([self.ds[x][0] for x in idxs])
|
||||
title_probs = [self.probs[x,y] for x in idxs]
|
||||
|
||||
return plots(self.ds.denorm(imgs), rows=1, titles=title_probs)
|
||||
# if idxs is empty return false
|
||||
else:
|
||||
return False;
|
||||
|
||||
def most_by_mask(self, mask, y, mult):
|
||||
""" Extracts the first 4 most correct/incorrect indexes from the ordered list of probabilities
|
||||
|
||||
Arguments:
|
||||
mask (numpy.ndarray): the mask of probabilities specific to the selected class; a boolean array with shape (num_of_samples,) which contains True where class==selected_class, and False everywhere else
|
||||
y (int): the selected class
|
||||
mult (int): sets the ordering; -1 descending, 1 ascending
|
||||
|
||||
Returns:
|
||||
idxs (ndarray): An array of indexes of length 4
|
||||
"""
|
||||
idxs = np.where(mask)[0]
|
||||
cnt = min(4, len(idxs))
|
||||
return idxs[np.argsort(mult * self.probs[idxs,y])[:cnt]]
|
||||
|
||||
def most_uncertain_by_mask(self, mask, y):
|
||||
""" Extracts the first 4 most uncertain indexes from the ordered list of probabilities
|
||||
|
||||
Arguments:
|
||||
mask (numpy.ndarray): the mask of probabilities specific to the selected class; a boolean array with shape (num_of_samples,) which contains True where class==selected_class, and False everywhere else
|
||||
y (int): the selected class
|
||||
|
||||
Returns:
|
||||
idxs (ndarray): An array of indexes of length 4
|
||||
"""
|
||||
idxs = np.where(mask)[0]
|
||||
# the most uncertain samples will have abs(probs-1/num_classes) close to 0;
|
||||
return idxs[np.argsort(np.abs(self.probs[idxs,y]-(1/self.num_classes)))[:4]]
|
||||
|
||||
def most_by_correct(self, y, is_correct):
|
||||
""" Extracts the predicted classes which correspond to the selected class (y) and to the specific case (prediction is correct - is_true=True, prediction is wrong - is_true=False)
|
||||
|
||||
Arguments:
|
||||
y (int): the selected class
|
||||
is_correct (boolean): a boolean flag (True, False) which specify the what to look for. Ex: True - most correct samples, False - most incorrect samples
|
||||
|
||||
Returns:
|
||||
idxs (numpy.ndarray): An array of indexes (numpy.ndarray)
|
||||
"""
|
||||
# mult=-1 when the is_correct flag is true -> when we want to display the most correct classes we will make a descending sorting (argsort) because we want that the biggest probabilities to be displayed first.
|
||||
# When is_correct is false, we want to display the most incorrect classes, so we want an ascending sorting since our interest is in the smallest probabilities.
|
||||
mult = -1 if is_correct==True else 1
|
||||
return self.most_by_mask(((self.preds == self.ds.y)==is_correct)
|
||||
& (self.ds.y == y), y, mult)
|
||||
|
||||
def plot_by_correct(self, y, is_correct):
|
||||
""" Plots the images which correspond to the selected class (y) and to the specific case (prediction is correct - is_true=True, prediction is wrong - is_true=False)
|
||||
|
||||
Arguments:
|
||||
y (int): the selected class
|
||||
is_correct (boolean): a boolean flag (True, False) which specify the what to look for. Ex: True - most correct samples, False - most incorrect samples
|
||||
"""
|
||||
return self.plot_val_with_title(self.most_by_correct(y, is_correct), y)
|
||||
|
||||
def most_by_uncertain(self, y):
|
||||
""" Extracts the predicted classes which correspond to the selected class (y) and have probabilities nearest to 1/number_of_classes (eg. 0.5 for 2 classes, 0.33 for 3 classes) for the selected class.
|
||||
|
||||
Arguments:
|
||||
y (int): the selected class
|
||||
|
||||
Returns:
|
||||
idxs (numpy.ndarray): An array of indexes (numpy.ndarray)
|
||||
"""
|
||||
return self.most_uncertain_by_mask((self.ds.y == y), y)
|
||||
|
||||
def plot_most_correct(self, y):
|
||||
""" Plots the images which correspond to the selected class (y) and are most correct.
|
||||
|
||||
Arguments:
|
||||
y (int): the selected class
|
||||
"""
|
||||
return self.plot_by_correct(y, True)
|
||||
def plot_most_incorrect(self, y):
|
||||
""" Plots the images which correspond to the selected class (y) and are most incorrect.
|
||||
|
||||
Arguments:
|
||||
y (int): the selected class
|
||||
"""
|
||||
return self.plot_by_correct(y, False)
|
||||
def plot_most_uncertain(self, y):
|
||||
""" Plots the images which correspond to the selected class (y) and are most uncertain i.e have probabilities nearest to 1/number_of_classes.
|
||||
|
||||
Arguments:
|
||||
y (int): the selected class
|
||||
"""
|
||||
return self.plot_val_with_title(self.most_by_uncertain(y), y)
|
||||
@@ -0,0 +1,190 @@
|
||||
from .torch_imports import *
|
||||
from .core import *
|
||||
from functools import wraps
|
||||
import torch.nn.functional as F
|
||||
from torch.autograd import Variable
|
||||
|
||||
IS_TORCH_04 = LooseVersion(torch.__version__) >= LooseVersion('0.4')
|
||||
|
||||
def dropout_mask(x, sz, dropout):
|
||||
""" Applies a dropout mask whose size is determined by passed argument 'sz'.
|
||||
Args:
|
||||
x (nn.Variable): A torch Variable object
|
||||
sz (tuple(int, int, int)): The expected size of the new tensor
|
||||
dropout (float): The dropout fraction to apply
|
||||
|
||||
This method uses the bernoulli distribution to decide which activations to keep.
|
||||
Additionally, the sampled activations is rescaled is using the factor 1/(1 - dropout).
|
||||
|
||||
In the example given below, one can see that approximately .8 fraction of the
|
||||
returned tensors are zero. Rescaling with the factor 1/(1 - 0.8) returns a tensor
|
||||
with 5's in the unit places.
|
||||
|
||||
The official link to the pytorch bernoulli function is here:
|
||||
http://pytorch.org/docs/master/torch.html#torch.bernoulli
|
||||
|
||||
Examples:
|
||||
>>> a_Var = torch.autograd.Variable(torch.Tensor(2, 3, 4).uniform_(0, 1), requires_grad=False)
|
||||
>>> a_Var
|
||||
Variable containing:
|
||||
(0 ,.,.) =
|
||||
0.6890 0.5412 0.4303 0.8918
|
||||
0.3871 0.7944 0.0791 0.5979
|
||||
0.4575 0.7036 0.6186 0.7217
|
||||
(1 ,.,.) =
|
||||
0.8354 0.1690 0.1734 0.8099
|
||||
0.6002 0.2602 0.7907 0.4446
|
||||
0.5877 0.7464 0.4257 0.3386
|
||||
[torch.FloatTensor of size 2x3x4]
|
||||
>>> a_mask = dropout_mask(a_Var.data, (1,a_Var.size(1),a_Var.size(2)), dropout=0.8)
|
||||
>>> a_mask
|
||||
(0 ,.,.) =
|
||||
0 5 0 0
|
||||
0 0 0 5
|
||||
5 0 5 0
|
||||
[torch.FloatTensor of size 1x3x4]
|
||||
"""
|
||||
return x.new(*sz).bernoulli_(1-dropout)/(1-dropout)
|
||||
|
||||
|
||||
class LockedDropout(nn.Module):
|
||||
def __init__(self, p=0.5):
|
||||
super().__init__()
|
||||
self.p=p
|
||||
|
||||
def forward(self, x):
|
||||
if not self.training or not self.p: return x
|
||||
m = dropout_mask(x.data, (1, x.size(1), x.size(2)), self.p)
|
||||
return Variable(m, requires_grad=False) * x
|
||||
|
||||
|
||||
class WeightDrop(torch.nn.Module):
|
||||
"""A custom torch layer that serves as a wrapper on another torch layer.
|
||||
Primarily responsible for updating the weights in the wrapped module based
|
||||
on a specified dropout.
|
||||
"""
|
||||
def __init__(self, module, dropout, weights=['weight_hh_l0']):
|
||||
""" Default constructor for the WeightDrop module
|
||||
|
||||
Args:
|
||||
module (torch.nn.Module): A pytorch layer being wrapped
|
||||
dropout (float): a dropout value to apply
|
||||
weights (list(str)): the parameters of the wrapped **module**
|
||||
which should be fractionally dropped.
|
||||
"""
|
||||
super().__init__()
|
||||
self.module,self.weights,self.dropout = module,weights,dropout
|
||||
self._setup()
|
||||
|
||||
def _setup(self):
|
||||
""" for each string defined in self.weights, the corresponding
|
||||
attribute in the wrapped module is referenced, then deleted, and subsequently
|
||||
registered as a new parameter with a slightly modified name.
|
||||
|
||||
Args:
|
||||
None
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
if isinstance(self.module, torch.nn.RNNBase): self.module.flatten_parameters = noop
|
||||
for name_w in self.weights:
|
||||
w = getattr(self.module, name_w)
|
||||
del self.module._parameters[name_w]
|
||||
self.module.register_parameter(name_w + '_raw', nn.Parameter(w.data))
|
||||
|
||||
|
||||
def _setweights(self):
|
||||
""" Uses pytorch's built-in dropout function to apply dropout to the parameters of
|
||||
the wrapped module.
|
||||
|
||||
Args:
|
||||
None
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
for name_w in self.weights:
|
||||
raw_w = getattr(self.module, name_w + '_raw')
|
||||
w = torch.nn.functional.dropout(raw_w, p=self.dropout, training=self.training)
|
||||
if hasattr(self.module, name_w):
|
||||
delattr(self.module, name_w)
|
||||
setattr(self.module, name_w, w)
|
||||
|
||||
def forward(self, *args):
|
||||
""" updates weights and delegates the propagation of the tensor to the wrapped module's
|
||||
forward method
|
||||
|
||||
Args:
|
||||
*args: supplied arguments
|
||||
|
||||
Returns:
|
||||
tensor obtained by running the forward method on the wrapped module.
|
||||
"""
|
||||
self._setweights()
|
||||
return self.module.forward(*args)
|
||||
|
||||
class EmbeddingDropout(nn.Module):
|
||||
|
||||
""" Applies dropout in the embedding layer by zeroing out some elements of the embedding vector.
|
||||
Uses the dropout_mask custom layer to achieve this.
|
||||
|
||||
Args:
|
||||
embed (torch.nn.Embedding): An embedding torch layer
|
||||
words (torch.nn.Variable): A torch variable
|
||||
dropout (float): dropout fraction to apply to the embedding weights
|
||||
scale (float): additional scaling to apply to the modified embedding weights
|
||||
|
||||
Returns:
|
||||
tensor of size: (batch_size x seq_length x embedding_size)
|
||||
|
||||
Example:
|
||||
|
||||
>> embed = torch.nn.Embedding(10,3)
|
||||
>> words = Variable(torch.LongTensor([[1,2,4,5] ,[4,3,2,9]]))
|
||||
>> words.size()
|
||||
(2,4)
|
||||
>> embed_dropout_layer = EmbeddingDropout(embed)
|
||||
>> dropout_out_ = embed_dropout_layer(embed, words, dropout=0.40)
|
||||
>> dropout_out_
|
||||
Variable containing:
|
||||
(0 ,.,.) =
|
||||
1.2549 1.8230 1.9367
|
||||
0.0000 -0.0000 0.0000
|
||||
2.2540 -0.1299 1.5448
|
||||
0.0000 -0.0000 -0.0000
|
||||
|
||||
(1 ,.,.) =
|
||||
2.2540 -0.1299 1.5448
|
||||
-4.0457 2.4815 -0.2897
|
||||
0.0000 -0.0000 0.0000
|
||||
1.8796 -0.4022 3.8773
|
||||
[torch.FloatTensor of size 2x4x3]
|
||||
"""
|
||||
|
||||
def __init__(self, embed):
|
||||
super().__init__()
|
||||
self.embed = embed
|
||||
|
||||
def forward(self, words, dropout=0.1, scale=None):
|
||||
if dropout:
|
||||
size = (self.embed.weight.size(0),1)
|
||||
mask = Variable(dropout_mask(self.embed.weight.data, size, dropout))
|
||||
masked_embed_weight = mask * self.embed.weight
|
||||
else: masked_embed_weight = self.embed.weight
|
||||
|
||||
if scale: masked_embed_weight = scale * masked_embed_weight
|
||||
|
||||
padding_idx = self.embed.padding_idx
|
||||
if padding_idx is None: padding_idx = -1
|
||||
|
||||
|
||||
if IS_TORCH_04:
|
||||
X = F.embedding(words,
|
||||
masked_embed_weight, padding_idx, self.embed.max_norm,
|
||||
self.embed.norm_type, self.embed.scale_grad_by_freq, self.embed.sparse)
|
||||
else:
|
||||
X = self.embed._backend.Embedding.apply(words,
|
||||
masked_embed_weight, padding_idx, self.embed.max_norm,
|
||||
self.embed.norm_type, self.embed.scale_grad_by_freq, self.embed.sparse)
|
||||
|
||||
return X
|
||||
@@ -0,0 +1,5 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.autograd import Variable
|
||||
from .core import *
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from multiprocessing import set_start_method
|
||||
set_start_method('spawn')
|
||||
|
||||
+609
@@ -0,0 +1,609 @@
|
||||
from .imports import *
|
||||
from .layer_optimizer import *
|
||||
from enum import IntEnum
|
||||
from timeit import default_timer as timer
|
||||
import copy
|
||||
|
||||
|
||||
class Callback:
|
||||
'''
|
||||
An abstract class that all callback(e.g., LossRecorder) classes extends from.
|
||||
Must be extended before usage.
|
||||
'''
|
||||
def on_train_begin(self): pass
|
||||
def on_batch_begin(self): pass
|
||||
def on_phase_begin(self): pass
|
||||
def on_epoch_end(self, metrics): pass
|
||||
def on_phase_end(self): pass
|
||||
def on_batch_end(self, metrics): pass
|
||||
def on_train_end(self): pass
|
||||
|
||||
# Useful for maintaining status of a long-running job.
|
||||
#
|
||||
# Usage:
|
||||
# learn.fit(0.01, 1, callbacks = [LoggingCallback(save_path="/tmp/log")])
|
||||
class LoggingCallback(Callback):
|
||||
'''
|
||||
A class useful for maintaining status of a long-running job.
|
||||
e.g.: learn.fit(0.01, 1, callbacks = [LoggingCallback(save_path="/tmp/log")])
|
||||
'''
|
||||
def __init__(self, save_path):
|
||||
super().__init__()
|
||||
self.save_path=save_path
|
||||
def on_train_begin(self):
|
||||
self.batch = 0
|
||||
self.epoch = 0
|
||||
self.phase = 0
|
||||
self.f = open(self.save_path, "a", 1)
|
||||
self.log("\ton_train_begin")
|
||||
def on_batch_begin(self):
|
||||
self.log(str(self.batch)+"\ton_batch_begin")
|
||||
def on_phase_begin(self):
|
||||
self.log(str(self.phase)+"\ton_phase_begin")
|
||||
def on_epoch_end(self, metrics):
|
||||
self.log(str(self.epoch)+"\ton_epoch_end: "+str(metrics))
|
||||
self.epoch += 1
|
||||
def on_phase_end(self):
|
||||
self.log(str(self.phase)+"\ton_phase_end")
|
||||
self.phase+=1
|
||||
def on_batch_end(self, metrics):
|
||||
self.log(str(self.batch)+"\ton_batch_end: "+str(metrics))
|
||||
self.batch += 1
|
||||
def on_train_end(self):
|
||||
self.log("\ton_train_end")
|
||||
self.f.close()
|
||||
def log(self, string):
|
||||
self.f.write(time.strftime("%Y-%m-%dT%H:%M:%S")+"\t"+string+"\n")
|
||||
|
||||
class LossRecorder(Callback):
|
||||
'''
|
||||
Saves and displays loss functions and other metrics.
|
||||
Default sched when none is specified in a learner.
|
||||
'''
|
||||
def __init__(self, layer_opt, save_path='', record_mom=False, metrics=[]):
|
||||
super().__init__()
|
||||
self.layer_opt=layer_opt
|
||||
self.init_lrs=np.array(layer_opt.lrs)
|
||||
self.save_path, self.record_mom, self.metrics = save_path, record_mom, metrics
|
||||
|
||||
def on_train_begin(self):
|
||||
self.losses,self.lrs,self.iterations,self.epochs,self.times = [],[],[],[],[]
|
||||
self.start_at = timer()
|
||||
self.val_losses, self.rec_metrics = [], []
|
||||
if self.record_mom:
|
||||
self.momentums = []
|
||||
self.iteration = 0
|
||||
self.epoch = 0
|
||||
|
||||
def on_epoch_end(self, metrics):
|
||||
self.epoch += 1
|
||||
self.epochs.append(self.iteration)
|
||||
self.times.append(timer() - self.start_at)
|
||||
self.save_metrics(metrics)
|
||||
|
||||
def on_batch_end(self, loss):
|
||||
self.iteration += 1
|
||||
self.lrs.append(self.layer_opt.lr)
|
||||
self.iterations.append(self.iteration)
|
||||
if isinstance(loss, list):
|
||||
self.losses.append(loss[0])
|
||||
self.save_metrics(loss[1:])
|
||||
else: self.losses.append(loss)
|
||||
if self.record_mom: self.momentums.append(self.layer_opt.mom)
|
||||
|
||||
def save_metrics(self,vals):
|
||||
self.val_losses.append(delistify(vals[0]))
|
||||
if len(vals) > 2: self.rec_metrics.append(vals[1:])
|
||||
elif len(vals) == 2: self.rec_metrics.append(vals[1])
|
||||
|
||||
def plot_loss(self, n_skip=10, n_skip_end=5):
|
||||
'''
|
||||
plots loss function as function of iterations.
|
||||
When used in Jupyternotebook, plot will be displayed in notebook. Else, plot will be displayed in console and both plot and loss are saved in save_path.
|
||||
'''
|
||||
if not in_ipynb(): plt.switch_backend('agg')
|
||||
plt.plot(self.iterations[n_skip:-n_skip_end], self.losses[n_skip:-n_skip_end])
|
||||
if not in_ipynb():
|
||||
plt.savefig(os.path.join(self.save_path, 'loss_plot.png'))
|
||||
np.save(os.path.join(self.save_path, 'losses.npy'), self.losses[10:])
|
||||
|
||||
def plot_lr(self):
|
||||
'''Plots learning rate in jupyter notebook or console, depending on the enviroment of the learner.'''
|
||||
if not in_ipynb():
|
||||
plt.switch_backend('agg')
|
||||
if self.record_mom:
|
||||
fig, axs = plt.subplots(1,2,figsize=(12,4))
|
||||
for i in range(0,2): axs[i].set_xlabel('iterations')
|
||||
axs[0].set_ylabel('learning rate')
|
||||
axs[1].set_ylabel('momentum')
|
||||
axs[0].plot(self.iterations,self.lrs)
|
||||
axs[1].plot(self.iterations,self.momentums)
|
||||
else:
|
||||
plt.xlabel("iterations")
|
||||
plt.ylabel("learning rate")
|
||||
plt.plot(self.iterations, self.lrs)
|
||||
if not in_ipynb():
|
||||
plt.savefig(os.path.join(self.save_path, 'lr_plot.png'))
|
||||
|
||||
|
||||
class LR_Updater(LossRecorder):
|
||||
'''
|
||||
Abstract class where all Learning Rate updaters inherit from. (e.g., CirularLR)
|
||||
Calculates and updates new learning rate and momentum at the end of each batch.
|
||||
Have to be extended.
|
||||
'''
|
||||
def on_train_begin(self):
|
||||
super().on_train_begin()
|
||||
self.update_lr()
|
||||
if self.record_mom:
|
||||
self.update_mom()
|
||||
|
||||
def on_batch_end(self, loss):
|
||||
res = super().on_batch_end(loss)
|
||||
self.update_lr()
|
||||
if self.record_mom:
|
||||
self.update_mom()
|
||||
return res
|
||||
|
||||
def update_lr(self):
|
||||
new_lrs = self.calc_lr(self.init_lrs)
|
||||
self.layer_opt.set_lrs(new_lrs)
|
||||
|
||||
def update_mom(self):
|
||||
new_mom = self.calc_mom()
|
||||
self.layer_opt.set_mom(new_mom)
|
||||
|
||||
@abstractmethod
|
||||
def calc_lr(self, init_lrs): raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def calc_mom(self): raise NotImplementedError
|
||||
|
||||
|
||||
class LR_Finder(LR_Updater):
|
||||
'''
|
||||
Helps you find an optimal learning rate for a model, as per suggetion of 2015 CLR paper.
|
||||
Learning rate is increased in linear or log scale, depending on user input, and the result of the loss funciton is retained and can be plotted later.
|
||||
'''
|
||||
def __init__(self, layer_opt, nb, end_lr=10, linear=False, metrics = []):
|
||||
self.linear, self.stop_dv = linear, True
|
||||
ratio = end_lr/layer_opt.lr
|
||||
self.lr_mult = (ratio/nb) if linear else ratio**(1/nb)
|
||||
super().__init__(layer_opt,metrics=metrics)
|
||||
|
||||
def on_train_begin(self):
|
||||
super().on_train_begin()
|
||||
self.best=1e9
|
||||
|
||||
def calc_lr(self, init_lrs):
|
||||
mult = self.lr_mult*self.iteration if self.linear else self.lr_mult**self.iteration
|
||||
return init_lrs * mult
|
||||
|
||||
def on_batch_end(self, metrics):
|
||||
loss = metrics[0] if isinstance(metrics,list) else metrics
|
||||
if self.stop_dv and (math.isnan(loss) or loss>self.best*4):
|
||||
return True
|
||||
if (loss<self.best and self.iteration>10): self.best=loss
|
||||
return super().on_batch_end(metrics)
|
||||
|
||||
def plot(self, n_skip=10, n_skip_end=5):
|
||||
'''
|
||||
Plots the loss function with respect to learning rate, in log scale.
|
||||
'''
|
||||
plt.ylabel("validation loss")
|
||||
plt.xlabel("learning rate (log scale)")
|
||||
plt.plot(self.lrs[n_skip:-(n_skip_end+1)], self.losses[n_skip:-(n_skip_end+1)])
|
||||
plt.xscale('log')
|
||||
plt.savefig(os.path.join(self.save_path, 'lr_loss_plot.png'))
|
||||
|
||||
class LR_Finder2(LR_Finder):
|
||||
"""
|
||||
A variant of lr_find() that helps find the best learning rate. It doesn't do
|
||||
an epoch but a fixed num of iterations (which may be more or less than an epoch
|
||||
depending on your data).
|
||||
"""
|
||||
def __init__(self, layer_opt, nb, end_lr=10, linear=False, metrics=[], stop_dv=True):
|
||||
self.nb, self.metrics = nb, metrics
|
||||
super().__init__(layer_opt, nb, end_lr, linear, metrics)
|
||||
self.stop_dv = stop_dv
|
||||
|
||||
def on_batch_end(self, loss):
|
||||
if self.iteration == self.nb:
|
||||
return True
|
||||
return super().on_batch_end(loss)
|
||||
|
||||
def plot(self, n_skip=10, n_skip_end=5, smoothed=True):
|
||||
if self.metrics is None: self.metrics = []
|
||||
n_plots = len(self.metrics)+2
|
||||
fig, axs = plt.subplots(n_plots,figsize=(6,4*n_plots))
|
||||
for i in range(0,n_plots): axs[i].set_xlabel('learning rate')
|
||||
axs[0].set_ylabel('training loss')
|
||||
axs[1].set_ylabel('validation loss')
|
||||
for i,m in enumerate(self.metrics):
|
||||
axs[i+2].set_ylabel(m.__name__)
|
||||
if len(self.metrics) == 1:
|
||||
values = self.rec_metrics
|
||||
else:
|
||||
values = [rec[i] for rec in self.rec_metrics]
|
||||
if smoothed: values = smooth_curve(values,0.98)
|
||||
axs[i+2].plot(self.lrs[n_skip:-n_skip_end], values[n_skip:-n_skip_end])
|
||||
plt_val_l = smooth_curve(self.val_losses, 0.98) if smoothed else self.val_losses
|
||||
axs[0].plot(self.lrs[n_skip:-n_skip_end],self.losses[n_skip:-n_skip_end])
|
||||
axs[1].plot(self.lrs[n_skip:-n_skip_end],plt_val_l[n_skip:-n_skip_end])
|
||||
|
||||
class CosAnneal(LR_Updater):
|
||||
''' Learning rate scheduler that implements a cosine annealation schedule. '''
|
||||
def __init__(self, layer_opt, nb, on_cycle_end=None, cycle_mult=1):
|
||||
self.nb,self.on_cycle_end,self.cycle_mult = nb,on_cycle_end,cycle_mult
|
||||
super().__init__(layer_opt)
|
||||
|
||||
def on_train_begin(self):
|
||||
self.cycle_iter,self.cycle_count=0,0
|
||||
super().on_train_begin()
|
||||
|
||||
def calc_lr(self, init_lrs):
|
||||
if self.iteration<self.nb/20:
|
||||
self.cycle_iter += 1
|
||||
return init_lrs/100.
|
||||
|
||||
cos_out = np.cos(np.pi*(self.cycle_iter)/self.nb) + 1
|
||||
self.cycle_iter += 1
|
||||
if self.cycle_iter==self.nb:
|
||||
self.cycle_iter = 0
|
||||
self.nb *= self.cycle_mult
|
||||
if self.on_cycle_end: self.on_cycle_end(self, self.cycle_count)
|
||||
self.cycle_count += 1
|
||||
return init_lrs / 2 * cos_out
|
||||
|
||||
|
||||
class CircularLR(LR_Updater):
|
||||
'''
|
||||
A learning rate updater that implements the CircularLearningRate (CLR) scheme.
|
||||
Learning rate is increased then decreased linearly.
|
||||
'''
|
||||
def __init__(self, layer_opt, nb, div=4, cut_div=8, on_cycle_end=None, momentums=None):
|
||||
self.nb,self.div,self.cut_div,self.on_cycle_end = nb,div,cut_div,on_cycle_end
|
||||
if momentums is not None:
|
||||
self.moms = momentums
|
||||
super().__init__(layer_opt, record_mom=(momentums is not None))
|
||||
|
||||
def on_train_begin(self):
|
||||
self.cycle_iter,self.cycle_count=0,0
|
||||
super().on_train_begin()
|
||||
|
||||
def calc_lr(self, init_lrs):
|
||||
cut_pt = self.nb//self.cut_div
|
||||
if self.cycle_iter>cut_pt:
|
||||
pct = 1 - (self.cycle_iter - cut_pt)/(self.nb - cut_pt)
|
||||
else: pct = self.cycle_iter/cut_pt
|
||||
res = init_lrs * (1 + pct*(self.div-1)) / self.div
|
||||
self.cycle_iter += 1
|
||||
if self.cycle_iter==self.nb:
|
||||
self.cycle_iter = 0
|
||||
if self.on_cycle_end: self.on_cycle_end(self, self.cycle_count)
|
||||
self.cycle_count += 1
|
||||
return res
|
||||
|
||||
def calc_mom(self):
|
||||
cut_pt = self.nb//self.cut_div
|
||||
if self.cycle_iter>cut_pt:
|
||||
pct = (self.cycle_iter - cut_pt)/(self.nb - cut_pt)
|
||||
else: pct = 1 - self.cycle_iter/cut_pt
|
||||
res = self.moms[1] + pct * (self.moms[0] - self.moms[1])
|
||||
return res
|
||||
|
||||
class CircularLR_beta(LR_Updater):
|
||||
def __init__(self, layer_opt, nb, div=10, pct=10, on_cycle_end=None, momentums=None):
|
||||
self.nb,self.div,self.pct,self.on_cycle_end = nb,div,pct,on_cycle_end
|
||||
self.cycle_nb = int(nb * (1-pct/100) / 2)
|
||||
if momentums is not None:
|
||||
self.moms = momentums
|
||||
super().__init__(layer_opt, record_mom=(momentums is not None))
|
||||
|
||||
def on_train_begin(self):
|
||||
self.cycle_iter,self.cycle_count=0,0
|
||||
super().on_train_begin()
|
||||
|
||||
def calc_lr(self, init_lrs):
|
||||
if self.cycle_iter>2 * self.cycle_nb:
|
||||
pct = (self.cycle_iter - 2*self.cycle_nb)/(self.nb - 2*self.cycle_nb)
|
||||
res = init_lrs * (1 + (pct * (1-100)/100)) / self.div
|
||||
elif self.cycle_iter>self.cycle_nb:
|
||||
pct = 1 - (self.cycle_iter - self.cycle_nb)/self.cycle_nb
|
||||
res = init_lrs * (1 + pct*(self.div-1)) / self.div
|
||||
else:
|
||||
pct = self.cycle_iter/self.cycle_nb
|
||||
res = init_lrs * (1 + pct*(self.div-1)) / self.div
|
||||
self.cycle_iter += 1
|
||||
if self.cycle_iter==self.nb:
|
||||
self.cycle_iter = 0
|
||||
if self.on_cycle_end: self.on_cycle_end(self, self.cycle_count)
|
||||
self.cycle_count += 1
|
||||
return res
|
||||
|
||||
def calc_mom(self):
|
||||
if self.cycle_iter>2*self.cycle_nb:
|
||||
res = self.moms[0]
|
||||
elif self.cycle_iter>self.cycle_nb:
|
||||
pct = 1 - (self.cycle_iter - self.cycle_nb)/self.cycle_nb
|
||||
res = self.moms[0] + pct * (self.moms[1] - self.moms[0])
|
||||
else:
|
||||
pct = self.cycle_iter/self.cycle_nb
|
||||
res = self.moms[0] + pct * (self.moms[1] - self.moms[0])
|
||||
return res
|
||||
|
||||
|
||||
class SaveBestModel(LossRecorder):
|
||||
|
||||
""" Save weights of the best model based during training.
|
||||
If metrics are provided, the first metric in the list is used to
|
||||
find the best model.
|
||||
If no metrics are provided, the loss is used.
|
||||
|
||||
Args:
|
||||
model: the fastai model
|
||||
lr: indicate to use test images; otherwise use validation images
|
||||
name: the name of filename of the weights without '.h5'
|
||||
|
||||
Usage:
|
||||
Briefly, you have your model 'learn' variable and call fit.
|
||||
>>> learn.fit(lr, 2, cycle_len=2, cycle_mult=1, best_save_name='mybestmodel')
|
||||
....
|
||||
>>> learn.load('mybestmodel')
|
||||
|
||||
For more details see http://forums.fast.ai/t/a-code-snippet-to-save-the-best-model-during-training/12066
|
||||
|
||||
"""
|
||||
def __init__(self, model, layer_opt, metrics, name='best_model'):
|
||||
super().__init__(layer_opt)
|
||||
self.name = name
|
||||
self.model = model
|
||||
self.best_loss = None
|
||||
self.best_acc = None
|
||||
self.save_method = self.save_when_only_loss if metrics==None else self.save_when_acc
|
||||
|
||||
def save_when_only_loss(self, metrics):
|
||||
loss = metrics[0]
|
||||
if self.best_loss == None or loss < self.best_loss:
|
||||
self.best_loss = loss
|
||||
self.model.save(f'{self.name}')
|
||||
|
||||
def save_when_acc(self, metrics):
|
||||
loss, acc = metrics[0], metrics[1]
|
||||
if self.best_acc == None or acc > self.best_acc:
|
||||
self.best_acc = acc
|
||||
self.best_loss = loss
|
||||
self.model.save(f'{self.name}')
|
||||
elif acc == self.best_acc and loss < self.best_loss:
|
||||
self.best_loss = loss
|
||||
self.model.save(f'{self.name}')
|
||||
|
||||
def on_epoch_end(self, metrics):
|
||||
super().on_epoch_end(metrics)
|
||||
self.save_method(metrics)
|
||||
|
||||
|
||||
class WeightDecaySchedule(Callback):
|
||||
def __init__(self, layer_opt, batch_per_epoch, cycle_len, cycle_mult, n_cycles, norm_wds=False, wds_sched_mult=None):
|
||||
"""
|
||||
Implements the weight decay schedule as mentioned in https://arxiv.org/abs/1711.05101
|
||||
|
||||
:param layer_opt: The LayerOptimizer
|
||||
:param batch_per_epoch: Num batches in 1 epoch
|
||||
:param cycle_len: Num epochs in initial cycle. Subsequent cycle_len = previous cycle_len * cycle_mult
|
||||
:param cycle_mult: Cycle multiplier
|
||||
:param n_cycles: Number of cycles to be executed
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
self.layer_opt = layer_opt
|
||||
self.batch_per_epoch = batch_per_epoch
|
||||
self.init_wds = np.array(layer_opt.wds) # Weights as set by user
|
||||
self.init_lrs = np.array(layer_opt.lrs) # Learning rates as set by user
|
||||
self.new_wds = None # Holds the new weight decay factors, calculated in on_batch_begin()
|
||||
self.iteration = 0
|
||||
self.epoch = 0
|
||||
self.wds_sched_mult = wds_sched_mult
|
||||
self.norm_wds = norm_wds
|
||||
self.wds_history = list()
|
||||
|
||||
# Pre calculating the number of epochs in the cycle of current running epoch
|
||||
self.epoch_to_num_cycles, i = dict(), 0
|
||||
for cycle in range(n_cycles):
|
||||
for _ in range(cycle_len):
|
||||
self.epoch_to_num_cycles[i] = cycle_len
|
||||
i += 1
|
||||
cycle_len *= cycle_mult
|
||||
|
||||
def on_train_begin(self):
|
||||
self.iteration = 0
|
||||
self.epoch = 0
|
||||
|
||||
def on_batch_begin(self):
|
||||
# Prepare for decay of weights
|
||||
|
||||
# Default weight decay (as provided by user)
|
||||
wdn = self.init_wds
|
||||
|
||||
# Weight decay multiplier (The 'eta' in the paper). Optional.
|
||||
wdm = 1.0
|
||||
if self.wds_sched_mult is not None:
|
||||
wdm = self.wds_sched_mult(self)
|
||||
|
||||
# Weight decay normalized. Optional.
|
||||
if self.norm_wds:
|
||||
wdn = wdn / np.sqrt(self.batch_per_epoch * self.epoch_to_num_cycles[self.epoch])
|
||||
|
||||
# Final wds
|
||||
self.new_wds = wdm * wdn
|
||||
|
||||
# Set weight_decay with zeros so that it is not applied in Adam, we will apply it outside in on_batch_end()
|
||||
self.layer_opt.set_wds_out(self.new_wds)
|
||||
# We have to save the existing weights before the optimizer changes the values
|
||||
self.iteration += 1
|
||||
|
||||
def on_epoch_end(self, metrics):
|
||||
self.epoch += 1
|
||||
|
||||
class DecayType(IntEnum):
|
||||
''' Data class, each decay type is assigned a number. '''
|
||||
NO = 1
|
||||
LINEAR = 2
|
||||
COSINE = 3
|
||||
EXPONENTIAL = 4
|
||||
POLYNOMIAL = 5
|
||||
|
||||
class DecayScheduler():
|
||||
'''Given initial and endvalue, this class generates the next value depending on decay type and number of iterations. (by calling next_val().) '''
|
||||
|
||||
def __init__(self, dec_type, num_it, start_val, end_val=None, extra=None):
|
||||
self.dec_type, self.nb, self.start_val, self.end_val, self.extra = dec_type, num_it, start_val, end_val, extra
|
||||
self.it = 0
|
||||
if self.end_val is None and not (self.dec_type in [1,4]): self.end_val = 0
|
||||
|
||||
def next_val(self):
|
||||
self.it += 1
|
||||
if self.dec_type == DecayType.NO:
|
||||
return self.start_val
|
||||
elif self.dec_type == DecayType.LINEAR:
|
||||
pct = self.it/self.nb
|
||||
return self.start_val + pct * (self.end_val-self.start_val)
|
||||
elif self.dec_type == DecayType.COSINE:
|
||||
cos_out = np.cos(np.pi*(self.it)/self.nb) + 1
|
||||
return self.end_val + (self.start_val-self.end_val) / 2 * cos_out
|
||||
elif self.dec_type == DecayType.EXPONENTIAL:
|
||||
ratio = self.end_val / self.start_val
|
||||
return self.start_val * (ratio ** (self.it/self.nb))
|
||||
elif self.dec_type == DecayType.POLYNOMIAL:
|
||||
return self.end_val + (self.start_val-self.end_val) * (1 - self.it/self.nb)**self.extra
|
||||
|
||||
|
||||
class TrainingPhase():
|
||||
'''
|
||||
Object with training information for each phase, when multiple phases are involved during training.
|
||||
Used in fit_opt_sched in learner.py
|
||||
'''
|
||||
def __init__(self, epochs=1, opt_fn=optim.SGD, lr=1e-2, lr_decay=DecayType.NO, momentum=0.9,
|
||||
momentum_decay=DecayType.NO, beta=None, wds=None, wd_loss=True):
|
||||
"""
|
||||
Creates an object containing all the relevant informations for one part of a model training.
|
||||
|
||||
Args
|
||||
epochs: number of epochs to train like this
|
||||
opt_fn: an optimizer (example optim.Adam)
|
||||
lr: one learning rate or a tuple of the form (start_lr,end_lr)
|
||||
each of those can be a list/numpy array for differential learning rates
|
||||
lr_decay: a DecayType object specifying how the learning rate should change
|
||||
momentum: one momentum (or beta1 in case of Adam), or a tuple of the form (start_mom,end_mom)
|
||||
momentum_decay: a DecayType object specifying how the momentum should change
|
||||
beta: beta2 parameter of Adam or alpha parameter of RMSProp
|
||||
wds: weight decay (can be an array for differential wds)
|
||||
"""
|
||||
self.epochs, self.opt_fn, self.lr, self.momentum, self.beta, self.wds = epochs, opt_fn, lr, momentum, beta, wds
|
||||
if isinstance(lr_decay,tuple): self.lr_decay, self.extra_lr = lr_decay
|
||||
else: self.lr_decay, self.extra_lr = lr_decay, None
|
||||
if isinstance(momentum_decay,tuple): self.mom_decay, self.extra_mom = momentum_decay
|
||||
else: self.mom_decay, self.extra_mom = momentum_decay, None
|
||||
self.wd_loss = wd_loss
|
||||
|
||||
def phase_begin(self, layer_opt, nb_batches):
|
||||
self.layer_opt = layer_opt
|
||||
if isinstance(self.lr, tuple): start_lr,end_lr = self.lr
|
||||
else: start_lr, end_lr = self.lr, None
|
||||
self.lr_sched = DecayScheduler(self.lr_decay, nb_batches * self.epochs, start_lr, end_lr, extra=self.extra_lr)
|
||||
if isinstance(self.momentum, tuple): start_mom,end_mom = self.momentum
|
||||
else: start_mom, end_mom = self.momentum, None
|
||||
self.mom_sched = DecayScheduler(self.mom_decay, nb_batches * self.epochs, start_mom, end_mom, extra=self.extra_mom)
|
||||
self.layer_opt.set_opt_fn(self.opt_fn)
|
||||
self.layer_opt.set_lrs(start_lr)
|
||||
self.layer_opt.set_mom(start_mom)
|
||||
if self.beta is not None: self.layer_opt.set_beta(self.beta)
|
||||
if self.wds is not None:
|
||||
if not isinstance(self.wds, Iterable): self.wds=[self.wds]
|
||||
if len(self.wds)==1: self.wds=self.wds*len(self.layer_opt.layer_groups)
|
||||
if self.wd_loss: self.layer_opt.set_wds(self.wds)
|
||||
else: self.layer_opt.set_wds_out(self.wds)
|
||||
|
||||
def update(self):
|
||||
new_lr, new_mom = self.lr_sched.next_val(), self.mom_sched.next_val()
|
||||
self.layer_opt.set_lrs(new_lr)
|
||||
self.layer_opt.set_mom(new_mom)
|
||||
|
||||
|
||||
class OptimScheduler(LossRecorder):
|
||||
'''Learning rate Scheduler for training involving multiple phases.'''
|
||||
|
||||
def __init__(self, layer_opt, phases, nb_batches, stop_div = False):
|
||||
self.phases, self.nb_batches, self.stop_div = phases, nb_batches, stop_div
|
||||
super().__init__(layer_opt, record_mom=True)
|
||||
|
||||
def on_train_begin(self):
|
||||
super().on_train_begin()
|
||||
self.phase,self.best=0,1e9
|
||||
|
||||
def on_batch_end(self, metrics):
|
||||
loss = metrics[0] if isinstance(metrics,list) else metrics
|
||||
if self.stop_div and (math.isnan(loss) or loss>self.best*4):
|
||||
return True
|
||||
if (loss<self.best and self.iteration>10): self.best=loss
|
||||
super().on_batch_end(metrics)
|
||||
self.phases[self.phase].update()
|
||||
|
||||
def on_phase_begin(self):
|
||||
self.phases[self.phase].phase_begin(self.layer_opt, self.nb_batches[self.phase])
|
||||
|
||||
def on_phase_end(self):
|
||||
self.phase += 1
|
||||
|
||||
def plot_lr(self, show_text=True, show_moms=True):
|
||||
"""Plots the lr rate/momentum schedule"""
|
||||
phase_limits = [0]
|
||||
for nb_batch, phase in zip(self.nb_batches, self.phases):
|
||||
phase_limits.append(phase_limits[-1] + nb_batch * phase.epochs)
|
||||
if not in_ipynb():
|
||||
plt.switch_backend('agg')
|
||||
np_plts = 2 if show_moms else 1
|
||||
fig, axs = plt.subplots(1,np_plts,figsize=(6*np_plts,4))
|
||||
if not show_moms: axs = [axs]
|
||||
for i in range(np_plts): axs[i].set_xlabel('iterations')
|
||||
axs[0].set_ylabel('learning rate')
|
||||
axs[0].plot(self.iterations,self.lrs)
|
||||
if show_moms:
|
||||
axs[1].set_ylabel('momentum')
|
||||
axs[1].plot(self.iterations,self.momentums)
|
||||
if show_text:
|
||||
for i, phase in enumerate(self.phases):
|
||||
text = phase.opt_fn.__name__
|
||||
if phase.wds is not None: text+='\nwds='+str(phase.wds)
|
||||
if phase.beta is not None: text+='\nbeta='+str(phase.beta)
|
||||
for k in range(np_plts):
|
||||
if i < len(self.phases)-1:
|
||||
draw_line(axs[k], phase_limits[i+1])
|
||||
draw_text(axs[k], (phase_limits[i]+phase_limits[i+1])/2, text)
|
||||
if not in_ipynb():
|
||||
plt.savefig(os.path.join(self.save_path, 'lr_plot.png'))
|
||||
|
||||
def plot(self, n_skip=10, n_skip_end=5, linear=None):
|
||||
if linear is None: linear = self.phases[-1].lr_decay == DecayType.LINEAR
|
||||
plt.ylabel("loss")
|
||||
plt.plot(self.lrs[n_skip:-n_skip_end], self.losses[n_skip:-n_skip_end])
|
||||
if linear: plt.xlabel("learning rate")
|
||||
else:
|
||||
plt.xlabel("learning rate (log scale)")
|
||||
plt.xscale('log')
|
||||
|
||||
def draw_line(ax,x):
|
||||
xmin, xmax, ymin, ymax = ax.axis()
|
||||
ax.plot([x,x],[ymin,ymax], color='red', linestyle='dashed')
|
||||
|
||||
def draw_text(ax,x, text):
|
||||
xmin, xmax, ymin, ymax = ax.axis()
|
||||
ax.text(x,(ymin+ymax)/2,text, horizontalalignment='center', verticalalignment='center', fontsize=14, alpha=0.5)
|
||||
|
||||
def smooth_curve(vals, beta):
|
||||
avg_val = 0
|
||||
smoothed = []
|
||||
for (i,v) in enumerate(vals):
|
||||
avg_val = beta * avg_val + (1-beta) * v
|
||||
smoothed.append(avg_val/(1-beta**(i+1)))
|
||||
return smoothed
|
||||
@@ -0,0 +1,475 @@
|
||||
from .imports import *
|
||||
|
||||
from sklearn_pandas import DataFrameMapper
|
||||
from sklearn.preprocessing import LabelEncoder, Imputer, StandardScaler
|
||||
from pandas.api.types import is_string_dtype, is_numeric_dtype
|
||||
from sklearn.ensemble import forest
|
||||
from sklearn.tree import export_graphviz
|
||||
|
||||
|
||||
def set_plot_sizes(sml, med, big):
|
||||
plt.rc('font', size=sml) # controls default text sizes
|
||||
plt.rc('axes', titlesize=sml) # fontsize of the axes title
|
||||
plt.rc('axes', labelsize=med) # fontsize of the x and y labels
|
||||
plt.rc('xtick', labelsize=sml) # fontsize of the tick labels
|
||||
plt.rc('ytick', labelsize=sml) # fontsize of the tick labels
|
||||
plt.rc('legend', fontsize=sml) # legend fontsize
|
||||
plt.rc('figure', titlesize=big) # fontsize of the figure title
|
||||
|
||||
def parallel_trees(m, fn, n_jobs=8):
|
||||
return list(ProcessPoolExecutor(n_jobs).map(fn, m.estimators_))
|
||||
|
||||
def draw_tree(t, df, size=10, ratio=0.6, precision=0):
|
||||
""" Draws a representation of a random forest in IPython.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
t: The tree you wish to draw
|
||||
df: The data used to train the tree. This is used to get the names of the features.
|
||||
"""
|
||||
s=export_graphviz(t, out_file=None, feature_names=df.columns, filled=True,
|
||||
special_characters=True, rotate=True, precision=precision)
|
||||
IPython.display.display(graphviz.Source(re.sub('Tree {',
|
||||
f'Tree {{ size={size}; ratio={ratio}', s)))
|
||||
|
||||
def combine_date(years, months=1, days=1, weeks=None, hours=None, minutes=None,
|
||||
seconds=None, milliseconds=None, microseconds=None, nanoseconds=None):
|
||||
years = np.asarray(years) - 1970
|
||||
months = np.asarray(months) - 1
|
||||
days = np.asarray(days) - 1
|
||||
types = ('<M8[Y]', '<m8[M]', '<m8[D]', '<m8[W]', '<m8[h]',
|
||||
'<m8[m]', '<m8[s]', '<m8[ms]', '<m8[us]', '<m8[ns]')
|
||||
vals = (years, months, days, weeks, hours, minutes, seconds,
|
||||
milliseconds, microseconds, nanoseconds)
|
||||
return sum(np.asarray(v, dtype=t) for t, v in zip(types, vals)
|
||||
if v is not None)
|
||||
|
||||
def get_sample(df,n):
|
||||
""" Gets a random sample of n rows from df, without replacement.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df: A pandas data frame, that you wish to sample from.
|
||||
n: The number of rows you wish to sample.
|
||||
|
||||
Returns:
|
||||
--------
|
||||
return value: A random sample of n rows of df.
|
||||
|
||||
Examples:
|
||||
---------
|
||||
>>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})
|
||||
>>> df
|
||||
col1 col2
|
||||
0 1 a
|
||||
1 2 b
|
||||
2 3 a
|
||||
|
||||
>>> get_sample(df, 2)
|
||||
col1 col2
|
||||
1 2 b
|
||||
2 3 a
|
||||
"""
|
||||
idxs = sorted(np.random.permutation(len(df))[:n])
|
||||
return df.iloc[idxs].copy()
|
||||
|
||||
def add_datepart(df, fldname, drop=True, time=False):
|
||||
"""add_datepart converts a column of df from a datetime64 to many columns containing
|
||||
the information from the date. This applies changes inplace.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df: A pandas data frame. df gain several new columns.
|
||||
fldname: A string that is the name of the date column you wish to expand.
|
||||
If it is not a datetime64 series, it will be converted to one with pd.to_datetime.
|
||||
drop: If true then the original date column will be removed.
|
||||
time: If true time features: Hour, Minute, Second will be added.
|
||||
|
||||
Examples:
|
||||
---------
|
||||
|
||||
>>> df = pd.DataFrame({ 'A' : pd.to_datetime(['3/11/2000', '3/12/2000', '3/13/2000'], infer_datetime_format=False) })
|
||||
>>> df
|
||||
|
||||
A
|
||||
0 2000-03-11
|
||||
1 2000-03-12
|
||||
2 2000-03-13
|
||||
|
||||
>>> add_datepart(df, 'A')
|
||||
>>> df
|
||||
|
||||
AYear AMonth AWeek ADay ADayofweek ADayofyear AIs_month_end AIs_month_start AIs_quarter_end AIs_quarter_start AIs_year_end AIs_year_start AElapsed
|
||||
0 2000 3 10 11 5 71 False False False False False False 952732800
|
||||
1 2000 3 10 12 6 72 False False False False False False 952819200
|
||||
2 2000 3 11 13 0 73 False False False False False False 952905600
|
||||
"""
|
||||
fld = df[fldname]
|
||||
fld_dtype = fld.dtype
|
||||
if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):
|
||||
fld_dtype = np.datetime64
|
||||
|
||||
if not np.issubdtype(fld_dtype, np.datetime64):
|
||||
df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)
|
||||
targ_pre = re.sub('[Dd]ate$', '', fldname)
|
||||
attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',
|
||||
'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']
|
||||
if time: attr = attr + ['Hour', 'Minute', 'Second']
|
||||
for n in attr: df[targ_pre + n] = getattr(fld.dt, n.lower())
|
||||
df[targ_pre + 'Elapsed'] = fld.astype(np.int64) // 10 ** 9
|
||||
if drop: df.drop(fldname, axis=1, inplace=True)
|
||||
|
||||
def is_date(x): return np.issubdtype(x.dtype, np.datetime64)
|
||||
|
||||
def train_cats(df):
|
||||
"""Change any columns of strings in a panda's dataframe to a column of
|
||||
catagorical values. This applies the changes inplace.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df: A pandas dataframe. Any columns of strings will be changed to
|
||||
categorical values.
|
||||
|
||||
Examples:
|
||||
---------
|
||||
|
||||
>>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})
|
||||
>>> df
|
||||
col1 col2
|
||||
0 1 a
|
||||
1 2 b
|
||||
2 3 a
|
||||
|
||||
note the type of col2 is string
|
||||
|
||||
>>> train_cats(df)
|
||||
>>> df
|
||||
|
||||
col1 col2
|
||||
0 1 a
|
||||
1 2 b
|
||||
2 3 a
|
||||
|
||||
now the type of col2 is category
|
||||
"""
|
||||
for n,c in df.items():
|
||||
if is_string_dtype(c): df[n] = c.astype('category').cat.as_ordered()
|
||||
|
||||
def apply_cats(df, trn):
|
||||
"""Changes any columns of strings in df into categorical variables using trn as
|
||||
a template for the category codes.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df: A pandas dataframe. Any columns of strings will be changed to
|
||||
categorical values. The category codes are determined by trn.
|
||||
|
||||
trn: A pandas dataframe. When creating a category for df, it looks up the
|
||||
what the category's code were in trn and makes those the category codes
|
||||
for df.
|
||||
|
||||
Examples:
|
||||
---------
|
||||
>>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})
|
||||
>>> df
|
||||
col1 col2
|
||||
0 1 a
|
||||
1 2 b
|
||||
2 3 a
|
||||
|
||||
note the type of col2 is string
|
||||
|
||||
>>> train_cats(df)
|
||||
>>> df
|
||||
|
||||
col1 col2
|
||||
0 1 a
|
||||
1 2 b
|
||||
2 3 a
|
||||
|
||||
now the type of col2 is category {a : 1, b : 2}
|
||||
|
||||
>>> df2 = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['b', 'a', 'a']})
|
||||
>>> apply_cats(df2, df)
|
||||
|
||||
col1 col2
|
||||
0 1 b
|
||||
1 2 a
|
||||
2 3 a
|
||||
|
||||
now the type of col is category {a : 1, b : 2}
|
||||
"""
|
||||
for n,c in df.items():
|
||||
if (n in trn.columns) and (trn[n].dtype.name=='category'):
|
||||
df[n] = pd.Categorical(c, categories=trn[n].cat.categories, ordered=True)
|
||||
|
||||
def fix_missing(df, col, name, na_dict):
|
||||
""" Fill missing data in a column of df with the median, and add a {name}_na column
|
||||
which specifies if the data was missing.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df: The data frame that will be changed.
|
||||
|
||||
col: The column of data to fix by filling in missing data.
|
||||
|
||||
name: The name of the new filled column in df.
|
||||
|
||||
na_dict: A dictionary of values to create na's of and the value to insert. If
|
||||
name is not a key of na_dict the median will fill any missing data. Also
|
||||
if name is not a key of na_dict and there is no missing data in col, then
|
||||
no {name}_na column is not created.
|
||||
|
||||
|
||||
Examples:
|
||||
---------
|
||||
>>> df = pd.DataFrame({'col1' : [1, np.NaN, 3], 'col2' : [5, 2, 2]})
|
||||
>>> df
|
||||
col1 col2
|
||||
0 1 5
|
||||
1 nan 2
|
||||
2 3 2
|
||||
|
||||
>>> fix_missing(df, df['col1'], 'col1', {})
|
||||
>>> df
|
||||
col1 col2 col1_na
|
||||
0 1 5 False
|
||||
1 2 2 True
|
||||
2 3 2 False
|
||||
|
||||
|
||||
>>> df = pd.DataFrame({'col1' : [1, np.NaN, 3], 'col2' : [5, 2, 2]})
|
||||
>>> df
|
||||
col1 col2
|
||||
0 1 5
|
||||
1 nan 2
|
||||
2 3 2
|
||||
|
||||
>>> fix_missing(df, df['col2'], 'col2', {})
|
||||
>>> df
|
||||
col1 col2
|
||||
0 1 5
|
||||
1 nan 2
|
||||
2 3 2
|
||||
|
||||
|
||||
>>> df = pd.DataFrame({'col1' : [1, np.NaN, 3], 'col2' : [5, 2, 2]})
|
||||
>>> df
|
||||
col1 col2
|
||||
0 1 5
|
||||
1 nan 2
|
||||
2 3 2
|
||||
|
||||
>>> fix_missing(df, df['col1'], 'col1', {'col1' : 500})
|
||||
>>> df
|
||||
col1 col2 col1_na
|
||||
0 1 5 False
|
||||
1 500 2 True
|
||||
2 3 2 False
|
||||
"""
|
||||
if is_numeric_dtype(col):
|
||||
if pd.isnull(col).sum() or (name in na_dict):
|
||||
df[name+'_na'] = pd.isnull(col)
|
||||
filler = na_dict[name] if name in na_dict else col.median()
|
||||
df[name] = col.fillna(filler)
|
||||
na_dict[name] = filler
|
||||
return na_dict
|
||||
|
||||
def numericalize(df, col, name, max_n_cat):
|
||||
""" Changes the column col from a categorical type to it's integer codes.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df: A pandas dataframe. df[name] will be filled with the integer codes from
|
||||
col.
|
||||
|
||||
col: The column you wish to change into the categories.
|
||||
name: The column name you wish to insert into df. This column will hold the
|
||||
integer codes.
|
||||
|
||||
max_n_cat: If col has more categories than max_n_cat it will not change the
|
||||
it to its integer codes. If max_n_cat is None, then col will always be
|
||||
converted.
|
||||
|
||||
Examples:
|
||||
---------
|
||||
>>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})
|
||||
>>> df
|
||||
col1 col2
|
||||
0 1 a
|
||||
1 2 b
|
||||
2 3 a
|
||||
|
||||
note the type of col2 is string
|
||||
|
||||
>>> train_cats(df)
|
||||
>>> df
|
||||
|
||||
col1 col2
|
||||
0 1 a
|
||||
1 2 b
|
||||
2 3 a
|
||||
|
||||
now the type of col2 is category { a : 1, b : 2}
|
||||
|
||||
>>> numericalize(df, df['col2'], 'col3', None)
|
||||
|
||||
col1 col2 col3
|
||||
0 1 a 1
|
||||
1 2 b 2
|
||||
2 3 a 1
|
||||
"""
|
||||
if not is_numeric_dtype(col) and ( max_n_cat is None or col.nunique()>max_n_cat):
|
||||
df[name] = col.cat.codes+1
|
||||
|
||||
def scale_vars(df, mapper):
|
||||
warnings.filterwarnings('ignore', category=sklearn.exceptions.DataConversionWarning)
|
||||
if mapper is None:
|
||||
map_f = [([n],StandardScaler()) for n in df.columns if is_numeric_dtype(df[n])]
|
||||
mapper = DataFrameMapper(map_f).fit(df)
|
||||
df[mapper.transformed_names_] = mapper.transform(df)
|
||||
return mapper
|
||||
|
||||
def proc_df(df, y_fld=None, skip_flds=None, ignore_flds=None, do_scale=False, na_dict=None,
|
||||
preproc_fn=None, max_n_cat=None, subset=None, mapper=None):
|
||||
""" proc_df takes a data frame df and splits off the response variable, and
|
||||
changes the df into an entirely numeric dataframe.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df: The data frame you wish to process.
|
||||
|
||||
y_fld: The name of the response variable
|
||||
|
||||
skip_flds: A list of fields that dropped from df.
|
||||
|
||||
ignore_flds: A list of fields that are ignored during processing.
|
||||
|
||||
do_scale: Standardizes each column in df. Takes Boolean Values(True,False)
|
||||
|
||||
na_dict: a dictionary of na columns to add. Na columns are also added if there
|
||||
are any missing values.
|
||||
|
||||
preproc_fn: A function that gets applied to df.
|
||||
|
||||
max_n_cat: The maximum number of categories to break into dummy values, instead
|
||||
of integer codes.
|
||||
|
||||
subset: Takes a random subset of size subset from df.
|
||||
|
||||
mapper: If do_scale is set as True, the mapper variable
|
||||
calculates the values used for scaling of variables during training time (mean and standard deviation).
|
||||
|
||||
Returns:
|
||||
--------
|
||||
[x, y, nas, mapper(optional)]:
|
||||
|
||||
x: x is the transformed version of df. x will not have the response variable
|
||||
and is entirely numeric.
|
||||
|
||||
y: y is the response variable
|
||||
|
||||
nas: returns a dictionary of which nas it created, and the associated median.
|
||||
|
||||
mapper: A DataFrameMapper which stores the mean and standard deviation of the corresponding continuous
|
||||
variables which is then used for scaling of during test-time.
|
||||
|
||||
Examples:
|
||||
---------
|
||||
>>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})
|
||||
>>> df
|
||||
col1 col2
|
||||
0 1 a
|
||||
1 2 b
|
||||
2 3 a
|
||||
|
||||
note the type of col2 is string
|
||||
|
||||
>>> train_cats(df)
|
||||
>>> df
|
||||
|
||||
col1 col2
|
||||
0 1 a
|
||||
1 2 b
|
||||
2 3 a
|
||||
|
||||
now the type of col2 is category { a : 1, b : 2}
|
||||
|
||||
>>> x, y, nas = proc_df(df, 'col1')
|
||||
>>> x
|
||||
|
||||
col2
|
||||
0 1
|
||||
1 2
|
||||
2 1
|
||||
|
||||
>>> data = DataFrame(pet=["cat", "dog", "dog", "fish", "cat", "dog", "cat", "fish"],
|
||||
children=[4., 6, 3, 3, 2, 3, 5, 4],
|
||||
salary=[90, 24, 44, 27, 32, 59, 36, 27])
|
||||
|
||||
>>> mapper = DataFrameMapper([(:pet, LabelBinarizer()),
|
||||
([:children], StandardScaler())])
|
||||
|
||||
>>>round(fit_transform!(mapper, copy(data)), 2)
|
||||
|
||||
8x4 Array{Float64,2}:
|
||||
1.0 0.0 0.0 0.21
|
||||
0.0 1.0 0.0 1.88
|
||||
0.0 1.0 0.0 -0.63
|
||||
0.0 0.0 1.0 -0.63
|
||||
1.0 0.0 0.0 -1.46
|
||||
0.0 1.0 0.0 -0.63
|
||||
1.0 0.0 0.0 1.04
|
||||
0.0 0.0 1.0 0.21
|
||||
"""
|
||||
if not ignore_flds: ignore_flds=[]
|
||||
if not skip_flds: skip_flds=[]
|
||||
if subset: df = get_sample(df,subset)
|
||||
else: df = df.copy()
|
||||
ignored_flds = df.loc[:, ignore_flds]
|
||||
df.drop(ignore_flds, axis=1, inplace=True)
|
||||
if preproc_fn: preproc_fn(df)
|
||||
if y_fld is None: y = None
|
||||
else:
|
||||
if not is_numeric_dtype(df[y_fld]): df[y_fld] = df[y_fld].cat.codes
|
||||
y = df[y_fld].values
|
||||
skip_flds += [y_fld]
|
||||
df.drop(skip_flds, axis=1, inplace=True)
|
||||
|
||||
if na_dict is None: na_dict = {}
|
||||
for n,c in df.items(): na_dict = fix_missing(df, c, n, na_dict)
|
||||
if do_scale: mapper = scale_vars(df, mapper)
|
||||
for n,c in df.items(): numericalize(df, c, n, max_n_cat)
|
||||
df = pd.get_dummies(df, dummy_na=True)
|
||||
df = pd.concat([ignored_flds, df], axis=1)
|
||||
res = [df, y, na_dict]
|
||||
if do_scale: res = res + [mapper]
|
||||
return res
|
||||
|
||||
def rf_feat_importance(m, df):
|
||||
return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}
|
||||
).sort_values('imp', ascending=False)
|
||||
|
||||
def set_rf_samples(n):
|
||||
""" Changes Scikit learn's random forests to give each tree a random sample of
|
||||
n random rows.
|
||||
"""
|
||||
forest._generate_sample_indices = (lambda rs, n_samples:
|
||||
forest.check_random_state(rs).randint(0, n_samples, n))
|
||||
|
||||
def reset_rf_samples():
|
||||
""" Undoes the changes produced by set_rf_samples.
|
||||
"""
|
||||
forest._generate_sample_indices = (lambda rs, n_samples:
|
||||
forest.check_random_state(rs).randint(0, n_samples, n_samples))
|
||||
|
||||
def get_nn_mappers(df, cat_vars, contin_vars):
|
||||
# Replace nulls with 0 for continuous, "" for categorical.
|
||||
for v in contin_vars: df[v] = df[v].fillna(df[v].max()+100,)
|
||||
for v in cat_vars: df[v].fillna('#NA#', inplace=True)
|
||||
|
||||
# list of tuples, containing variable and instance of a transformer for that variable
|
||||
# for categoricals, use LabelEncoder to map to integers. For continuous, standardize
|
||||
cat_maps = [(o, LabelEncoder()) for o in cat_vars]
|
||||
contin_maps = [([o], StandardScaler()) for o in contin_vars]
|
||||
return DataFrameMapper(cat_maps).fit(df), DataFrameMapper(contin_maps).fit(df)
|
||||
@@ -0,0 +1,83 @@
|
||||
"""
|
||||
From the paper:
|
||||
Averaging Weights Leads to Wider Optima and Better Generalization
|
||||
Pavel Izmailov, Dmitrii Podoprikhin, Timur Garipov, Dmitry Vetrov, Andrew Gordon Wilson
|
||||
https://arxiv.org/abs/1803.05407
|
||||
2018
|
||||
|
||||
Author's implementation: https://github.com/timgaripov/swa
|
||||
"""
|
||||
|
||||
import torch
|
||||
from .sgdr import *
|
||||
from .core import *
|
||||
|
||||
|
||||
class SWA(Callback):
|
||||
def __init__(self, model, swa_model, swa_start):
|
||||
super().__init__()
|
||||
self.model,self.swa_model,self.swa_start=model,swa_model,swa_start
|
||||
|
||||
def on_train_begin(self):
|
||||
self.epoch = 0
|
||||
self.swa_n = 0
|
||||
|
||||
def on_epoch_end(self, metrics):
|
||||
if (self.epoch + 1) >= self.swa_start:
|
||||
self.update_average_model()
|
||||
self.swa_n += 1
|
||||
|
||||
self.epoch += 1
|
||||
|
||||
def update_average_model(self):
|
||||
# update running average of parameters
|
||||
model_params = self.model.parameters()
|
||||
swa_params = self.swa_model.parameters()
|
||||
for model_param, swa_param in zip(model_params, swa_params):
|
||||
swa_param.data *= self.swa_n
|
||||
swa_param.data += model_param.data
|
||||
swa_param.data /= (self.swa_n + 1)
|
||||
|
||||
def collect_bn_modules(module, bn_modules):
|
||||
if isinstance(module, torch.nn.modules.batchnorm._BatchNorm):
|
||||
bn_modules.append(module)
|
||||
|
||||
def fix_batchnorm(swa_model, train_dl):
|
||||
"""
|
||||
During training, batch norm layers keep track of a running mean and
|
||||
variance of the previous layer's activations. Because the parameters
|
||||
of the SWA model are computed as the average of other models' parameters,
|
||||
the SWA model never sees the training data itself, and therefore has no
|
||||
opportunity to compute the correct batch norm statistics. Before performing
|
||||
inference with the SWA model, we perform a single pass over the training data
|
||||
to calculate an accurate running mean and variance for each batch norm layer.
|
||||
"""
|
||||
bn_modules = []
|
||||
swa_model.apply(lambda module: collect_bn_modules(module, bn_modules))
|
||||
|
||||
if not bn_modules: return
|
||||
|
||||
swa_model.train()
|
||||
|
||||
for module in bn_modules:
|
||||
module.running_mean = torch.zeros_like(module.running_mean)
|
||||
module.running_var = torch.ones_like(module.running_var)
|
||||
|
||||
momenta = [m.momentum for m in bn_modules]
|
||||
|
||||
inputs_seen = 0
|
||||
|
||||
for (*x,y) in iter(train_dl):
|
||||
xs = V(x)
|
||||
batch_size = xs[0].size(0)
|
||||
|
||||
momentum = batch_size / (inputs_seen + batch_size)
|
||||
for module in bn_modules:
|
||||
module.momentum = momentum
|
||||
|
||||
res = swa_model(*xs)
|
||||
|
||||
inputs_seen += batch_size
|
||||
|
||||
for module, momentum in zip(bn_modules, momenta):
|
||||
module.momentum = momentum
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
from .core import *
|
||||
from .learner import *
|
||||
from .lm_rnn import *
|
||||
from torch.utils.data.sampler import Sampler
|
||||
import spacy
|
||||
from spacy.symbols import ORTH
|
||||
|
||||
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
|
||||
def tokenize(s): return re_tok.sub(r' \1 ', s).split()
|
||||
|
||||
def texts_labels_from_folders(path, folders):
|
||||
texts,labels = [],[]
|
||||
for idx,label in enumerate(folders):
|
||||
for fname in glob(os.path.join(path, label, '*.*')):
|
||||
texts.append(open(fname, 'r').read())
|
||||
labels.append(idx)
|
||||
return texts, np.array(labels).astype(np.int64)
|
||||
|
||||
def numericalize_tok(tokens, max_vocab=50000, min_freq=0, unk_tok="_unk_", pad_tok="_pad_", bos_tok="_bos_", eos_tok="_eos_"):
|
||||
"""Takes in text tokens and returns int2tok and tok2int converters
|
||||
|
||||
Arguments:
|
||||
tokens(list): List of tokens. Can be a list of strings, or a list of lists of strings.
|
||||
max_vocab(int): Number of tokens to return in the vocab (sorted by frequency)
|
||||
min_freq(int): Minimum number of instances a token must be present in order to be preserved.
|
||||
unk_tok(str): Token to use when unknown tokens are encountered in the source text.
|
||||
pad_tok(str): Token to use when padding sequences.
|
||||
"""
|
||||
if isinstance(tokens, str):
|
||||
raise ValueError("Expected to receive a list of tokens. Received a string instead")
|
||||
if isinstance(tokens[0], list):
|
||||
tokens = [p for o in tokens for p in o]
|
||||
freq = Counter(tokens)
|
||||
int2tok = [o for o,c in freq.most_common(max_vocab) if c>min_freq]
|
||||
unk_id = 3
|
||||
int2tok.insert(0, bos_tok)
|
||||
int2tok.insert(1, pad_tok)
|
||||
int2tok.insert(2, eos_tok)
|
||||
int2tok.insert(unk_id, unk_tok)
|
||||
tok2int = collections.defaultdict(lambda:unk_id, {v:k for k,v in enumerate(int2tok)})
|
||||
return int2tok, tok2int
|
||||
|
||||
class Tokenizer():
|
||||
def __init__(self, lang='en'):
|
||||
self.re_br = re.compile(r'<\s*br\s*/?>', re.IGNORECASE)
|
||||
self.tok = spacy.load(lang)
|
||||
for w in ('<eos>','<bos>','<unk>'):
|
||||
self.tok.tokenizer.add_special_case(w, [{ORTH: w}])
|
||||
|
||||
def sub_br(self,x): return self.re_br.sub("\n", x)
|
||||
|
||||
def spacy_tok(self,x):
|
||||
return [t.text for t in self.tok.tokenizer(self.sub_br(x))]
|
||||
|
||||
re_rep = re.compile(r'(\S)(\1{3,})')
|
||||
re_word_rep = re.compile(r'(\b\w+\W+)(\1{3,})')
|
||||
|
||||
@staticmethod
|
||||
def replace_rep(m):
|
||||
TK_REP = 'tk_rep'
|
||||
c,cc = m.groups()
|
||||
return f' {TK_REP} {len(cc)+1} {c} '
|
||||
|
||||
@staticmethod
|
||||
def replace_wrep(m):
|
||||
TK_WREP = 'tk_wrep'
|
||||
c,cc = m.groups()
|
||||
return f' {TK_WREP} {len(cc.split())+1} {c} '
|
||||
|
||||
@staticmethod
|
||||
def do_caps(ss):
|
||||
TOK_UP,TOK_SENT,TOK_MIX = ' t_up ',' t_st ',' t_mx '
|
||||
res = []
|
||||
prev='.'
|
||||
re_word = re.compile('\w')
|
||||
re_nonsp = re.compile('\S')
|
||||
for s in re.findall(r'\w+|\W+', ss):
|
||||
res += ([TOK_UP,s.lower()] if (s.isupper() and (len(s)>2))
|
||||
# else [TOK_SENT,s.lower()] if (s.istitle() and re_word.search(prev))
|
||||
else [s.lower()])
|
||||
# if re_nonsp.search(s): prev = s
|
||||
return ''.join(res)
|
||||
|
||||
def proc_text(self, s):
|
||||
s = self.re_rep.sub(Tokenizer.replace_rep, s)
|
||||
s = self.re_word_rep.sub(Tokenizer.replace_wrep, s)
|
||||
s = Tokenizer.do_caps(s)
|
||||
s = re.sub(r'([/#])', r' \1 ', s)
|
||||
s = re.sub(' {2,}', ' ', s)
|
||||
return self.spacy_tok(s)
|
||||
|
||||
@staticmethod
|
||||
def proc_all(ss, lang):
|
||||
tok = Tokenizer(lang)
|
||||
return [tok.proc_text(s) for s in ss]
|
||||
|
||||
@staticmethod
|
||||
def proc_all_mp(ss, lang='en'):
|
||||
ncpus = num_cpus()//2
|
||||
with ProcessPoolExecutor(ncpus) as e:
|
||||
return sum(e.map(Tokenizer.proc_all, ss, [lang]*len(ss)), [])
|
||||
|
||||
|
||||
class TextDataset(Dataset):
|
||||
def __init__(self, x, y, backwards=False, sos=None, eos=None):
|
||||
self.x,self.y,self.backwards,self.sos,self.eos = x,y,backwards,sos,eos
|
||||
|
||||
def __getitem__(self, idx):
|
||||
x = self.x[idx]
|
||||
if self.backwards: x = list(reversed(x))
|
||||
if self.eos is not None: x = x + [self.eos]
|
||||
if self.sos is not None: x = [self.sos]+x
|
||||
return np.array(x),self.y[idx]
|
||||
|
||||
def __len__(self): return len(self.x)
|
||||
|
||||
|
||||
class SortSampler(Sampler):
|
||||
def __init__(self, data_source, key): self.data_source,self.key = data_source,key
|
||||
def __len__(self): return len(self.data_source)
|
||||
def __iter__(self):
|
||||
return iter(sorted(range(len(self.data_source)), key=self.key, reverse=True))
|
||||
|
||||
|
||||
class SortishSampler(Sampler):
|
||||
"""Returns an iterator that traverses the the data in randomly ordered batches that are approximately the same size.
|
||||
The max key size batch is always returned in the first call because of pytorch cuda memory allocation sequencing.
|
||||
Without that max key returned first multiple buffers may be allocated when the first created isn't large enough
|
||||
to hold the next in the sequence.
|
||||
"""
|
||||
def __init__(self, data_source, key, bs):
|
||||
self.data_source,self.key,self.bs = data_source,key,bs
|
||||
|
||||
def __len__(self): return len(self.data_source)
|
||||
|
||||
def __iter__(self):
|
||||
idxs = np.random.permutation(len(self.data_source))
|
||||
sz = self.bs*50
|
||||
ck_idx = [idxs[i:i+sz] for i in range(0, len(idxs), sz)]
|
||||
sort_idx = np.concatenate([sorted(s, key=self.key, reverse=True) for s in ck_idx])
|
||||
sz = self.bs
|
||||
ck_idx = [sort_idx[i:i+sz] for i in range(0, len(sort_idx), sz)]
|
||||
max_ck = np.argmax([self.key(ck[0]) for ck in ck_idx]) # find the chunk with the largest key,
|
||||
ck_idx[0],ck_idx[max_ck] = ck_idx[max_ck],ck_idx[0] # then make sure it goes first.
|
||||
sort_idx = np.concatenate(np.random.permutation(ck_idx[1:]))
|
||||
sort_idx = np.concatenate((ck_idx[0], sort_idx))
|
||||
return iter(sort_idx)
|
||||
|
||||
|
||||
class LanguageModelLoader():
|
||||
""" Returns a language model iterator that iterates through batches that are of length N(bptt,5)
|
||||
The first batch returned is always bptt+25; the max possible width. This is done because of they way that pytorch
|
||||
allocates cuda memory in order to prevent multiple buffers from being created as the batch width grows.
|
||||
"""
|
||||
def __init__(self, nums, bs, bptt, backwards=False):
|
||||
self.bs,self.bptt,self.backwards = bs,bptt,backwards
|
||||
self.data = self.batchify(nums)
|
||||
self.i,self.iter = 0,0
|
||||
self.n = len(self.data)
|
||||
|
||||
def __iter__(self):
|
||||
self.i,self.iter = 0,0
|
||||
while self.i < self.n-1 and self.iter<len(self):
|
||||
if self.i == 0:
|
||||
seq_len = self.bptt + 5 * 5
|
||||
else:
|
||||
bptt = self.bptt if np.random.random() < 0.95 else self.bptt / 2.
|
||||
seq_len = max(5, int(np.random.normal(bptt, 5)))
|
||||
res = self.get_batch(self.i, seq_len)
|
||||
self.i += seq_len
|
||||
self.iter += 1
|
||||
yield res
|
||||
|
||||
def __len__(self): return self.n // self.bptt - 1
|
||||
|
||||
def batchify(self, data):
|
||||
nb = data.shape[0] // self.bs
|
||||
data = np.array(data[:nb*self.bs])
|
||||
data = data.reshape(self.bs, -1).T
|
||||
if self.backwards: data=data[::-1]
|
||||
return T(data)
|
||||
|
||||
def get_batch(self, i, seq_len):
|
||||
source = self.data
|
||||
seq_len = min(seq_len, len(source) - 1 - i)
|
||||
return source[i:i+seq_len], source[i+1:i+1+seq_len].view(-1)
|
||||
|
||||
|
||||
class LanguageModel(BasicModel):
|
||||
def get_layer_groups(self):
|
||||
m = self.model[0]
|
||||
return [*zip(m.rnns, m.dropouths), (self.model[1], m.dropouti)]
|
||||
|
||||
|
||||
class LanguageModelData():
|
||||
def __init__(self, path, pad_idx, n_tok, trn_dl, val_dl, test_dl=None, **kwargs):
|
||||
self.path,self.pad_idx,self.n_tok = path,pad_idx,n_tok
|
||||
self.trn_dl,self.val_dl,self.test_dl = trn_dl,val_dl,test_dl
|
||||
|
||||
def get_model(self, opt_fn, emb_sz, n_hid, n_layers, **kwargs):
|
||||
m = get_language_model(self.n_tok, emb_sz, n_hid, n_layers, self.pad_idx, **kwargs)
|
||||
model = LanguageModel(to_gpu(m))
|
||||
return RNN_Learner(self, model, opt_fn=opt_fn)
|
||||
|
||||
|
||||
class RNN_Learner(Learner):
|
||||
def __init__(self, data, models, **kwargs):
|
||||
super().__init__(data, models, **kwargs)
|
||||
|
||||
def _get_crit(self, data): return F.cross_entropy
|
||||
def fit(self, *args, **kwargs): return super().fit(*args, **kwargs, seq_first=True)
|
||||
|
||||
def save_encoder(self, name): save_model(self.model[0], self.get_model_path(name))
|
||||
def load_encoder(self, name): load_model(self.model[0], self.get_model_path(name))
|
||||
|
||||
|
||||
class TextModel(BasicModel):
|
||||
def get_layer_groups(self):
|
||||
m = self.model[0]
|
||||
return [(m.encoder, m.dropouti), *zip(m.rnns, m.dropouths), (self.model[1])]
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import os
|
||||
from distutils.version import LooseVersion
|
||||
import torch, torchvision, torchtext
|
||||
from torch import nn, cuda, backends, FloatTensor, LongTensor, optim
|
||||
import torch.nn.functional as F
|
||||
from torch.autograd import Variable
|
||||
from torch.utils.data import Dataset, TensorDataset
|
||||
from torch.nn.init import kaiming_uniform, kaiming_normal
|
||||
from torchvision.transforms import Compose
|
||||
from torchvision.models import resnet18, resnet34, resnet50, resnet101, resnet152
|
||||
from torchvision.models import vgg16_bn, vgg19_bn
|
||||
from torchvision.models import densenet121, densenet161, densenet169, densenet201
|
||||
|
||||
from .models.resnext_50_32x4d import resnext_50_32x4d
|
||||
from .models.resnext_101_32x4d import resnext_101_32x4d
|
||||
from .models.resnext_101_64x4d import resnext_101_64x4d
|
||||
from .models.wrn_50_2f import wrn_50_2f
|
||||
from .models.inceptionresnetv2 import InceptionResnetV2
|
||||
from .models.inceptionv4 import inceptionv4
|
||||
from .models.nasnet import nasnetalarge
|
||||
from .models.fa_resnet import *
|
||||
|
||||
import warnings
|
||||
warnings.filterwarnings('ignore', message='Implicit dimension choice', category=UserWarning)
|
||||
|
||||
def children(m): return m if isinstance(m, (list, tuple)) else list(m.children())
|
||||
def save_model(m, p): torch.save(m.state_dict(), p)
|
||||
def load_model(m, p):
|
||||
sd = torch.load(p, map_location=lambda storage, loc: storage)
|
||||
names = set(m.state_dict().keys())
|
||||
for n in list(sd.keys()): # list "detatches" the iterator
|
||||
if n not in names and n+'_raw' in names:
|
||||
if n+'_raw' not in sd: sd[n+'_raw'] = sd[n]
|
||||
del sd[n]
|
||||
m.load_state_dict(sd)
|
||||
|
||||
def load_pre(pre, f, fn):
|
||||
m = f()
|
||||
path = os.path.dirname(__file__)
|
||||
if pre: load_model(m, f'{path}/weights/{fn}.pth')
|
||||
return m
|
||||
|
||||
def _fastai_model(name, paper_title, paper_href):
|
||||
def add_docs_wrapper(f):
|
||||
f.__doc__ = f"""{name} model from
|
||||
`"{paper_title}" <{paper_href}>`_
|
||||
|
||||
Args:
|
||||
pre (bool): If True, returns a model pre-trained on ImageNet
|
||||
"""
|
||||
return f
|
||||
return add_docs_wrapper
|
||||
|
||||
@_fastai_model('Inception 4', 'Inception-v4, Inception-ResNet and the Impact of Residual Connections on Learning',
|
||||
'https://arxiv.org/pdf/1602.07261.pdf')
|
||||
def inception_4(pre): return children(inceptionv4(pretrained=pre))[0]
|
||||
|
||||
@_fastai_model('Inception 4', 'Inception-v4, Inception-ResNet and the Impact of Residual Connections on Learning',
|
||||
'https://arxiv.org/pdf/1602.07261.pdf')
|
||||
def inceptionresnet_2(pre): return load_pre(pre, InceptionResnetV2, 'inceptionresnetv2-d579a627')
|
||||
|
||||
@_fastai_model('ResNeXt 50', 'Aggregated Residual Transformations for Deep Neural Networks',
|
||||
'https://arxiv.org/abs/1611.05431')
|
||||
def resnext50(pre): return load_pre(pre, resnext_50_32x4d, 'resnext_50_32x4d')
|
||||
|
||||
@_fastai_model('ResNeXt 101_32', 'Aggregated Residual Transformations for Deep Neural Networks',
|
||||
'https://arxiv.org/abs/1611.05431')
|
||||
def resnext101(pre): return load_pre(pre, resnext_101_32x4d, 'resnext_101_32x4d')
|
||||
|
||||
@_fastai_model('ResNeXt 101_64', 'Aggregated Residual Transformations for Deep Neural Networks',
|
||||
'https://arxiv.org/abs/1611.05431')
|
||||
def resnext101_64(pre): return load_pre(pre, resnext_101_64x4d, 'resnext_101_64x4d')
|
||||
|
||||
@_fastai_model('Wide Residual Networks', 'Wide Residual Networks',
|
||||
'https://arxiv.org/pdf/1605.07146.pdf')
|
||||
def wrn(pre): return load_pre(pre, wrn_50_2f, 'wrn_50_2f')
|
||||
|
||||
@_fastai_model('Densenet-121', 'Densely Connected Convolutional Networks',
|
||||
'https://arxiv.org/pdf/1608.06993.pdf')
|
||||
def dn121(pre): return children(densenet121(pre))[0]
|
||||
|
||||
@_fastai_model('Densenet-169', 'Densely Connected Convolutional Networks',
|
||||
'https://arxiv.org/pdf/1608.06993.pdf')
|
||||
def dn161(pre): return children(densenet161(pre))[0]
|
||||
|
||||
@_fastai_model('Densenet-161', 'Densely Connected Convolutional Networks',
|
||||
'https://arxiv.org/pdf/1608.06993.pdf')
|
||||
def dn169(pre): return children(densenet169(pre))[0]
|
||||
|
||||
@_fastai_model('Densenet-201', 'Densely Connected Convolutional Networks',
|
||||
'https://arxiv.org/pdf/1608.06993.pdf')
|
||||
def dn201(pre): return children(densenet201(pre))[0]
|
||||
|
||||
@_fastai_model('Vgg-16 with batch norm added', 'Very Deep Convolutional Networks for Large-Scale Image Recognition',
|
||||
'https://arxiv.org/pdf/1409.1556.pdf')
|
||||
def vgg16(pre): return children(vgg16_bn(pre))[0]
|
||||
|
||||
@_fastai_model('Vgg-19 with batch norm added', 'Very Deep Convolutional Networks for Large-Scale Image Recognition',
|
||||
'https://arxiv.org/pdf/1409.1556.pdf')
|
||||
def vgg19(pre): return children(vgg19_bn(pre))[0]
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
import math
|
||||
import torch
|
||||
from torch.autograd import Variable
|
||||
from cupy.cuda import function
|
||||
from cupy.cuda.compiler import _NVRTCProgram
|
||||
from collections import namedtuple
|
||||
|
||||
##Adapted from the code here https://github.com/salesforce/pytorch-qrnn, implementation of the QRNN by
|
||||
##Bradbury, James and Merity, Stephen and Xiong, Caiming and Socher, Richard
|
||||
##https://arxiv.org/abs/1611.01576
|
||||
|
||||
kernel = '''
|
||||
extern "C"
|
||||
__global__ void recurrent_forget_mult(float *dst, const float *f, const float *x, int SEQ, int BATCH, int HIDDEN)
|
||||
{
|
||||
/*
|
||||
Note: destination is assumed to be one timestep longer than f or x where dst[0] = h_{-1}
|
||||
This means dst array has a separate index than that of f or x
|
||||
*/
|
||||
int hid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int bid = blockIdx.y * blockDim.y + threadIdx.y;
|
||||
if(hid >= HIDDEN || bid >= BATCH)
|
||||
return;
|
||||
//
|
||||
for (int ts = 0 + 1; ts < SEQ + 1; ts++) {
|
||||
// Good sanity check for debugging - only perform additions to a zeroed chunk of memory
|
||||
// Addition seems atomic or near atomic - you should get incorrect answers if doubling up via threads
|
||||
// Note: the index i needs to be offset by one as f[0] (f_t) is used for dst[1] (h_t) etc
|
||||
|
||||
// To move timesteps, we step HIDDEN * BATCH
|
||||
// To move batches, we move HIDDEN
|
||||
// To move neurons, we move +- 1
|
||||
// Note: dst[dst_i] = ts * 100 + bid * 10 + hid; is useful for debugging
|
||||
|
||||
int i = (ts - 1) * HIDDEN * BATCH + bid * HIDDEN + hid;
|
||||
int dst_i = (ts - 0) * HIDDEN * BATCH + bid * HIDDEN + hid;
|
||||
int dst_iminus1 = (ts - 1) * HIDDEN * BATCH + bid * HIDDEN + hid;
|
||||
dst[dst_i] = f[i] * x[i];
|
||||
dst[dst_i] += (1 - f[i]) * dst[dst_iminus1];
|
||||
}
|
||||
}
|
||||
|
||||
extern "C"
|
||||
__global__ void bwd_recurrent_forget_mult(const float *h, const float *f, const float *x, const float *gh, float *gf, float *gx, float *ghinit, int SEQ, int BATCH, int HIDDEN)
|
||||
{
|
||||
/*
|
||||
Note: h is assumed to be one timestep longer than f, x, gf, gx, or gh where dst[0] = h_{-1}
|
||||
This means dst array has a separate index than that of f or x
|
||||
*/
|
||||
int hid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int bid = blockIdx.y * blockDim.y + threadIdx.y;
|
||||
if(hid >= HIDDEN || bid >= BATCH)
|
||||
return;
|
||||
//
|
||||
double running_f = 0;
|
||||
for (int ts = SEQ - 1 + 1; ts >= 0 + 1; ts--) {
|
||||
int i = (ts - 1) * HIDDEN * BATCH + bid * HIDDEN + hid;
|
||||
int dst_i = (ts - 0) * HIDDEN * BATCH + bid * HIDDEN + hid;
|
||||
int dst_iminus1 = (ts - 1) * HIDDEN * BATCH + bid * HIDDEN + hid;
|
||||
//
|
||||
running_f += gh[dst_iminus1];
|
||||
// Gradient of X
|
||||
gx[i] = f[i] * running_f;
|
||||
// Gradient of F
|
||||
gf[i] = (x[i] - h[dst_iminus1]) * running_f;
|
||||
//
|
||||
// The line below is likely more numerically stable than (1 - f[i]) * running_f;
|
||||
running_f = running_f - f[i] * running_f;
|
||||
}
|
||||
ghinit[bid * HIDDEN + hid] = running_f;
|
||||
}
|
||||
'''
|
||||
|
||||
###
|
||||
|
||||
class CPUForgetMult(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super(CPUForgetMult, self).__init__()
|
||||
|
||||
def forward(self, f, x, hidden_init=None):
|
||||
result = []
|
||||
###
|
||||
forgets = f.split(1, dim=0)
|
||||
prev_h = hidden_init
|
||||
for i, h in enumerate((f * x).split(1, dim=0)):
|
||||
if prev_h is not None: h = h + (1 - forgets[i]) * prev_h
|
||||
# h is (1, batch, hidden) when it needs to be (batch_hidden)
|
||||
# Calling squeeze will result in badness if batch size is 1
|
||||
h = h.view(h.size()[1:])
|
||||
result.append(h)
|
||||
prev_h = h
|
||||
###
|
||||
return torch.stack(result)
|
||||
|
||||
|
||||
class GPUForgetMult(torch.autograd.Function):
|
||||
configured_gpus = {}
|
||||
ptx = None
|
||||
def __init__(self):
|
||||
super(GPUForgetMult, self).__init__()
|
||||
|
||||
def compile(self):
|
||||
if self.ptx is None:
|
||||
|
||||
program = _NVRTCProgram(kernel.encode(), 'recurrent_forget_mult.cu'.encode())
|
||||
GPUForgetMult.ptx = program.compile()
|
||||
|
||||
if torch.cuda.current_device() not in GPUForgetMult.configured_gpus:
|
||||
m = function.Module()
|
||||
m.load(bytes(self.ptx.encode()))
|
||||
|
||||
self.forget_mult = m.get_function('recurrent_forget_mult')
|
||||
self.bwd_forget_mult = m.get_function('bwd_recurrent_forget_mult')
|
||||
|
||||
Stream = namedtuple('Stream', ['ptr'])
|
||||
self.stream = Stream(ptr=torch.cuda.current_stream().cuda_stream)
|
||||
|
||||
GPUForgetMult.configured_gpus[torch.cuda.current_device()] = (self.forget_mult, self.bwd_forget_mult, self.stream)
|
||||
|
||||
self.forget_mult, self.bwd_forget_mult, self.stream = GPUForgetMult.configured_gpus[torch.cuda.current_device()]
|
||||
|
||||
def forward(self, f, x, hidden_init=None):
|
||||
self.compile()
|
||||
seq_size, batch_size, hidden_size = f.size()
|
||||
result = f.new(seq_size + 1, batch_size, hidden_size)
|
||||
# We only zero the result array (result[0]) if we don't set a hidden initial state
|
||||
# All other values (result[1:]) are overwritten by default
|
||||
if hidden_init is not None: result[0, :, :] = hidden_init
|
||||
else: result = result.zero_()
|
||||
###
|
||||
grid_hidden_size = min(hidden_size, 512)
|
||||
grid = (math.ceil(hidden_size / grid_hidden_size), batch_size)
|
||||
self.forget_mult(grid=grid, block=(grid_hidden_size, 1), args=[result.data_ptr(), f.data_ptr(), x.data_ptr(), seq_size, batch_size, hidden_size], stream=self.stream)
|
||||
self.save_for_backward(f, x, hidden_init)
|
||||
self.result = result
|
||||
return result[1:, :, :]
|
||||
|
||||
def backward(self, grad_h):
|
||||
self.compile()
|
||||
f, x, hidden_init = self.saved_tensors
|
||||
h = self.result
|
||||
###
|
||||
seq_size, batch_size, hidden_size = f.size()
|
||||
# Zeroing is not necessary as these will be overwritten
|
||||
grad_f = f.new(*f.size())
|
||||
grad_x = f.new(*f.size())
|
||||
grad_h_init = f.new(batch_size, hidden_size)
|
||||
###
|
||||
grid_hidden_size = min(hidden_size, 512)
|
||||
grid = (math.ceil(hidden_size / grid_hidden_size), batch_size)
|
||||
self.bwd_forget_mult(grid=grid, block=(grid_hidden_size, 1), args=[h.data_ptr(), f.data_ptr(), x.data_ptr(), grad_h.data_ptr(), grad_f.data_ptr(), grad_x.data_ptr(), grad_h_init.data_ptr(), seq_size, batch_size, hidden_size], stream=self.stream)
|
||||
###
|
||||
if hidden_init is not None:
|
||||
return grad_f, grad_x, grad_h_init
|
||||
return grad_f, grad_x
|
||||
|
||||
|
||||
class ForgetMult(torch.nn.Module):
|
||||
r"""ForgetMult computes a simple recurrent equation:
|
||||
h_t = f_t * x_t + (1 - f_t) * h_{t-1}
|
||||
|
||||
This equation is equivalent to dynamic weighted averaging.
|
||||
|
||||
Inputs: X, hidden
|
||||
- X (seq_len, batch, input_size): tensor containing the features of the input sequence.
|
||||
- F (seq_len, batch, input_size): tensor containing the forget gate values, assumed in range [0, 1].
|
||||
- hidden_init (batch, input_size): tensor containing the initial hidden state for the recurrence (h_{t-1}).
|
||||
- use_cuda: If True, use the fast element-wise CUDA kernel for recurrence. If False, uses naive for loop. Default: True.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super(ForgetMult, self).__init__()
|
||||
|
||||
def forward(self, f, x, hidden_init=None, use_cuda=True):
|
||||
# Use CUDA by default unless it's available
|
||||
use_cuda = use_cuda and torch.cuda.is_available()
|
||||
# Ensure the user is aware when ForgetMult is not GPU version as it's far faster
|
||||
if use_cuda: assert f.is_cuda and x.is_cuda, 'GPU ForgetMult with fast element-wise CUDA kernel requested but tensors not on GPU'
|
||||
###
|
||||
# Avoiding 'RuntimeError: expected a Variable argument, but got NoneType' when hidden_init is None
|
||||
if hidden_init is None: return GPUForgetMult()(f, x) if use_cuda else CPUForgetMult()(f, x)
|
||||
return GPUForgetMult()(f, x, hidden_init) if use_cuda else CPUForgetMult()(f, x, hidden_init)
|
||||
@@ -0,0 +1,172 @@
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.autograd import Variable
|
||||
from .forget_mult import ForgetMult
|
||||
|
||||
|
||||
##Adapted from the code here https://github.com/salesforce/pytorch-qrnn, implementation of the QRNN by
|
||||
##Bradbury, James and Merity, Stephen and Xiong, Caiming and Socher, Richard
|
||||
##https://arxiv.org/abs/1611.01576
|
||||
|
||||
class QRNNLayer(nn.Module):
|
||||
r"""Applies a single layer Quasi-Recurrent Neural Network (QRNN) to an input sequence.
|
||||
|
||||
Args:
|
||||
input_size: The number of expected features in the input x.
|
||||
hidden_size: The number of features in the hidden state h. If not specified, the input size is used.
|
||||
save_prev_x: Whether to store previous inputs for use in future convolutional windows (i.e. for a continuing sequence such as in language modeling). If true, you must call reset to remove cached previous values of x. Default: False.
|
||||
window: Defines the size of the convolutional window (how many previous tokens to look when computing the QRNN values). Supports 1 and 2. Default: 1.
|
||||
zoneout: Whether to apply zoneout (i.e. failing to update elements in the hidden state) to the hidden state updates. Default: 0.
|
||||
output_gate: If True, performs QRNN-fo (applying an output gate to the output). If False, performs QRNN-f. Default: True.
|
||||
use_cuda: If True, uses fast custom CUDA kernel. If False, uses naive for loop. Default: True.
|
||||
|
||||
Inputs: X, hidden
|
||||
- X (seq_len, batch, input_size): tensor containing the features of the input sequence.
|
||||
- hidden (batch, hidden_size): tensor containing the initial hidden state for the QRNN.
|
||||
|
||||
Outputs: output, h_n
|
||||
- output (seq_len, batch, hidden_size): tensor containing the output of the QRNN for each timestep.
|
||||
- h_n (batch, hidden_size): tensor containing the hidden state for t=seq_len
|
||||
"""
|
||||
|
||||
def __init__(self, input_size, hidden_size=None, save_prev_x=False, zoneout=0, window=1, output_gate=True, use_cuda=True):
|
||||
super(QRNNLayer, self).__init__()
|
||||
|
||||
assert window in [1, 2], "This QRNN implementation currently only handles convolutional window of size 1 or size 2"
|
||||
self.window = window
|
||||
self.input_size = input_size
|
||||
self.hidden_size = hidden_size if hidden_size else input_size
|
||||
self.zoneout = zoneout
|
||||
self.save_prev_x = save_prev_x
|
||||
self.prevX = None
|
||||
self.output_gate = output_gate
|
||||
self.use_cuda = use_cuda
|
||||
|
||||
# One large matmul with concat is faster than N small matmuls and no concat
|
||||
self.linear = nn.Linear(self.window * self.input_size, 3 * self.hidden_size if self.output_gate else 2 * self.hidden_size)
|
||||
|
||||
def reset(self):
|
||||
# If you are saving the previous value of x, you should call this when starting with a new state
|
||||
self.prevX = None
|
||||
|
||||
def forward(self, X, hidden=None):
|
||||
seq_len, batch_size, _ = X.size()
|
||||
|
||||
source = None
|
||||
if self.window == 1:
|
||||
source = X
|
||||
elif self.window == 2:
|
||||
# Construct the x_{t-1} tensor with optional x_{-1}, otherwise a zeroed out value for x_{-1}
|
||||
Xm1 = []
|
||||
Xm1.append(self.prevX if self.prevX is not None else X[:1, :, :] * 0)
|
||||
# Note: in case of len(X) == 1, X[:-1, :, :] results in slicing of empty tensor == bad
|
||||
if len(X) > 1:
|
||||
Xm1.append(X[:-1, :, :])
|
||||
Xm1 = torch.cat(Xm1, 0)
|
||||
# Convert two (seq_len, batch_size, hidden) tensors to (seq_len, batch_size, 2 * hidden)
|
||||
source = torch.cat([X, Xm1], 2)
|
||||
|
||||
# Matrix multiplication for the three outputs: Z, F, O
|
||||
Y = self.linear(source)
|
||||
# Convert the tensor back to (batch, seq_len, len([Z, F, O]) * hidden_size)
|
||||
if self.output_gate:
|
||||
Y = Y.view(seq_len, batch_size, 3 * self.hidden_size)
|
||||
Z, F, O = Y.chunk(3, dim=2)
|
||||
else:
|
||||
Y = Y.view(seq_len, batch_size, 2 * self.hidden_size)
|
||||
Z, F = Y.chunk(2, dim=2)
|
||||
###
|
||||
Z = torch.nn.functional.tanh(Z)
|
||||
F = torch.nn.functional.sigmoid(F)
|
||||
|
||||
# If zoneout is specified, we perform dropout on the forget gates in F
|
||||
# If an element of F is zero, that means the corresponding neuron keeps the old value
|
||||
if self.zoneout:
|
||||
if self.training:
|
||||
mask = Variable(F.data.new(*F.size()).bernoulli_(1 - self.zoneout), requires_grad=False)
|
||||
F = F * mask
|
||||
else:
|
||||
F *= 1 - self.zoneout
|
||||
|
||||
# Ensure the memory is laid out as expected for the CUDA kernel
|
||||
# This is a null op if the tensor is already contiguous
|
||||
Z = Z.contiguous()
|
||||
F = F.contiguous()
|
||||
# The O gate doesn't need to be contiguous as it isn't used in the CUDA kernel
|
||||
|
||||
# Forget Mult
|
||||
# For testing QRNN without ForgetMult CUDA kernel, C = Z * F may be useful
|
||||
C = ForgetMult()(F, Z, hidden, use_cuda=self.use_cuda)
|
||||
|
||||
# Apply (potentially optional) output gate
|
||||
if self.output_gate:
|
||||
H = torch.nn.functional.sigmoid(O) * C
|
||||
else:
|
||||
H = C
|
||||
|
||||
# In an optimal world we may want to backprop to x_{t-1} but ...
|
||||
if self.window > 1 and self.save_prev_x:
|
||||
self.prevX = Variable(X[-1:, :, :].data, requires_grad=False)
|
||||
|
||||
return H, C[-1:, :, :]
|
||||
|
||||
|
||||
class QRNN(torch.nn.Module):
|
||||
r"""Applies a multiple layer Quasi-Recurrent Neural Network (QRNN) to an input sequence.
|
||||
|
||||
Args:
|
||||
input_size: The number of expected features in the input x.
|
||||
hidden_size: The number of features in the hidden state h. If not specified, the input size is used.
|
||||
num_layers: The number of QRNN layers to produce.
|
||||
layers: List of preconstructed QRNN layers to use for the QRNN module (optional).
|
||||
save_prev_x: Whether to store previous inputs for use in future convolutional windows (i.e. for a continuing sequence such as in language modeling). If true, you must call reset to remove cached previous values of x. Default: False.
|
||||
window: Defines the size of the convolutional window (how many previous tokens to look when computing the QRNN values). Supports 1 and 2. Default: 1.
|
||||
zoneout: Whether to apply zoneout (i.e. failing to update elements in the hidden state) to the hidden state updates. Default: 0.
|
||||
output_gate: If True, performs QRNN-fo (applying an output gate to the output). If False, performs QRNN-f. Default: True.
|
||||
use_cuda: If True, uses fast custom CUDA kernel. If False, uses naive for loop. Default: True.
|
||||
|
||||
Inputs: X, hidden
|
||||
- X (seq_len, batch, input_size): tensor containing the features of the input sequence.
|
||||
- hidden (layers, batch, hidden_size): tensor containing the initial hidden state for the QRNN.
|
||||
|
||||
Outputs: output, h_n
|
||||
- output (seq_len, batch, hidden_size): tensor containing the output of the QRNN for each timestep.
|
||||
- h_n (layers, batch, hidden_size): tensor containing the hidden state for t=seq_len
|
||||
"""
|
||||
|
||||
def __init__(self, input_size, hidden_size,
|
||||
num_layers=1, bias=True, batch_first=False,
|
||||
dropout=0, bidirectional=False, layers=None, **kwargs):
|
||||
assert bidirectional == False, 'Bidirectional QRNN is not yet supported'
|
||||
assert batch_first == False, 'Batch first mode is not yet supported'
|
||||
assert bias == True, 'Removing underlying bias is not yet supported'
|
||||
|
||||
super(QRNN, self).__init__()
|
||||
|
||||
self.layers = torch.nn.ModuleList(layers if layers else [QRNNLayer(input_size if l == 0 else hidden_size, hidden_size, **kwargs) for l in range(num_layers)])
|
||||
|
||||
self.input_size = input_size
|
||||
self.hidden_size = hidden_size
|
||||
self.num_layers = len(layers) if layers else num_layers
|
||||
self.bias = bias
|
||||
self.batch_first = batch_first
|
||||
self.dropout = dropout
|
||||
self.bidirectional = bidirectional
|
||||
|
||||
def reset(self):
|
||||
r'''If your convolutional window is greater than 1, you must reset at the beginning of each new sequence'''
|
||||
[layer.reset() for layer in self.layers]
|
||||
|
||||
def forward(self, input, hidden=None):
|
||||
next_hidden = []
|
||||
|
||||
for i, layer in enumerate(self.layers):
|
||||
input, hn = layer(input, None if hidden is None else hidden[i])
|
||||
next_hidden.append(hn)
|
||||
|
||||
if self.dropout != 0 and i < len(self.layers) - 1:
|
||||
input = torch.nn.functional.dropout(input, p=self.dropout, training=self.training, inplace=False)
|
||||
|
||||
next_hidden = torch.cat(next_hidden, 0).view(self.num_layers, *next_hidden[0].size()[-2:])
|
||||
|
||||
return input, next_hidden
|
||||
@@ -0,0 +1,735 @@
|
||||
from .imports import *
|
||||
from .layer_optimizer import *
|
||||
from enum import IntEnum
|
||||
|
||||
def scale_min(im, targ, interpolation=cv2.INTER_AREA):
|
||||
""" Scale the image so that the smallest axis is of size targ.
|
||||
|
||||
Arguments:
|
||||
im (array): image
|
||||
targ (int): target size
|
||||
"""
|
||||
r,c,*_ = im.shape
|
||||
ratio = targ/min(r,c)
|
||||
sz = (scale_to(c, ratio, targ), scale_to(r, ratio, targ))
|
||||
return cv2.resize(im, sz, interpolation=interpolation)
|
||||
|
||||
def zoom_cv(x,z):
|
||||
""" Zoom the center of image x by a factor of z+1 while retaining the original image size and proportion. """
|
||||
if z==0: return x
|
||||
r,c,*_ = x.shape
|
||||
M = cv2.getRotationMatrix2D((c/2,r/2),0,z+1.)
|
||||
return cv2.warpAffine(x,M,(c,r))
|
||||
|
||||
def stretch_cv(x,sr,sc,interpolation=cv2.INTER_AREA):
|
||||
""" Stretches image x horizontally by sr+1, and vertically by sc+1 while retaining the original image size and proportion. """
|
||||
if sr==0 and sc==0: return x
|
||||
r,c,*_ = x.shape
|
||||
x = cv2.resize(x, None, fx=sr+1, fy=sc+1, interpolation=interpolation)
|
||||
nr,nc,*_ = x.shape
|
||||
cr = (nr-r)//2; cc = (nc-c)//2
|
||||
return x[cr:r+cr, cc:c+cc]
|
||||
|
||||
def dihedral(x, dih):
|
||||
""" Perform any of 8 permutations of 90-degrees rotations or flips for image x. """
|
||||
x = np.rot90(x, dih%4)
|
||||
return x if dih<4 else np.fliplr(x)
|
||||
|
||||
def lighting(im, b, c):
|
||||
""" Adjust image balance and contrast """
|
||||
if b==0 and c==1: return im
|
||||
mu = np.average(im)
|
||||
return np.clip((im-mu)*c+mu+b,0.,1.).astype(np.float32)
|
||||
|
||||
def rotate_cv(im, deg, mode=cv2.BORDER_CONSTANT, interpolation=cv2.INTER_AREA):
|
||||
""" Rotate an image by deg degrees
|
||||
|
||||
Arguments:
|
||||
deg (float): degree to rotate.
|
||||
"""
|
||||
r,c,*_ = im.shape
|
||||
M = cv2.getRotationMatrix2D((c//2,r//2),deg,1)
|
||||
return cv2.warpAffine(im,M,(c,r), borderMode=mode, flags=cv2.WARP_FILL_OUTLIERS+interpolation)
|
||||
|
||||
def no_crop(im, min_sz=None, interpolation=cv2.INTER_AREA):
|
||||
""" Return a squared resized image """
|
||||
r,c,*_ = im.shape
|
||||
if min_sz is None: min_sz = min(r,c)
|
||||
return cv2.resize(im, (min_sz, min_sz), interpolation=interpolation)
|
||||
|
||||
def center_crop(im, min_sz=None):
|
||||
""" Return a center crop of an image """
|
||||
r,c,*_ = im.shape
|
||||
if min_sz is None: min_sz = min(r,c)
|
||||
start_r = math.ceil((r-min_sz)/2)
|
||||
start_c = math.ceil((c-min_sz)/2)
|
||||
return crop(im, start_r, start_c, min_sz)
|
||||
|
||||
def googlenet_resize(im, targ, min_area_frac, min_aspect_ratio, max_aspect_ratio, flip_hw_p, interpolation=cv2.INTER_AREA):
|
||||
""" Randomly crop an image with an aspect ratio and returns a squared resized image of size targ
|
||||
|
||||
References:
|
||||
1. https://arxiv.org/pdf/1409.4842.pdf
|
||||
2. https://arxiv.org/pdf/1802.07888.pdf
|
||||
"""
|
||||
h,w,*_ = im.shape
|
||||
area = h*w
|
||||
for _ in range(10):
|
||||
targetArea = random.uniform(min_area_frac, 1.0) * area
|
||||
aspectR = random.uniform(min_aspect_ratio, max_aspect_ratio)
|
||||
ww = int(np.sqrt(targetArea * aspectR) + 0.5)
|
||||
hh = int(np.sqrt(targetArea / aspectR) + 0.5)
|
||||
if flip_hw_p:
|
||||
ww, hh = hh, ww
|
||||
if hh <= h and ww <= w:
|
||||
x1 = 0 if w == ww else random.randint(0, w - ww)
|
||||
y1 = 0 if h == hh else random.randint(0, h - hh)
|
||||
out = im[y1:y1 + hh, x1:x1 + ww]
|
||||
out = cv2.resize(out, (targ, targ), interpolation=interpolation)
|
||||
return out
|
||||
out = scale_min(im, targ, interpolation=interpolation)
|
||||
out = center_crop(out)
|
||||
return out
|
||||
|
||||
def cutout(im, n_holes, length):
|
||||
""" Cut out n_holes number of square holes of size length in image at random locations. Holes may overlap. """
|
||||
r,c,*_ = im.shape
|
||||
mask = np.ones((r, c), np.int32)
|
||||
for n in range(n_holes):
|
||||
y = np.random.randint(length / 2, r - length / 2)
|
||||
x = np.random.randint(length / 2, c - length / 2)
|
||||
|
||||
y1 = int(np.clip(y - length / 2, 0, r))
|
||||
y2 = int(np.clip(y + length / 2, 0, r))
|
||||
x1 = int(np.clip(x - length / 2, 0, c))
|
||||
x2 = int(np.clip(x + length / 2, 0, c))
|
||||
mask[y1: y2, x1: x2] = 0.
|
||||
|
||||
mask = mask[:,:,None]
|
||||
im = im * mask
|
||||
return im
|
||||
|
||||
def scale_to(x, ratio, targ):
|
||||
'''Calculate dimension of an image during scaling with aspect ratio'''
|
||||
return max(math.floor(x*ratio), targ)
|
||||
|
||||
def crop(im, r, c, sz):
|
||||
'''
|
||||
crop image into a square of size sz,
|
||||
'''
|
||||
return im[r:r+sz, c:c+sz]
|
||||
|
||||
def det_dihedral(dih): return lambda x: dihedral(x, dih)
|
||||
def det_stretch(sr, sc): return lambda x: stretch_cv(x, sr, sc)
|
||||
def det_lighting(b, c): return lambda x: lighting(x, b, c)
|
||||
def det_rotate(deg): return lambda x: rotate_cv(x, deg)
|
||||
def det_zoom(zoom): return lambda x: zoom_cv(x, zoom)
|
||||
|
||||
def rand0(s): return random.random()*(s*2)-s
|
||||
|
||||
|
||||
class TfmType(IntEnum):
|
||||
""" Type of transformation.
|
||||
Parameters
|
||||
IntEnum: predefined types of transformations
|
||||
NO: the default, y does not get transformed when x is transformed.
|
||||
PIXEL: x and y are images and should be transformed in the same way.
|
||||
Example: image segmentation.
|
||||
COORD: y are coordinates (i.e bounding boxes)
|
||||
CLASS: y are class labels (same behaviour as PIXEL, except no normalization)
|
||||
"""
|
||||
NO = 1
|
||||
PIXEL = 2
|
||||
COORD = 3
|
||||
CLASS = 4
|
||||
|
||||
|
||||
class Denormalize():
|
||||
""" De-normalizes an image, returning it to original format.
|
||||
"""
|
||||
def __init__(self, m, s):
|
||||
self.m=np.array(m, dtype=np.float32)
|
||||
self.s=np.array(s, dtype=np.float32)
|
||||
def __call__(self, x): return x*self.s+self.m
|
||||
|
||||
|
||||
class Normalize():
|
||||
""" Normalizes an image to zero mean and unit standard deviation, given the mean m and std s of the original image """
|
||||
def __init__(self, m, s, tfm_y=TfmType.NO):
|
||||
self.m=np.array(m, dtype=np.float32)
|
||||
self.s=np.array(s, dtype=np.float32)
|
||||
self.tfm_y=tfm_y
|
||||
|
||||
def __call__(self, x, y=None):
|
||||
x = (x-self.m)/self.s
|
||||
if self.tfm_y==TfmType.PIXEL and y is not None: y = (y-self.m)/self.s
|
||||
return x,y
|
||||
|
||||
class ChannelOrder():
|
||||
'''
|
||||
changes image array shape from (h, w, 3) to (3, h, w).
|
||||
tfm_y decides the transformation done to the y element.
|
||||
'''
|
||||
def __init__(self, tfm_y=TfmType.NO): self.tfm_y=tfm_y
|
||||
|
||||
def __call__(self, x, y):
|
||||
x = np.rollaxis(x, 2)
|
||||
#if isinstance(y,np.ndarray) and (len(y.shape)==3):
|
||||
if self.tfm_y==TfmType.PIXEL: y = np.rollaxis(y, 2)
|
||||
elif self.tfm_y==TfmType.CLASS: y = y[...,0]
|
||||
return x,y
|
||||
|
||||
|
||||
def to_bb(YY, y="deprecated"):
|
||||
"""Convert mask YY to a bounding box, assumes 0 as background nonzero object"""
|
||||
cols,rows = np.nonzero(YY)
|
||||
if len(cols)==0: return np.zeros(4, dtype=np.float32)
|
||||
top_row = np.min(rows)
|
||||
left_col = np.min(cols)
|
||||
bottom_row = np.max(rows)
|
||||
right_col = np.max(cols)
|
||||
return np.array([left_col, top_row, right_col, bottom_row], dtype=np.float32)
|
||||
|
||||
|
||||
def coords2px(y, x):
|
||||
""" Transforming coordinates to pixels.
|
||||
|
||||
Arguments:
|
||||
y : np array
|
||||
vector in which (y[0], y[1]) and (y[2], y[3]) are the
|
||||
the corners of a bounding box.
|
||||
x : image
|
||||
an image
|
||||
Returns:
|
||||
Y : image
|
||||
of shape x.shape
|
||||
"""
|
||||
rows = np.rint([y[0], y[0], y[2], y[2]]).astype(int)
|
||||
cols = np.rint([y[1], y[3], y[1], y[3]]).astype(int)
|
||||
r,c,*_ = x.shape
|
||||
Y = np.zeros((r, c))
|
||||
Y[rows, cols] = 1
|
||||
return Y
|
||||
|
||||
|
||||
class Transform():
|
||||
""" A class that represents a transform.
|
||||
|
||||
All other transforms should subclass it. All subclasses should override
|
||||
do_transform.
|
||||
|
||||
Arguments
|
||||
---------
|
||||
tfm_y : TfmType
|
||||
type of transform
|
||||
"""
|
||||
def __init__(self, tfm_y=TfmType.NO):
|
||||
self.tfm_y=tfm_y
|
||||
self.store = threading.local()
|
||||
|
||||
def set_state(self): pass
|
||||
def __call__(self, x, y):
|
||||
self.set_state()
|
||||
x,y = ((self.transform(x),y) if self.tfm_y==TfmType.NO
|
||||
else self.transform(x,y) if self.tfm_y in (TfmType.PIXEL, TfmType.CLASS)
|
||||
else self.transform_coord(x,y))
|
||||
return x, y
|
||||
|
||||
def transform_coord(self, x, y): return self.transform(x),y
|
||||
|
||||
def transform(self, x, y=None):
|
||||
x = self.do_transform(x,False)
|
||||
return (x, self.do_transform(y,True)) if y is not None else x
|
||||
|
||||
@abstractmethod
|
||||
def do_transform(self, x, is_y): raise NotImplementedError
|
||||
|
||||
|
||||
class CoordTransform(Transform):
|
||||
""" A coordinate transform. """
|
||||
|
||||
@staticmethod
|
||||
def make_square(y, x):
|
||||
r,c,*_ = x.shape
|
||||
y1 = np.zeros((r, c))
|
||||
y = y.astype(np.int)
|
||||
y1[y[0]:y[2], y[1]:y[3]] = 1.
|
||||
return y1
|
||||
|
||||
def map_y(self, y0, x):
|
||||
y = CoordTransform.make_square(y0, x)
|
||||
y_tr = self.do_transform(y, True)
|
||||
return to_bb(y_tr)
|
||||
|
||||
def transform_coord(self, x, ys):
|
||||
yp = partition(ys, 4)
|
||||
y2 = [self.map_y(y,x) for y in yp]
|
||||
x = self.do_transform(x, False)
|
||||
return x, np.concatenate(y2)
|
||||
|
||||
|
||||
class AddPadding(CoordTransform):
|
||||
""" A class that represents adding paddings to an image.
|
||||
|
||||
The default padding is border_reflect
|
||||
Arguments
|
||||
---------
|
||||
pad : int
|
||||
size of padding on top, bottom, left and right
|
||||
mode:
|
||||
type of cv2 padding modes. (e.g., constant, reflect, wrap, replicate. etc. )
|
||||
"""
|
||||
def __init__(self, pad, mode=cv2.BORDER_REFLECT, tfm_y=TfmType.NO):
|
||||
super().__init__(tfm_y)
|
||||
self.pad,self.mode = pad,mode
|
||||
|
||||
def do_transform(self, im, is_y):
|
||||
return cv2.copyMakeBorder(im, self.pad, self.pad, self.pad, self.pad, self.mode)
|
||||
|
||||
class CenterCrop(CoordTransform):
|
||||
""" A class that represents a Center Crop.
|
||||
|
||||
This transforms (optionally) transforms x,y at with the same parameters.
|
||||
Arguments
|
||||
---------
|
||||
sz: int
|
||||
size of the crop.
|
||||
tfm_y : TfmType
|
||||
type of y transformation.
|
||||
"""
|
||||
def __init__(self, sz, tfm_y=TfmType.NO, sz_y=None):
|
||||
super().__init__(tfm_y)
|
||||
self.min_sz,self.sz_y = sz,sz_y
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
return center_crop(x, self.sz_y if is_y else self.min_sz)
|
||||
|
||||
|
||||
class RandomCrop(CoordTransform):
|
||||
""" A class that represents a Random Crop transformation.
|
||||
|
||||
This transforms (optionally) transforms x,y at with the same parameters.
|
||||
Arguments
|
||||
---------
|
||||
targ: int
|
||||
target size of the crop.
|
||||
tfm_y: TfmType
|
||||
type of y transformation.
|
||||
"""
|
||||
def __init__(self, targ_sz, tfm_y=TfmType.NO, sz_y=None):
|
||||
super().__init__(tfm_y)
|
||||
self.targ_sz,self.sz_y = targ_sz,sz_y
|
||||
|
||||
def set_state(self):
|
||||
self.store.rand_r = random.uniform(0, 1)
|
||||
self.store.rand_c = random.uniform(0, 1)
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
r,c,*_ = x.shape
|
||||
sz = self.sz_y if is_y else self.targ_sz
|
||||
start_r = np.floor(self.store.rand_r*(r-sz)).astype(int)
|
||||
start_c = np.floor(self.store.rand_c*(c-sz)).astype(int)
|
||||
return crop(x, start_r, start_c, sz)
|
||||
|
||||
|
||||
class NoCrop(CoordTransform):
|
||||
""" A transformation that resize to a square image without cropping.
|
||||
|
||||
This transforms (optionally) resizes x,y at with the same parameters.
|
||||
Arguments:
|
||||
targ: int
|
||||
target size of the crop.
|
||||
tfm_y (TfmType): type of y transformation.
|
||||
"""
|
||||
def __init__(self, sz, tfm_y=TfmType.NO, sz_y=None):
|
||||
super().__init__(tfm_y)
|
||||
self.sz,self.sz_y = sz,sz_y
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
if is_y: return no_crop(x, self.sz_y, cv2.INTER_AREA if self.tfm_y == TfmType.PIXEL else cv2.INTER_NEAREST)
|
||||
else : return no_crop(x, self.sz, cv2.INTER_AREA )
|
||||
|
||||
|
||||
class Scale(CoordTransform):
|
||||
""" A transformation that scales the min size to sz.
|
||||
|
||||
Arguments:
|
||||
sz: int
|
||||
target size to scale minimum size.
|
||||
tfm_y: TfmType
|
||||
type of y transformation.
|
||||
"""
|
||||
def __init__(self, sz, tfm_y=TfmType.NO, sz_y=None):
|
||||
super().__init__(tfm_y)
|
||||
self.sz,self.sz_y = sz,sz_y
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
if is_y: return scale_min(x, self.sz_y, cv2.INTER_AREA if self.tfm_y == TfmType.PIXEL else cv2.INTER_NEAREST)
|
||||
else : return scale_min(x, self.sz, cv2.INTER_AREA )
|
||||
|
||||
|
||||
class RandomScale(CoordTransform):
|
||||
""" Scales an image so that the min size is a random number between [sz, sz*max_zoom]
|
||||
|
||||
This transforms (optionally) scales x,y at with the same parameters.
|
||||
Arguments:
|
||||
sz: int
|
||||
target size
|
||||
max_zoom: float
|
||||
float >= 1.0
|
||||
p : float
|
||||
a probability for doing the random sizing
|
||||
tfm_y: TfmType
|
||||
type of y transform
|
||||
"""
|
||||
def __init__(self, sz, max_zoom, p=0.75, tfm_y=TfmType.NO, sz_y=None):
|
||||
super().__init__(tfm_y)
|
||||
self.sz,self.max_zoom,self.p,self.sz_y = sz,max_zoom,p,sz_y
|
||||
|
||||
def set_state(self):
|
||||
min_z = 1.
|
||||
max_z = self.max_zoom
|
||||
if isinstance(self.max_zoom, collections.Iterable):
|
||||
min_z, max_z = self.max_zoom
|
||||
self.store.mult = random.uniform(min_z, max_z) if random.random()<self.p else 1
|
||||
self.store.new_sz = int(self.store.mult*self.sz)
|
||||
if self.sz_y is not None: self.store.new_sz_y = int(self.store.mult*self.sz_y)
|
||||
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
if is_y: return scale_min(x, self.store.new_sz_y, cv2.INTER_AREA if self.tfm_y == TfmType.PIXEL else cv2.INTER_NEAREST)
|
||||
else : return scale_min(x, self.store.new_sz, cv2.INTER_AREA )
|
||||
|
||||
|
||||
class RandomRotate(CoordTransform):
|
||||
""" Rotates images and (optionally) target y.
|
||||
|
||||
Rotating coordinates is treated differently for x and y on this
|
||||
transform.
|
||||
Arguments:
|
||||
deg (float): degree to rotate.
|
||||
p (float): probability of rotation
|
||||
mode: type of border
|
||||
tfm_y (TfmType): type of y transform
|
||||
"""
|
||||
def __init__(self, deg, p=0.75, mode=cv2.BORDER_REFLECT, tfm_y=TfmType.NO):
|
||||
super().__init__(tfm_y)
|
||||
self.deg,self.p = deg,p
|
||||
if tfm_y == TfmType.COORD or tfm_y == TfmType.CLASS:
|
||||
self.modes = (mode,cv2.BORDER_CONSTANT)
|
||||
else:
|
||||
self.modes = (mode,mode)
|
||||
|
||||
def set_state(self):
|
||||
self.store.rdeg = rand0(self.deg)
|
||||
self.store.rp = random.random()<self.p
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
if self.store.rp: x = rotate_cv(x, self.store.rdeg,
|
||||
mode= self.modes[1] if is_y else self.modes[0],
|
||||
interpolation=cv2.INTER_NEAREST if is_y else cv2.INTER_AREA)
|
||||
return x
|
||||
|
||||
|
||||
class RandomDihedral(CoordTransform):
|
||||
"""
|
||||
Rotates images by random multiples of 90 degrees and/or reflection.
|
||||
Please reference D8(dihedral group of order eight), the group of all symmetries of the square.
|
||||
"""
|
||||
def set_state(self):
|
||||
self.store.rot_times = random.randint(0,3)
|
||||
self.store.do_flip = random.random()<0.5
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
x = np.rot90(x, self.store.rot_times)
|
||||
return np.fliplr(x).copy() if self.store.do_flip else x
|
||||
|
||||
|
||||
class RandomFlip(CoordTransform):
|
||||
def __init__(self, tfm_y=TfmType.NO, p=0.5):
|
||||
super().__init__(tfm_y=tfm_y)
|
||||
self.p=p
|
||||
|
||||
def set_state(self): self.store.do_flip = random.random()<self.p
|
||||
def do_transform(self, x, is_y): return np.fliplr(x).copy() if self.store.do_flip else x
|
||||
|
||||
|
||||
class RandomLighting(Transform):
|
||||
def __init__(self, b, c, tfm_y=TfmType.NO):
|
||||
super().__init__(tfm_y)
|
||||
self.b,self.c = b,c
|
||||
|
||||
def set_state(self):
|
||||
self.store.b_rand = rand0(self.b)
|
||||
self.store.c_rand = rand0(self.c)
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
if is_y and self.tfm_y != TfmType.PIXEL: return x
|
||||
b = self.store.b_rand
|
||||
c = self.store.c_rand
|
||||
c = -1/(c-1) if c<0 else c+1
|
||||
x = lighting(x, b, c)
|
||||
return x
|
||||
|
||||
class RandomRotateZoom(CoordTransform):
|
||||
"""
|
||||
Selects between a rotate, zoom, stretch, or no transform.
|
||||
Arguments:
|
||||
deg - maximum degrees of rotation.
|
||||
zoom - maximum fraction of zoom.
|
||||
stretch - maximum fraction of stretch.
|
||||
ps - probabilities for each transform. List of length 4. The order for these probabilities is as listed respectively (4th probability is 'no transform'.
|
||||
"""
|
||||
def __init__(self, deg, zoom, stretch, ps=None, mode=cv2.BORDER_REFLECT, tfm_y=TfmType.NO):
|
||||
super().__init__(tfm_y)
|
||||
if ps is None: ps = [0.25,0.25,0.25,0.25]
|
||||
assert len(ps) == 4, 'does not have 4 probabilities for p, it has %d' % len(ps)
|
||||
self.transforms = RandomRotate(deg, p=1, mode=mode, tfm_y=tfm_y), RandomZoom(zoom, tfm_y=tfm_y), RandomStretch(stretch,tfm_y=tfm_y)
|
||||
self.pass_t = PassThru()
|
||||
self.cum_ps = np.cumsum(ps)
|
||||
assert self.cum_ps[3]==1, 'probabilites do not sum to 1; they sum to %d' % self.cum_ps[3]
|
||||
|
||||
def set_state(self):
|
||||
self.store.trans = self.pass_t
|
||||
self.store.choice = self.cum_ps[3]*random.random()
|
||||
for i in range(len(self.transforms)):
|
||||
if self.store.choice < self.cum_ps[i]:
|
||||
self.store.trans = self.transforms[i]
|
||||
break
|
||||
self.store.trans.set_state()
|
||||
|
||||
def do_transform(self, x, is_y): return self.store.trans.do_transform(x, is_y)
|
||||
|
||||
class RandomZoom(CoordTransform):
|
||||
def __init__(self, zoom_max, zoom_min=0, mode=cv2.BORDER_REFLECT, tfm_y=TfmType.NO):
|
||||
super().__init__(tfm_y)
|
||||
self.zoom_max, self.zoom_min = zoom_max, zoom_min
|
||||
|
||||
def set_state(self):
|
||||
self.store.zoom = self.zoom_min+(self.zoom_max-self.zoom_min)*random.random()
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
return zoom_cv(x, self.store.zoom)
|
||||
|
||||
class RandomStretch(CoordTransform):
|
||||
def __init__(self, max_stretch, tfm_y=TfmType.NO):
|
||||
super().__init__(tfm_y)
|
||||
self.max_stretch = max_stretch
|
||||
|
||||
def set_state(self):
|
||||
self.store.stretch = self.max_stretch*random.random()
|
||||
self.store.stretch_dir = random.randint(0,1)
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
if self.store.stretch_dir==0: x = stretch_cv(x, self.store.stretch, 0)
|
||||
else: x = stretch_cv(x, 0, self.store.stretch)
|
||||
return x
|
||||
|
||||
class PassThru(CoordTransform):
|
||||
def do_transform(self, x, is_y):
|
||||
return x
|
||||
|
||||
class RandomBlur(Transform):
|
||||
"""
|
||||
Adds a gaussian blur to the image at chance.
|
||||
Multiple blur strengths can be configured, one of them is used by random chance.
|
||||
"""
|
||||
|
||||
def __init__(self, blur_strengths=5, probability=0.5, tfm_y=TfmType.NO):
|
||||
# Blur strength must be an odd number, because it is used as a kernel size.
|
||||
super().__init__(tfm_y)
|
||||
self.blur_strengths = (np.array(blur_strengths, ndmin=1) * 2) - 1
|
||||
if np.any(self.blur_strengths < 0):
|
||||
raise ValueError("all blur_strengths must be > 0")
|
||||
self.probability = probability
|
||||
self.apply_transform = False
|
||||
|
||||
def set_state(self):
|
||||
self.store.apply_transform = random.random() < self.probability
|
||||
kernel_size = np.random.choice(self.blur_strengths)
|
||||
self.store.kernel = (kernel_size, kernel_size)
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
return cv2.GaussianBlur(src=x, ksize=self.store.kernel, sigmaX=0) if self.apply_transform else x
|
||||
|
||||
class Cutout(Transform):
|
||||
""" Randomly masks squares of size length on the image.
|
||||
https://arxiv.org/pdf/1708.04552.pdf
|
||||
|
||||
Arguments:
|
||||
n_holes: number of squares
|
||||
length: size of the square
|
||||
p: probability to apply cutout
|
||||
tfm_y: type of y transform
|
||||
"""
|
||||
def __init__(self, n_holes, length, p=0.5, tfm_y=TfmType.NO):
|
||||
super().__init__(tfm_y)
|
||||
self.n_holes, self.length, self.p = n_holes, length, p
|
||||
|
||||
def set_state(self):
|
||||
self.apply_transform = random.random() < self.p
|
||||
|
||||
def do_transform(self, img, is_y):
|
||||
return cutout(img, self.n_holes, self.length) if self.apply_transform else img
|
||||
|
||||
class GoogleNetResize(CoordTransform):
|
||||
""" Randomly crops an image with an aspect ratio and returns a squared resized image of size targ
|
||||
|
||||
Arguments:
|
||||
targ_sz: int
|
||||
target size
|
||||
min_area_frac: float < 1.0
|
||||
minimum area of the original image for cropping
|
||||
min_aspect_ratio : float
|
||||
minimum aspect ratio
|
||||
max_aspect_ratio : float
|
||||
maximum aspect ratio
|
||||
flip_hw_p : float
|
||||
probability for flipping magnitudes of height and width
|
||||
tfm_y: TfmType
|
||||
type of y transform
|
||||
"""
|
||||
|
||||
def __init__(self, targ_sz,
|
||||
min_area_frac=0.08, min_aspect_ratio=0.75, max_aspect_ratio=1.333, flip_hw_p=0.5,
|
||||
tfm_y=TfmType.NO, sz_y=None):
|
||||
super().__init__(tfm_y)
|
||||
self.targ_sz, self.tfm_y, self.sz_y = targ_sz, tfm_y, sz_y
|
||||
self.min_area_frac, self.min_aspect_ratio, self.max_aspect_ratio, self.flip_hw_p = min_area_frac, min_aspect_ratio, max_aspect_ratio, flip_hw_p
|
||||
|
||||
def set_state(self):
|
||||
# if self.random_state: random.seed(self.random_state)
|
||||
self.store.fp = random.random()<self.flip_hw_p
|
||||
|
||||
def do_transform(self, x, is_y):
|
||||
sz = self.sz_y if is_y else self.targ_sz
|
||||
if is_y:
|
||||
interpolation = cv2.INTER_NEAREST if self.tfm_y in (TfmType.COORD, TfmType.CLASS) else cv2.INTER_AREA
|
||||
else:
|
||||
interpolation = cv2.INTER_AREA
|
||||
return googlenet_resize(x, sz, self.min_area_frac, self.min_aspect_ratio, self.max_aspect_ratio, self.store.fp, interpolation=interpolation)
|
||||
|
||||
|
||||
def compose(im, y, fns):
|
||||
""" Apply a collection of transformation functions :fns: to images """
|
||||
for fn in fns:
|
||||
#pdb.set_trace()
|
||||
im, y =fn(im, y)
|
||||
return im if y is None else (im, y)
|
||||
|
||||
|
||||
class CropType(IntEnum):
|
||||
""" Type of image cropping. """
|
||||
RANDOM = 1
|
||||
CENTER = 2
|
||||
NO = 3
|
||||
GOOGLENET = 4
|
||||
|
||||
crop_fn_lu = {CropType.RANDOM: RandomCrop, CropType.CENTER: CenterCrop, CropType.NO: NoCrop, CropType.GOOGLENET: GoogleNetResize}
|
||||
|
||||
class Transforms():
|
||||
def __init__(self, sz, tfms, normalizer, denorm, crop_type=CropType.CENTER,
|
||||
tfm_y=TfmType.NO, sz_y=None):
|
||||
if sz_y is None: sz_y = sz
|
||||
self.sz,self.denorm,self.norm,self.sz_y = sz,denorm,normalizer,sz_y
|
||||
crop_tfm = crop_fn_lu[crop_type](sz, tfm_y, sz_y)
|
||||
self.tfms = tfms
|
||||
self.tfms.append(crop_tfm)
|
||||
if normalizer is not None: self.tfms.append(normalizer)
|
||||
self.tfms.append(ChannelOrder(tfm_y))
|
||||
|
||||
def __call__(self, im, y=None): return compose(im, y, self.tfms)
|
||||
def __repr__(self): return str(self.tfms)
|
||||
|
||||
|
||||
def image_gen(normalizer, denorm, sz, tfms=None, max_zoom=None, pad=0, crop_type=None,
|
||||
tfm_y=None, sz_y=None, pad_mode=cv2.BORDER_REFLECT, scale=None):
|
||||
"""
|
||||
Generate a standard set of transformations
|
||||
|
||||
Arguments
|
||||
---------
|
||||
normalizer :
|
||||
image normalizing function
|
||||
denorm :
|
||||
image denormalizing function
|
||||
sz :
|
||||
size, sz_y = sz if not specified.
|
||||
tfms :
|
||||
iterable collection of transformation functions
|
||||
max_zoom : float,
|
||||
maximum zoom
|
||||
pad : int,
|
||||
padding on top, left, right and bottom
|
||||
crop_type :
|
||||
crop type
|
||||
tfm_y :
|
||||
y axis specific transformations
|
||||
sz_y :
|
||||
y size, height
|
||||
pad_mode :
|
||||
cv2 padding style: repeat, reflect, etc.
|
||||
|
||||
Returns
|
||||
-------
|
||||
type : ``Transforms``
|
||||
transformer for specified image operations.
|
||||
|
||||
See Also
|
||||
--------
|
||||
Transforms: the transformer object returned by this function
|
||||
"""
|
||||
if tfm_y is None: tfm_y=TfmType.NO
|
||||
if tfms is None: tfms=[]
|
||||
elif not isinstance(tfms, collections.Iterable): tfms=[tfms]
|
||||
if sz_y is None: sz_y = sz
|
||||
if scale is None:
|
||||
scale = [RandomScale(sz, max_zoom, tfm_y=tfm_y, sz_y=sz_y) if max_zoom is not None
|
||||
else Scale(sz, tfm_y, sz_y=sz_y)]
|
||||
elif not is_listy(scale): scale = [scale]
|
||||
if pad: scale.append(AddPadding(pad, mode=pad_mode))
|
||||
if crop_type!=CropType.GOOGLENET: tfms=scale+tfms
|
||||
return Transforms(sz, tfms, normalizer, denorm, crop_type,
|
||||
tfm_y=tfm_y, sz_y=sz_y)
|
||||
|
||||
def noop(x):
|
||||
"""dummy function for do-nothing.
|
||||
equivalent to: lambda x: x"""
|
||||
return x
|
||||
|
||||
transforms_basic = [RandomRotate(10), RandomLighting(0.05, 0.05)]
|
||||
transforms_side_on = transforms_basic + [RandomFlip()]
|
||||
transforms_top_down = transforms_basic + [RandomDihedral()]
|
||||
|
||||
imagenet_stats = A([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
|
||||
"""Statistics pertaining to image data from image net. mean and std of the images of each color channel"""
|
||||
inception_stats = A([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
|
||||
inception_models = (inception_4, inceptionresnet_2)
|
||||
|
||||
def tfms_from_stats(stats, sz, aug_tfms=None, max_zoom=None, pad=0, crop_type=CropType.RANDOM,
|
||||
tfm_y=None, sz_y=None, pad_mode=cv2.BORDER_REFLECT, norm_y=True, scale=None):
|
||||
""" Given the statistics of the training image sets, returns separate training and validation transform functions
|
||||
"""
|
||||
if aug_tfms is None: aug_tfms=[]
|
||||
tfm_norm = Normalize(*stats, tfm_y=tfm_y if norm_y else TfmType.NO) if stats is not None else None
|
||||
tfm_denorm = Denormalize(*stats) if stats is not None else None
|
||||
val_crop = CropType.CENTER if crop_type in (CropType.RANDOM,CropType.GOOGLENET) else crop_type
|
||||
val_tfm = image_gen(tfm_norm, tfm_denorm, sz, pad=pad, crop_type=val_crop,
|
||||
tfm_y=tfm_y, sz_y=sz_y, scale=scale)
|
||||
trn_tfm = image_gen(tfm_norm, tfm_denorm, sz, pad=pad, crop_type=crop_type,
|
||||
tfm_y=tfm_y, sz_y=sz_y, tfms=aug_tfms, max_zoom=max_zoom, pad_mode=pad_mode, scale=scale)
|
||||
return trn_tfm, val_tfm
|
||||
|
||||
|
||||
def tfms_from_model(f_model, sz, aug_tfms=None, max_zoom=None, pad=0, crop_type=CropType.RANDOM,
|
||||
tfm_y=None, sz_y=None, pad_mode=cv2.BORDER_REFLECT, norm_y=True, scale=None):
|
||||
""" Returns separate transformers of images for training and validation.
|
||||
Transformers are constructed according to the image statistics given b y the model. (See tfms_from_stats)
|
||||
|
||||
Arguments:
|
||||
f_model: model, pretrained or not pretrained
|
||||
"""
|
||||
stats = inception_stats if f_model in inception_models else imagenet_stats
|
||||
return tfms_from_stats(stats, sz, aug_tfms, max_zoom=max_zoom, pad=pad, crop_type=crop_type,
|
||||
tfm_y=tfm_y, sz_y=sz_y, pad_mode=pad_mode, norm_y=norm_y, scale=scale)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
|
||||
class Cutout(object):
|
||||
"""Randomly mask out one or more patches from an image.
|
||||
|
||||
Args:
|
||||
n_holes (int): Number of patches to cut out of each image.
|
||||
length (int): The length (in pixels) of each square patch.
|
||||
"""
|
||||
def __init__(self, n_holes, length):
|
||||
self.n_holes = n_holes
|
||||
self.length = length
|
||||
|
||||
def __call__(self, img):
|
||||
"""
|
||||
Args:
|
||||
img (Tensor): Tensor image of size (C, H, W).
|
||||
Returns:
|
||||
Tensor: Image with n_holes of dimension length x length cut out of it.
|
||||
"""
|
||||
h = img.size(1)
|
||||
w = img.size(2)
|
||||
|
||||
mask = np.ones((h, w), np.float32)
|
||||
|
||||
for n in range(self.n_holes):
|
||||
y = np.random.randint(h)
|
||||
x = np.random.randint(w)
|
||||
|
||||
y1 = np.clip(y - self.length / 2, 0, h)
|
||||
y2 = np.clip(y + self.length / 2, 0, h)
|
||||
x1 = np.clip(x - self.length / 2, 0, w)
|
||||
x2 = np.clip(x + self.length / 2, 0, w)
|
||||
|
||||
mask[y1: y2, x1: x2] = 0.
|
||||
|
||||
mask = torch.from_numpy(mask)
|
||||
mask = mask.expand_as(img)
|
||||
img = img * mask
|
||||
|
||||
return img
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
import math, os, json, sys, re, numpy as np, pickle, PIL, scipy
|
||||
from PIL import Image
|
||||
from glob import glob
|
||||
from matplotlib import pyplot as plt
|
||||
from operator import itemgetter, attrgetter, methodcaller
|
||||
from collections import OrderedDict
|
||||
import itertools
|
||||
from itertools import chain
|
||||
|
||||
import pandas as pd
|
||||
from numpy.random import random, permutation, randn, normal, uniform, choice
|
||||
from numpy import newaxis
|
||||
from scipy import misc, ndimage
|
||||
from scipy.ndimage.interpolation import zoom
|
||||
from scipy.ndimage import imread
|
||||
from sklearn.metrics import confusion_matrix
|
||||
from sklearn.preprocessing import OneHotEncoder
|
||||
from sklearn.manifold import TSNE
|
||||
import bcolz
|
||||
|
||||
from IPython.lib.display import FileLink
|
||||
|
||||
import keras
|
||||
from keras import backend as K
|
||||
from keras.utils.data_utils import get_file
|
||||
from keras.utils import np_utils
|
||||
from keras.utils.np_utils import to_categorical
|
||||
from keras.models import Sequential, Model
|
||||
from keras.layers import Input, Embedding, Reshape, merge, LSTM, Bidirectional
|
||||
from keras.layers import TimeDistributed, Activation, SimpleRNN, GRU
|
||||
from keras.layers import Flatten, Dense, Dropout, Lambda
|
||||
from keras.regularizers import l2, l1
|
||||
from keras.layers.normalization import BatchNormalization
|
||||
from keras.optimizers import SGD, RMSprop, Adam
|
||||
from keras.layers import deserialize as layer_from_config
|
||||
from keras.metrics import categorical_crossentropy, categorical_accuracy
|
||||
from keras.layers.convolutional import *
|
||||
from keras.preprocessing import image, sequence
|
||||
from keras.preprocessing.text import Tokenizer
|
||||
from vgg16 import Vgg16
|
||||
np.set_printoptions(precision=4, linewidth=100)
|
||||
|
||||
|
||||
to_bw = np.array([0.299, 0.587, 0.114])
|
||||
|
||||
def gray(img): return np.rollaxis(img, 0, 1).dot(to_bw)
|
||||
def to_plot(img): return np.rollaxis(img, 0, 1).astype(np.uint8)
|
||||
def plot(img): plt.imshow(to_plot(img))
|
||||
|
||||
def floor(x): return int(math.floor(x))
|
||||
def ceil(x): return int(math.ceil(x))
|
||||
|
||||
def plots(ims, figsize=(12,6), rows=1, interp=False, titles=None):
|
||||
if type(ims[0]) is np.ndarray:
|
||||
ims = np.array(ims).astype(np.uint8)
|
||||
if (ims.shape[-1] != 3): ims = ims.transpose((0,2,3,1))
|
||||
f = plt.figure(figsize=figsize)
|
||||
|
||||
for i in range(len(ims)):
|
||||
sp = f.add_subplot(rows, len(ims)//rows, i+1)
|
||||
sp.axis('Off')
|
||||
if titles is not None: sp.set_title(titles[i], fontsize=16)
|
||||
plt.imshow(ims[i], interpolation=None if interp else 'none')
|
||||
|
||||
|
||||
def do_clip(arr, mx):
|
||||
clipped = np.clip(arr, (1-mx)/1, mx)
|
||||
return clipped/clipped.sum(axis=1)[:, np.newaxis]
|
||||
|
||||
|
||||
def wrap_config(layer):
|
||||
return {'class_name': layer.__class__.__name__, 'config': layer.get_config()}
|
||||
|
||||
def copy_layer(layer): return layer_from_config(wrap_config(layer))
|
||||
|
||||
def copy_layers(layers): return [copy_layer(layer) for layer in layers]
|
||||
|
||||
def copy_weights(from_layers, to_layers):
|
||||
for from_layer,to_layer in zip(from_layers, to_layers):
|
||||
to_layer.set_weights(from_layer.get_weights())
|
||||
|
||||
def save_array(fname, arr):
|
||||
c=bcolz.carray(arr, rootdir=fname, mode='w')
|
||||
c.flush()
|
||||
|
||||
def load_array(fname): return bcolz.open(fname)[:]
|
||||
|
||||
def get_classes(path):
|
||||
batches = get_batches(path+'train', shuffle=False, batch_size=1)
|
||||
val_batches = get_batches(path+'valid', shuffle=False, batch_size=1)
|
||||
test_batches = get_batches(path+'test', shuffle=False, batch_size=1)
|
||||
return (val_batches.classes, batches.classes, onehot(val_batches.classes), onehot(batches.classes),
|
||||
val_batches.filenames, batches.filenames, test_batches.filenames)
|
||||
|
||||
def limit_mem():
|
||||
K.get_session().close()
|
||||
cfg = K.tf.ConfigProto()
|
||||
cfg.gpu_options.allow_growth = True
|
||||
K.set_session(K.tf.Session(config=cfg))
|
||||
|
||||
class MixIterator(object):
|
||||
def __init__(self, iters):
|
||||
self.iters = iters
|
||||
self.multi = type(iters) is list
|
||||
if self.multi:
|
||||
self.N = sum([it[0].N for it in self.iters])
|
||||
else:
|
||||
self.N = sum([it.N for it in self.iters])
|
||||
|
||||
def reset(self):
|
||||
for it in self.iters: it.reset()
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next(self, *args, **kwargs):
|
||||
if self.multi:
|
||||
nexts = [[next(it) for it in o] for o in self.iters]
|
||||
n0 = np.concatenate([n[0] for n in nexts])
|
||||
n1 = np.concatenate([n[1] for n in nexts])
|
||||
return (n0, n1)
|
||||
else:
|
||||
nexts = [next(it) for it in self.iters]
|
||||
n0 = np.concatenate([n[0] for n in nexts])
|
||||
n1 = np.concatenate([n[1] for n in nexts])
|
||||
return (n0, n1)
|
||||
Reference in New Issue
Block a user