Skip to content

Commit 870f8ad

Browse files
jccaicedojccaicedo
committed
Fixing multiple unit tests
Merge branch 'master' of github.com:jccaicedo/DeepProfiler
2 parents de78708 + 217e672 commit 870f8ad

File tree

16 files changed

+118
-50
lines changed

16 files changed

+118
-50
lines changed

‎deepprofiler/__main__.py‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
type=click.Path("r"))
2323
@click.option("--config", default=None,
2424
help="Path to existing config file",
25-
type=click.Path("r"))
25+
type=click.STRING)
2626
@click.option("--cores", default=0,
2727
help="Number of CPU cores for parallel processing (all=0)",
2828
type=click.INT)
@@ -52,13 +52,8 @@ def cli(context, root, config, exp, cores, gpu):
5252
"summaries": root+"/outputs/" + exp + "/summaries/",
5353
"features": root+"/outputs/" + exp + "/features/"
5454
}
55-
if config is not None:
56-
context.obj["config"] = {}
57-
context.obj["config"]["paths"] = {}
58-
context.obj["config"]["paths"]["config"] = config
59-
dirs["config"] = os.path.dirname(os.path.abspath(config))
60-
else:
61-
config = dirs["config"] + "/config.json"
55+
56+
config = dirs["config"] + "/" + config
6257
context.obj["cores"] = cores
6358
context.obj["gpu"] = gpu
6459

‎deepprofiler/learning/profiling.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
class Profile(object):
1313

14-
def __init__(self, config, dset):
14+
def __init__(self, config, dset, gpu):
1515
self.config = config
1616
self.dset = dset
1717
self.num_channels = len(self.config["dataset"]["images"]["channels"])
@@ -24,7 +24,6 @@ def __init__(self, config, dset):
2424
.ModelClass(config, dset, gpu, self.crop_generator, self.profile_crop_generator)
2525
self.profile_crop_generator = self.profile_crop_generator(config, dset)
2626

27-
2827
def configure(self):
2928
# Main session configuration
3029
configuration = tf.ConfigProto()
@@ -45,7 +44,6 @@ def configure(self):
4544
self.config["profile"]["feature_layer"]).output)
4645
self.feat_extractor.summary()
4746

48-
4947
def check(self, meta):
5048
output_folder = self.config["paths"]["features"]
5149
output_file = self.config["paths"]["features"] + "/{}_{}_{}.npz"

‎plugins/crop_generators/repeat_channel_crop_generator.py‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,36 @@
11
import numpy as np
2-
2+
import tensorflow as tf
33
import deepprofiler.imaging.cropping
4-
4+
from keras.applications import inception_resnet_v2
55

66
def repeat_channels(crops, labels, repeats):
7+
crops = crops[np.newaxis, :]
8+
labels = labels[np.newaxis, :]
9+
print('crops begin', crops.shape)
10+
print('crops begin', labels.shape)
11+
print(repeats)
712
crops = np.reshape(crops, (crops.shape[0] * crops.shape[3], crops.shape[1], crops.shape[2], 1))
13+
print(crops.shape)
814
crops = np.tile(crops, (1, 1, 1, repeats))
15+
print(crops.shape, crops.__class__)
16+
print(labels.shape)
917
labels = np.tile(labels, (repeats, 1))
18+
print(labels.shape)
19+
#crops = tf.image.resize_images(crops, size=(299, 299))
20+
print(crops.shape, crops.__class__)
1021
return crops, labels
22+
#return inception_resnet_v2.preprocess_input(crops), labels
23+
#def crop_transform(crop_ph, image_size):
24+
#crops_shape = crops.shape
25+
#resized_crops = tf.image.resize_images(crops, size=(299, 299))
26+
#print(resized_crops.shape)
27+
#reordered_channels = tf.transpose(resized_crops, [3, 0, 1, 2])
28+
#print(reordered_channels.shape)
29+
#reshaped_data = tf.reshape(reordered_channels, shape=[-1, 299, 299, 1])
30+
#print(reshaped_data.shape)
31+
#rgb_data = tf.image.grayscale_to_rgb(reordered_channels)
32+
#print(rgb_data.shape)
33+
#return inception_resnet_v2.preprocess_input(rgb_data), labels
1134

1235

1336
class GeneratorClass(deepprofiler.imaging.cropping.CropGenerator):
@@ -21,6 +44,7 @@ def generate(self, sess, global_step=0):
2144
crops = data[0]
2245
labels = data[1] # TODO: enable multiple targets
2346
crops, labels = repeat_channels(crops, labels, self.config["dataset"]["images"]["channel_repeats"])
47+
crops = crops.eval(session=sess)
2448
global_step += 1
2549
yield (crops, labels)
2650

@@ -30,5 +54,22 @@ class SingleImageGeneratorClass(deepprofiler.imaging.cropping.SingleImageCropGen
3054
def generate(self, session, global_step=0):
3155
crops = self.image_pool
3256
labels = self.label_pool
33-
crops, labels = repeat_channels(crops, labels, self.config["dataset"]["images"]["channel_repeats"])
34-
yield [crops, labels]
57+
#new_crops = np.array([[[[]]]])
58+
#new_labels = np.array([[[[]]]])
59+
for i in range(crops.shape[0]):
60+
crop, label = repeat_channels(crops[i, :, :, :], labels[i, :], self.config["dataset"]["images"]["channel_repeats"])
61+
#crop = crop.eval(session=session)
62+
crop = session.run(tf.image.resize_images(crop, size=(299, 299)))
63+
if i == 0:
64+
new_crops = crop
65+
new_labels = label
66+
else:
67+
new_crops = np.concatenate((new_crops, crop), axis=0)
68+
new_labels = np.concatenate((new_labels, label), axis=0)
69+
70+
#print('last shape', crop.shape, label.shape)
71+
#print('generator shapes', new_crops.shape, new_labels.shape)
72+
#new_crops = new_crops.eval(session=session)
73+
#crops, labels = repeat_channels(crops, labels, self.config["dataset"]["images"]["channel_repeats"])
74+
#crops = crops.eval(session=session)
75+
yield [new_crops, new_labels]

‎plugins/models/autoencoder.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,6 @@ def define_model(config, dset):
6666

6767

6868
class ModelClass(DeepProfilerModel):
69-
def __init__(self, config, dset, generator, val_generator):
70-
super(ModelClass, self).__init__(config, dset, generator, val_generator)
69+
def __init__(self, config, dset, gpu, generator, val_generator):
70+
super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
7171
self.feature_model, self.encoder, self.decoder, self.optimizer, self.loss = define_model(config, dset)

‎plugins/models/cnn.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,6 @@ def define_model(config, dset):
5252

5353

5454
class ModelClass(DeepProfilerModel):
55-
def __init__(self, config, dset, generator, val_generator):
56-
super(ModelClass, self).__init__(config, dset, generator, val_generator)
55+
def __init__(self, config, dset, gpu, generator, val_generator):
56+
super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
5757
self.feature_model, self.optimizer, self.loss = define_model(config, dset)

‎plugins/models/densenet.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ def define_model(config, dset):
4747
return model, optimizer, loss_func
4848

4949
class ModelClass(DeepProfilerModel):
50-
def __init__(self, config, dset, generator, val_generator):
51-
super(ModelClass, self).__init__(config, dset, generator, val_generator)
50+
def __init__(self, config, dset, gpu, generator, val_generator):
51+
super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
5252
self.feature_model, self.optimizer, self.loss = define_model(config, dset)
5353

‎plugins/models/gan.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,8 +140,8 @@ def train(self, epochs, steps_per_epoch, init_epoch):
140140

141141

142142
class ModelClass(DeepProfilerModel):
143-
def __init__(self, config, dset, generator, val_generator):
144-
super(ModelClass, self).__init__(config, dset, generator, val_generator)
143+
def __init__(self, config, dset, gpu, generator, val_generator):
144+
super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
145145
self.gan = GAN(config, self.train_crop_generator, self.val_crop_generator)
146146
self.feature_model = self.gan.discriminator
147147

‎plugins/models/vae.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,6 @@ def vae_loss(x, x_decoded_mean):
8888

8989

9090
class ModelClass(DeepProfilerModel):
91-
def __init__(self, config, dset, generator, val_generator):
92-
super(ModelClass, self).__init__(config, dset, generator, val_generator)
91+
def __init__(self, config, dset, gpu, generator, val_generator):
92+
super(ModelClass, self).__init__(config, dset, gpu, generator, val_generator)
9393
self.feature_model, self.encoder, self.generator, self.optimizer, self.loss = define_model(config, dset)

‎tests/deepprofiler/learning/test_model.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,19 +111,22 @@ def val_crop_generator(config):
111111
generator = module.SingleImageGeneratorClass
112112
return generator
113113

114+
@pytest.fixture(scope="function")
115+
def gpu():
116+
return '0'
114117

115118
@pytest.fixture(scope="function")
116-
def model(config, dataset, crop_generator, val_crop_generator):
119+
def model(config, dataset, gpu, crop_generator, val_crop_generator):
117120
def create():
118121
module = importlib.import_module("plugins.models.{}".format(config["train"]["model"]["name"]))
119122
importlib.invalidate_caches()
120-
dpmodel = module.ModelClass(config, dataset, crop_generator, val_crop_generator)
123+
dpmodel = module.ModelClass(config, dataset, gpu, crop_generator, val_crop_generator)
121124
return dpmodel
122125
return create
123126

124127

125-
def test_init(config, dataset, crop_generator, val_crop_generator):
126-
dpmodel = DeepProfilerModel(config, dataset, crop_generator, val_crop_generator)
128+
def test_init(config, dataset, gpu, crop_generator, val_crop_generator):
129+
dpmodel = DeepProfilerModel(config, dataset, gpu, crop_generator, val_crop_generator)
127130
assert dpmodel.feature_model is None
128131
assert dpmodel.config == config
129132
assert dpmodel.dset == dataset

‎tests/deepprofiler/learning/test_profiling.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,12 @@ def locations(out_dir, metadata, config, make_struct):
9898
})
9999
locs.to_csv(path, index=False)
100100

101+
102+
@pytest.fixture(scope="function")
103+
def gpu():
104+
return '0'
105+
106+
101107
@pytest.fixture(scope="function")
102108
def checkpoint(config, dataset):
103109
crop_generator = importlib.import_module(
@@ -107,20 +113,20 @@ def checkpoint(config, dataset):
107113
"plugins.crop_generators.{}".format(config["train"]["model"]["crop_generator"])) \
108114
.SingleImageGeneratorClass
109115
dpmodel = importlib.import_module("plugins.models.{}".format(config["train"]["model"]["name"])) \
110-
.ModelClass(config, dataset, crop_generator, profile_crop_generator)
116+
.ModelClass(config, dataset, gpu, crop_generator, profile_crop_generator)
111117
dpmodel.feature_model.compile(dpmodel.optimizer, dpmodel.loss)
112118
filename = os.path.join(config["paths"]["checkpoints"], config["profile"]["checkpoint"])
113119
dpmodel.feature_model.save_weights(filename)
114120
return filename
115121

116122

117123
@pytest.fixture(scope="function")
118-
def profile(config, dataset):
119-
return deepprofiler.learning.profiling.Profile(config, dataset)
124+
def profile(config, dataset, gpu):
125+
return deepprofiler.learning.profiling.Profile(config, dataset, gpu)
120126

121127

122-
def test_init(config, dataset):
123-
prof = deepprofiler.learning.profiling.Profile(config, dataset)
128+
def test_init(config, dataset, gpu):
129+
prof = deepprofiler.learning.profiling.Profile(config, dataset, gpu)
124130
test_num_channels = len(config["dataset"]["images"]["channels"])
125131
assert prof.config == config
126132
assert prof.dset == dataset
@@ -152,8 +158,8 @@ def test_extract_features(profile, metadata, locations, checkpoint):
152158
assert os.path.isfile(output_file)
153159

154160

155-
def test_profile(config, dataset, data, locations, checkpoint):
156-
deepprofiler.learning.profiling.profile(config, dataset)
161+
def test_profile(config, dataset, data, locations, checkpoint, gpu):
162+
deepprofiler.learning.profiling.profile(config, dataset, gpu)
157163
for index, row in dataset.meta.data.iterrows():
158164
output_file = config["paths"]["features"] + "/{}_{}_{}.npz" \
159165
.format(row["Metadata_Plate"], row["Metadata_Well"], row["Metadata_Site"])

0 commit comments

Comments
 (0)