-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathutil.py
229 lines (202 loc) · 7.84 KB
/
util.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
import numpy as np
from PIL import Image
from skimage import transform,data,img_as_ubyte
from random import shuffle
import matplotlib.pyplot as plt
import math
def get_data(size, data_type):
'''
Read in images and labels on the training set
Arguments:
size: size of images
data_type: train or val
Return:
imgs: images readed in
labels: labels readed in
count: for data_random
begin_index: for data_random
index: for data_random
'''
img_dir = '/home/guhanxue/jupyter/root/demo/tensorflow-triplet-loss/data/face/CASIA-WebFace-Align-96/'
if(data_type == 'train'):
f = open(img_dir + "train.txt","r")
elif(data_type == 'val'):
f = open(img_dir + "val.txt","r")
elif(data_type == 'ghx_train'):
f = open(img_dir + "ghx_train.txt","r")
elif(data_type == 'ghx_val'):
f = open(img_dir + "ghx_val.txt","r")
#imgs: sized 112*96*3
imgs = np.zeros((size,112, 96, 3), dtype = 'uint8')
labels = np.zeros((size,1), dtype = 'int32')
count = [0]*20000
i = 0
for k in range(size):
line = f.readline().strip('\n')
p, label = line.split(' ')
img_path = img_dir + p
index=int(label)
#count[i]: the number of pictures labeled in i
count[index] = count[index] + 1
# read images
img = plt.imread(img_path).astype('uint8')
labels[i] = label
imgs[i] = img
i += 1
#begin_index: the first index of differnet labels
#index: the largest label of the images
begin_index = [0]*(index+1)
for i in range(index+1):
if (i==0):
begin_index[i]=0
else:
begin_index [i] = begin_index[i-1]+count[i-1]
f.close()
return imgs, labels, count, begin_index, index
def data_random(imgs,index,labels,count,begin_index,batch_size,size):
'''
Data shuffle for training and valuation
Arguments:
imgs: 112*96*3
index: the largest label
labels: labels
count:the number of different labels
begin_index: first index of different labels
batch_size: number of images per batch
size: number of images in total
Return:
train_imags: images list after shuffling
train_table: labels list after shuffling
'''
#number_of_batchpair: we need two batch each time
number_of_batchpair = math.floor(size/batch_size/2)
#half of the batch is the positive samples, and the rest is the negative samples
#every_batch_same: the number of positive samples in each batch
every_batch_same = math.floor(batch_size/2)
#every_batch_different: the number of negative samples in each batch
every_batch_different = batch_size - every_batch_same
train_index = 0
train_imags = np.zeros((size,112, 96, 3), dtype = 'uint8')
train_table = np.zeros((size,1), dtype = 'int32')
import random
for i in range (number_of_batchpair):
#choose the postive pairs
for j in range(every_batch_same):
#type_same: random choose the label of positive pairs
type_same = random.randint(0,index)
#if this type don't have any pictures, choose another one
while (count[type_same]==0):
type_same = random.randint(0,index)
#choose two different images from the same type
choose1 = random.randint(0,count[type_same]-1)
choose2 = random.randint(0,count[type_same]-1)
while (choose2==choose1):
choose2 = random.randint(0,count[type_same])
#find the index of the images in orginal imags, labels
index1 = begin_index[type_same] + choose1
index2 = begin_index[type_same] + choose2
#store the samples
train_imags[train_index,:,:,:] = imgs[index1,:,:,:]
train_table[train_index] = labels[index1]
train_imags[batch_size+train_index,:,:,:] = imgs[index2,:,:,:]
train_table[batch_size+train_index] = labels[index2]
train_index = train_index+1;
#choose different pairs
for j in range(every_batch_different):
#choose two types
type_different1 = random.randint(0,index)
type_different2 = random.randint(0,index)
#choose another type
while (count[type_different1]==0):
type_different1 = random.randint(0,index)
while (count[type_different2]==0):
type_different2 = random.randint(0,index)
while (type_different1==type_different2):
type_different2 = random.randint(0,index)
#choose two pictures
choose1 = random.randint(0,count[type_different1]-1)
choose2 = random.randint(0,count[type_different2]-1)
#calculate the index
index1 = begin_index[type_different1] + choose1
index2 = begin_index[type_different2] + choose2
#assign the value
train_imags[train_index,:,:,:] = imgs[index1,:,:,:]
train_table[train_index] = labels [index1]
train_imags[batch_size+train_index,:,:,:] = imgs[index2,:,:,:]
train_table[batch_size+train_index] = labels[index2]
train_index = train_index+1;
#find the next two batch
train_index = train_index + batch_size;
return train_imags, train_table
def data_shuffle(imgs, labels):
'''
Random shuffle
Arguments:
imgs: images list
labels: labels list
Return:
imgs: images list after shuffling
labels: labels list after shuffling
'''
index = [i for i in range(len(imgs))]
shuffle(index)
imgs = imgs[index, :, :, :]
labels = labels[index, :]
return imgs, labels
def get_minibatch(image, label, batch_size, now_batch, total_batch):
'''
Get minibath and rescale the images to 96*96*3
Arguments:
image: images list
labels: labels list
batch_size: number of images per batch
now_batch: index of the current batch
total_batch: index upper bound for batches
Return:
imgs_scaled: minibatch of images after scaling
label_batch: minibatch of labels
'''
if now_batch < total_batch-1:
image_batch = image[now_batch*batch_size:(now_batch+1)*batch_size]
label_batch = label[now_batch*batch_size:(now_batch+1)*batch_size]
else:
image_batch = image[now_batch*batch_size:]
label_batch = label[now_batch*batch_size:]
imgs_scaled = np.zeros((image_batch.shape[0],96,96,3))
for i in range (image_batch.shape[0]):
imgs_scaled[i] = transform.rescale(image_batch[i], [0.857,1])
return imgs_scaled, label_batch.reshape(label_batch.shape[0])
def tf_distance(distance, labels, batch_size):
'''
Calculate the distance of positive paris and negative paris in the embedding space
Arguments:
distance: distance list of all paris
labels: labels of paris(positive or negative)
batch_size: number of images per batch
Return:
neg: total distance of negative pair in a batch
pos: total distance of positive pair in a batch
neg_c: number of negative pairs
pos_c: number of positive pairs
min_neg_dis: minimum negative distance in a batch
max_pos_dis: maximum positive distance in a batch
'''
neg = 0.0
neg_c = 0
# 正样本距离
pos = 0.0
pos_c = 0
min_neg_dis = 1000.0
max_pos_dis = 0.0
for i in range(batch_size):
if(labels[i] < 0.5):
neg += distance[i]
neg_c += 1
if(distance[i] < min_neg_dis):
min_neg_dis = distance[i]
else:
pos += distance[i]
pos_c += 1
if(distance[i] > max_pos_dis):
max_pos_dis = distance[i]
return neg, pos, neg_c, pos_c, min_neg_dis, max_pos_dis