-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathbaseline_CIFAR10_sup.py
164 lines (136 loc) · 8.38 KB
/
baseline_CIFAR10_sup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
import torch
import random
from torchvision import datasets
from torchvision.transforms import ToTensor
from torchsummary import summary
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
import os
from Networks.create_lenet import create_lenet
from Train.train_CNP_images import train_sup
from Utils.simple_models import KNN_classifier, LR_classifier, SVM_classifier
from Utils.data_loader import load_supervised_data_as_matrix,load_supervised_data_as_generator
from Utils.helper_results import test_model_accuracy_with_best_checkpoint, plot_loss
if __name__ == "__main__":
random.seed(1234)
dataset = "CIFAR10"
##### K-nearest neighbour #####
print("KNN, LR and SVM")
accuracies_dir_txt_knn = "saved_models/" + dataset + "/supervised/accuracies/KNN.txt"
accuracies_dir_txt_lr = "saved_models/" + dataset + "/supervised/accuracies/LR.txt"
accuracies_dir_txt_SVM = "saved_models/" + dataset + "/supervised/accuracies/SVM.txt"
ks = [1,2,3,5,7,10]
cs = [1e-2,1,1e2,1e4,1e6,1e8,1e10]
max_iter = 1000
num_training_samples = [400, 800, 1600, 2400, 3200, 4000, 24000, 40000]
optimal_k = np.zeros(len(num_training_samples))
accuracies_knn = np.zeros(len(num_training_samples))
optimal_c = np.zeros(len(num_training_samples))
accuracies_lr = np.zeros(len(num_training_samples))
optimal_c_SVM = np.zeros(len(num_training_samples))
accuracies_SVM = np.zeros(len(num_training_samples))
for i,num_samples in enumerate(num_training_samples):
if i == 0: # at the iteration over the different number of training samples
# create directories for the accuracy if they don't exist yet
dir_to_create = os.path.dirname(accuracies_dir_txt_knn)
os.makedirs(dir_to_create, exist_ok=True)
dir_to_create = os.path.dirname(accuracies_dir_txt_lr)
os.makedirs(dir_to_create, exist_ok=True)
dir_to_create = os.path.dirname(accuracies_dir_txt_SVM)
os.makedirs(dir_to_create, exist_ok=True)
X_train, y_train, X_validation, y_validation, X_test, y_test = load_supervised_data_as_matrix(num_samples,dataset=dataset)
accuracies_knn[i], optimal_k[i] = KNN_classifier(X_train, y_train, X_validation, y_validation, X_test, y_test, ks=ks)
accuracies_lr[i], optimal_c[i] = LR_classifier(X_train, y_train, X_validation, y_validation, X_test, y_test, cs=cs, max_iter=max_iter)
accuracies_SVM[i], optimal_c_SVM[i] = SVM_classifier(X_train, y_train, X_validation, y_validation, X_test, y_test, cs=cs, max_iter=max_iter)
# write the accuracy to the text file
with open(accuracies_dir_txt_knn, 'a+') as f:
text = str(num_samples) +", " + str(accuracies_knn[i]) + "\n"
f.write(text)
with open(accuracies_dir_txt_lr, 'a+') as f:
text = str(num_samples) +", " + str(accuracies_lr[i]) + "\n"
f.write(text)
with open(accuracies_dir_txt_SVM, 'a+') as f:
text = str(num_samples) +", " + str(accuracies_SVM[i]) + "\n"
f.write(text)
##### LeNet #####
# use GPU if available
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
# type of model
dropout = True
model_name = "LeNet" + ("_dropout" if dropout else "") # one of ["CNP", "ConvCNP", "ConvCNPXL"]
print(model_name)
cheat_validation= True# use a large validation set even if the trainign data is small
# for continued supervised training
train = True
load = False
save = False
evaluate = True
if load:
epoch_start = 20 # which epoch to start from
else:
epoch_start = 0
# training parameters
num_training_samples = [400, 800, 1600, 2400, 3200, 4000, 24000, 40000]
for model_size in ["small","medium","large"]:
for i,num_samples in enumerate(num_training_samples):
if num_samples <= 200:
batch_size = 64
learning_rate = 5e-3
epochs = 200
save_freq = 20
else:
batch_size = 64
learning_rate = 1e-3
epochs = 200
save_freq = 20
# load the supervised set
train_data, validation_data, test_data, img_height, img_width, num_channels, num_classes = load_supervised_data_as_generator(batch_size, num_samples,cheat_validation=cheat_validation,dataset=dataset)
# create the model
model = create_lenet(model_size,dropout,num_channels=num_channels,num_classes=num_classes)
model.to(device)
# define the directories
model_save_dir = ["saved_models/" + dataset + "/supervised" + ("_cheat_validation/" if cheat_validation else "/") + str(num_samples) + "S/", model_name, "/", model_name, "_",model_size, "", "E", ".pth"]
train_loss_dir_txt = "saved_models/" + dataset + "/supervised" + ("_cheat_validation/" if cheat_validation else "/") + str(num_samples) + "S/" + model_name + "/loss/" + model_name + "_" + model_size + "_train.txt"
validation_loss_dir_txt = "saved_models/" + dataset + "/supervised" + ("_cheat_validation/" if cheat_validation else "/") + str(num_samples) + "S/" + model_name + "/loss/" + model_name + "_" + model_size + "_validation.txt"
loss_dir_plot = "saved_models/" + dataset + "/supervised" + ("_cheat_validation/" if cheat_validation else "/") + str(num_samples) + "S/"+ model_name + "/loss/" + model_name + "_" + model_size + ".svg"
accuracies_dir_txt = "saved_models/" + dataset + "/supervised" + ("_cheat_validation/" if cheat_validation else "/") + "accuracies/" + model_name + "_" + model_size + ".txt"
# create directories for the checkpoints and loss files if they don't exist yet
dir_to_create = "".join(model_save_dir[:3]) + "loss/"
os.makedirs(dir_to_create, exist_ok=True)
if load:
load_dir = model_save_dir.copy()
load_dir[-3] = str(epoch_start)
load_dir = "".join(load_dir)
if train:
# check if the loss file is valid
with open(train_loss_dir_txt, 'r') as f:
nbr_losses = len(f.read().split())
assert nbr_losses == epoch_start, "The number of lines in the loss file does not correspond to the number of epochs"
# load the model
model.load_state_dict(torch.load(load_dir, map_location=device))
else:
# if train from scratch, check if a loss file already exists (it should not, so remove it if necessary)
assert not (os.path.isfile(train_loss_dir_txt)), "The corresponding loss file already exists, please remove it to train from scratch"
if train:
avg_loss_per_epoch = train_sup(train_data, model, epochs, model_save_dir, train_loss_dir_txt,
validation_data=validation_data,
validation_loss_dir_txt=validation_loss_dir_txt, is_CNP = False,
save_freq=save_freq,
epoch_start=epoch_start, device=device, learning_rate=learning_rate)
plot_loss([train_loss_dir_txt, validation_loss_dir_txt], loss_dir_plot)
if save:
save_dir = model_save_dir.copy()
save_dir[-3] = str(epoch_start + epochs)
save_dir = "".join(save_dir)
torch.save(model.state_dict(), save_dir)
if evaluate:
if i == 0: # at the iteration over the different number of training samples
# create directories for the accuracy if they don't exist yet
dir_to_create = os.path.dirname(accuracies_dir_txt)
os.makedirs(dir_to_create, exist_ok=True)
num_context_points = img_height * img_width * num_channels
accuracy = test_model_accuracy_with_best_checkpoint(model,model_save_dir,validation_loss_dir_txt,test_data,device,convolutional=False,num_context_points=num_context_points, save_freq=save_freq, is_CNP=False)
# write the accuracy to the text file
with open(accuracies_dir_txt, 'a+') as f:
text = str(num_samples) +", " + str(accuracy) + "\n"
f.write(text)