-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy path02_pytorch_nnclassification.py
514 lines (382 loc) · 84.7 KB
/
02_pytorch_nnclassification.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
# -*- coding: utf-8 -*-
"""02_pytorch_NNClassification.ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/163rKmTTVN6m4blbWzR2mHaCZBoYzq60q
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sklearn
from sklearn.datasets import make_circles
from sklearn.model_selection import train_test_split
"""## Make Classification Data and get ready
The `make_circles()` method from Scikit-Learn to generate two circles with different coloured dots.
"""
n_samples = 2000
X, y = make_circles(n_samples,
noise = 0.03,
random_state = 42)
len(X), len(y)
print(f"First 5 samples of X: \n{X[:5]}\n")
print(f"First 5 samples of y: \n{y[:5]}")
# Make a data frame of circle data
circles = pd.DataFrame({'X1': X[:, 0],
'X2': X[:, 1],
'label': y})
circles.head()
plt.scatter(x = X[:, 0],
y = X[:, 1],
c = y,
cmap = plt.cm.RdYlBu)
plt.xlabel('X1')
plt.ylabel('X2')
plt.show()
X.shape, y.shape
# Check different labels
circles.label.value_counts()
"""Turn data into tensors"""
X = torch.from_numpy(X).type(torch.float)
y = torch.from_numpy(y).type(torch.float)
X[:5], y[:5]
type(X), X.dtype, y.dtype
torch.manual_seed(42)
X_train, X_test, y_train, y_test = train_test_split(X, y, shuffle = True, test_size = 0.2, random_state = 42)
len(X_train), len(X_test)
"""## Building a Model
Build a model for classification
To do so, we need or have to
1. Setup device agnostic code, to run our code on an accelerator(GPU), if there is one
2. Construct a model
3. Define a loss function and optimizer
4. Create a training and testing loop
### Device Agnostic Code
"""
# Make a device agnostic code
device = 'cuda' if torch.cuda.is_available() else 'cpu'
device
"""### Construct a Model
Let's create a model class that:
1. Subclasses `nn.Module` (almost all PyTorch models are subclasses of nn.Module).
2. Creates 2 `nn.Linear` layers in the constructor capable of handling the input and output shapes of X and y.
3. Defines a `forward()` method containing the forward pass computation of the model.
4. Instantiates the model class and sends it to the target device.
"""
class NNCircleClassifier(nn.Module):
def __init__(self, in_features, h1, out_features):
super().__init__()
self.layer1 = nn.Linear(in_features, h1)
self.layer2 = nn.Linear(h1, out_features)
# self.two_linear_layers = nn.Sequential(
# nn.Linear(in_features = 2, outfeatures = 5),
# nn.Linear(in_features = 5, out_features = 1)
# )
def forward(self, x):
x = (self.layer1(x))
x = (self.layer2(x))
return x
model_0 = NNCircleClassifier(in_features = 2, h1 = 5, out_features = 1).to(device)
model_0
model_0.state_dict()
# model_0 = nn.Sequential(
# nn.Linear(in_features = 2, outfeatures = 5),
# nn.Linear(in_features = 5, out_features = 1)
# ).to(device)
"""### Device Loss Function and Optimizer
"""
# loss_fn = nn.BCELoss() --> this requires inputs to have gone through the sigmoid activation function prior to input going through BCE
loss_fn = nn.BCEWithLogitsLoss()
optimizer = torch.optim.SGD(params = model_0.parameters(),
lr = 0.01)
"""Define Accuracy Function"""
# Calculate accuracy
def accuracy_fn(y_true, y_pred):
correct = torch.eq(y_true, y_pred).sum().item()
acc = (correct/len(y_pred))*100
return acc
"""### Training Model
1. Set up training mode
2. Forward Pass
3. Calculate the Loss
4. Optimizer zero grad
5. Loss Backward
6. Optimizer Step
Going from *raw logits* ⟶ *prediction probabilities* ⟶ *prediction labels*
Our model outputs are going to be raw logits.
We have to convert them into **prediction probs** by passing them into some kind of activation function (e.g. sigmoid for BCE and softmax for for multiclass).
Then we we can convert *prediction probs* ⟶ *prediction labels* by either rounding them or taking the `argmax()`
"""
model_0.eval()
with torch.inference_mode():
y_logits = model_0((X_test.to(device))[:5])
y_logits
y_test[:5]
"""Use the sigmoid activation function on output logits, to turn them into pred probs"""
y_pred_probs = torch.sigmoid(y_logits)
y_pred_probs
"""For our *pred prob* values, we need to perform a range-style rounding on them:
* `y_pred_probs` `> =` `0.5` ⟹ `y=1` (class 1)
* `y_pred_probs` `< ` `0.5` ⟹ `y=0` (class 0)
"""
y_preds = torch.round(y_pred_probs)
# in one go
y_pred_labels = torch.round(torch.sigmoid(model_0(X_test.to(device)[:5])))
y_pred_labels
"""Building a Training Loop and a Testing Loop"""
torch.manual_seed(42)
epochs = 100
# batch_size = 32
X_train, y_train = X_train.to(device), y_train.to(device)
X_test, y_test = X_test.to(device), y_test.to(device)
epoch_count = []
train_losses = []
test_losses = []
train_accs = []
test_accs = []
for epoch in range(epochs):
model_0.train()
# 1. Forward Pass
y_logits = model_0.forward(X_train).squeeze()
y_pred = torch.round(torch.sigmoid(y_logits))
# 2. Calculate Loss / accuracy
# loss = nn.BCELoss(torch.sigmoids(y_logits), # nn.BCE expects prediction probabilities as input
# y_train)
loss = loss_fn(y_logits,
y_train) # remember nn.BCEwithLogits expects raw logits as it performs sigmoid by itself
acc = accuracy_fn(y_true = y_train,
y_pred = y_pred)
epoch_count.append(epoch)
train_losses.append(loss)
train_accs.append(acc)
# 3. Optimizer zero grad
optimizer.zero_grad()
# 4. Loss Backward
loss.backward()
# 5. Optimizer step
optimizer.step()
model_0.eval()
with torch.inference_mode():
# 1. Forward Pass
test_logits = model_0(X_test).squeeze()
test_preds = torch.round(torch.sigmoid(test_logits))
# 2. Calculate Loss/accuracy
test_loss = loss_fn(test_logits,
y_test)
test_acc = accuracy_fn(y_true = y_test,
y_pred = test_preds)
test_losses.append(test_loss)
test_accs.append(test_acc)
if epoch % 10 == 0:
print(f"Epoch: {epoch} | Train Loss: {loss: .5f}, | Train Acc: {acc: .2f}% | Test loss: {test_loss: .5f}, | Test Acc: {test_acc: .2f}%")
import matplotlib.pyplot as plt
def plot_decision_boundary(model, X, y):
X, y = X.to("cpu"), y.to("cpu") # CPU works better with NumPy and Matplotlib
# setup prediction boundaries
x1_min, x1_max = X[:, 0].min() - 0.1, X[:,0].max() + 0.1
x2_min, x2_max = X[:, 1].min() - 0.1, X[:,1].max() + 0.1
x1_ = np.linspace(x1_min, x1_max, 101) # creates evenly spaced 101 points between min and max
x2_ = np.linspace(x2_min, x2_max, 101)
x1x1, x2x2 = np.meshgrid(x1_, x2_) # creates a meshgrid --> useful for contour plot
xx1_ravel = x1x1.ravel()
xx2_ravel = x2x2.ravel()
# np.column_stack((x1_, x2_)) --> will simply add x2_ beside x1 and size is 101
# np.column_stack(xx1_ravel, xx2_ravel) --> will add each element of x2_ beside each element of x1_ and size is 10201
# because size of xx1 is 10201 which is flattened version of meshgrid
# Make features
X_to_pred_on = torch.from_numpy(np.column_stack((xx1_ravel, xx2_ravel))).float().to(device) # taking each x1 and x2 to create feature values
# make predictions
model.eval()
with torch.inference_mode():
y_logits = model(X_to_pred_on)
# Test for multiclass of binary
if (len(torch.unique(y)) > 2):
y_pred = torch.softmax(y_logits, dim = 1).argmax(dim=1)
else:
y_pred = torch.round(torch.sigmoid(y_logits)) #binary
y_pred = y_pred.reshape(x1x1.shape).detach().cpu().numpy()
plt.contourf(x1x1, x2x2, y_pred, cmap = plt.cm.RdYlBu, alpha = 0.7)
plt.scatter(X[:,0], X[:,1], c = y, s = 40, cmap = plt.cm.RdYlBu)
plt.xlim(x1x1.min(), x1x1.max())
plt.ylim(x2x2.min(), x2x2.max())
plt.figure(figsize = (12, 6))
plt.subplot(1, 2, 1)
plt.title("Train")
plot_decision_boundary(model_0, X_train, y_train)
plt.subplot(1, 2, 2)
plt.title("Test")
plot_decision_boundary(model_0, X_test, y_test)
"""## Improving our model
* Add more layers
* Add more nuerons in hidden units
* Change the activation functions
* Change the learning rate
* Change the loss function
"""
class CircleModelV1(nn.Module):
def __init__(self):
super().__init__()
self.layer1 = nn.Linear(in_features = 2, out_features = 10)
self.layer2 = nn.Linear(in_features = 10, out_features = 10)
self.layer3 = nn.Linear(in_features = 10, out_features = 1)
def forward(self, x):
x = self.layer1(x)
x = F.relu(x)
x = self.layer2(x)
x = F.relu(x)
x = self.layer3(x)
return x
model_1 = CircleModelV1().to(device)
model_1
model_1.state_dict()
# loss_fn = nn.BCELoss() --> this requires inputs to have gone through the sigmoid activation function prior to input going through BCE
loss_fn = nn.BCEWithLogitsLoss()
optimizer = torch.optim.SGD(params = model_1.parameters(),
lr = 0.1)
# Calculate accuracy
def accuracy_fn(y_true, y_pred):
correct = torch.eq(y_true, y_pred).sum().item()
acc = (correct/len(y_pred))*100
return acc
torch.manual_seed(42)
epochs = 1000
X_train, y_train = X_train.to(device), y_train.to(device)
X_test, y_test = X_test.to(device), y_test.to(device)
epoch_count = []
train_losses = []
test_losses = []
train_accs = []
test_accs = []
for epoch in range(epochs):
model_1.train()
# 1. Forward Pass
y_logits = model_1.forward(X_train).squeeze()
y_pred = torch.round(torch.sigmoid(y_logits))
# 2. Calculate Loss / accuracy
# loss = nn.BCELoss(torch.sigmoids(y_logits), # nn.BCE expects prediction probabilities as input
# y_train)
loss = loss_fn(y_logits,
y_train) # remember nn.BCEwithLogits expects raw logits as it performs sigmoid by itself
acc = accuracy_fn(y_true = y_train,
y_pred = y_pred)
epoch_count.append(epoch)
train_losses.append(loss)
train_accs.append(acc)
# 3. Optimizer zero grad
optimizer.zero_grad()
# 4. Loss Backward
loss.backward()
# 5. Optimizer step
optimizer.step()
model_1.eval()
with torch.inference_mode():
# 1. Forward Pass
test_logits = model_1(X_test).squeeze()
test_preds = torch.round(torch.sigmoid(test_logits))
# 2. Calculate Loss/accuracy
test_loss = loss_fn(test_logits,
y_test)
test_acc = accuracy_fn(y_true = y_test,
y_pred = test_preds)
test_losses.append(test_loss)
test_accs.append(test_acc)
if epoch % 50 == 0:
print(f"Epoch: {epoch} | Train Loss: {loss: .5f}, | Train Acc: {acc: .2f}% | Test loss: {test_loss: .5f}, | Test Acc: {test_acc: .2f}%")
plt.figure(figsize = (12, 6))
plt.subplot(1, 2, 1)
plt.title("Train")
plot_decision_boundary(model_1, X_train, y_train)
plt.subplot(1, 2, 2)
plt.title("Test")
plot_decision_boundary(model_1, X_test, y_test)
"""## The missing piece - Non- Linearity
Neural Networks, rather than us telling the mdel what to learn, we give it the tools to discover patterns in data and it tries to figure out the patterns on its own
And these tools are linear & non-linear functions.
"""
class CircleModelV2(nn.Module):
def __init__(self):
super().__init__()
self.layer1 = nn.Linear(in_features = 2, out_features = 15)
self.layer2 = nn.Linear(in_features = 15, out_features = 10)
self.layer3 = nn.Linear(in_features = 10, out_features = 1)
def forward(self, x):
x = self.layer1(x)
x = F.relu(x)
x = self.layer2(x)
x = F.relu(x)
x = self.layer3(x)
return x
model_2 = CircleModelV2().to(device)
model_2
# loss_fn = nn.BCELoss() --> this requires inputs to have gone through the sigmoid activation function prior to input going through BCE
loss_fn = nn.BCEWithLogitsLoss()
optimizer = torch.optim.SGD(params = model_2.parameters(),
lr = 0.1)
# Calculate accuracy
def accuracy_fn(y_true, y_pred):
correct = torch.eq(y_true, y_pred).sum().item()
acc = (correct/len(y_pred))*100
return acc
torch.manual_seed(42)
torch.cuda.manual_seed(42)
epochs = 1000
X_train, y_train = X_train.to(device), y_train.to(device)
X_test, y_test = X_test.to(device), y_test.to(device)
epoch_count = []
train_losses = []
test_losses = []
train_accs = []
test_accs = []
for epoch in range(epochs):
model_2.train()
# 1. Forward Pass
y_logits = model_2.forward(X_train).squeeze()
y_pred = torch.round(torch.sigmoid(y_logits))
# 2. Calculate Loss / accuracy
loss = loss_fn(y_logits,
y_train)
acc = accuracy_fn(y_true = y_train,
y_pred = y_pred)
epoch_count.append(epoch)
train_losses.append(loss)
train_accs.append(acc)
# 3. Optimizer zero grad
optimizer.zero_grad()
# 4. Loss Backward
loss.backward()
# 5. Optimizer step
optimizer.step()
model_2.eval()
with torch.inference_mode():
# 1. Forward Pass
test_logits = model_2(X_test).squeeze()
test_preds = torch.round(torch.sigmoid(test_logits))
# 2. Calculate Loss/accuracy
test_loss = loss_fn(test_logits,
y_test)
test_acc = accuracy_fn(y_true = y_test,
y_pred = test_preds)
test_losses.append(test_loss)
test_accs.append(test_acc)
if epoch % 50 == 0:
print(f"Epoch: {epoch} | Train Loss: {loss: .5f}, | Train Acc: {acc: .2f}% | Test loss: {test_loss: .5f}, | Test Acc: {test_acc: .2f}%")
plt.figure(figsize = (12, 6))
plt.subplot(1, 2, 1)
plt.title("Train")
plot_decision_boundary(model_2, X_train, y_train)
plt.subplot(1, 2, 2)
plt.title("Test")
plot_decision_boundary(model_2, X_test, y_test)
plt.figure(figsize = (18, 6))
plt.subplot(1, 3, 1)
plt.title("1 H_layer with 5 units and NO activations")
plot_decision_boundary(model_0, X_train, y_train)
plt.subplot(1, 3, 2)
plt.title("2 H_layers with 10 units each and NO activations")
plot_decision_boundary(model_1, X_train, y_train)
plt.subplot(1, 3, 3)
plt.title("3 H_layers with 15 and 10 units respectively with activations")
plot_decision_boundary(model_2, X_train, y_train)