-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathann_torch.py
135 lines (127 loc) · 5.45 KB
/
ann_torch.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
#coding https://mattmazur.com/2015/03/17/a-step-by-step-backpropagation-example/ with pytorch, checking with iterative version at https://github.com/animesh/ann/blob/master/ann/Program.cs with following output
#Iteration = 1 Error = 0.298371108760003 Outputs = 0.751365069552316 0.772928465321463
#Iteration = 2 Error = 0.291027773693599 Outputs = 0.742088111190782 0.775284968294459 ...
#https://youtu.be/FHdlXe1bSe4?t=319
input=[0.05,0.10]
inpw=[[0.15,0.20],[0.25,0.3]]
hidw=[[0.4,0.45],[0.5,0.55]]
output=[0.01,0.99]
bias=[0.35,0.6]
lr=0.5
#mamba create -n torch -c nvidia -c pytorch -c conda-forge pytorch torchvision torchaudio cudatoolkit=11.6
#https://pub.towardsai.net/the-fundamental-mathematics-of-machine-learning-39c2418d19c6
import torch
import torch.nn as nn
import torch.optim as optim
# Define a simple neural network
class SimpleNN(nn.Module):
def __init__(self):
super(SimpleNN, self).__init__()
self.fc1 = nn.Linear(2, 3)
self.fc2 = nn.Linear(3, 1)
def forward(self, x):
x = torch.relu(self.fc1(x))
x = torch.sigmoid(self.fc2(x))
return x
# Example data
X = torch.tensor([[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0]])
y = torch.tensor([[0.0], [1.0], [1.0], [0.0]])
# Initialize model, loss function, and optimizer
model = SimpleNN()
criterion = nn.BCELoss()
optimizer = optim.SGD(model.parameters(), lr=0.1)
# Training loop
for epoch in range(10000):
optimizer.zero_grad()
output = model(X)
loss = criterion(output, y)
loss.backward()
optimizer.step()
print("Finished Training")
print(output)
#OUTPUT: Finished Training tensor([[0.0259],[0.8772],[0.8772],[0.1237]], grad_fn=<SigmoidBackward0>)
import torch
print("PyTorch ",torch.__version__)
from datetime import datetime
startTime = datetime.now()
print("Start time:", startTime)
if torch.cuda.is_available():
print(torch.cuda.get_device_name(torch.cuda.current_device()))
print('Allocated:', round(torch.cuda.memory_allocated(0)/1024**3,1), 'GB')
print('Cached: ', round(torch.cuda.memory_reserved(0)/1024**3,1), 'GB')
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import SGD
class mANN(nn.Module):
def __init__(self):
super().__init__()
self.iw00 = nn.Parameter(torch.tensor(inpw[0][0]), requires_grad=True)
self.iw01 = nn.Parameter(torch.tensor(inpw[0][1]), requires_grad=True)
self.iw10 = nn.Parameter(torch.tensor(inpw[1][0]), requires_grad=True)
self.iw11 = nn.Parameter(torch.tensor(inpw[1][1]), requires_grad=True)
self.bi0 = nn.Parameter(torch.tensor(bias[0]), requires_grad=True)
self.hw00 = nn.Parameter(torch.tensor(hidw[0][0]), requires_grad=True)
self.hw01 = nn.Parameter(torch.tensor(hidw[0][1]), requires_grad=True)
self.hw10 = nn.Parameter(torch.tensor(hidw[1][0]), requires_grad=True)
self.hw11 = nn.Parameter(torch.tensor(hidw[1][1]), requires_grad=True)
self.bi1 = nn.Parameter(torch.tensor(bias[1]), requires_grad=True)
def forward(self, input):
input_to_top_relu = input * self.iw00 + self.bi0
top_relu_output = F.relu(input_to_top_relu)
scaled_top_relu_output = top_relu_output * self.iw01
input_to_bottom_relu = input * self.iw10 + self.bi0
bottom_relu_output = F.relu(input_to_bottom_relu)
scaled_bottom_relu_output = bottom_relu_output * self.iw11
input_to_final_relu = scaled_top_relu_output + scaled_bottom_relu_output + self.bi1
output = F.relu(input_to_final_relu)
return output
model = mANN()
output_values = model(input)
inputs = torch.tensor(input)
labels = torch.tensor(output)
optimizer = SGD(model.parameters(), lr=lr)
print("Final bias, before optimization: " + str(model.final_bias.data) + "\n")
for epoch in range(2):
total_loss = 0
for iteration in range(len(inputs)):
input_i = inputs[iteration]
label_i = labels[iteration]
output_i = model(input_i)
loss = (output_i - label_i)**2
loss.backward()
total_loss += float(loss)
if (total_loss < 0.0001):
print("Num steps: " + str(epoch))
break
optimizer.step()
optimizer.zero_grad()
print("Step: " + str(epoch) + " Final Bias: " + str(model.final_bias.data) + "\n")
print("Total loss: " + str(total_loss))
print("Final bias, after optimization: " + str(model.final_bias.data))
iter=0
while iter<0:
iter+=1
h = torch.sigmoid(x.matmul(w1.transpose(0,1))+b[0])
y_pred = torch.sigmoid(h.matmul(w2.transpose(0,1))+b[1])
print("iteration:",iter,"MSE: ",0.5*(((y_pred - y).pow(2)).sum()))
grad=(y_pred - y)*(1-y_pred)*y_pred # numerically unstable?
w1-=lr*w2.matmul(grad.reshape(-1, 1))*h*(1-h).reshape(-1, 1).matmul(x) # though it seems like descent is faster if first layer done in end? #FILO
print(w1)
#w2-=lr*np.outer(grad,h)
print(w2)
print("Time taken:", datetime.now() - startTime)
#https://alphasignalai.beehiiv.com/p/gpulevel-inference-cpu?utm_source=alphasignalai.beehiiv.com&utm_medium=newsletter&utm_campaign=gpu-level-inference-on-your-cpu
from torch.utils.tensorboard import SummaryWriter
import torchvision
# Init writer and model
writer = SummaryWriter('runs/demo')
model = torchvision.models.resnet50()
dummy_data, _ = load_dataset()
# Add model graph
writer.add_graph(model, dummy_data)
# Fake training loop for demo
for epoch in range(5):
loss = epoch * 0.1 # Simulated loss
writer.add_scalar('train_loss', loss, epoch)
# Close writer
writer.close()