From 48b8ec43a1518c87b07304a2a768c61853d2ace8 Mon Sep 17 00:00:00 2001
From: Vinh Khuc <knvinh@gmail.com>
Date: Sun, 12 Feb 2017 22:24:20 -0800
Subject: [PATCH] Neural networks

---
 3_neural_net.py        | 71 +++++++++++++++++++++++++++++++++++++++
 4_modern_neural_net.py | 75 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 146 insertions(+)
 create mode 100644 3_neural_net.py
 create mode 100644 4_modern_neural_net.py

diff --git a/3_neural_net.py b/3_neural_net.py
new file mode 100644
index 0000000..decf299
--- /dev/null
+++ b/3_neural_net.py
@@ -0,0 +1,71 @@
+import numpy as np
+
+import torch
+from torch.autograd import Variable
+from torch import optim
+
+from data_util import load_mnist
+
+
+def build_model(input_dim, output_dim):
+    model = torch.nn.Sequential()
+    model.add_module("linear_1", torch.nn.Linear(input_dim, 512, bias=False))
+    model.add_module("sigmoid_1", torch.nn.Sigmoid())
+    model.add_module("linear_2", torch.nn.Linear(512, output_dim, bias=False))
+    model.add_module("softmax", torch.nn.Softmax())
+    return model
+
+
+def train(model, loss, optimizer, x, y):
+    x = Variable(x, requires_grad=False)
+    y = Variable(y, requires_grad=False)
+
+    # Reset gradient
+    optimizer.zero_grad()
+
+    # Forward
+    fx = model.forward(x)
+    output = loss.forward(fx, y)
+
+    # Backward
+    output.backward()
+
+    # Update parameters
+    optimizer.step()
+
+    return output.data[0]
+
+
+def predict(model, x):
+    var_x = Variable(x, requires_grad=False)
+    output = model.forward(var_x)
+    return output.data.numpy().argmax(axis=1)
+
+
+def main():
+    torch.manual_seed(42)
+    trX, teX, trY, teY = load_mnist(onehot=False)
+    trX = torch.from_numpy(trX).float()
+    teX = torch.from_numpy(teX).float()
+    trY = torch.from_numpy(trY).long()
+
+    n_examples, n_features = trX.size()
+    n_classes = 10
+    model = build_model(n_features, n_classes)
+    loss = torch.nn.CrossEntropyLoss(size_average=True)
+    optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
+    batch_size = 100
+
+    for i in range(100):
+        cost = 0.
+        num_batches = n_examples / batch_size
+        for k in range(num_batches):
+            start, end = k * batch_size, (k + 1) * batch_size
+            cost += train(model, loss, optimizer, trX[start:end], trY[start:end])
+        predY = predict(model, teX)
+        print("Epoch %d, cost = %f, acc = %.2f%%"
+              % (i + 1, cost / num_batches, 100. * np.mean(predY == teY)))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/4_modern_neural_net.py b/4_modern_neural_net.py
new file mode 100644
index 0000000..91a042b
--- /dev/null
+++ b/4_modern_neural_net.py
@@ -0,0 +1,75 @@
+import numpy as np
+
+import torch
+from torch.autograd import Variable
+from torch import optim
+
+from data_util import load_mnist
+
+
+def build_model(input_dim, output_dim):
+    model = torch.nn.Sequential()
+    model.add_module("linear_1", torch.nn.Linear(input_dim, 512, bias=False))
+    model.add_module("relu_1", torch.nn.ReLU())
+    model.add_module("dropout_1", torch.nn.Dropout(0.2))
+    model.add_module("linear_2", torch.nn.Linear(512, 512, bias=False))
+    model.add_module("relu_2", torch.nn.ReLU())
+    model.add_module("dropout_2", torch.nn.Dropout(0.2))
+    model.add_module("linear_3", torch.nn.Linear(512, output_dim, bias=False))
+    model.add_module("softmax", torch.nn.Softmax())
+    return model
+
+
+def train(model, loss, optimizer, x, y):
+    x = Variable(x, requires_grad=False)
+    y = Variable(y, requires_grad=False)
+
+    # Reset gradient
+    optimizer.zero_grad()
+
+    # Forward
+    fx = model.forward(x)
+    output = loss.forward(fx, y)
+
+    # Backward
+    output.backward()
+
+    # Update parameters
+    optimizer.step()
+
+    return output.data[0]
+
+
+def predict(model, x):
+    var_x = Variable(x, requires_grad=False)
+    output = model.forward(var_x)
+    return output.data.numpy().argmax(axis=1)
+
+
+def main():
+    torch.manual_seed(42)
+    trX, teX, trY, teY = load_mnist(onehot=False)
+    trX = torch.from_numpy(trX).float()
+    teX = torch.from_numpy(teX).float()
+    trY = torch.from_numpy(trY).long()
+
+    n_examples, n_features = trX.size()
+    n_classes = 10
+    model = build_model(n_features, n_classes)
+    loss = torch.nn.CrossEntropyLoss(size_average=True)
+    optimizer = optim.RMSprop(model.parameters(), lr=0.01)
+    batch_size = 100
+
+    for i in range(100):
+        cost = 0.
+        num_batches = n_examples / batch_size
+        for k in range(num_batches):
+            start, end = k * batch_size, (k + 1) * batch_size
+            cost += train(model, loss, optimizer, trX[start:end], trY[start:end])
+        predY = predict(model, teX)
+        print("Epoch %d, cost = %f, acc = %.2f%%"
+              % (i + 1, cost / num_batches, 100. * np.mean(predY == teY)))
+
+
+if __name__ == "__main__":
+    main()