xgboost

dataset.py

@@ -47,16 +47,26 @@ def import_data_TREND(split:float, shuffle:bool, extra_args:Dict[str, bool]):
     data_anthropique = data_anthropique[indicies]
 
     if use_fourier_transform:
-        data_selected = np.stack([np.fft.fft(data_selected).real, np.fft.fft(data_selected).imag], axis=1)
-        data_anthropique = np.stack([np.fft.fft(data_anthropique).real, np.fft.fft(data_anthropique).imag], axis=1)
+        data_selected_fft = np.fft.fft(data_selected)
+        data_anthropique_fft = np.fft.fft(data_anthropique)
+
+        data_selected_fft_r = data_selected_fft.real/np.expand_dims(np.maximum(np.max(data_selected_fft.real, axis=-1), -np.min(data_selected_fft.real, axis=-1)), axis=-1)
+        data_selected_fft_i = data_selected_fft.imag/np.expand_dims(np.maximum(np.max(data_selected_fft.imag, axis=-1), -np.min(data_selected_fft.imag, axis=-1)), axis=-1)
+        data_selected = np.stack([data_selected, data_selected_fft_r, data_selected_fft_i], axis=1)
+
+        data_anthropique_fft_r = data_anthropique_fft.real/np.expand_dims(np.maximum(np.max(data_anthropique_fft.real, axis=-1), -np.min(data_anthropique_fft.real, axis=-1)), axis=-1)
+        data_anthropique_fft_i = data_anthropique_fft.imag/np.expand_dims(np.maximum(np.max(data_anthropique_fft.imag, axis=-1), -np.min(data_anthropique_fft.imag, axis=-1)), axis=-1)
+        data_anthropique = np.stack([data_anthropique, data_anthropique_fft_r, data_anthropique_fft_i], axis=1)
+
         data_train = np.concatenate([data_selected[:int(data_size*(1-split))], data_anthropique[:int(data_size*(1-split))]], axis=0)
         data_test = np.concatenate([data_selected[int(data_size*(1-split)):], data_anthropique[int(data_size*(1-split)):]], axis=0)
 
     else:
         data_train = np.expand_dims(np.concatenate([data_selected[:int(data_size*(1-split))], data_anthropique[:int(data_size*(1-split))]]), axis=1)
         data_test = np.expand_dims(np.concatenate([data_selected[int(data_size*(1-split)):], data_anthropique[int(data_size*(1-split)):]]), axis=1)
-        data_train = data_train - np.expand_dims(np.mean(data_train, axis=-1), axis=-1) #We normalize the input
-        data_test = data_test - np.expand_dims(np.mean(data_test, axis=-1), axis=-1)
+
+    data_train = data_train - np.expand_dims(np.mean(data_train, axis=-1), axis=-1) #We normalize the input
+    data_test = data_test - np.expand_dims(np.mean(data_test, axis=-1), axis=-1)
 
     labels_train = np.expand_dims(np.concatenate([np.ones((int(data_size*(1-split)),)), np.zeros((int(data_size*(1-split)),))]), axis=1)
     labels_test = np.expand_dims(np.concatenate([np.ones((int(data_size*(split)),)), np.zeros((int(data_size*(split)),))]), axis=1)

main.py

@@ -13,6 +13,7 @@
 from dataset import import_dataset
 from model import SimpleModel
 from utils import create_batch_tensorboard
+import xgboost as xgb
 
 dataset_name = "trend"
 # Get cpu or gpu device for training.
@@ -31,7 +32,7 @@
     print("Unknown exception")
 
 
-comment = "trend_model"
+comment = "xgb_model"
 tensorboard_log_dir = "./Models/"+ dataset_name + "/" + comment + "-" + datetime.now().strftime("%Y-%m-%d-%H-%M-%S") # + str(len(os.listdir("./Models/"+ dataset_name))) To count the experiments
 
 
@@ -44,14 +45,21 @@
 
 #Define loss funct and optimizer
 criterion = torch.nn.BCELoss()
-optimizer = torch.optim.Adam(model.parameters(), lr=5e-4, weight_decay=1e-4)
+optimizer = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-4)
 lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=1)
 batch_size = 10
 
 #Import dataset
-dataset_args = {"use_fourier_transform":False}
+dataset_args = {"use_fourier_transform":True}
 data_train, data_test = import_dataset(dataset_name, split=0.2, shuffle=True, extra_args=dataset_args)
 
+learning_rate_range = np.arange(0.01, 1, 0.05)
+for lr in learning_rate_range:
+    xgb_classifier = xgb.XGBClassifier(eta=lr)
+    xgb_classifier.fit(data_train[0][:, 2, :], data_train[1])
+    print("lr", lr, xgb_classifier.score(data_train[0][:, 2, :], data_train[1]))
+    print(xgb_classifier.score(data_test[0][:, 2, :], data_test[1]))
+
 print(data_train[0].shape, data_train[1].shape)
 for k in range(1):
     print(data_train[1][k])
@@ -113,7 +121,7 @@ def train_epoch(epoch:int, data, data_labels, is_testing:bool=False):
             lr_scheduler.step()
             # Gather data and report
 
-        if (i % 30 == 29) or (i * batch_size >= size):
+        if (i % 30 == 29) or ((i+1) * batch_size >= size):
             loss, current = loss.item(), i * batch_size + len(inputs) 
             key = "test" if is_testing else "train"
             print(f"{key}:{epoch}, loss: {mean_loss}, accuracy: {mean_accuracy}  [{current}/{size}]")
@@ -122,13 +130,13 @@ def train_epoch(epoch:int, data, data_labels, is_testing:bool=False):
             mean_counter = 0
 
 #training
-nb_epoch = 30 
+nb_epoch = 40 
 for i in range(nb_epoch):
     print(f"epoch: {i}, lr: {lr_scheduler.get_last_lr()}")
     train_epoch(i, data_train[0], data_train[1])
     train_epoch(i, data_test[0], data_test[1], is_testing=True)
     if i % 5 == 0:
-        torch.save(model.state_dict, tensorboard_log_dir + "/checkpoint" + str(nb_epoch) + ".pth")
+        torch.save(model.state_dict, tensorboard_log_dir + "/checkpoint" + str(i) + ".pth")
 
 
 writer.flush()

model.py

@@ -1,15 +1,22 @@
 import torch.nn as F
+import torch
 
 class SimpleModel(F.Module):
     def __init__(self):
         super(SimpleModel, self).__init__()
         self.layers = []
 
-        self.conv1 = F.Conv1d(1, 128, kernel_size=15, padding=7)
-        self.layers.append(self.conv1)
+        self.conv1_sig = F.Conv1d(1, 64, kernel_size=15, padding=7)
+        self.layers.append(self.conv1_sig)
 
-        self.batch_norm1 = F.BatchNorm1d(128)
-        self.layers.append(self.batch_norm1)
+        self.conv1_fft = F.Conv1d(2, 64, kernel_size=15, padding=7)
+        self.layers.append(self.conv1_fft)
+
+        self.batch_norm1_sig = F.BatchNorm1d(64)
+        self.layers.append(self.batch_norm1_sig)
+
+        self.batch_norm1_fft = F.BatchNorm1d(64)
+        self.layers.append(self.batch_norm1_fft)
 
         self.conv2 = F.Conv1d(128, 128, kernel_size=7, padding=3)
         self.layers.append(self.conv2)
@@ -47,8 +54,13 @@ def __init__(self):
         self.sigmoid = F.Sigmoid()
 
     def forward(self, x):
-        x = self.maxpool(self.conv1(x))
-        x = self.dropout(self.activation(self.batch_norm1(x)))
+        x_sig = self.maxpool(self.conv1_sig(x[:, :1, :]))
+        x_sig = self.dropout(self.activation(self.batch_norm1_sig(x_sig)))
+
+        x_fft = self.maxpool(self.conv1_fft(x[:, 1:, :]))
+        x_fft = self.dropout(self.activation(self.batch_norm1_fft(x_fft)))
+
+        x = torch.cat([x_sig, x_fft], axis=1)
 
         x = self.dropout(self.activation(self.activation(self.batch_norm2(self.conv2(x))) + x))
         x = self.maxpool(x)
@@ -57,7 +69,7 @@ def forward(self, x):
         x = self.maxpool(x)
 
         x = self.dropout(self.activation(self.activation(self.batch_norm4(self.conv4(x))) + x))
-#        x = self.maxpool(x)
+        #x = self.maxpool(x)
 
         x = self.flatten(x)
         x = self.activation(self.dense1(x))