Merge pull request #3 from AkiRusProd/dev

Autograd update

Author: AkiRusProd

.gitignore

@@ -137,4 +137,7 @@ dmypy.json
 *.png
 *.jpeg
 *.zip
-*.nt
+*.nt
+
+datasets/
+saved models/

README.md

@@ -1,5 +1,5 @@
 # numpy-nn-model
-Сustom CPU/GPU torch style machine learning framework with automatic differentiation for creating neural networks, implemented on numpy with cupy.
+Сustom CPU/GPU torch style machine learning framework with tape-based automatic differentiation for creating neural networks, implemented on numpy with cupy.
 
 ## Some information and features:
 
@@ -9,16 +9,17 @@
 1) Sigmoid
 2) Tanh
 3) Softmax
-4) Softplus
-5) Softsign
-6) Swish
-7) Mish
-8) TanhExp
-9) ReLU
-10) LeakyReLU
-11) ELU
-12) SELU
-13) GELU
+4) LogSoftmax
+5) Softplus
+6) Softsign
+7) Swish
+8) Mish
+9) TanhExp
+10) ReLU
+11) LeakyReLU
+12) ELU
+13) SELU
+14) GELU
 
 *[See Activation Functions...](neunet/nn/activations.py)*
 
@@ -137,9 +138,10 @@ Some [examples](examples/) were trained on the [MNIST](https://pjreddie.com/proj
 5. *[Generative Adversarial Network](examples/gan.py)*     
 6. *[Recurrent Digits Classifier](examples/recurrent_digits_classifier.py)*    
 7. *[Recurrent Sequences Classifier](examples/recurrent_sequences_classifier.py)*    
-8. *[Variational Autoencoder](examples/vae.py)*    
-9. *[Vector Quantized Variational Autoencoder](examples/vqvae.py)* 
-10. *[Word2Vec](examples/word2vec.py)*
+8. *[Seq2Seq Transformer](examples/seq2seq.py)*
+9. *[Variational Autoencoder](examples/vae.py)*    
+10. *[Vector Quantized Variational Autoencoder](examples/vqvae.py)* 
+11. *[Word2Vec](examples/word2vec.py)*
 
 
 
@@ -329,6 +331,44 @@ Code:
 *[Model Example](examples/recurrent_digits_classifier.py)*
 </details>
 
+<details>
+<summary>Seq2Seq Transformer</summary>
+
+#### Examples of translated sentences of validation set:  
+
+>Example №1  
+*Input sentence: These four people are standing outdoors, with 3 dogs.  
+Decoded sentence: Vier Personen stehen im Freien mit drei Hunden.  
+Target sentence: Diese vier Personen stehen mit 3 Hunden im Freien.*  
+
+>Example №2  
+*Input sentence: A man in a martial arts uniform in midair.  
+Decoded sentence: Ein Mann in Uniform befindet sich in der Luft.  
+Target sentence: Ein Mann in einem Karateanzug in der Luft.*  
+
+>Example №3  
+*Input sentence: A long-haired, male musician is playing on a piano.  
+Decoded sentence: Ein langhaariger Mann spielt Klavier auf einem Klavier.   
+Target sentence: Ein Musiker mit langen Haaren spielt Keyboard.*  
+
+>Example №4  
+*Input sentence: A child is laying on a beige rug laughing.  
+Decoded sentence: Ein Kind liegt auf einem beigen Teppich.  
+Target sentence: Ein Kind liegt auf einem beigefarbenen Teppich und lacht.*  
+
+>Example №5  
+*Input sentence: A dark-haired bearded man in glasses and a Hawaiian shirt is sitting on the grass.     
+Decoded sentence: Ein bärtiger Mann mit Brille und einem dunkelhaarigen Mann sitzt im Gras.  
+Target sentence: Ein dunkelhaariger Mann mit Bart, Brille und Hawaiihemd sitzt auf dem Gras.*  
+</details>
+
+#### Attention plots of the first sentence:
+
+<p align="center">
+<img src="generated images/attention plots.jpg" width=100% height=100%>
+</p>
+
+
 <details>
 <summary>Variational Autoencoder (VAE)</summary>
 
@@ -800,8 +840,6 @@ Native implementation Example | Neural network Example
 
 
 ### TODO:
-- [ ] Fix bugs (if found), clean up and refactor the code
-- [ ] Add more Tensor operations
-- [ ] Add more layers, activations, optimizers, loss functions, etc
-- [ ] Add more examples
-- [x] Add CuPy support
+- [x] Add Seq2Seq Transformer example
+- [ ] Add GPT example
+- [ ] Add lr schedulers

data_loader.py

@@ -1,10 +1,13 @@
+import os
+import tarfile
 import zipfile
 from pathlib import Path
 
 import numpy as np
 from tqdm import tqdm
 
-from mnist_data_downloader import download_data
+from mnist_data_downloader import download_mnist_data
+from multi30k_data_downloader import download_multi30k_data
 
 
 def prepare_mnist_data(data):
@@ -24,39 +27,39 @@ def load_mnist(path="datasets/mnist/"):
         train_url = "https://pjreddie.com/media/files/mnist_train.csv"
         test_url = "https://pjreddie.com/media/files/mnist_test.csv"
 
-        download_data(train_url, path + "mnist_train.csv")
-        download_data(test_url, path + "mnist_test.csv")
+        download_mnist_data(train_url, path + "mnist_train.csv")
+        download_mnist_data(test_url, path + "mnist_test.csv")
 
-    training_data = Path(path).joinpath("mnist_train.csv").open("r").readlines()
+    train_data = Path(path).joinpath("mnist_train.csv").open("r").readlines()
     test_data = Path(path).joinpath("mnist_test.csv").open("r").readlines()
 
 
     if not (Path(path) / "mnist_train.npy").exists() or not (Path(path) / "mnist_test.npy").exists():
-        training_inputs, training_targets = prepare_mnist_data(training_data)
-        training_inputs = np.asfarray(training_inputs)
+        train_inputs, train_targets = prepare_mnist_data(train_data)
+        train_inputs = np.asfarray(train_inputs)
 
         test_inputs, test_targets = prepare_mnist_data(test_data)
         test_inputs = np.asfarray(test_inputs)
 
-        np.save(path + "mnist_train.npy", training_inputs)
+        np.save(path + "mnist_train.npy", train_inputs)
         np.save(path + "mnist_test.npy", test_inputs)
 
-        np.save(path + "mnist_train_targets.npy", training_targets)
+        np.save(path + "mnist_train_targets.npy", train_targets)
         np.save(path + "mnist_test_targets.npy", test_targets)
     else:
-        training_inputs = np.load(path + "mnist_train.npy")
+        train_inputs = np.load(path + "mnist_train.npy")
         test_inputs = np.load(path + "mnist_test.npy")
 
-        training_targets = np.load(path + "mnist_train_targets.npy")
+        train_targets = np.load(path + "mnist_train_targets.npy")
         test_targets = np.load(path + "mnist_test_targets.npy")
 
-    training_dataset = training_inputs
+    train_dataset = train_inputs
     test_dataset = test_inputs
 
-    return training_dataset, test_dataset, training_targets, test_targets
+    return train_dataset, test_dataset, train_targets, test_targets
+
 
 
-import os
 
 
 def prepare_utkface_data(path, image_size = (3, 32, 32)):
@@ -69,16 +72,16 @@ def prepare_utkface_data(path, image_size = (3, 32, 32)):
     images = os.listdir(path)
     random.shuffle(images)
 
-    training_inputs = []
+    train_inputs = []
     for image in tqdm(images, desc = 'preparing data'):
         image = Image.open(path + "/" + image)
         image = image.resize((image_size[1], image_size[2]))
         image = np.asarray(image)
         image = image.transpose(2, 0, 1)
         image = image / 127.5 - 1
-        training_inputs.append(image)
+        train_inputs.append(image)
 
-    return np.array(training_inputs)
+    return np.array(train_inputs)
 
 
 def load_utkface(path="datasets/utkface/", image_size=(3, 32, 32)):
@@ -92,9 +95,70 @@ def load_utkface(path="datasets/utkface/", image_size=(3, 32, 32)):
 
     save_path = path / 'UTKFace.npy'
     if not save_path.exists():
-        training_inputs = prepare_utkface_data(path / 'UTKFace', image_size)
-        np.save(save_path, training_inputs)
+        train_inputs = prepare_utkface_data(path / 'UTKFace', image_size)
+        np.save(save_path, train_inputs)
     else:
-        training_inputs = np.load(save_path)
+        train_inputs = np.load(save_path)
+
+    return train_inputs
+
+
+
+def load_multi30k(path="datasets/multi30k/"):
+    #References: https://pytorch.org/text/stable/_modules/torchtext/datasets/multi30k.html
+    urls = {
+        "train": r"https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/training.tar.gz",
+        "valid": r"https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/validation.tar.gz",
+        "test": r"https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/mmt16_task1_test.tar.gz",
+    }
+
+    filenames = ["mmt16_task1_test.tar.gz", "training.tar.gz", "validation.tar.gz"]
+
+    path = Path(path)
+    if not path.exists():
+        path.mkdir(parents=True)
+
+        download_multi30k_data(urls.values(), path, filenames)
+
+        for filename in filenames:
+            tar = tarfile.open(Path(path) / filename)
+            tar.extractall(path)
+            tar.close()
+
+            print(f'Extracted {filename}')
+
+
+    ret = []
+    filenames = ["train", "val", "test"]
+
+    for filename in filenames:
+
+        examples = []
+
+        en_path = os.path.join(path, filename + '.en')
+        de_path = os.path.join(path, filename + '.de')
+
+        en_file = [l.strip() for l in open(en_path, 'r', encoding='utf-8')]
+        de_file = [l.strip() for l in open(de_path, 'r', encoding='utf-8')]
+
+        assert len(en_file) == len(de_file)
+
+        for i in range(len(en_file)):
+            if en_file[i] == '' or de_file[i] == '':
+                continue
+            en_seq, de_seq = en_file[i], de_file[i]
+
+            examples.append({'en': en_seq, 'de': de_seq})
+    
+        ret.append(examples)
+
+    train_dataset, valid_dataset, test_dataset = ret
+    return train_dataset, valid_dataset, test_dataset
+
+
+
+
+
+
 
-    return training_inputs
+