Source features support for V2.0 (#2090)

Author: anderleich

.github/workflows/push.yml

@@ -42,6 +42,16 @@ jobs:
           -src_vocab /tmp/onmt.vocab.src \
           -tgt_vocab /tmp/onmt.vocab.tgt \
           && rm -rf /tmp/sample
+    - name: Test vocabulary build with features
+      run: |
+        python onmt/bin/build_vocab.py \
+          -config data/features_data.yaml \
+          -save_data /tmp/onmt_feat \
+          -src_vocab /tmp/onmt_feat.vocab.src \
+          -tgt_vocab /tmp/onmt_feat.vocab.tgt \
+          -src_feats_vocab '{"feat0": "/tmp/onmt_feat.vocab.feat0"}' \
+          -n_sample -1 \
+          && rm -rf /tmp/sample
     - name: Test field/transform dump
       run: |
         # The dumped fields are used later when testing tools
@@ -169,6 +179,26 @@ jobs:
           -state_dim 256 \
           -n_steps 10 \
           -n_node 64
+    - name: Testing training with features
+      run: |
+        python onmt/bin/train.py \
+          -config data/features_data.yaml \
+          -src_vocab /tmp/onmt_feat.vocab.src \
+          -tgt_vocab /tmp/onmt_feat.vocab.tgt \
+          -src_feats_vocab '{"feat0": "/tmp/onmt_feat.vocab.feat0"}' \
+          -src_vocab_size 1000 -tgt_vocab_size 1000 \
+          -rnn_size 2 -batch_size 10 \
+          -word_vec_size 5 -rnn_size 10 \
+          -report_every 5 -train_steps 10 \
+          -save_model /tmp/onmt.model \
+          -save_checkpoint_steps 10
+    - name: Testing translation with features
+      run: |
+        python translate.py \
+            -model /tmp/onmt.model_step_10.pt \
+            -src data/data_features/src-test.txt \
+            -src_feats "{'feat0': 'data/data_features/src-test.feat0'}" \
+            -verbose
     - name: Test RNN translation
       run: |
         head data/src-test.txt > /tmp/src-test.txt

data/data_features/src-test.feat0

@@ -0,0 +1 @@
+C B A B

data/data_features/src-test.txt

@@ -0,0 +1 @@
+she is a hard-working.

data/data_features/src-train.feat0

@@ -0,0 +1,3 @@
+A A A A B A A A C
+A B C D E
+C B A B

data/data_features/src-train.txt

@@ -0,0 +1,3 @@
+however, according to the logs, she is a hard-working.
+however, according to the logs,
+she is a hard-working.

data/data_features/src-val.feat0

@@ -0,0 +1 @@
+C B A B

data/data_features/src-val.txt

@@ -0,0 +1 @@
+she is a hard-working.

data/data_features/tgt-train.txt

@@ -0,0 +1,3 @@
+however, according to the logs, she is a hard-working.
+however, according to the logs,
+she is a hard-working.

data/data_features/tgt-val.txt

@@ -0,0 +1 @@
+she is a hard-working.

data/features_data.yaml

@@ -0,0 +1,11 @@
+# Corpus opts:
+data:
+    corpus_1:
+        path_src: data/data_features/src-train.txt
+        path_tgt: data/data_features/tgt-train.txt
+        src_feats:
+            feat0: data/data_features/src-train.feat0
+        transforms: [filterfeats, inferfeats]
+    valid:
+        path_src: data/data_features/src-val.txt
+        path_tgt: data/data_features/tgt-val.txt