Merge pull request #261 from yczhang1017/main

szarnyasg · web-flow · commit 944657b03e67 · 2022-03-31T12:48:49.000+02:00
diff --git a/tigergraph/README.md b/tigergraph/README.md
@@ -73,52 +73,43 @@ This section explains how to set up the database for the benchmarking **using Do
 The instruction assumes that you are starting in the `tigergraph` subfolder of the project root directory (`ldbc_snb_interactive_impls/tigergraph`).
 
 #### Prepare the dataset
-Set the following environment variables based on your data source:
-
-To use the test data set SF-0.003, run:
+Set the following environment variables based on your data source (the example below uses the test data set SF-0.003):
 ```bash
 export TIGERGRAPH_DATA_DIR=`pwd`/test-data/social_network
 ```
+#### Start database and load data
+:warning: Be careful -- this stops the currently running (containerized) TigerGraph database and deletes all of its data.
 
-#### Start the database
 To start the database, run the following [script](./scripts/start.sh):
-```
+```bash
+./scripts/stop.sh # if you have an existing TG database
+# wait several seconds for docker to reset 
 ./scripts/start.sh
 ```
 
-It will start a single node TigerGraph database and all required services.
+It will start a single node TigerGraph database and all required services. Note the license in the container is a trial license supporting at most 100GB data. For benchmarks on SF-100 and larger, you need to obtain a license after running `start.sh`. We have an example command in the end of `start.sh`.
 
-You can verify the readiness of TigerGraph by visiting it's console in the browser: `http://localhost:14240/`
-
-#### Load the data
 To set up the database, run the following [script](./scripts/setup.sh):
 ```bash
 ./scripts/setup.sh
 ```
-It leverages the fact, that the TigerGraph container has the `$TIGERGRAPH_DATA_DIR`, `setup` and `queries` directories mounted as volumes.
-(The configuration is stored in [vars.sh](./scripts/vars.sh).)
+This step may take a while (several minutes), as it is responsible for defining the queries, loading jobs, loading the data and installing the queries. After the data is ready, you can explore the graph using TigerGraph GraphStudio in the browser: `http://localhost:14240/`. By default, the docker terminal can be accessed via `ssh tigergraph@localhost -p 14022` with password tigergraph, or using Docker command `docker exec --user tigergraph -it snb-interactive-tigergraph bash`.
 
-If you have your data located elsewhere, please update `vars.sh` or run the setup script with parameters:
+The above scripts can be executed with a single command:
 ```bash
-./scripts/setup.sh <<dataset dir>> <<query dir>>
+scripts/load-in-one-step.sh
 ```
 
-This step may take a while (several minutes), as it is responsible for defining the queries, loading jobs, loading the data
-and installing (optimizing and compiling on the server) the queries.
-
 ## Running the benchmark
 
-To run the scripts of benchmark framework, edit the `driver/{create-validation-parameters,validate,benchmark}.properties` files,
-then run their script, one of:
+To run the scripts of benchmark framework, edit the `driver/{create-validation-parameters,validate,benchmark}.properties` files, then run their script, one of: 
 
 ```bash
 driver/create-validation-parameters.sh
 driver/validate.sh
 driver/benchmark.sh
 ```
 
-:warning: Our DateTime library does not support dateTime precision to milliseconds. We use INT for datetime right now.
-:warning: SNB data sets of **different scale factors require different configurations** for the benchmark runs. Therefore, make sure you use the correct values (update_interleave and query frequencies) based on the files provided in the [`sf-properties/` directory](../sf-properties).
-
-> **Warning:** Note that if the default workload contains updates which are persisted in the database. Therefore, the database needs to be re-loaded between steps – otherwise repeated updates would insert duplicate entries.*
+SNB data sets of **different scale factors require different configurations** for the benchmark runs. Therefore, make sure you use the correct values (update_interleave and query frequencies) based on the files provided in the [`sf-properties/` directory](../sf-properties).
 
+* The default workload contains updates which are persisted in the database. Therefore, **the database needs to be reloaded or restored from backup before each run**. Use the provided `scripts/backup-database.sh` and `scripts/restore-database.sh` scripts to achieve this.
diff --git a/tigergraph/queries/interactiveInsert2.gsql b/tigergraph/queries/interactiveInsert2.gsql
@@ -1,5 +1,5 @@
 CREATE OR REPLACE QUERY interactiveInsert2(UINT personId, UINT postId, INT creationDate) {
-    INSERT INTO Person (PRIMARY_ID) VALUES (personId);
-    INSERT INTO Post (PRIMARY_ID) VALUES (postId);
+    INSERT INTO Person (id) VALUES (personId);
+    INSERT INTO Post (id) VALUES (postId);
     INSERT INTO LIKES (FROM, TO, creationDate) VALUES (personId Person, postId Post, creationDate);
 }
diff --git a/tigergraph/queries/interactiveInsert3.gsql b/tigergraph/queries/interactiveInsert3.gsql
@@ -1,5 +1,5 @@
 CREATE OR REPLACE QUERY interactiveInsert3(UINT personId, UINT commentId, INT creationDate) {
-    INSERT INTO Person (PRIMARY_ID) VALUES (personId);
-    INSERT INTO Comment (PRIMARY_ID) VALUES (commentId);
+    INSERT INTO Person (id) VALUES (personId);
+    INSERT INTO Comment (id) VALUES (commentId);
     INSERT INTO LIKES (FROM, TO, creationDate) VALUES(personId Person, commentId Comment, creationDate);
 }
diff --git a/tigergraph/queries/interactiveInsert4.gsql b/tigergraph/queries/interactiveInsert4.gsql
@@ -1,6 +1,6 @@
 CREATE OR REPLACE QUERY interactiveInsert4(UINT forumId, STRING forumTitle, UINT moderatorPersonId, INT creationDate, SET<VERTEX<Tag>> tagIds) {
     INSERT INTO Forum VALUES(forumId, forumTitle, creationDate);
-    INSERT INTO Person (PRIMARY_ID) VALUES(moderatorPersonId);
+    INSERT INTO Person (id) VALUES(moderatorPersonId);
     INSERT INTO HAS_MODERATOR VALUES(forumId, moderatorPersonId Person);
     tmp = {tagIds};
     tmp =
diff --git a/tigergraph/queries/interactiveInsert5.gsql b/tigergraph/queries/interactiveInsert5.gsql
@@ -1,5 +1,5 @@
 CREATE OR REPLACE QUERY interactiveInsert5(UINT personId, UINT forumId, INT joinDate) {
-    INSERT INTO Forum (PRIMARY_ID) VALUES (forumId);
-    INSERT INTO Person (PRIMARY_ID) VALUES (personId);
+    INSERT INTO Forum (id) VALUES (forumId);
+    INSERT INTO Person (id) VALUES (personId);
     INSERT INTO HAS_MEMBER VALUES(forumId Forum, personId Person, joinDate);
 }
diff --git a/tigergraph/queries/interactiveInsert6.gsql b/tigergraph/queries/interactiveInsert6.gsql
@@ -14,8 +14,8 @@
 
       //IS_LOCATED_IN country is inserted as an attribute
       INSERT INTO Post VALUES(postId, imageFile, creationDate, locationIP, browserUsed, language, content, length, countryId); 
-      INSERT INTO Person (PRIMARY_ID) VALUES(authorPersonId);
-      INSERT INTO Forum (PRIMARY_ID) VALUES(forumId);
+      INSERT INTO Person (id) VALUES(authorPersonId);
+      INSERT INTO Forum (id) VALUES(forumId);
       INSERT INTO HAS_CREATOR VALUES(postId Post, authorPersonId Person);
       INSERT INTO CONTAINER_OF VALUES(forumId Forum, postId Post);
 
diff --git a/tigergraph/queries/interactiveInsert7.gsql b/tigergraph/queries/interactiveInsert7.gsql
@@ -12,14 +12,14 @@ CREATE OR REPLACE QUERY interactiveInsert7(
         SET<VERTEX<Tag>> tagIds) {
 
     INSERT INTO Comment VALUES(commentId, creationDate, locationIP, browserUsed, content, length, countryId);
-    INSERT INTO Person (PRIMARY_ID) VAlUES (authorPersonId);
+    INSERT INTO Person (id) VAlUES (authorPersonId);
     INSERT INTO HAS_CREATOR VALUES(commentId Comment, authorPersonId Person);
     IF replyToPostId > 0 THEN
-        INSERT INTO Post (PRIMARY_ID) VALUES(replyToPostId);
+        INSERT INTO Post (id) VALUES(replyToPostId);
         INSERT INTO REPLY_OF (FROM, TO) VALUES(commentId, replyToPostId Post);
     END;
     IF replyToCommentId > 0 THEN
-        INSERT INTO Comment (PRIMARY_ID) VALUES(replyToCommentId);
+        INSERT INTO Comment (id) VALUES(replyToCommentId);
         INSERT INTO REPLY_OF (FROM, TO) VALUES(commentId, replyToCommentId Comment);
     END;
     tmp = {tagIds};
diff --git a/tigergraph/queries/interactiveInsert8.gsql b/tigergraph/queries/interactiveInsert8.gsql
@@ -1,5 +1,5 @@
 CREATE OR REPLACE QUERY interactiveInsert8(UINT person1Id, UINT person2Id, INT creationDate) {
-    INSERT INTO Person (PRIMARY_ID) VALUES (person1Id);
-    INSERT INTO Person (PRIMARY_ID) VALUES (person2Id);
+    INSERT INTO Person (id) VALUES (person1Id);
+    INSERT INTO Person (id) VALUES (person2Id);
     INSERT INTO KNOWS (FROM, TO, creationDate) VALUES(person1Id, person2Id, creationDate);
 }
diff --git a/tigergraph/scripts/backup-database.sh b/tigergraph/scripts/backup-database.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd ..
+
+. scripts/vars.sh
+
+echo "==============================================================================="
+echo "Backup the TIGERGRAPH database"
+echo "-------------------------------------------------------------------------------"
+echo "TIGERGRAPH_VERSION: ${TIGERGRAPH_VERSION}"
+echo "TIGERGRAPH_CONTAINER_NAME: ${TIGERGRAPH_CONTAINER_NAME}"
+echo "==============================================================================="
+
+docker exec --user tigergraph --interactive --tty ${TIGERGRAPH_CONTAINER_NAME} bash -c \
+  "export PATH=/home/tigergraph/tigergraph/app/cmd:\$PATH; \
+  gadmin config set System.Backup.Local.Enable true; \
+  gadmin config set System.Backup.Local.Path /home/tigergraph/backup; \
+  gadmin config apply -y; 
+  export GSQL_USERNAME=tigergraph; \
+  export GSQL_PASSWORD=tigergraph; \
+  gbar backup -t snb-backup"
diff --git a/tigergraph/scripts/load-in-one-step.sh b/tigergraph/scripts/load-in-one-step.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd ..
+
+. scripts/vars.sh
+
+scripts/stop.sh
+sleep 4
+scripts/start.sh
+scripts/setup.sh
diff --git a/tigergraph/scripts/restore-database.sh b/tigergraph/scripts/restore-database.sh
@@ -8,7 +8,16 @@ cd ..
 
 . scripts/vars.sh
 
-scripts/stop.sh
-sleep 4
-scripts/start.sh
-scripts/setup.sh
+echo "==============================================================================="
+echo "Restore the TIGERGRAPH database"
+echo "-------------------------------------------------------------------------------"
+echo "TIGERGRAPH_VERSION: ${TIGERGRAPH_VERSION}"
+echo "TIGERGRAPH_CONTAINER_NAME: ${TIGERGRAPH_CONTAINER_NAME}"
+echo "==============================================================================="
+
+docker exec --user tigergraph --interactive --tty ${TIGERGRAPH_CONTAINER_NAME} bash -c \
+  "export PATH=/home/tigergraph/tigergraph/app/cmd:\$PATH; \
+  export GSQL_USERNAME=tigergraph; \
+  export GSQL_PASSWORD=tigergraph; \
+  export tag=\$(gbar list | grep snb-backup | cut -d' ' -f1);
+  gbar restore \$tag -y"
diff --git a/tigergraph/scripts/start.sh b/tigergraph/scripts/start.sh
@@ -62,5 +62,10 @@ until docker exec --user tigergraph --interactive --tty ${TIGERGRAPH_CONTAINER_N
   sleep 1
 done
 
+echo "Trial license is used. The license support benchmarks of SF-30 and smaller ones."
+# Note: for SF-100 and larger you need to provide a license here
+# echo "Applying the services."
+# docker exec --user tigergraph --interactive --tty ${TIGERGRAPH_CONTAINER_NAME} /home/tigergraph/tigergraph/app/cmd/gadmin license set [license_string]
+
 echo
 echo "All done."
diff --git a/tigergraph/setup/create_schema.gsql b/tigergraph/setup/create_schema.gsql
@@ -10,22 +10,22 @@
 #   For Comment, Post, Forum, Person, the order of attributes is different.
 
 ## Message
-CREATE VERTEX Comment (PRIMARY_ID id UINT, creationDate INT, locationIP STRING, browserUsed STRING, content STRING, length UINT, countryId UINT) WITH primary_id_as_attribute="TRUE"
-CREATE VERTEX Post (PRIMARY_ID id UINT, imageFile STRING, creationDate INT, locationIP STRING, browserUsed STRING, language STRING, content STRING, length UINT, countryId UINT) WITH primary_id_as_attribute="TRUE"
+CREATE VERTEX Comment (id UINT PRIMARY KEY, creationDate INT, locationIP STRING, browserUsed STRING, content STRING, length UINT, countryId UINT)
+CREATE VERTEX Post (id UINT PRIMARY KEY, imageFile STRING, creationDate INT, locationIP STRING, browserUsed STRING, language STRING, content STRING, length UINT, countryId UINT)
 
 ## Organisation
-CREATE VERTEX Company (PRIMARY_ID id UINT, name STRING, url STRING) WITH primary_id_as_attribute="TRUE"
-CREATE VERTEX University (PRIMARY_ID id UINT, name STRING, url STRING) WITH primary_id_as_attribute="TRUE"
+CREATE VERTEX Company (id UINT PRIMARY KEY, name STRING, url STRING)
+CREATE VERTEX University (id UINT PRIMARY KEY, name STRING, url STRING)
 
 ## Place
-CREATE VERTEX City (PRIMARY_ID id UINT, name STRING, url STRING) WITH primary_id_as_attribute="TRUE"
-CREATE VERTEX Country (PRIMARY_ID id UINT, name STRING, url STRING) WITH primary_id_as_attribute="TRUE"
-CREATE VERTEX Continent (PRIMARY_ID id UINT, name STRING, url STRING) WITH primary_id_as_attribute="TRUE"
-
-CREATE VERTEX Forum (PRIMARY_ID id UINT, title STRING, creationDate INT) WITH primary_id_as_attribute="TRUE"
-CREATE VERTEX Person (PRIMARY_ID id UINT, firstName STRING, lastName STRING, gender STRING, birthday INT, creationDate INT, locationIP STRING, browserUsed STRING, speaks SET<STRING>, email SET<STRING>) WITH primary_id_as_attribute="TRUE"
-CREATE VERTEX Tag (PRIMARY_ID id UINT, name STRING, url STRING) WITH primary_id_as_attribute="TRUE"
-CREATE VERTEX TagClass (PRIMARY_ID id UINT, name STRING, url STRING) WITH primary_id_as_attribute="TRUE"
+CREATE VERTEX City (id UINT PRIMARY KEY, name STRING, url STRING)
+CREATE VERTEX Country (id UINT PRIMARY KEY, name STRING, url STRING)
+CREATE VERTEX Continent (id UINT PRIMARY KEY, name STRING, url STRING)
+
+CREATE VERTEX Forum (id UINT PRIMARY KEY, title STRING, creationDate INT)
+CREATE VERTEX Person (id UINT PRIMARY KEY, firstName STRING, lastName STRING, gender STRING, birthday INT, creationDate INT, locationIP STRING, browserUsed STRING, speaks SET<STRING>, email SET<STRING>)
+CREATE VERTEX Tag (id UINT PRIMARY KEY, name STRING, url STRING)
+CREATE VERTEX TagClass (id UINT PRIMARY KEY, name STRING, url STRING)
 
 # Edges
 CREATE DIRECTED EDGE CONTAINER_OF (FROM Forum, TO Post) WITH REVERSE_EDGE="CONTAINER_OF_REVERSE"

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`CREATE OR REPLACE QUERY interactiveInsert2(UINT personId, UINT postId, INT creationDate) {`
`2`		`- INSERT INTO Person (PRIMARY_ID) VALUES (personId);`
`3`		`- INSERT INTO Post (PRIMARY_ID) VALUES (postId);`
	`2`	`+ INSERT INTO Person (id) VALUES (personId);`
	`3`	`+ INSERT INTO Post (id) VALUES (postId);`
`4`	`4`	`INSERT INTO LIKES (FROM, TO, creationDate) VALUES (personId Person, postId Post, creationDate);`
`5`	`5`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`CREATE OR REPLACE QUERY interactiveInsert5(UINT personId, UINT forumId, INT joinDate) {`
`2`		`- INSERT INTO Forum (PRIMARY_ID) VALUES (forumId);`
`3`		`- INSERT INTO Person (PRIMARY_ID) VALUES (personId);`
	`2`	`+ INSERT INTO Forum (id) VALUES (forumId);`
	`3`	`+ INSERT INTO Person (id) VALUES (personId);`
`4`	`4`	`INSERT INTO HAS_MEMBER VALUES(forumId Forum, personId Person, joinDate);`
`5`	`5`	`}`