Merge branch 'main' into feature/mssql-queries

Author: GLaDAP

.circleci/config.yml

@@ -25,7 +25,7 @@ jobs:
           # Install Java 11 (Ubuntu 22.04 has Java 17 by default)
           name: Install OpenJDK 11
           command: |
-            sudo apt-get update && sudo apt-get install openjdk-11-jdk
+            sudo apt-get update && sudo apt-get install -y openjdk-11-jdk
             sudo update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
             sudo update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
             java -version

postgres/scripts/install-dependencies.sh

@@ -6,10 +6,10 @@ set -o pipefail
 cd "$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )"
 
 if [[ ! -z $(which yum) ]]; then
-    sudo yum install -y postgresql-devel
+    sudo yum install -y postgresql-devel postgresql
 elif [[ ! -z $(which apt-get) ]]; then
     sudo apt-get update
-    sudo apt-get install -y libpq5
+    sudo apt-get install -y libpq5 postgresql-client
 else
     echo "Operating system not supported, please install the dependencies manually"
 fi

snb-interactive-pre-generated-data-sets.md

@@ -38,6 +38,13 @@ This command works on both standalone files (`.tar.zst`) and chunked ones (`.tar
 * https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/interactive/snb-out-sf100-projected-fk.tar.zst
 * https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/interactive/snb-out-sf300-projected-fk.tar.zst
 
+### Raw (up to SF30)
+
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/bi-sf1-raw.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/bi-sf3-raw.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/bi-sf10-raw.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/bi-sf30-raw.tar.zst
+
 ### Update streams
 
 * https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/interactive-updates/interactive-updates-sf1.tar.zst
@@ -50,7 +57,19 @@ This command works on both standalone files (`.tar.zst`) and chunked ones (`.tar
 * https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/interactive-updates/interactive-updates-sf3000.tar.zst
 * https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/interactive-updates/interactive-updates-sf10000.tar.zst
 
-### Parameters
+### Factor tables
+
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/factors/factors-sf1.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/factors/factors-sf3.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/factors/factors-sf10.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/factors/factors-sf30.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/factors/factors-sf100.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/factors/factors-sf300.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/factors/factors-sf1000.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/factors/factors-sf3000.tar.zst
+* https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/bi-pre-audit/factors/factors-sf10000.tar.zst
+
+### Parameters (deprecated)
 
 * https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/interactive/snb-interactive-parameters-sf1.tar.zst
 * https://pub-383410a98aef4cb686f0c7601eddd25f.r2.dev/interactive/snb-interactive-parameters-sf3.tar.zst

umbra/output/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

umbra/queries/interactive-update-1-add-person.sql

@@ -15,13 +15,13 @@ VALUES
 (
     :creationDate
   , :personId
-  , :personFirstName
-  , :personLastName
-  , :gender
+  , :personFirstName::varchar
+  , :personLastName::varchar
+  , :gender::varchar
   , :birthday
-  , :locationIP
-  , :browserUsed
+  , :locationIP::varchar
+  , :browserUsed::varchar
   , :cityId
-  , :languages
-  , :emails
+  , :languages::varchar
+  , :emails::varchar
 );

umbra/scripts/install-dependencies.sh

@@ -5,13 +5,4 @@ set -o pipefail
 
 cd "$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )"
 
-if [[ ! -z $(which yum) ]]; then
-    sudo yum install -y postgresql-devel postgresql
-elif [[ ! -z $(which apt-get) ]]; then
-    sudo apt-get update
-    sudo apt-get install -y libpq5 postgresql-client
-else
-    echo "Operating system not supported, please install the dependencies manually"
-fi
-
-pip3 install --user --progress-bar off psycopg
+../../postgres/scripts/install-dependencies.sh