Update analytics scheme and testing

Pipfile

@@ -29,7 +29,6 @@ black = "==19.10b0"
 psycopg2 = "*"
 
 [dev-packages]
-
 coverage = "*"
 
 [requires]
@@ -40,3 +39,4 @@ docker= "docker-compose up -d"
 lint = "./scripts/lint"
 start = "uvicorn src.main:app --reload"
 test = "pytest"
+db = "./scripts/create_table.py"

src/models.py

@@ -47,37 +47,52 @@ class RedisEvent(CustomModel):
     key: bytes | str
     value: bytes | str
 
+    def __init__(self, **data):
+        super().__init__(**data)
+        self.key = data["key"]
+        self.value = data["value"]
+
+
+class AnalyticsValue(CustomModel):
+    key: str
+    value: str
+    timestamp: datetime
+
+    def __init__(self, **data):
+        super().__init__(**data)
+        self.key = data["key"]
+        self.value = data["value"]
+        self.timestamp = datetime.fromtimestamp(data["timestamp"])
+
 
 class AnalyticsTxn(CustomModel):
     product: Product
     pennkey: Optional[str] = None
-    timestamp: datetime
-    data: list[RedisEvent]
+    data: list[AnalyticsValue]
 
     # init with JSON data
     def __init__(self, **data):
         super().__init__(**data)
-        self.timestamp = datetime.fromtimestamp(data["timestamp"])
-        self.data = [RedisEvent(**event) for event in data["data"]]
         self.product = Product(data["product"])
         self.pennkey = data.get("pennkey")
+        self.data = [AnalyticsValue(**value) for value in data["data"]]
 
     def get_redis_key(self):
         return f"{self.product}.{self.hash_as_key()}"
 
     def build_redis_data(self) -> list[RedisEvent]:
         return [
             RedisEvent(
-                key=f"{self.get_redis_key()}.{event.hash_as_key()}",
+                key=f"{self.get_redis_key()}.{value.hash_as_key()}",
                 value=json.dumps(
                     {
                         "product": str(self.product),
                         "pennkey": self.pennkey,
-                        "timestamp": self.timestamp.timestamp(),
-                        "datapoint": event.key,
-                        "value": event.value,
+                        "timestamp": value.timestamp.timestamp(),
+                        "datapoint": value.key,
+                        "value": value.value,
                     }
                 ),
             )
-            for event in self.data
+            for value in self.data
         ]

tests/test_load.py

@@ -5,7 +5,8 @@
 from datetime import datetime
 
 import requests
-from test_token import get_tokens
+
+from tests.test_token import get_tokens
 
 
 # Runtime should be less that 3 seconds for most laptops
@@ -16,32 +17,79 @@
 THREADS = 16
 
 
-def make_request():
-    access_token, _ = get_tokens()
-
-    url = "http://localhost:8000/analytics"
+def make_request(token: str):
+    url = "http://localhost:80/analytics/"
     payload = json.dumps(
         {
             "product": random.randint(1, 10),
             "pennkey": "test_usr",
-            "timestamp": int(datetime.now().timestamp()),
             "data": [
-                {"key": "user.click", "value": str(random.randint(1, 1000))},
-                {"key": "user.drag", "value": str(random.randint(1, 1000))},
-                {"key": "data.dowload", "value": str(random.randint(1, 1000))},
-                {"key": "user.drive", "value": str(random.randint(1, 1000))},
-                {"key": "user.play", "value": str(random.randint(1, 1000))},
-                {"key": "user.sit", "value": str(random.randint(1, 1000))},
-                {"key": "user.stand", "value": str(random.randint(1, 1000))},
-                {"key": "user.bike", "value": str(random.randint(1, 1000))},
-                {"key": "user.flip", "value": str(random.randint(1, 1000))},
-                {"key": "food.eat", "value": str(random.randint(1, 1000))},
+                {
+                    "key": "user.click",
+                    "value": str(random.randint(1, 1000)),
+                    "timestamp": int(datetime.now().timestamp())
+                    + random.randint(1, 1000),
+                },
+                {
+                    "key": "user.drag",
+                    "value": str(random.randint(1, 1000)),
+                    "timestamp": int(datetime.now().timestamp())
+                    + random.randint(1, 1000),
+                },
+                {
+                    "key": "data.dowload",
+                    "value": str(random.randint(1, 1000)),
+                    "timestamp": int(datetime.now().timestamp())
+                    + random.randint(1, 1000),
+                },
+                {
+                    "key": "user.drive",
+                    "value": str(random.randint(1, 1000)),
+                    "timestamp": int(datetime.now().timestamp())
+                    + random.randint(1, 1000),
+                },
+                {
+                    "key": "user.play",
+                    "value": str(random.randint(1, 1000)),
+                    "timestamp": int(datetime.now().timestamp())
+                    + random.randint(1, 1000),
+                },
+                {
+                    "key": "user.sit",
+                    "value": str(random.randint(1, 1000)),
+                    "timestamp": int(datetime.now().timestamp())
+                    + random.randint(1, 1000),
+                },
+                {
+                    "key": "user.stand",
+                    "value": str(random.randint(1, 1000)),
+                    "timestamp": int(datetime.now().timestamp())
+                    + random.randint(1, 1000),
+                },
+                {
+                    "key": "user.bike",
+                    "value": str(random.randint(1, 1000)),
+                    "timestamp": int(datetime.now().timestamp())
+                    + random.randint(1, 1000),
+                },
+                {
+                    "key": "user.flip",
+                    "value": str(random.randint(1, 1000)),
+                    "timestamp": int(datetime.now().timestamp())
+                    + random.randint(1, 1000),
+                },
+                {
+                    "key": "food.eat",
+                    "value": str(random.randint(1, 1000)),
+                    "timestamp": int(datetime.now().timestamp())
+                    + random.randint(1, 1000),
+                },
             ],
         }
     )
     headers = {
         "Content-Type": "application/json",
-        "Authorization": f"Bearer {access_token}",
+        "Authorization": f"Bearer {token}",
     }
 
     try:
@@ -55,8 +103,10 @@ def make_request():
 
 def run_threads():
     with ThreadPoolExecutor(max_workers=THREADS) as executor:
+        # Fetch access token once to avoid repeated requests.
+        access_token, _ = get_tokens()
         for _ in range(NUMBER_OF_REQUESTS):
-            executor.submit(make_request)
+            executor.submit(make_request, access_token)
 
 
 def test_load():

tests/test_redis.py

@@ -1,3 +1,6 @@
+import random
+from datetime import datetime
+
 import pytest
 
 from src.models import RedisEvent
@@ -7,8 +10,16 @@
 @pytest.mark.asyncio
 async def test_redis():
     data = [
-        {"key": "test_key", "value": "test_value"},
-        {"key": "test_key2", "value": "test_value2"},
+        {
+            "key": "test_key",
+            "value": "test_value",
+            "timestamp": int(datetime.now().timestamp()) + random.randint(1, 1000),
+        },
+        {
+            "key": "test_key2",
+            "value": "test_value2",
+            "timestamp": int(datetime.now().timestamp()) + random.randint(1, 1000),
+        },
     ]
 
     payload = [RedisEvent(**d) for d in data]

tests/test_token.py

@@ -1,31 +1,38 @@
 # Test to generate jwt token from Penn Labs platforms
 import os
 
-import requests
-
 from src.auth import verify_jwt
 
 
-ATTEST_URL = "https://platform.pennlabs.org/identity/attest/"
-
+# ATTEST_URL = "https://platform.pennlabs.org/identity/attest/"
 # Using Penn Basics DLA Account for testing, will not work if you don't have that in .env
-CLIENT_ID: str = os.environ.get("CLIENT_ID") or ""
-CLIENT_SECRET: str = os.environ.get("CLIENT_SECRET") or ""
+# CLIENT_ID: str = os.environ.get("CLIENT_ID") or ""
+# CLIENT_SECRET: str = os.environ.get("CLIENT_SECRET") or ""
+
+
+# With the new URL to Verify JWT's (See https://github.com/pennlabs/labs-analytics/pull/38)
+# The testing JWT generated with client_id and client_secret is no longer
+# correctly decoded. Hence, provide a valid JWT issued by Platform for a given user.
+# This will allow the JWT to be correctly decoded.
+TESTING_JWT: str = os.environ.get("TESTING_JWT") or ""
 
 
 def test_env_vars():
     assert os.environ.get("CLIENT_ID") is not None
     assert os.environ.get("CLIENT_SECRET") is not None
+    assert os.environ.get("TESTING_JWT") is not None
 
 
 def get_tokens():
-    response = requests.post(ATTEST_URL, auth=(CLIENT_ID, CLIENT_SECRET))
-    if response.status_code == 200:
-        content = response.json()
-        token = content["access"]
-        refresh = content["refresh"]
-        return (token, refresh)
-    return ("", "")
+    return (TESTING_JWT, "")
+    # response = requests.post(ATTEST_URL, auth=(CLIENT_ID, CLIENT_SECRET))
+    # if response.status_code == 200:
+    #     content = response.json()
+    #     token = content["access"]
+    #     refresh = content["refresh"]
+    #     return (TESTING_JWT, refresh)
+
+    # return ("", "")
 
 
 def test_get_tokens():