Add retry loop on set state, fix #524

Author: slinkydeveloper

packages/restate-sdk-testcontainers/jest.config.js

@@ -2,6 +2,6 @@
 export default {
   testEnvironment: "node",
   transform: {
-    "^.+\.tsx?$": ["ts-jest",{}],
+    "^.+.tsx?$": ["ts-jest", {}],
   },
-};
+};

packages/restate-sdk-testcontainers/src/restate_test_environment.ts

@@ -23,6 +23,7 @@ import {
 import { tableFromIPC } from "apache-arrow";
 import * as http2 from "http2";
 import type * as net from "net";
+import { setTimeout } from "node:timers/promises";
 
 // Prepare the restate server
 async function prepareRestateEndpoint(
@@ -303,37 +304,46 @@ export class StateProxy<TState extends TypedState> {
     entries: [key: string, value: Uint8Array][],
     version?: string
   ) {
-    const res = await fetch(
-      `${this.adminAPIBaseUrl}/services/${this.service}/state`,
-      {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify(
-          {
-            version,
-            object_key: this.serviceKey,
-            new_state: Object.fromEntries(entries),
+    let lastFailure: Error | undefined = undefined;
+    for (let i = 0; i < 10; i++) {
+      const res = await fetch(
+        `${this.adminAPIBaseUrl}/services/${this.service}/state`,
+        {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
           },
-          (key, value) => {
-            if (value instanceof Uint8Array) {
-              return Array.from(value);
-            } else {
-              // eslint-disable-next-line @typescript-eslint/no-unsafe-return
-              return value;
+          body: JSON.stringify(
+            {
+              version,
+              object_key: this.serviceKey,
+              new_state: Object.fromEntries(entries),
+            },
+            (key, value) => {
+              if (value instanceof Uint8Array) {
+                return Array.from(value);
+              } else {
+                // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+                return value;
+              }
             }
-          }
-        ),
+          ),
+        }
+      );
+
+      if (res.ok) {
+        return;
       }
-    );
 
-    if (!res.ok) {
       const badResponse = await res.text();
-      throw new Error(
+      lastFailure = new Error(
         `Error ${res.status} during modify state: ${badResponse}`
       );
+
+      await setTimeout(1000);
     }
+
+    throw lastFailure;
   }
 }
 

packages/restate-sdk-testcontainers/test/test.ts

@@ -1,6 +1,6 @@
 import { RestateTestEnvironment } from "../src/restate_test_environment";
 import * as restate from "@restatedev/restate-sdk";
-import { setTimeout} from "node:timers/promises";
+import { setTimeout } from "node:timers/promises";
 import * as clients from "@restatedev/restate-sdk-clients";
 
 const exampleService = restate.service({
@@ -74,8 +74,6 @@ describe("ExampleObject", () => {
     expect(await state.getAll()).toStrictEqual({});
     expect(await state.get("count")).toBeNull();
 
-    await setTimeout(5000)
-
     // Setting state is an eventually consistent operation, so retrying might be needed
     await state.set("count", 123);
     const greet = await restateIngress