Merge pull request #247 from lightninglabs/uint64-strings

refactor: use strings for GRPC uint64 fields

Author: jamaljsr

app/package.json

@@ -20,7 +20,7 @@
     "@emotion/react": "11.4.0",
     "@emotion/styled": "11.3.0",
     "@improbable-eng/grpc-web": "0.14.0",
-    "big.js": "5.2.2",
+    "big.js": "6.1.1",
     "bootstrap": "4.5.0",
     "copy-to-clipboard": "3.3.1",
     "d3": "6.3.1",
@@ -58,7 +58,7 @@
     "@testing-library/jest-dom": "5.11.5",
     "@testing-library/react": "11.1.1",
     "@testing-library/user-event": "12.2.0",
-    "@types/big.js": "4.0.5",
+    "@types/big.js": "6.1.1",
     "@types/d3": "6.2.0",
     "@types/debug": "4.1.5",
     "@types/file-saver": "2.0.1",

app/scripts/build-protos.js

@@ -36,7 +36,9 @@ const protoSources = async () => {
     throw new Error(`go.mod did not match pattern ${POOL_VERSION_PATTERN}`);
   }
 
-  console.log(`Found lnd version ${lndVersion[1]} and loop version ${loopVersion[1]}.`);
+  console.log(
+    `Found:\n LND ${lndVersion[1]}\n Loop ${loopVersion[1]}\n Pool ${poolVersion[1]}`,
+  );
   return {
     lnd: `lightningnetwork/lnd/${lndVersion[1]}/lnrpc/rpc.proto`,
     loop: `lightninglabs/loop/${loopVersion[1]}/looprpc/client.proto`,
@@ -63,7 +65,7 @@ const download = async () => {
   for ([name, urlPath] of Object.entries(await protoSources())) {
     const url = `https://raw.githubusercontent.com/${urlPath}`;
     const filePath = join(appPath, '..', 'proto', `${name}.proto`);
-    mkdirSync(dirname(filePath), {recursive: true});
+    mkdirSync(dirname(filePath), { recursive: true });
     console.log(`${url}`);
     console.log(` -> ${filePath}`);
     const content = await new Promise((resolve, reject) => {
@@ -78,6 +80,26 @@ const download = async () => {
   }
 };
 
+/**
+ * Adds "[jstype = JS_STRING]" to uint64 fields to indicate that they should be
+ * represented as strings to avoid Number overflow issues
+ */
+const sanitize = async () => {
+  const filePaths = Object.keys(filePatches).map(name =>
+    join(appPath, '..', 'proto', `${name}.proto`),
+  );
+  for (path of filePaths) {
+    let content = (await fs.readFile(path)).toString();
+    content = content.replace(/^\s*(repeated)? u?int64 ((?!jstype).)*$/gm, match => {
+      // add the jstype descriptor
+      return /^.*];$/.test(match)
+        ? match.replace(/\s*];$/, `, jstype = JS_STRING];`)
+        : match.replace(/;$/, ` [jstype = JS_STRING];`);
+    });
+    await fs.writeFile(path, content);
+  }
+};
+
 /**
  * Executes the `protoc` compiler to convert *.proto files into TS & JS code
  */
@@ -117,11 +139,6 @@ const patch = async () => {
   console.log('\nPatching generated JS files');
 
   for (const filename of Object.keys(filePatches)) {
-    const patch = [
-      '/* eslint-disable */',
-      `var proto = { ${filePatches[filename]} };`,
-      '',
-    ].join('\n');
     const path = join(
       appPath,
       'src',
@@ -132,7 +149,15 @@ const patch = async () => {
 
     console.log(` - ${path}`);
     let content = await fs.readFile(path);
+
+    // apply the webpack patch
+    const patch = [
+      '/* eslint-disable */',
+      `var proto = { ${filePatches[filename]} };`,
+      '',
+    ].join('\n');
     content = `${patch}\n${content}`;
+
     await fs.writeFile(path, content);
   }
 };
@@ -143,6 +168,7 @@ const patch = async () => {
 const main = async () => {
   try {
     await download();
+    await sanitize();
     await generate();
     await patch();
   } catch (error) {

app/src/__stories__/ProcessingSwaps.stories.tsx

@@ -1,6 +1,7 @@
 import React, { useEffect } from 'react';
 import { observable } from 'mobx';
 import * as LOOP from 'types/generated/loop_pb';
+import Big from 'big.js';
 import { loopListSwaps } from 'util/tests/sampleData';
 import { useStore } from 'store';
 import { Swap } from 'store/models';
@@ -47,7 +48,7 @@ const mockSwap = (type: number, state: number, id?: string) => {
   swap.id = `${id || ''}${swap.id}`;
   swap.type = type;
   swap.state = state;
-  swap.lastUpdateTime = Date.now() * 1000 * 1000;
+  swap.lastUpdateTime = Big(Date.now() * 1000 * 1000);
   return swap;
 };
 // create a list of swaps to use for stories
@@ -84,7 +85,7 @@ export const LoopInProgress = () => {
       await delay(2000);
       swap.state = SUCCESS;
       await delay(2000);
-      swap.initiationTime = 0;
+      swap.initiationTime = Big(0);
     };
 
     startTransitions();
@@ -106,7 +107,7 @@ export const LoopOutProgress = () => {
       await delay(2000);
       swap.state = SUCCESS;
       await delay(2000);
-      swap.initiationTime = 0;
+      swap.initiationTime = Big(0);
     };
 
     startTransitions();

app/src/__tests__/components/loop/LoopPage.spec.tsx

@@ -3,6 +3,7 @@ import { runInAction } from 'mobx';
 import { SwapStatus } from 'types/generated/loop_pb';
 import { grpc } from '@improbable-eng/grpc-web';
 import { fireEvent, waitFor } from '@testing-library/react';
+import Big from 'big.js';
 import { saveAs } from 'file-saver';
 import { formatSats } from 'util/formatters';
 import { renderWithProviders } from 'util/tests';
@@ -70,9 +71,9 @@ describe('LoopPage component', () => {
     const { findByText } = render();
     // convert from numeric timestamp to string (1586390353623905000 -> '4/15/2020')
     const formatDate = (s: SwapStatus.AsObject) =>
-      new Date(s.initiationTime / 1000 / 1000).toLocaleDateString();
+      new Date(+Big(s.initiationTime).div(1000).div(1000)).toLocaleDateString();
     const [swap1, swap2] = loopListSwaps.swapsList.sort(
-      (a, b) => b.initiationTime - a.initiationTime,
+      (a, b) => +Big(b.initiationTime).sub(a.initiationTime),
     );
 
     expect(await findByText(formatDate(swap1))).toBeInTheDocument();

app/src/__tests__/components/loop/ProcessingSwaps.spec.tsx

@@ -2,6 +2,7 @@ import React from 'react';
 import { runInAction } from 'mobx';
 import * as LOOP from 'types/generated/loop_pb';
 import { fireEvent } from '@testing-library/react';
+import Big from 'big.js';
 import { renderWithProviders } from 'util/tests';
 import { loopListSwaps } from 'util/tests/sampleData';
 import { createStore, Store } from 'store';
@@ -27,7 +28,7 @@ describe('ProcessingSwaps component', () => {
     swap.id = `${id || ''}${swap.id}`;
     swap.type = type;
     swap.state = state;
-    swap.lastUpdateTime = Date.now() * 1000 * 1000;
+    swap.lastUpdateTime = Big(Date.now() * 1000 * 1000);
     runInAction(() => {
       store.swapStore.swaps.set(swap.id, swap);
     });

app/src/__tests__/components/pool/AccountSection.spec.tsx

@@ -117,7 +117,7 @@ describe('AccountSection', () => {
     });
 
     expect(store.accountStore.activeTraderKey).toBe(hex(poolInitAccount.traderKey));
-    expect(store.fundNewAccountView.amount).toBe(0);
+    expect(+store.fundNewAccountView.amount).toBe(0);
     expect(store.fundNewAccountView.confTarget).toBe(DEFAULT_CONF_TARGET);
     expect(store.fundNewAccountView.expireBlocks).toBe(DEFAULT_EXPIRE_BLOCKS);
   });
@@ -206,7 +206,7 @@ describe('AccountSection', () => {
       expect(getByText('Account')).toBeInTheDocument();
     });
 
-    expect(+store.accountStore.activeAccount.totalBalance).toBe(
+    expect(store.accountStore.activeAccount.totalBalance.toString()).toBe(
       poolDepositAccount.account.value,
     );
     expect(store.fundAccountView.amount).toBe(0);
@@ -322,7 +322,7 @@ describe('AccountSection', () => {
     });
 
     expect(req!.traderKey).toBe(b64(store.accountStore.activeAccount.traderKey));
-    expect(req!.outputWithFee?.feeRateSatPerKw).toBe(2500);
+    expect(req!.outputWithFee?.feeRateSatPerKw).toBe('2500');
     expect(req!.outputWithFee?.address).toBe('abc123');
   });
 
@@ -375,7 +375,7 @@ describe('AccountSection', () => {
     });
 
     expect(req!.accountKey).toBe(b64(store.accountStore.activeAccount.traderKey));
-    expect(req!.feeRateSatPerKw).toBe(31250);
+    expect(req!.feeRateSatPerKw).toBe('31250');
     expect(req!.relativeExpiry).toBe(2016);
   });
 });

app/src/__tests__/components/pool/BatchStats.spec.tsx

@@ -1,6 +1,7 @@
 import React from 'react';
 import { runInAction } from 'mobx';
 import { act, waitFor } from '@testing-library/react';
+import Big from 'big.js';
 import { formatSats } from 'util/formatters';
 import { renderWithProviders } from 'util/tests';
 import { createStore, Store } from 'store';
@@ -25,7 +26,7 @@ describe('BatchStats', () => {
     jest.useFakeTimers();
     runInAction(() => {
       const nowSecs = Math.ceil(Date.now() / 1000);
-      store.batchStore.nextBatchTimestamp = nowSecs + 90;
+      store.batchStore.nextBatchTimestamp = Big(nowSecs + 90);
     });
     const { getByText } = render();
     expect(getByText('Next Batch')).toBeInTheDocument();

app/src/__tests__/components/pool/OrderFormSection.spec.tsx

@@ -71,12 +71,12 @@ describe('OrderFormSection', () => {
     });
 
     fireEvent.click(getByText('Place Bid Order'));
-    expect(bid!.details.amt).toBe(1000000);
+    expect(bid!.details.amt).toBe('1000000');
     expect(bid!.details.rateFixed).toBe(4960);
     expect(bid!.details.minUnitsMatch).toBe(1);
     expect(bid!.leaseDurationBlocks).toBe(2016);
     expect(bid!.minNodeTier).toBe(1);
-    expect(bid!.details.maxBatchFeeRateSatPerKw).toBe(253);
+    expect(bid!.details.maxBatchFeeRateSatPerKw).toBe('253');
   });
 
   it('should submit an ask order', async () => {
@@ -95,11 +95,11 @@ describe('OrderFormSection', () => {
     });
 
     fireEvent.click(getByText('Place Ask Order'));
-    expect(ask!.details.amt).toBe(1000000);
+    expect(ask!.details.amt).toBe('1000000');
     expect(ask!.details.rateFixed).toBe(4960);
     expect(ask!.details.minUnitsMatch).toBe(1);
     expect(ask!.leaseDurationBlocks).toBe(2016);
-    expect(ask!.details.maxBatchFeeRateSatPerKw).toBe(253);
+    expect(ask!.details.maxBatchFeeRateSatPerKw).toBe('253');
   });
 
   it('should submit an order with a different lease duration', async () => {
@@ -119,12 +119,12 @@ describe('OrderFormSection', () => {
     });
 
     fireEvent.click(getByText('Place Bid Order'));
-    expect(bid!.details.amt).toBe(1000000);
+    expect(bid!.details.amt).toBe('1000000');
     expect(bid!.details.rateFixed).toBe(2480);
     expect(bid!.details.minUnitsMatch).toBe(1);
     expect(bid!.leaseDurationBlocks).toBe(4032);
     expect(bid!.minNodeTier).toBe(1);
-    expect(bid!.details.maxBatchFeeRateSatPerKw).toBe(253);
+    expect(bid!.details.maxBatchFeeRateSatPerKw).toBe('253');
   });
 
   it('should reset the form after placing an order', async () => {

app/src/__tests__/store/accountStore.spec.ts

@@ -46,8 +46,8 @@ describe('AccountStore', () => {
   });
 
   it('should return sorted accounts', async () => {
-    const a = new Account(rootStore, { ...poolInitAccount, value: 300 });
-    const b = new Account(rootStore, { ...poolInitAccount, value: 100 });
+    const a = new Account(rootStore, { ...poolInitAccount, value: '300' });
+    const b = new Account(rootStore, { ...poolInitAccount, value: '100' });
     const c = new Account(rootStore, {
       ...poolInitAccount,
       expirationHeight: 5000,
@@ -76,8 +76,8 @@ describe('AccountStore', () => {
   });
 
   it('should excluded closed accounts in sorted accounts', async () => {
-    const a = new Account(rootStore, { ...poolInitAccount, value: 300 });
-    const b = new Account(rootStore, { ...poolInitAccount, value: 100 });
+    const a = new Account(rootStore, { ...poolInitAccount, value: '300' });
+    const b = new Account(rootStore, { ...poolInitAccount, value: '100' });
     const c = new Account(rootStore, {
       ...poolInitAccount,
       expirationHeight: 5000,
@@ -158,7 +158,7 @@ describe('AccountStore', () => {
 
   it('should create a new Account', async () => {
     expect(store.accounts.size).toEqual(0);
-    await store.createAccount(3000000, 4032);
+    await store.createAccount(Big(3000000), 4032);
     expect(store.accounts.size).toEqual(1);
     expect(store.activeAccount).toBeDefined();
   });
@@ -168,7 +168,7 @@ describe('AccountStore', () => {
       throw new Error('test-err');
     });
     expect(rootStore.appView.alerts.size).toBe(0);
-    await store.createAccount(3000000, 4032);
+    await store.createAccount(Big(3000000), 4032);
     await waitFor(() => {
       expect(rootStore.appView.alerts.size).toBe(1);
       expect(values(rootStore.appView.alerts)[0].message).toBe('test-err');
@@ -216,7 +216,9 @@ describe('AccountStore', () => {
   it('should deposit funds into an account', async () => {
     await store.fetchAccounts();
     const txid = await store.deposit(1);
-    expect(+store.activeAccount.totalBalance).toBe(poolDepositAccount.account?.value);
+    expect(store.activeAccount.totalBalance.toString()).toBe(
+      poolDepositAccount.account?.value,
+    );
     expect(txid).toEqual(poolDepositAccount.depositTxid);
   });
 
@@ -236,7 +238,9 @@ describe('AccountStore', () => {
   it('should withdraw funds from an account', async () => {
     await store.fetchAccounts();
     const txid = await store.withdraw(1);
-    expect(+store.activeAccount.totalBalance).toBe(poolWithdrawAccount.account?.value);
+    expect(store.activeAccount.totalBalance.toString()).toBe(
+      poolWithdrawAccount.account?.value,
+    );
     expect(txid).toEqual(poolWithdrawAccount.withdrawTxid);
   });
 

app/src/__tests__/store/buildSwapView.spec.ts

@@ -111,15 +111,15 @@ describe('BuildSwapView', () => {
   });
 
   it('should ensure amount is greater than the min terms', async () => {
-    store.setAmount(Big(loopInTerms.minSwapAmount - 100));
+    store.setAmount(Big(loopInTerms.minSwapAmount).sub(100));
     await store.getTerms();
-    expect(+store.amountForSelected).toBe(loopInTerms.minSwapAmount);
+    expect(store.amountForSelected.toString()).toBe(loopInTerms.minSwapAmount);
   });
 
   it('should ensure amount is less than the max terms', async () => {
     store.setAmount(Big(loopInTerms.maxSwapAmount + 100));
     await store.getTerms();
-    expect(+store.amountForSelected).toBe(loopInTerms.maxSwapAmount);
+    expect(store.amountForSelected.toString()).toBe(loopInTerms.maxSwapAmount);
   });
 
   it('should validate the conf target', async () => {
@@ -293,7 +293,7 @@ describe('BuildSwapView', () => {
     store.setDirection(SwapDirection.OUT);
     store.setAmount(Big(600));
 
-    let deadline = 0;
+    let deadline = '';
     // mock the grpc unary function in order to capture the supplied deadline
     // passed in with the API request
     injectIntoGrpcUnary((desc, props) => {
@@ -303,7 +303,7 @@ describe('BuildSwapView', () => {
     // run a loop on mainnet and verify the deadline
     rootStore.nodeStore.network = 'mainnet';
     store.requestSwap();
-    await waitFor(() => expect(deadline).toBeGreaterThan(0));
+    await waitFor(() => expect(+deadline).toBeGreaterThan(0));
 
     // inject again for the next swap
     injectIntoGrpcUnary((desc, props) => {
@@ -313,7 +313,7 @@ describe('BuildSwapView', () => {
     // run a loop on regtest and verify the deadline
     rootStore.nodeStore.network = 'regtest';
     store.requestSwap();
-    await waitFor(() => expect(deadline).toEqual(0));
+    await waitFor(() => expect(+deadline).toEqual(0));
   });
 
   it('should handle errors when performing a loop', async () => {
@@ -370,7 +370,12 @@ describe('BuildSwapView', () => {
   describe('min/max swap limits', () => {
     const addChannel = (capacity: number, localBalance: number) => {
       const remoteBalance = capacity - localBalance;
-      const lndChan = { ...lndChannel, capacity, localBalance, remoteBalance };
+      const lndChan = {
+        ...lndChannel,
+        capacity: `${capacity}`,
+        localBalance: `${localBalance}`,
+        remoteBalance: `${remoteBalance}`,
+      };
       const channel = Channel.create(rootStore, lndChan);
       channel.chanId = `${channel.chanId}${rootStore.channelStore.channels.size}`;
       channel.remotePubkey = `${channel.remotePubkey}${rootStore.channelStore.channels.size}`;