[rush-amazon-s3-build-cache-plugin] verify that retry logic works

Author: ricsam

build-tests/rush-amazon-s3-build-cache-plugin-integration-test/.gitignore

@@ -1 +1,2 @@
 s3data/.minio.sys
+s3data/rush-build-cache/test

build-tests/rush-amazon-s3-build-cache-plugin-integration-test/README.md

@@ -16,3 +16,32 @@ In this folder run `docker-compose down`
 # build the code: rushx build
 rushx read-s3-object
 ```
+
+# Test retries
+
+To test that requests can be retried start the proxy server which will fail every second request:
+
+```bash
+rushx start-proxy-server
+```
+
+Update the build-cache.json file:
+```json
+{
+  "cacheProvider": "amazon-s3",
+  "amazonS3Configuration": {
+    "s3Endpoint": "http://localhost:9002",
+    "s3Region": "us-east-1",
+    "s3Prefix": "rush-build-cache/test",
+    "isCacheWriteAllowed": true
+  }
+}
+```
+
+Run the rush rebuild command
+
+```bash
+cd apps
+cd rush
+RUSH_BUILD_CACHE_CREDENTIAL="minio:minio123" node lib/start-dev.js --debug rebuild --verbose
+```

build-tests/rush-amazon-s3-build-cache-plugin-integration-test/package.json

@@ -7,7 +7,8 @@
   "scripts": {
     "build": "heft build --clean",
     "_phase:build": "heft build --clean",
-    "read-s3-object": "node ./lib/readObject.js"
+    "read-s3-object": "node ./lib/readObject.js",
+    "start-proxy-server": "node ./lib/startProxyServer.js"
   },
   "devDependencies": {
     "@rushstack/eslint-config": "workspace:*",
@@ -16,6 +17,8 @@
     "@rushstack/node-core-library": "workspace:*",
     "@types/node": "12.20.24",
     "eslint": "~8.7.0",
-    "typescript": "~4.5.2"
+    "typescript": "~4.5.2",
+    "http-proxy": "~1.18.1",
+    "@types/http-proxy": "~1.17.8"
   }
 }

build-tests/rush-amazon-s3-build-cache-plugin-integration-test/src/startProxyServer.ts

@@ -0,0 +1,27 @@
+import * as httpProxy from 'http-proxy';
+import * as http from 'http';
+
+const proxy: httpProxy = httpProxy.createProxyServer({});
+
+const hasFailed: { [k: string]: boolean } = {};
+
+let requestCount: number = 0;
+const server: http.Server = http.createServer((req, res) => {
+  requestCount += 1;
+
+  if (req.url && requestCount % 2 === 0 && !hasFailed[req.url]) {
+    console.log('failing', req.url);
+    hasFailed[req.url] = true;
+    res.statusCode = 500;
+    res.end();
+    return;
+  } else if (req.url) {
+    console.log('proxying', req.url);
+  }
+
+  proxy.web(req, res, {
+    target: 'http://127.0.0.1:9000'
+  });
+});
+
+server.listen(9002);

common/config/rush/nonbrowser-approved-packages.json

@@ -414,6 +414,10 @@
       "name": "html-webpack-plugin",
       "allowedCategories": [ "libraries", "tests" ]
     },
+    {
+      "name": "http-proxy",
+      "allowedCategories": [ "tests" ]
+    },
     {
       "name": "https-proxy-agent",
       "allowedCategories": [ "libraries" ]

common/config/rush/pnpm-lock.yaml

@@ -402,9 +402,8 @@ export class AmazonS3Client {
   }
 
   private async _sendCacheRequestWithRetries<T>(sendRequest: () => Promise<T>): Promise<T> {
-    const terminal: ITerminal = this._terminal;
-
     type TryResponse = { hasNetworkError: false; response: T } | { hasNetworkError: true; error: unknown };
+
     const trySendRequest: () => Promise<TryResponse> = async (): Promise<TryResponse> => {
       try {
         const response: T = await sendRequest();
@@ -422,20 +421,20 @@ export class AmazonS3Client {
 
     const response: TryResponse = await trySendRequest();
 
+    const log: (...messageParts: (string | IColorableSequence)[]) => void = this._writeDebugLine.bind(this);
+
     if (response.hasNetworkError) {
       if (storageRetryOptions && storageRetryOptions.maxTries > 1) {
-        terminal.writeVerboseLine(
-          'Network request failed. Will retry request as specified in storageRetryOptions'
-        );
-        const { retryDelayInMs, retryPolicyType, maxTries, maxRetryDelayInMs } = storageRetryOptions;
+        log('Network request failed. Will retry request as specified in storageRetryOptions');
         async function retry(retryAttempt: number): Promise<T> {
+          const { retryDelayInMs, retryPolicyType, maxTries, maxRetryDelayInMs } = storageRetryOptions;
           let delay: number = retryDelayInMs;
           if (retryPolicyType === StorageRetryPolicyType.EXPONENTIAL) {
             delay = retryDelayInMs * Math.pow(2, retryAttempt - 1);
           }
           delay = Math.min(maxRetryDelayInMs, delay);
 
-          terminal.writeVerboseLine(`Will retry request in ${delay}s...`);
+          log(`Will retry request in ${delay}s...`);
           await new Promise<void>((resolve) => {
             setTimeout(() => {
               resolve();
@@ -446,10 +445,10 @@ export class AmazonS3Client {
 
           if (response.hasNetworkError) {
             if (retryAttempt < maxTries - 1) {
-              terminal.writeVerboseLine('The retried request failed, will try again');
+              log('The retried request failed, will try again');
               return retry(retryAttempt + 1);
             } else {
-              terminal.writeVerboseLine(
+              log(
                 'The retried request failed and has reached the maxTries limit, the cloud service is not accessible'
               );
               throw response.error;
@@ -460,7 +459,7 @@ export class AmazonS3Client {
         }
         return retry(1);
       } else {
-        terminal.writeVerboseLine('Network request failed and storageRetryOptions is not specified');
+        log('Network request failed and storageRetryOptions is not specified');
         throw response.error;
       }
     }