feat: batch api requests

biome.json

@@ -31,7 +31,7 @@
       "lib/**",
       "node_modules/**",
       "test/snapshot/**",
-      "test/fixtures/**",
+      "e2e/fixtures/**",
       "types/cdxgen.d.ts"
     ]
   }

e2e/scan/eol.test.ts

@@ -0,0 +1,95 @@
+import { strictEqual } from 'node:assert';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, it } from 'node:test';
+import { fileURLToPath } from 'node:url';
+import { runCommand } from '@oclif/test';
+
+describe('scan:eol e2e', () => {
+  const __dirname = path.dirname(fileURLToPath(import.meta.url));
+  const testDir = path.resolve(__dirname, '../fixtures/npm/simple');
+  const reportPath = path.join(testDir, 'nes.eol.json');
+
+  async function cleanupReport() {
+    try {
+      await fs.unlink(reportPath);
+    } catch {
+      // Ignore if file doesn't exist
+    }
+  }
+
+  beforeEach(async () => {
+    // Skip tests if no backend URL is provided
+    if (!process.env.E2E_BACKEND_URL) {
+      console.log('Skipping E2E tests: No backend URL provided');
+      return;
+    }
+
+    // Set up environment
+    process.env.GRAPHQL_HOST = process.env.E2E_BACKEND_URL;
+
+    // Ensure test directory exists and is clean
+    await fs.mkdir(testDir, { recursive: true });
+    await cleanupReport();
+  });
+
+  afterEach(cleanupReport);
+
+  it('scans a directory for EOL components', async () => {
+    const cmd = `scan:eol --dir ${testDir}`;
+    const output = await runCommand(cmd);
+
+    // Log any errors for debugging
+    if (output.error) {
+      console.error('Command failed with error:', output.error);
+      console.error('Error details:', output.stderr);
+    }
+
+    // Verify command executed successfully
+    strictEqual(output.error, undefined, 'Command should execute without errors');
+
+    // Verify output contains expected content
+    const stdout = output.stdout;
+
+    strictEqual(stdout.includes('Here are the results of the scan:'), true, 'Should show results header');
+
+    // Since we know [email protected] is EOL, verify it's detected
+    strictEqual(stdout.includes('pkg:npm/[email protected]'), true, 'Should detect bootstrap package');
+    strictEqual(stdout.includes('End of Life (EOL)'), true, 'Should show EOL status');
+  });
+
+  it('saves report when --save flag is used', async () => {
+    const cmd = `scan:eol --dir ${testDir} --save`;
+    const output = await runCommand(cmd);
+
+    // Log any errors for debugging
+    if (output.error) {
+      console.error('Command failed with error:', output.error);
+      console.error('Error details:', output.stderr);
+    }
+
+    console.log('output', output.stdout);
+    // Verify command executed successfully
+    strictEqual(output.error, undefined, 'Command should execute without errors');
+
+    // Verify report was saved
+    const reportExists = await fs
+      .access(reportPath)
+      .then(() => true)
+      .catch(() => false);
+
+    strictEqual(reportExists, true, 'Report file should be created');
+
+    // Verify report content
+    const reportContent = await fs.readFile(reportPath, 'utf-8');
+    const report = JSON.parse(reportContent);
+
+    // Verify structure
+    strictEqual(Array.isArray(report.components), true, 'Report should contain components array');
+
+    // Verify content (we know [email protected] should be EOL)
+    const bootstrap = report.components.find((c) => c.purl === 'pkg:npm/[email protected]');
+    strictEqual(bootstrap !== undefined, true, 'Report should contain bootstrap package');
+    strictEqual(bootstrap?.info?.isEol, true, 'Bootstrap should be marked as EOL');
+  });
+});

package.json

@@ -28,7 +28,9 @@
     "readme": "npm run ci:fix && npm run build && npm exec oclif readme",
     "test": "globstar -- node --import tsx --test \"test/**/*.test.ts\"",
     "typecheck": "tsc --noEmit",
-    "version": "oclif readme && git add README.md"
+    "version": "oclif readme && git add README.md",
+    "test:e2e": "E2E_BACKEND_URL=${E2E_BACKEND_URL:-https://api.nes.herodevs.com} globstar -- node --import tsx --test \"e2e/**/*.test.ts\"",
+    "test:e2e:local": "E2E_BACKEND_URL=http://localhost:3000 npm run test:e2e"
   },
   "keywords": [
     "herodevs",

src/api/nes/nes.client.ts

@@ -1,7 +1,8 @@
 import type * as apollo from '@apollo/client/core/index.js';
 
 import { ApolloClient } from '../../api/client.ts';
-import type { ScanResult } from '../../api/types/nes.types.ts';
+import type { ScanResult, ScanResultComponent } from '../../api/types/nes.types.ts';
+import { debugLogger } from '../../service/log.svc.ts';
 import { SbomScanner } from '../../service/nes/nes.svc.ts';
 
 export interface NesClient {
@@ -29,14 +30,78 @@ export class NesApolloClient implements NesClient {
   }
 }
 
-/**
- * Uses the purls from the sbom to run the scan.
- */
-export async function submitScan(purls: string[]): Promise<ScanResult> {
+async function submitScan(purls: string[]): Promise<ScanResult> {
   // NOTE: GRAPHQL_HOST is set in `./bin/dev.js` or tests
   const host = process.env.GRAPHQL_HOST || 'https://api.nes.herodevs.com';
   const path = process.env.GRAPHQL_PATH || '/graphql';
   const url = host + path;
   const client = new NesApolloClient(url);
   return client.scan.sbom(purls);
 }
+
+function combineScanResults(results: ScanResult[]): ScanResult {
+  const combinedResults: ScanResult = {
+    components: new Map<string, ScanResultComponent>(),
+    message: '',
+    success: true,
+    warnings: [],
+  };
+
+  for (const result of results) {
+    for (const component of result.components.values()) {
+      combinedResults.components.set(component.purl, component);
+    }
+    combinedResults.warnings.push(...result.warnings);
+    combinedResults.success = combinedResults.success && result.success;
+  }
+
+  return combinedResults;
+}
+
+export function createBatches(items: string[], batchSize: number): string[][] {
+  return Array.from({ length: Math.ceil(items.length / batchSize) }, (_, i) =>
+    items.slice(i * batchSize, (i + 1) * batchSize),
+  );
+}
+
+export async function batchSubmitPurls(purls: string[], batchSize = 1000): Promise<ScanResult> {
+  try {
+    const batches = createBatches(purls, batchSize);
+    debugLogger('Processing %d batches', batches.length);
+
+    const results = await Promise.allSettled(
+      batches.map((batch, index) => {
+        debugLogger('Starting batch %d', index + 1);
+        return submitScan(batch);
+      }),
+    );
+
+    const successfulResults: ScanResult[] = [];
+    const errors: string[] = [];
+
+    for (const [index, result] of results.entries()) {
+      if (result.status === 'fulfilled') {
+        debugLogger('Batch %d completed successfully', index + 1);
+        successfulResults.push(result.value);
+      } else {
+        debugLogger('Batch %d failed: %s', index + 1, result.reason);
+        errors.push(`Batch ${index + 1}: ${result.reason}`);
+      }
+    }
+
+    if (successfulResults.length === 0) {
+      throw new Error(`All batches failed:\n${errors.join('\n')}`);
+    }
+
+    const combinedResults = combineScanResults(successfulResults);
+    if (errors.length > 0) {
+      combinedResults.success = false;
+      combinedResults.message = `Errors encountered:\n${errors.join('\n')}`;
+    }
+
+    return combinedResults;
+  } catch (error) {
+    debugLogger('Fatal error in batchSubmitPurls: %s', error);
+    throw new Error(`Failed to process purls: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}

src/commands/scan/eol.ts

@@ -1,6 +1,7 @@
 import fs from 'node:fs';
+import path from 'node:path';
 import { Command, Flags, ux } from '@oclif/core';
-import { submitScan } from '../../api/nes/nes.client.ts';
+import { batchSubmitPurls } from '../../api/nes/nes.client.ts';
 import type { ScanResult, ScanResultComponent } from '../../api/types/nes.types.ts';
 import type { Sbom } from '../../service/eol/cdx.svc.ts';
 import { getErrorMessage, isErrnoException } from '../../service/error.svc.ts';
@@ -77,8 +78,9 @@ export default class ScanEol extends Command {
 
   private async getScan(flags: Record<string, string>, config: Command['config']): Promise<ScanResult> {
     if (flags.purls) {
+      ux.action.start(`Scanning purls from ${flags.purls}`);
       const purls = this.getPurlsFromFile(flags.purls);
-      return submitScan(purls);
+      return batchSubmitPurls(purls);
     }
 
     const sbom = await ScanSbom.loadSbom(flags, config);
@@ -106,7 +108,7 @@ export default class ScanEol extends Command {
       this.error(`Failed to extract purls from sbom. ${getErrorMessage(error)}`);
     }
     try {
-      scan = await submitScan(purls);
+      scan = await batchSubmitPurls(purls);
     } catch (error) {
       this.error(`Failed to submit scan to NES from sbom. ${getErrorMessage(error)}`);
     }
@@ -126,7 +128,9 @@ export default class ScanEol extends Command {
 
   private async saveReport(components: ScanResultComponent[]): Promise<void> {
     try {
-      fs.writeFileSync('nes.eol.json', JSON.stringify({ components }, null, 2));
+      const { flags } = await this.parse(ScanEol);
+      const reportPath = path.join(flags.dir || process.cwd(), 'nes.eol.json');
+      fs.writeFileSync(reportPath, JSON.stringify({ components }, null, 2));
       this.log('Report saved to nes.eol.json');
     } catch (error) {
       if (isErrnoException(error)) {

test/api/nes.client.test.ts

@@ -0,0 +1,34 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+import { createBatches } from '../../src/api/nes/nes.client.ts';
+
+describe('createBatches', () => {
+  it('should handle empty array', () => {
+    const result = createBatches([], 1000);
+    assert.deepStrictEqual(result, []);
+  });
+
+  it('should create single batch when items length is less than batch size', () => {
+    const items = ['a', 'b', 'c'];
+    const result = createBatches(items, 5);
+    assert.deepStrictEqual(result, [['a', 'b', 'c']]);
+  });
+
+  it('should create single batch when items length equals batch size', () => {
+    const items = ['a', 'b', 'c', 'd', 'e'];
+    const result = createBatches(items, 5);
+    assert.deepStrictEqual(result, [['a', 'b', 'c', 'd', 'e']]);
+  });
+
+  it('should create multiple batches when items length exceeds batch size', () => {
+    const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'];
+    const result = createBatches(items, 3);
+    assert.deepStrictEqual(result, [['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i'], ['j']]);
+  });
+
+  it('should handle batch size of 1', () => {
+    const items = ['a', 'b', 'c'];
+    const result = createBatches(items, 1);
+    assert.deepStrictEqual(result, [['a'], ['b'], ['c']]);
+  });
+});