Deploy with Manifests from URLs (#251)

* added functionality, need to add/modify existing tests

* added tests

* updated readme

* prettier

Author: jaiveerk

README.md

@@ -51,7 +51,7 @@ Following are the key capabilities of this action:
   </tr>
   <tr>
     <td>manifests </br></br>(Required)</td>
-    <td>Path to the manifest files to be used for deployment. These can also be directories containing manifest files, in which case, all manifest files in the referenced directory at every depth will be deployed. Files not ending in .yml or .yaml will be ignored.</td>
+    <td>Path to the manifest files to be used for deployment. These can also be directories containing manifest files, in which case, all manifest files in the referenced directory at every depth will be deployed, or URLs to manifest files (like https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/controllers/nginx-deployment.yaml). Files and URLs not ending in .yml or .yaml will be ignored.</td>
   </tr>
     <tr>
     <td>strategy </br></br>(Required)</td>

jest.config.js

@@ -6,5 +6,6 @@ module.exports = {
    transform: {
       '^.+\\.ts$': 'ts-jest'
    },
-   verbose: true
+   verbose: true,
+   testTimeout: 9000
 }

src/run.ts

@@ -5,7 +5,7 @@ import {promote} from './actions/promote'
 import {reject} from './actions/reject'
 import {Action, parseAction} from './types/action'
 import {parseDeploymentStrategy} from './types/deploymentStrategy'
-import {getFilesFromDirectories} from './utilities/fileUtils'
+import {getFilesFromDirectoriesAndURLs} from './utilities/fileUtils'
 import {PrivateKubectl} from './types/privatekubectl'
 
 export async function run() {
@@ -26,7 +26,9 @@ export async function run() {
       .map((manifest) => manifest.trim()) // remove surrounding whitespace
       .filter((manifest) => manifest.length > 0) // remove any blanks
 
-   const fullManifestFilePaths = getFilesFromDirectories(manifestFilePaths)
+   const fullManifestFilePaths = await getFilesFromDirectoriesAndURLs(
+      manifestFilePaths
+   )
    const kubectlPath = await getKubectlPath()
    const namespace = core.getInput('namespace') || 'default'
    const isPrivateCluster =

src/types/errorable.ts

@@ -0,0 +1,48 @@
+export interface Succeeded<T> {
+   readonly succeeded: true
+   readonly result: T
+}
+
+export interface Failed {
+   readonly succeeded: false
+   readonly error: string
+}
+
+export type Errorable<T> = Succeeded<T> | Failed
+
+export function succeeded<T>(e: Errorable<T>): e is Succeeded<T> {
+   return e.succeeded
+}
+
+export function failed<T>(e: Errorable<T>): e is Failed {
+   return !e.succeeded
+}
+
+export function map<T, U>(e: Errorable<T>, fn: (t: T) => U): Errorable<U> {
+   if (failed(e)) {
+      return {succeeded: false, error: e.error}
+   }
+   return {succeeded: true, result: fn(e.result)}
+}
+
+export function combine<T>(es: Errorable<T>[]): Errorable<T[]> {
+   const failures = es.filter(failed)
+   if (failures.length > 0) {
+      return {
+         succeeded: false,
+         error: failures.map((f) => f.error).join('\n')
+      }
+   }
+
+   return {
+      succeeded: true,
+      result: es.map((e) => (e as Succeeded<T>).result)
+   }
+}
+
+export function getErrorMessage(error: unknown) {
+   if (error instanceof Error) {
+      return error.message
+   }
+   return String(error)
+}

src/utilities/fileUtils.test.ts

@@ -1,11 +1,45 @@
-import {getFilesFromDirectories} from './fileUtils'
+import {
+   getFilesFromDirectoriesAndURLs,
+   getTempDirectory,
+   urlFileKind,
+   writeYamlFromURLToFile
+} from './fileUtils'
 
+import * as yaml from 'js-yaml'
+import * as fs from 'fs'
 import * as path from 'path'
+import {succeeded} from '../types/errorable'
 
+const sampleYamlUrl =
+   'https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/controllers/nginx-deployment.yaml'
 describe('File utils', () => {
-   it('detects files in nested directories and ignores non-manifest files and empty dirs', () => {
+   test('correctly parses a yaml file from a URL', async () => {
+      const tempFile = await writeYamlFromURLToFile(sampleYamlUrl, 0)
+      const fileContents = fs.readFileSync(tempFile).toString()
+      const inputObjects = yaml.safeLoadAll(fileContents)
+      expect(inputObjects).toHaveLength(1)
+
+      for (const obj of inputObjects) {
+         expect(obj.metadata.name).toBe('nginx-deployment')
+         expect(obj.kind).toBe('Deployment')
+      }
+   })
+
+   it('fails when a bad URL is given among other files', async () => {
+      const badUrl = 'https://www.github.com'
+
       const testPath = path.join('test', 'unit', 'manifests')
-      const testSearch: string[] = getFilesFromDirectories([testPath])
+      await expect(
+         getFilesFromDirectoriesAndURLs([testPath, badUrl])
+      ).rejects.toThrow()
+   })
+
+   it('detects files in nested directories and ignores non-manifest files and empty dirs', async () => {
+      const testPath = path.join('test', 'unit', 'manifests')
+      const testSearch: string[] = await getFilesFromDirectoriesAndURLs([
+         testPath,
+         sampleYamlUrl
+      ])
 
       const expectedManifests = [
          'test/unit/manifests/manifest_test_dir/another_layer/deep-ingress.yaml',
@@ -17,13 +51,18 @@ describe('File utils', () => {
       ]
 
       // is there a more efficient way to test equality w random order?
-      expect(testSearch).toHaveLength(7)
+      expect(testSearch).toHaveLength(8)
       expectedManifests.forEach((fileName) => {
-         expect(testSearch).toContain(fileName)
+         if (fileName.startsWith('test/unit')) {
+            expect(testSearch).toContain(fileName)
+         } else {
+            expect(fileName.includes(urlFileKind)).toBe(true)
+            expect(fileName.startsWith(getTempDirectory()))
+         }
       })
    })
 
-   it('crashes when an invalid file is provided', () => {
+   it('crashes when an invalid file is provided', async () => {
       const badPath = path.join('test', 'unit', 'manifests', 'nonexistent.yaml')
       const goodPath = path.join(
          'test',
@@ -32,12 +71,12 @@ describe('File utils', () => {
          'manifest_test_dir'
       )
 
-      expect(() => {
-         getFilesFromDirectories([badPath, goodPath])
-      }).toThrowError()
+      expect(
+         getFilesFromDirectoriesAndURLs([badPath, goodPath])
+      ).rejects.toThrowError()
    })
 
-   it("doesn't duplicate files when nested dir included", () => {
+   it("doesn't duplicate files when nested dir included", async () => {
       const outerPath = path.join('test', 'unit', 'manifests')
       const fileAtOuter = path.join(
          'test',
@@ -53,11 +92,16 @@ describe('File utils', () => {
       )
 
       expect(
-         getFilesFromDirectories([outerPath, fileAtOuter, innerPath])
+         await getFilesFromDirectoriesAndURLs([
+            outerPath,
+            fileAtOuter,
+            innerPath
+         ])
       ).toHaveLength(7)
    })
-})
 
-// files that don't exist / nested files that don't exist / something else with non-manifest
-// lots of combinations of pointing to a directory and non yaml/yaml file
-// similarly named files in different folders
+   it('throws an error for an invalid URL', async () => {
+      const badUrl = 'https://www.github.com'
+      await expect(writeYamlFromURLToFile(badUrl, 0)).rejects.toBeTruthy()
+   })
+})

src/utilities/fileUtils.ts

@@ -1,8 +1,15 @@
 import * as fs from 'fs'
+import * as https from 'https'
 import * as path from 'path'
 import * as core from '@actions/core'
 import * as os from 'os'
+import * as yaml from 'js-yaml'
+import {Errorable, succeeded, failed, Failed} from '../types/errorable'
 import {getCurrentTime} from './timeUtils'
+import {isHttpUrl} from './githubUtils'
+import {K8sObject} from '../types/k8sObject'
+
+export const urlFileKind = 'urlfile'
 
 export function getTempDirectory(): string {
    return process.env['runner.tempDirectory'] || os.tmpdir()
@@ -62,12 +69,27 @@ function getManifestFileName(kind: string, name: string) {
    return path.join(tempDirectory, path.basename(filePath))
 }
 
-export function getFilesFromDirectories(filePaths: string[]): string[] {
+export async function getFilesFromDirectoriesAndURLs(
+   filePaths: string[]
+): Promise<string[]> {
    const fullPathSet: Set<string> = new Set<string>()
 
-   filePaths.forEach((fileName) => {
+   let fileCounter = 0
+   for (const fileName of filePaths) {
       try {
-         if (fs.lstatSync(fileName).isDirectory()) {
+         if (isHttpUrl(fileName)) {
+            try {
+               const tempFilePath: string = await writeYamlFromURLToFile(
+                  fileName,
+                  fileCounter++
+               )
+               fullPathSet.add(tempFilePath)
+            } catch (e) {
+               throw Error(
+                  `encountered error trying to pull YAML from URL ${fileName}: ${e}`
+               )
+            }
+         } else if (fs.lstatSync(fileName).isDirectory()) {
             recurisveManifestGetter(fileName).forEach((file) => {
                fullPathSet.add(file)
             })
@@ -86,9 +108,86 @@ export function getFilesFromDirectories(filePaths: string[]): string[] {
             `Exception occurred while reading the file ${fileName}: ${ex}`
          )
       }
+   }
+
+   const arr = Array.from(fullPathSet)
+   return arr
+}
+
+export async function writeYamlFromURLToFile(
+   url: string,
+   fileNumber: number
+): Promise<string> {
+   return new Promise((resolve, reject) => {
+      https
+         .get(url, async (response) => {
+            const code = response.statusCode ?? 0
+            if (code >= 400) {
+               reject(
+                  Error(
+                     `received response status ${response.statusMessage} from url ${url}`
+                  )
+               )
+            }
+
+            const targetPath = getManifestFileName(
+               urlFileKind,
+               fileNumber.toString()
+            )
+            // save the file to disk
+            const fileWriter = fs
+               .createWriteStream(targetPath)
+               .on('finish', () => {
+                  const verification = verifyYaml(targetPath, url)
+                  if (succeeded(verification)) {
+                     core.debug(
+                        `outputting YAML contents from ${url} to ${targetPath}: ${JSON.stringify(
+                           verification.result
+                        )}`
+                     )
+                     resolve(targetPath)
+                  } else {
+                     reject(verification.error)
+                  }
+               })
+
+            response.pipe(fileWriter)
+         })
+         .on('error', (error) => {
+            reject(error)
+         })
    })
+}
+
+function verifyYaml(filepath: string, url: string): Errorable<K8sObject[]> {
+   const fileContents = fs.readFileSync(filepath).toString()
+   let inputObjects
+   try {
+      inputObjects = yaml.safeLoadAll(fileContents)
+   } catch (e) {
+      return {
+         succeeded: false,
+         error: `failed to parse manifest from url ${url}: ${e}`
+      }
+   }
+
+   if (!inputObjects || inputObjects.length == 0) {
+      return {
+         succeeded: false,
+         error: `failed to parse manifest from url ${url}: no objects detected in manifest`
+      }
+   }
+
+   for (const obj of inputObjects) {
+      if (!obj.kind || !obj.apiVersion || !obj.metadata) {
+         return {
+            succeeded: false,
+            error: `failed to parse manifest from ${url}: missing fields`
+         }
+      }
+   }
 
-   return Array.from(fullPathSet)
+   return {succeeded: true, result: inputObjects}
 }
 
 function recurisveManifestGetter(dirName: string): string[] {