diff --git a/README.md b/README.md index 98e87ec4..038e31e6 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ import "github.com/bitfield/script" ``` -[![Magical gopher logo](img/magic.png)](https://bitfieldconsulting.com/golang/scripting) +[![Magical gopher logo](img/magic.png)](https://bitfieldconsulting.com/subscribe) + +[Subscribe to learn Go with me!](https://bitfieldconsulting.com/subscribe) # What is `script`? @@ -21,41 +23,11 @@ Shell scripts often compose a sequence of operations on a stream of data (a _pip > *This is one absolutely superb API design. Taking inspiration from shell pipes and turning it into a Go library with syntax this clean is really impressive.*\ > —[Simon Willison](https://news.ycombinator.com/item?id=30649524) -Read more: [Scripting with Go](https://bitfieldconsulting.com/golang/scripting) +# Quick start -# Quick start: Unix equivalents +Read my tutorial: [Scripting with Go](https://bitfieldconsulting.com/golang/scripting) -If you're already familiar with shell scripting and the Unix toolset, here is a rough guide to the equivalent `script` operation for each listed Unix command. - -| Unix / shell | `script` equivalent | -| ------------------ | ------------------- | -| (any program name) | [`Exec`](https://pkg.go.dev/github.com/bitfield/script#Exec) | -| `[ -f FILE ]` | [`IfExists`](https://pkg.go.dev/github.com/bitfield/script#IfExists) | -| `>` | [`WriteFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WriteFile) | -| `>>` | [`AppendFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.AppendFile) | -| `$*` | [`Args`](https://pkg.go.dev/github.com/bitfield/script#Args) | -| `base64` | [`DecodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.DecodeBase64) / [`EncodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.EncodeBase64) | -| `basename` | [`Basename`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Basename) | -| `cat` | [`File`](https://pkg.go.dev/github.com/bitfield/script#File) / [`Concat`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Concat) | -| `curl` | [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do) / [`Get`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Get) / [`Post`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Post) | -| `cut` | [`Column`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Column) | -| `dirname` | [`Dirname`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Dirname) | -| `echo` | [`Echo`](https://pkg.go.dev/github.com/bitfield/script#Echo) | -| `find` | [`FindFiles`](https://pkg.go.dev/github.com/bitfield/script#FindFiles) | -| `grep` | [`Match`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Match) / [`MatchRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.MatchRegexp) | -| `grep -v` | [`Reject`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Reject) / [`RejectRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.RejectRegexp) | -| `head` | [`First`](https://pkg.go.dev/github.com/bitfield/script#Pipe.First) | -| `jq` | [`JQ`](https://pkg.go.dev/github.com/bitfield/script#Pipe.JQ) | -| `ls` | [`ListFiles`](https://pkg.go.dev/github.com/bitfield/script#ListFiles) | -| `sed` | [`Replace`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Replace) / [`ReplaceRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ReplaceRegexp) | -| `sha256sum` | [`SHA256Sum`](https://pkg.go.dev/github.com/bitfield/script#Pipe.SHA256Sum) / [`SHA256Sums`](https://pkg.go.dev/github.com/bitfield/script#Pipe.SHA256Sums) | -| `tail` | [`Last`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Last) | -| `tee` | [`Tee`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Tee) | -| `uniq -c` | [`Freq`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Freq) | -| `wc -l` | [`CountLines`](https://pkg.go.dev/github.com/bitfield/script#Pipe.CountLines) | -| `xargs` | [`ExecForEach`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ExecForEach) | - -# Some examples +# Examples Let's see some simple examples. Suppose you want to read the contents of a file as a string: @@ -259,10 +231,72 @@ Let's try it out with some [sample data](testdata/access.log): 1 90.53.111.17 ``` +# Unix equivalents + +If you're already familiar with shell scripting and the Unix toolset, here is a rough guide to the equivalent `script` operation for each listed Unix command. + +| Unix / shell | `script` equivalent | +| ------------------ | ------------------- | +| (any program name) | [`Exec`](https://pkg.go.dev/github.com/bitfield/script#Exec) | +| `[ -f FILE ]` | [`IfExists`](https://pkg.go.dev/github.com/bitfield/script#IfExists) | +| `>` | [`WriteFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WriteFile) | +| `>>` | [`AppendFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.AppendFile) | +| `$*` | [`Args`](https://pkg.go.dev/github.com/bitfield/script#Args) | +| `base64` | [`DecodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.DecodeBase64) / [`EncodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.EncodeBase64) | +| `basename` | [`Basename`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Basename) | +| `cat` | [`File`](https://pkg.go.dev/github.com/bitfield/script#File) / [`Concat`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Concat) | +| `curl` | [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do) / [`Get`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Get) / [`Post`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Post) | +| `cut` | [`Column`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Column) | +| `dirname` | [`Dirname`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Dirname) | +| `echo` | [`Echo`](https://pkg.go.dev/github.com/bitfield/script#Echo) | +| `find` | [`FindFiles`](https://pkg.go.dev/github.com/bitfield/script#FindFiles) | +| `grep` | [`Match`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Match) / [`MatchRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.MatchRegexp) | +| `grep -v` | [`Reject`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Reject) / [`RejectRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.RejectRegexp) | +| `head` | [`First`](https://pkg.go.dev/github.com/bitfield/script#Pipe.First) | +| `jq` | [`JQ`](https://pkg.go.dev/github.com/bitfield/script#Pipe.JQ) | +| `ls` | [`ListFiles`](https://pkg.go.dev/github.com/bitfield/script#ListFiles) | +| `sed` | [`Replace`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Replace) / [`ReplaceRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ReplaceRegexp) | +| `sha256sum` | [`Hash`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Hash) / [`HashSums`](https://pkg.go.dev/github.com/bitfield/script#Pipe.HashSums) | +| `tail` | [`Last`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Last) | +| `tee` | [`Tee`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Tee) | +| `uniq -c` | [`Freq`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Freq) | +| `wc -l` | [`CountLines`](https://pkg.go.dev/github.com/bitfield/script#Pipe.CountLines) | +| `xargs` | [`ExecForEach`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ExecForEach) | + +# A `script` “interpreter” + +One of the nice things about shell scripts is that there's no build process: the script file itself is the “executable” (in fact, it's interpreted by the shell). Simon Willison (and GPT-4) contributed this elegant `script` interpreter, written in `bash`: + +* [`go-script`](https://til.simonwillison.net/bash/go-script) + +With `go-script`, you can run `script` one-liners directly: + +```sh +cat file.txt | ./goscript.sh -c 'script.Stdin().Column(1).Freq().First(10).Stdout()' +``` + +or create `.goscript` files that you can run using a “shebang” line: + +```sh +#!/tmp/goscript.sh +script.Stdin().Column(1).Freq().First(10).Stdout() +``` + # Documentation See [pkg.go.dev](https://pkg.go.dev/github.com/bitfield/script) for the full documentation, or read on for a summary. +[![The Power of Go: Tools cover image](img/tools.png)](https://bitfieldconsulting.com/books/tools) + +The `script` package originated as an exercise in my book [The Power of Go: Tools](https://bitfieldconsulting.com/books/tools): + +> *Not all software engineering is about writing applications. Developers also need tooling: programs and services to automate everyday tasks like configuring servers and containers, running builds and tests, deploying their applications, and so on. Why shouldn't we be able to use Go for that purpose, too?* +> +> *`script` is designed to make it easy to write Go programs that chain together operations into a pipeline, in the same way that shell scripts do, but with the robust type checking and error handling of a real programming language. You can use `script` to construct the sort of simple one‐off pipelines that would otherwise require the shell, or special‐purpose tools.* +> +> *So, when plain Go doesn’t provide a convenient way to solve a problem, you yourself can use it to implement a domain-specific “language” that does. In this case, we used Go to provide the language of Unix‐style pipelines. But we could have chosen any architecture we wanted to suit the problem. If Go doesn’t already provide the tool you need, use Go to build that tool, then use it.*\ +> —From the book + ## Sources These are functions that create a pipe with a given contents: @@ -317,6 +351,7 @@ Filters are methods on an existing pipe that also return a pipe, allowing you to | [`First`](https://pkg.go.dev/github.com/bitfield/script#Pipe.First) | first N lines of input | | [`Freq`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Freq) | frequency count of unique input lines, most frequent first | | [`Get`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Get) | response to HTTP GET on supplied URL | +| [`HashSums`](https://pkg.go.dev/github.com/bitfield/script#Pipe.HashSums) | hashes of each listed file | | [`Join`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Join) | replace all newlines with spaces | | [`JQ`](https://pkg.go.dev/github.com/bitfield/script#Pipe.JQ) | result of `jq` query | | [`Last`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Last) | last N lines of input| @@ -327,7 +362,6 @@ Filters are methods on an existing pipe that also return a pipe, allowing you to | [`RejectRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.RejectRegexp) | lines not matching given regexp | | [`Replace`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Replace) | matching text replaced with given string | | [`ReplaceRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ReplaceRegexp) | matching text replaced with given string | -| [`SHA256Sums`](https://pkg.go.dev/github.com/bitfield/script#Pipe.SHA256Sums) | SHA-256 hashes of each listed file | | [`Tee`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Tee) | input copied to supplied writers | Note that filters run concurrently, rather than producing nothing until each stage has fully read its input. This is convenient for executing long-running commands, for example. If you do need to wait for the pipeline to complete, call [`Wait`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Wait). @@ -340,9 +374,9 @@ Sinks are methods that return some data from a pipe, ending the pipeline and ext | ---- | ----------- | ------- | | [`AppendFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.AppendFile) | appended to file, creating if it doesn't exist | bytes written, error | | [`Bytes`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Bytes) | | data as `[]byte`, error +| [`Hash`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Hash) | | hash, error | | [`CountLines`](https://pkg.go.dev/github.com/bitfield/script#Pipe.CountLines) | |number of lines, error | | [`Read`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Read) | given `[]byte` | bytes read, error | -| [`SHA256Sum`](https://pkg.go.dev/github.com/bitfield/script#Pipe.SHA256Sum) | | SHA-256 hash, error | | [`Slice`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Slice) | | data as `[]string`, error | | [`Stdout`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Stdout) | standard output | bytes written, error | | [`String`](https://pkg.go.dev/github.com/bitfield/script#Pipe.String) | | data as `string`, error | @@ -353,6 +387,9 @@ Sinks are methods that return some data from a pipe, ending the pipeline and ext | Version | New | | ----------- | ------- | +| 0.24.1 | [`JQ`](https://pkg.go.dev/github.com/bitfield/script#Pipe.JQ) accepts JSONLines data | +| 0.24.0 | [`Hash`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Hash) | +| | [`HashSums`](https://pkg.go.dev/github.com/bitfield/script#Pipe.HashSums) | | 0.23.0 | [`WithEnv`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WithEnv) | | | [`DecodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.DecodeBase64) / [`EncodeBase64`](https://pkg.go.dev/github.com/bitfield/script#Pipe.EncodeBase64) | | | [`Wait`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Wait) returns error | @@ -366,9 +403,9 @@ See the [contributor's guide](CONTRIBUTING.md) for some helpful tips if you'd li # Links +- [Learn Go with me](https://bitfieldconsulting.com/subscribe) - [Scripting with Go](https://bitfieldconsulting.com/posts/scripting) - [Code Club: Script](https://www.youtube.com/watch?v=6S5EqzVwpEg) -- [Bitfield Consulting](https://bitfieldconsulting.com/) -- [Go books by John Arundel](https://bitfieldconsulting.com/books) +- [Books by John Arundel](https://bitfieldconsulting.com/books) Gopher image by [MariaLetta](https://github.com/MariaLetta/free-gophers-pack) diff --git a/img/tools.png b/img/tools.png new file mode 100644 index 00000000..1de2535d Binary files /dev/null and b/img/tools.png differ diff --git a/script.go b/script.go index e25b1c48..d7d1bc34 100644 --- a/script.go +++ b/script.go @@ -8,7 +8,9 @@ import ( "encoding/hex" "encoding/json" "fmt" + "hash" "io" + "io/fs" "math" "net/http" "os" @@ -75,8 +77,9 @@ func File(path string) *Pipe { } // FindFiles creates a pipe listing all the files in the directory dir and its -// subdirectories recursively, one per line, like Unix find(1). If dir doesn't -// exist or can't be read, the pipe's error status will be set. +// subdirectories recursively, one per line, like Unix find(1). +// Errors are ignored unless no files are found (in which case the pipe's error +// status will be set to the last error encountered). // // Each line of the output consists of a slash-separated path, starting with // the initial directory. For example, if the directory looks like this: @@ -91,17 +94,19 @@ func File(path string) *Pipe { // test/2.txt func FindFiles(dir string) *Pipe { var paths []string - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + var innerErr error + fs.WalkDir(os.DirFS(dir), ".", func(path string, d fs.DirEntry, err error) error { if err != nil { - return err + innerErr = err + return fs.SkipDir } - if !info.IsDir() { - paths = append(paths, path) + if !d.IsDir() { + paths = append(paths, filepath.Join(dir, path)) } return nil }) - if err != nil { - return NewPipe().WithError(err) + if innerErr != nil && len(paths) == 0 { + return NewPipe().WithError(innerErr) } return Slice(paths) } @@ -179,8 +184,12 @@ func Post(url string) *Pipe { return NewPipe().Post(url) } -// Slice creates a pipe containing each element of s, one per line. +// Slice creates a pipe containing each element of s, one per line. If s is +// empty or nil, then the pipe is empty. func Slice(s []string) *Pipe { + if len(s) == 0 { + return NewPipe() + } return Echo(strings.Join(s, "\n") + "\n") } @@ -650,6 +659,40 @@ func (p *Pipe) Get(url string) *Pipe { return p.Do(req) } +// Hash returns the hex-encoded hash of the entire contents of the +// pipe based on the provided hasher, or an error. +// To perform hashing on files, see [Pipe.HashSums]. +func (p *Pipe) Hash(hasher hash.Hash) (string, error) { + if p.Error() != nil { + return "", p.Error() + } + _, err := io.Copy(hasher, p) + if err != nil { + p.SetError(err) + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +// HashSums reads paths from the pipe, one per line, and produces the +// hex-encoded hash of each corresponding file based on the provided hasher, +// one per line. Any files that cannot be opened or read will be ignored. +// To perform hashing on the contents of the pipe, see [Pipe.Hash]. +func (p *Pipe) HashSums(hasher hash.Hash) *Pipe { + return p.FilterScan(func(line string, w io.Writer) { + f, err := os.Open(line) + if err != nil { + return // skip unopenable files + } + defer f.Close() + _, err = io.Copy(hasher, f) + if err != nil { + return // skip unreadable files + } + fmt.Fprintln(w, hex.EncodeToString(hasher.Sum(nil))) + }) +} + // Join joins all the lines in the pipe's contents into a single // space-separated string, which will always end with a newline. func (p *Pipe) Join() *Pipe { @@ -669,38 +712,50 @@ func (p *Pipe) Join() *Pipe { }) } -// JQ executes query on the pipe's contents (presumed to be JSON), producing -// the result. An invalid query will set the appropriate error on the pipe. +// JQ executes query on the pipe's contents (presumed to be valid JSON or +// [JSONLines] data), applying the query to each newline-delimited input value +// and producing results until the first error is encountered. An invalid query +// or value will set the appropriate error on the pipe. // // The exact dialect of JQ supported is that provided by // [github.com/itchyny/gojq], whose documentation explains the differences // between it and standard JQ. +// +// [JSONLines]: https://jsonlines.org/ func (p *Pipe) JQ(query string) *Pipe { + parsedQuery, err := gojq.Parse(query) + if err != nil { + return p.WithError(err) + } + code, err := gojq.Compile(parsedQuery) + if err != nil { + return p.WithError(err) + } return p.Filter(func(r io.Reader, w io.Writer) error { - q, err := gojq.Parse(query) - if err != nil { - return err - } - var input interface{} - err = json.NewDecoder(r).Decode(&input) - if err != nil { - return err - } - iter := q.Run(input) - for { - v, ok := iter.Next() - if !ok { - return nil - } - if err, ok := v.(error); ok { - return err - } - result, err := gojq.Marshal(v) + dec := json.NewDecoder(r) + for dec.More() { + var input any + err := dec.Decode(&input) if err != nil { return err } - fmt.Fprintln(w, string(result)) + iter := code.Run(input) + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return err + } + result, err := gojq.Marshal(v) + if err != nil { + return err + } + fmt.Fprintln(w, string(result)) + } } + return nil }) } @@ -816,36 +871,19 @@ func (p *Pipe) SetError(err error) { // SHA256Sum returns the hex-encoded SHA-256 hash of the entire contents of the // pipe, or an error. +// Deprecated: SHA256Sum has been deprecated by [Pipe.Hash]. To get the SHA-256 +// hash for the contents of the pipe, call `Hash(sha256.new())` func (p *Pipe) SHA256Sum() (string, error) { - if p.Error() != nil { - return "", p.Error() - } - hasher := sha256.New() - _, err := io.Copy(hasher, p) - if err != nil { - p.SetError(err) - return "", err - } - return hex.EncodeToString(hasher.Sum(nil)), p.Error() + return p.Hash(sha256.New()) } // SHA256Sums reads paths from the pipe, one per line, and produces the // hex-encoded SHA-256 hash of each corresponding file, one per line. Any files // that cannot be opened or read will be ignored. +// Deprecated: SHA256Sums has been deprecated by [Pipe.HashSums]. To get the SHA-256 +// hash for each file path in the pipe, call `HashSums(sha256.new())` func (p *Pipe) SHA256Sums() *Pipe { - return p.FilterScan(func(line string, w io.Writer) { - f, err := os.Open(line) - if err != nil { - return // skip unopenable files - } - defer f.Close() - h := sha256.New() - _, err = io.Copy(h, f) - if err != nil { - return // skip unreadable files - } - fmt.Fprintln(w, hex.EncodeToString(h.Sum(nil))) - }) + return p.HashSums(sha256.New()) } // Slice returns the pipe's contents as a slice of strings, one element per diff --git a/script_test.go b/script_test.go index e6742197..12f4d90a 100644 --- a/script_test.go +++ b/script_test.go @@ -3,8 +3,11 @@ package script_test import ( "bufio" "bytes" + "crypto/sha256" + "crypto/sha512" "errors" "fmt" + "hash" "io" "log" "net/http" @@ -740,7 +743,6 @@ func TestJQWithDotQueryPrettyPrintsInput(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -754,7 +756,6 @@ func TestJQWithFieldQueryProducesSelectedField(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -768,7 +769,6 @@ func TestJQWithArrayQueryProducesRequiredArray(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -782,7 +782,6 @@ func TestJQWithArrayInputAndElementQueryProducesSelectedElement(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) t.Error(cmp.Diff(want, got)) } } @@ -796,7 +795,32 @@ func TestJQHandlesGithubJSONWithRealWorldExampleQuery(t *testing.T) { t.Fatal(err) } if want != got { - t.Error(want, got) + t.Error(cmp.Diff(want, got)) + } +} + +func TestJQCorrectlyQueriesMultilineInputFields(t *testing.T) { + t.Parallel() + input := `{"a":1}` + "\n" + `{"a":2}` + want := "1\n2\n" + got, err := script.Echo(input).JQ(".a").String() + if err != nil { + t.Fatal(err) + } + if want != got { + t.Error(cmp.Diff(want, got)) + } +} + +func TestJQCorrectlyQueriesMultilineInputArrays(t *testing.T) { + t.Parallel() + input := `[1, 2, 3]` + "\n" + `[4, 5, 6]` + want := "1\n4\n" + got, err := script.Echo(input).JQ(".[0]").String() + if err != nil { + t.Fatal(err) + } + if want != got { t.Error(cmp.Diff(want, got)) } } @@ -810,6 +834,28 @@ func TestJQErrorsWithInvalidQuery(t *testing.T) { } } +func TestJQErrorsWithInvalidInput(t *testing.T) { + t.Parallel() + input := "invalid JSON value" + _, err := script.Echo(input).JQ(".").String() + if err == nil { + t.Error("want error from invalid JSON input, got nil") + } +} + +func TestJQProducesValidResultsUntilFirstError(t *testing.T) { + t.Parallel() + input := "[1]\ninvalid JSON value\n[2]" + want := "1\n" + got, err := script.Echo(input).JQ(".[0]").String() + if err == nil { + t.Error("want error from invalid JSON input, got nil") + } + if want != got { + t.Error(cmp.Diff(want, got)) + } +} + func TestLastDropsAllButLastNLinesOfInput(t *testing.T) { t.Parallel() input := "a\nb\nc\n" @@ -1127,7 +1173,7 @@ func TestSHA256Sums_OutputsCorrectHashForEachSpecifiedFile(t *testing.T) { want string }{ // To get the checksum run: openssl dgst -sha256 - {"testdata/sha256Sum.input.txt", "1870478d23b0b4db37735d917f4f0ff9393dd3e52d8b0efa852ab85536ddad8e\n"}, + {"testdata/hashSum.input.txt", "1870478d23b0b4db37735d917f4f0ff9393dd3e52d8b0efa852ab85536ddad8e\n"}, {"testdata/hello.txt", "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9\n"}, {"testdata/multiple_files", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"}, } @@ -1384,6 +1430,18 @@ func TestSliceProducesElementsOfSpecifiedSliceOnePerLine(t *testing.T) { } } +func TestSliceGivenEmptySliceProducesEmptyPipe(t *testing.T) { + t.Parallel() + want := "" + got, err := script.Slice([]string{}).String() + if err != nil { + t.Fatal(err) + } + if want != got { + t.Fatalf("want %q, got %q", want, got) + } +} + func TestStdoutReturnsErrorGivenReadErrorOnPipe(t *testing.T) { t.Parallel() brokenReader := iotest.ErrReader(errors.New("oh no")) @@ -2013,6 +2071,110 @@ func TestWithStdErr_IsConcurrencySafeAfterExec(t *testing.T) { } } +func TestHash_OutputsCorrectHash(t *testing.T) { + t.Parallel() + tcs := []struct { + name, input, want string + hasher hash.Hash + }{ + { + name: "for no data", + input: "", + hasher: sha256.New(), + want: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + { + name: "for short string with SHA 256 hasher", + input: "hello, world", + hasher: sha256.New(), + want: "09ca7e4eaa6e8ae9c7d261167129184883644d07dfba7cbfbc4c8a2e08360d5b", + }, + { + name: "for short string with SHA 512 hasher", + input: "hello, world", + hasher: sha512.New(), + want: "8710339dcb6814d0d9d2290ef422285c9322b7163951f9a0ca8f883d3305286f44139aa374848e4174f5aada663027e4548637b6d19894aec4fb6c46a139fbf9", + }, + { + name: "for string containing newline with SHA 256 hasher", + input: "The tao that can be told\nis not the eternal Tao", + hasher: sha256.New(), + want: "788542cb92d37f67e187992bdb402fdfb68228a1802947f74c6576e04790a688", + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + got, err := script.Echo(tc.input).Hash(tc.hasher) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("want %q, got %q", tc.want, got) + } + }) + } +} + +func TestHashSums_OutputsCorrectHashForEachSpecifiedFile(t *testing.T) { + t.Parallel() + tcs := []struct { + testFileName string + hasher hash.Hash + want string + }{ + // To get the checksum run: openssl dgst -sha256 + { + testFileName: "testdata/hashSum.input.txt", + hasher: sha256.New(), + want: "1870478d23b0b4db37735d917f4f0ff9393dd3e52d8b0efa852ab85536ddad8e\n", + }, + { + testFileName: "testdata/hashSum.input.txt", + hasher: sha512.New(), + want: "3543bd0d68129e860598ccabcee1beb6bb90d91105cea74a8e555588634ec6f6d6d02033139972da2dc4929b1fb61bd24c91c8e82054e9ae865cf7f70909be8c\n", + }, + { + testFileName: "testdata/hello.txt", + hasher: sha256.New(), + want: "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9\n", + }, + { + testFileName: "testdata/multiple_files", + hasher: sha256.New(), + want: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", + }, + } + for _, tc := range tcs { + got, err := script.ListFiles(tc.testFileName).HashSums(tc.hasher).String() + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("%q: want %q, got %q", tc.testFileName, tc.want, got) + } + } +} + +func TestHash_ReturnsErrorGivenReadErrorOnPipe(t *testing.T) { + t.Parallel() + brokenReader := iotest.ErrReader(errors.New("oh no")) + _, err := script.NewPipe().WithReader(brokenReader).Hash(sha256.New()) + if err == nil { + t.Fatal(nil) + } +} + +func TestHashSums_OutputsEmptyStringForFileThatCannotBeHashed(t *testing.T) { + got, err := script.Echo("file_does_not_exist.txt").HashSums(sha256.New()).String() + if err != nil { + t.Fatal(err) + } + want := "" + if got != want { + t.Errorf("want %q, got %q", want, got) + } +} + func ExampleArgs() { script.Args().Stdout() // prints command-line arguments @@ -2276,6 +2438,24 @@ func ExamplePipe_Get() { // You said: hello } +func ExamplePipe_Hash() { + sum, err := script.Echo("hello world").Hash(sha512.New()) + if err != nil { + panic(err) + } + fmt.Println(sum) + // Output: + // 309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f +} + +func ExamplePipe_HashSums() { + script.ListFiles("testdata/multiple_files").HashSums(sha256.New()).Stdout() + // Output: + // e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + // e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + // e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +} + func ExamplePipe_Join() { script.Echo("hello\nworld\n").Join().Stdout() // Output: diff --git a/script_unix_test.go b/script_unix_test.go index 40a98c89..71e2fae9 100644 --- a/script_unix_test.go +++ b/script_unix_test.go @@ -3,6 +3,8 @@ package script_test import ( + "os" + "path/filepath" "testing" "github.com/bitfield/script" @@ -106,6 +108,30 @@ func TestExecPipesDataToExternalCommandAndGetsExpectedOutput(t *testing.T) { } } +func TestFindFiles_DoesNotErrorWhenSubDirectoryIsNotReadable(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() + restrictedDirPath := filepath.Join(tmpDir, "a_restricted_dir") + if err := os.Mkdir(restrictedDirPath, 0o000); err != nil { + t.Fatal(err) + } + fileAPath := filepath.Join(tmpDir, "file_a.txt") + if err := os.WriteFile(fileAPath, []byte("hello world!"), os.ModePerm); err != nil { + t.Fatal(err) + } + got, err := script.FindFiles(tmpDir).String() + if err != nil { + t.Fatal(err) + } + want := fileAPath + "\n" + if err != nil { + t.Fatal(err) + } + if !cmp.Equal(want, got) { + t.Fatal(cmp.Diff(want, got)) + } +} + func ExampleExec_ok() { script.Exec("echo Hello, world!").Stdout() // Output: diff --git a/testdata/sha256Sum.input.txt b/testdata/hashSum.input.txt similarity index 100% rename from testdata/sha256Sum.input.txt rename to testdata/hashSum.input.txt