diff --git a/README.md b/README.md index 5322d38..fa8872c 100644 --- a/README.md +++ b/README.md @@ -25,28 +25,29 @@ If you're already familiar with shell scripting and the Unix toolset, here is a | Unix / shell | `script` equivalent | | ------------------ | ------------------- | -| (any program name) | [`Exec()`](https://pkg.go.dev/github.com/bitfield/script#Exec) | -| `[ -f FILE ]` | [`IfExists()`](https://pkg.go.dev/github.com/bitfield/script#IfExists) | -| `>` | [`WriteFile()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WriteFile) | -| `>>` | [`AppendFile()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.AppendFile) | -| `$*` | [`Args()`](https://pkg.go.dev/github.com/bitfield/script#Args) | -| `basename` | [`Basename()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Basename) | -| `cat` | [`File()`](https://pkg.go.dev/github.com/bitfield/script#File) / [`Concat()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Concat) | -| `cut` | [`Column()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Column) | -| `dirname` | [`Dirname()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Dirname) | -| `echo` | [`Echo()`](https://pkg.go.dev/github.com/bitfield/script#Echo) | -| `grep` | [`Match()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Match) / [`MatchRegexp()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.MatchRegexp) | -| `grep -v` | [`Reject()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Reject) / [`RejectRegexp()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.RejectRegexp) | -| `head` | [`First()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.First) | +| (any program name) | [`Exec`](https://pkg.go.dev/github.com/bitfield/script#Exec) | +| `[ -f FILE ]` | [`IfExists`](https://pkg.go.dev/github.com/bitfield/script#IfExists) | +| `>` | [`WriteFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.WriteFile) | +| `>>` | [`AppendFile`](https://pkg.go.dev/github.com/bitfield/script#Pipe.AppendFile) | +| `$*` | [`Args`](https://pkg.go.dev/github.com/bitfield/script#Args) | +| `basename` | [`Basename`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Basename) | +| `cat` | [`File`](https://pkg.go.dev/github.com/bitfield/script#File) / [`Concat`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Concat) | +| `curl` | [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do) / [`Get`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Get) / [`Post`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Post) | +| `cut` | [`Column`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Column) | +| `dirname` | [`Dirname`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Dirname) | +| `echo` | [`Echo`](https://pkg.go.dev/github.com/bitfield/script#Echo) | +| `grep` | [`Match`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Match) / [`MatchRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.MatchRegexp) | +| `grep -v` | [`Reject`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Reject) / [`RejectRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.RejectRegexp) | +| `head` | [`First`](https://pkg.go.dev/github.com/bitfield/script#Pipe.First) | | `find -type f` | [`FindFiles`](https://pkg.go.dev/github.com/bitfield/script#FindFiles) | | `jq` | [`JQ`](https://pkg.go.dev/github.com/bitfield/script#Pipe.JQ) | -| `ls` | [`ListFiles()`](https://pkg.go.dev/github.com/bitfield/script#ListFiles) | -| `sed` | [`Replace()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Replace) / [`ReplaceRegexp()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ReplaceRegexp) | -| `sha256sum` | [`SHA256Sum()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.SHA256Sum) / [`SHA256Sums()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.SHA256Sums) | -| `tail` | [`Last()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Last) | -| `uniq -c` | [`Freq()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Freq) | -| `wc -l` | [`CountLines()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.CountLines) | -| `xargs` | [`ExecForEach()`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ExecForEach) | +| `ls` | [`ListFiles`](https://pkg.go.dev/github.com/bitfield/script#ListFiles) | +| `sed` | [`Replace`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Replace) / [`ReplaceRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ReplaceRegexp) | +| `sha256sum` | [`SHA256Sum`](https://pkg.go.dev/github.com/bitfield/script#Pipe.SHA256Sum) / [`SHA256Sums`](https://pkg.go.dev/github.com/bitfield/script#Pipe.SHA256Sums) | +| `tail` | [`Last`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Last) | +| `uniq -c` | [`Freq`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Freq) | +| `wc -l` | [`CountLines`](https://pkg.go.dev/github.com/bitfield/script#Pipe.CountLines) | +| `xargs` | [`ExecForEach`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ExecForEach) | # Some examples @@ -62,7 +63,7 @@ That looks straightforward enough, but suppose you now want to count the lines i numLines, err := script.File("test.txt").CountLines() ``` -For something a bit more challenging, let's try counting the number of lines in the file that match the string "Error": +For something a bit more challenging, let's try counting the number of lines in the file that match the string `Error`: ```go numErrors, err := script.File("test.txt").Match("Error").CountLines() @@ -92,31 +93,91 @@ Maybe we're only interested in the first 10 matches. No problem: script.Args().Concat().Match("Error").First(10).Stdout() ``` -What's that? You want to append that output to a file instead of printing it to the terminal? _You've got some attitude, mister_. +What's that? You want to append that output to a file instead of printing it to the terminal? *You've got some attitude, mister*. But okay: ```go script.Args().Concat().Match("Error").First(10).AppendFile("/var/log/errors.txt") ``` -If the data is JSON, we can do better than simple string-matching. We can use [JQ](https://stedolan.github.io/jq/) queries: +We're not limited to getting data only from files or standard input. We can get it from HTTP requests too: ```go -script.File("commits.json").JQ(".[0] | {message: .commit.message, name: .commit.committer.name}").Stdout() +script.Get("https://wttr.in/London?format=3").Stdout() +// Output: +// London: 🌦 +13°C +``` + +That's great for simple GET requests, but suppose we want to *send* some data in the body of a POST request, for example. Here's how that works: + +```go +script.Echo(data).Post(URL).Stdout() +``` + +If we need to customise the HTTP behaviour in some way, such as using our own HTTP client, we can do that: + +```go +script.NewPipe().WithHTTPClient(&http.Client{ + Timeout: 10 * time.Second, +}).Get("https://example.com").Stdout() +``` + +Or maybe we need to set some custom header on the request. No problem. We can just create the request in the usual way, and set it up however we want. Then we pass it to `Do`, which will actually perform the request: + +```go +req, err := http.NewRequest(http.MethodGet, "http://example.com", nil) +req.Header.Add("Authorization", "Bearer "+token) +script.Do(req).Stdout() +``` + +The HTTP server could return some non-okay response, though; for example, “404 Not Found”. So what happens then? + +In general, when any pipe stage (such as `Do`) encounters an error, it produces no output to subsequent stages. And `script` treats HTTP response status codes outside the range 200-299 as errors. So the answer for the previous example is that we just won't *see* any output from this program if the server returns an error response. + +Instead, the pipe “remembers” any error that occurs, and we can retrieve it later by calling its `Error` method, or by using a *sink* method such as `String`, which returns an `error` value along with the result. + +`Stdout` also returns an error, plus the number of bytes successfully written (which we don't care about for this particular case). So we can check that error, which is always a good idea in Go: + +```go +_, err := script.Do(req).Stdout() +if err != nil { + log.Fatal(err) +} ``` -Suppose we want to execute some external program instead of doing the work ourselves. We can do that too: +If, as is common, the data we get from an HTTP request is in JSON format, we can use [JQ](https://stedolan.github.io/jq/) queries to interrogate it: + +```go +data, err := script.Do(req).JQ(".[0] | {message: .commit.message, name: .commit.committer.name}").String() +``` + +We can also run external programs and get their output: ```go script.Exec("ping 127.0.0.1").Stdout() ``` -But maybe we don't know the arguments yet; we might get them from the user, for example. We'd like to be able to run the external command repeatedly, each time passing it the next line of input. No worries: +Note that `Exec` runs the command concurrently: it doesn't wait for the command to complete before returning any output. That's good, because this `ping` command will run forever (or until we get bored). + +Instead, when we read from the pipe using `Stdout`, we see each line of output as it's produced: + +``` +PING 127.0.0.1 (127.0.0.1): 56 data bytes +64 bytes from 127.0.0.1: icmp_seq=0 ttl=64 time=0.056 ms +64 bytes from 127.0.0.1: icmp_seq=1 ttl=64 time=0.054 ms +... +``` + +In the `ping` example, we knew the exact arguments we wanted to send the command, and we just needed to run it once. But what if we don't know the arguments yet? We might get them from the user, for example. + +We might like to be able to run the external command repeatedly, each time passing it the next line of data from the pipe as an argument. No worries: ```go script.Args().ExecForEach("ping -c 1 {{.}}").Stdout() ``` -If there isn't a built-in operation that does what we want, we can just write our own: +That `{{.}}` is standard Go template syntax; it'll substitute each line of data from the pipe into the command line before it's executed. You can write as fancy a Go template expression as you want here (but this simple example probably covers most use cases). + +If there isn't a built-in operation that does what we want, we can just write our own, using `Filter`: ```go script.Echo("hello world").Filter(func (r io.Reader, w io.Writer) error { @@ -129,7 +190,11 @@ script.Echo("hello world").Filter(func (r io.Reader, w io.Writer) error { // filtered 11 bytes ``` -Notice that the "hello world" appeared before the "filtered n bytes". Filters run concurrently, so the pipeline can start producing output before the input has been fully read. +The `func` we supply to `Filter` takes just two parameters: a reader to read from, and a writer to write to. The reader reads the previous stages of the pipe, as you might expect, and anything written to the writer goes to the *next* stage of the pipe. + +If our `func` returns some error, then, just as with the `Do` example, the pipe's error status is set, and subsequent stages become a no-op. + +Filters run concurrently, so the pipeline can start producing output before the input has been fully read, as it did in the `ping` example. In fact, most built-in pipe methods, including `Exec`, are implemented *using* `Filter`. If we want to scan input line by line, we could do that with a `Filter` function that creates a `bufio.Scanner` on its input, but we don't need to: @@ -193,12 +258,15 @@ These are functions that create a pipe with a given contents: | Source | Contents | | -------- | ------------- | | [`Args`](https://pkg.go.dev/github.com/bitfield/script#Args) | command-line arguments +| [`Do`](https://pkg.go.dev/github.com/bitfield/script#Do) | HTTP response | [`Echo`](https://pkg.go.dev/github.com/bitfield/script#Echo) | a string | [`Exec`](https://pkg.go.dev/github.com/bitfield/script#Exec) | command output | [`File`](https://pkg.go.dev/github.com/bitfield/script#File) | file contents | [`FindFiles`](https://pkg.go.dev/github.com/bitfield/script#FindFiles) | recursive file listing +| [`Get`](https://pkg.go.dev/github.com/bitfield/script#Get) | HTTP response | [`IfExists`](https://pkg.go.dev/github.com/bitfield/script#IfExists) | do something only if some file exists | [`ListFiles`](https://pkg.go.dev/github.com/bitfield/script#ListFiles) | file listing (including wildcards) +| [`Post`](https://pkg.go.dev/github.com/bitfield/script#Post) | HTTP response | [`Slice`](https://pkg.go.dev/github.com/bitfield/script#Slice) | slice elements, one per line | [`Stdin`](https://pkg.go.dev/github.com/bitfield/script#Stdin) | standard input @@ -212,6 +280,7 @@ Filters are methods on an existing pipe that also return a pipe, allowing you to | [`Column`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Column) | Nth column of input | | [`Concat`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Concat) | contents of multiple files | | [`Dirname`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Dirname) | removes filename from each line, leaving only leading path components | +| [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do) | response to supplied HTTP request | | [`Echo`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Echo) | all input replaced by given string | | [`Exec`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Exec) | filtered through external command | | [`ExecForEach`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ExecForEach) | execute given command template for each line of input | @@ -220,18 +289,20 @@ Filters are methods on an existing pipe that also return a pipe, allowing you to | [`FilterScan`](https://pkg.go.dev/github.com/bitfield/script#Pipe.FilterScan) | user-supplied function filtering each line to a writer | | [`First`](https://pkg.go.dev/github.com/bitfield/script#Pipe.First) | first N lines of input | | [`Freq`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Freq) | frequency count of unique input lines, most frequent first | +| [`Get`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Get) | response to HTTP GET on supplied URL | | [`Join`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Join) | replace all newlines with spaces | | [`JQ`](https://pkg.go.dev/github.com/bitfield/script#Pipe.JQ) | result of `jq` query | | [`Last`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Last) | last N lines of input| | [`Match`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Match) | lines matching given string | | [`MatchRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.MatchRegexp) | lines matching given regexp | +| [`Post`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Post) | response to HTTP POST on supplied URL | | [`Reject`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Reject) | lines not matching given string | | [`RejectRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.RejectRegexp) | lines not matching given regexp | | [`Replace`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Replace) | matching text replaced with given string | | [`ReplaceRegexp`](https://pkg.go.dev/github.com/bitfield/script#Pipe.ReplaceRegexp) | matching text replaced with given string | | [`SHA256Sums`](https://pkg.go.dev/github.com/bitfield/script#Pipe.SHA256Sums) | SHA-256 hashes of each listed file | -Note that filters run concurrently, rather than producing nothing until each stage has fully read its input. This is convenient for executing long-running comands, for example. If you do need to wait for the pipeline to complete, call `Wait`. +Note that filters run concurrently, rather than producing nothing until each stage has fully read its input. This is convenient for executing long-running comands, for example. If you do need to wait for the pipeline to complete, call [`Wait`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Wait). ## Sinks @@ -254,6 +325,7 @@ Sinks are methods that return some data from a pipe, ending the pipeline and ext | Version | New | | ----------- | ------- | +| v0.21.0 | HTTP support: [`Do`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Do), [`Get`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Get), [`Post`](https://pkg.go.dev/github.com/bitfield/script#Pipe.Post) | | v0.20.0 | [`JQ`](https://pkg.go.dev/github.com/bitfield/script#Pipe.JQ) | # Contributing diff --git a/script.go b/script.go index d3a643d..8dc75bc 100644 --- a/script.go +++ b/script.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io" + "net/http" "os" "os/exec" "path/filepath" @@ -22,149 +23,20 @@ import ( "github.com/itchyny/gojq" ) -// ReadAutoCloser represents a pipe source that will be automatically closed -// once it has been fully read. -type ReadAutoCloser struct { - r io.ReadCloser -} - -// Read reads up to len(buf) bytes from the data source into buf. It returns the -// number of bytes read and any error encountered. At end of file, Read returns -// 0, io.EOF. In the EOF case, the data source will be closed. -func (a ReadAutoCloser) Read(buf []byte) (n int, err error) { - if a.r == nil { - return 0, io.EOF - } - n, err = a.r.Read(buf) - if err == io.EOF { - a.Close() - } - return n, err -} - -// Close closes the data source associated with a, and returns the result of -// that close operation. -func (a ReadAutoCloser) Close() error { - if a.r == nil { - return nil - } - return a.r.(io.Closer).Close() -} - -// NewReadAutoCloser returns an ReadAutoCloser wrapping the supplied Reader. If -// the Reader is not a Closer, it will be wrapped in a NopCloser to make it -// closable. -func NewReadAutoCloser(r io.Reader) ReadAutoCloser { - if _, ok := r.(io.Closer); !ok { - return ReadAutoCloser{io.NopCloser(r)} - } - rc, ok := r.(io.ReadCloser) - if !ok { - // This can never happen, but just in case it does... - panic("internal error: type assertion to io.ReadCloser failed") - } - return ReadAutoCloser{rc} -} - -// Pipe represents a pipe object with an associated ReadAutoCloser. +// Pipe represents a pipe object with an associated [ReadAutoCloser]. type Pipe struct { - Reader ReadAutoCloser - stdout io.Writer + // Reader is the underlying reader. + Reader ReadAutoCloser + stdout io.Writer + httpClient *http.Client // because pipe stages are concurrent, protect 'err' mu *sync.Mutex err error } -// NewPipe returns a pointer to a new empty pipe. -func NewPipe() *Pipe { - return &Pipe{ - Reader: ReadAutoCloser{}, - mu: &sync.Mutex{}, - err: nil, - stdout: os.Stdout, - } -} - -// Close closes the pipe's associated reader. This is a no-op if the reader is -// not also a Closer. -func (p *Pipe) Close() error { - return p.Reader.Close() -} - -// Error returns any error present on the pipe, or nil otherwise. -func (p *Pipe) Error() error { - if p.mu == nil { // uninitialised pipe - return nil - } - p.mu.Lock() - defer p.mu.Unlock() - return p.err -} - -var exitStatusPattern = regexp.MustCompile(`exit status (\d+)$`) - -// ExitStatus returns the integer exit status of a previous command, if the -// pipe's error status is set, and if the error matches the pattern "exit status -// %d". Otherwise, it returns zero. -func (p *Pipe) ExitStatus() int { - if p.Error() == nil { - return 0 - } - match := exitStatusPattern.FindStringSubmatch(p.Error().Error()) - if len(match) < 2 { - return 0 - } - status, err := strconv.Atoi(match[1]) - if err != nil { - // This seems unlikely, but... - return 0 - } - return status -} - -// Read reads up to len(b) bytes from the data source into b. It returns the -// number of bytes read and any error encountered. At end of file, or on a nil -// pipe, Read returns 0, io.EOF. -// -// Unlike most sinks, Read does not necessarily read the whole contents of the -// pipe. It will read as many bytes as it takes to fill the slice. -func (p *Pipe) Read(b []byte) (int, error) { - return p.Reader.Read(b) -} - -// SetError sets the specified error on the pipe. -func (p *Pipe) SetError(err error) { - if p.mu == nil { // uninitialised pipe - return - } - p.mu.Lock() - defer p.mu.Unlock() - p.err = err -} - -// WithReader sets the pipe's input to the specified reader. If necessary, the -// reader will be automatically closed once it has been completely read. -func (p *Pipe) WithReader(r io.Reader) *Pipe { - p.Reader = NewReadAutoCloser(r) - return p -} - -// WithStdout sets the pipe's standard output to the specified reader, instead -// of the default os.Stdout. -func (p *Pipe) WithStdout(w io.Writer) *Pipe { - p.stdout = w - return p -} - -// WithError sets the specified error on the pipe and returns the modified pipe. -func (p *Pipe) WithError(err error) *Pipe { - p.SetError(err) - return p -} - -// Args creates a pipe containing the program's command-line arguments, one per -// line. +// Args creates a pipe containing the program's command-line arguments from +// [os.Args], excluding the program name, one per line. func Args() *Pipe { var s strings.Builder for _, a := range os.Args[1:] { @@ -173,52 +45,52 @@ func Args() *Pipe { return Echo(s.String()) } -// Echo creates a pipe containing the supplied string. +// Do creates a pipe that makes the HTTP request req and produces the response. +// See [Pipe.Do] for how the HTTP response status is interpreted. +func Do(req *http.Request) *Pipe { + return NewPipe().Do(req) +} + +// Echo creates a pipe containing the string s. func Echo(s string) *Pipe { return NewPipe().WithReader(strings.NewReader(s)) } -// Exec runs an external command and creates a pipe containing its combined -// output (stdout and stderr). -// -// If the command had a non-zero exit status, the pipe's error status will also -// be set to the string "exit status X", where X is the integer exit status. -// -// For convenience, you can get this value directly as an integer by calling -// ExitStatus on the pipe. -// -// Even in the event of a non-zero exit status, the command's output will still -// be available in the pipe. This is often helpful for debugging. However, -// because String is a no-op if the pipe's error status is set, if you want -// output you will need to reset the error status before calling String. +// Exec creates a pipe that runs cmdLine as an external command and produces +// its combined output (interleaving standard output and standard error). See +// [Pipe.Exec] for error handling details. // -// Note that Exec can also be used as a filter, in which case the given command -// will read from the pipe as its standard input. -func Exec(s string) *Pipe { - return NewPipe().Exec(s) +// Use [Pipe.Exec] to send the contents of an existing pipe to the command's +// standard input. +func Exec(cmdLine string) *Pipe { + return NewPipe().Exec(cmdLine) } -// File creates a pipe that reads from the file at the specified path. -func File(name string) *Pipe { +// File creates a pipe that reads from the file path. +func File(path string) *Pipe { p := NewPipe() - f, err := os.Open(name) + f, err := os.Open(path) if err != nil { return p.WithError(err) } return p.WithReader(f) } -// FindFiles takes a directory path and creates a pipe listing all the files in -// the directory and its subdirectories recursively, one per line, like Unix -// `find -type f`. If the path doesn't exist or can't be read, the pipe's error -// status will be set. +// FindFiles creates a pipe listing all the files in the directory path and its +// subdirectories recursively, one per line, like Unix find(1). If path doesn't +// exist or can't be read, the pipe's error status will be set. // -// Each line of the output consists of a slash-separated pathname, starting with -// the initial directory. For example, if the starting directory is "test", and -// it contains 1.txt and 2.txt: +// Each line of the output consists of a slash-separated path, starting with +// the initial directory. For example, if the directory looks like this: // -// test/1.txt -// test/2.txt +// test/ +// 1.txt +// 2.txt +// +// the pipe's output will be: +// +// test/1.txt +// test/2.txt func FindFiles(path string) *Pipe { var fileNames []string walkFn := func(path string, info os.FileInfo, err error) error { @@ -236,28 +108,34 @@ func FindFiles(path string) *Pipe { return Slice(fileNames) } -// IfExists tests whether the specified file exists, and creates a pipe whose -// error status reflects the result. If the file doesn't exist, the pipe's error -// status will be set, and if the file does exist, the pipe will have no error -// status. This can be used to do some operation only if a given file exists: +// Get creates a pipe that makes an HTTP GET request to URL, and produces the +// response. See [Pipe.Do] for how the HTTP response status is interpreted. +func Get(URL string) *Pipe { + return NewPipe().Get(URL) +} + +// IfExists tests whether path exists, and creates a pipe whose error status +// reflects the result. If the file doesn't exist, the pipe's error status will +// be set, and if the file does exist, the pipe will have no error status. This +// can be used to do some operation only if a given file exists: // -// IfExists("/foo/bar").Exec("/usr/bin/something") -func IfExists(filename string) *Pipe { +// IfExists("/foo/bar").Exec("/usr/bin/something") +func IfExists(path string) *Pipe { p := NewPipe() - _, err := os.Stat(filename) + _, err := os.Stat(path) if err != nil { return p.WithError(err) } return p } -// ListFiles creates a pipe containing the files and directories matching the -// supplied path, one per line. The path can be the name of a directory -// (`/path/to/dir`), the name of a file (`/path/to/file`), or a glob (wildcard -// expression) conforming to the syntax accepted by filepath.Match (for example -// `/path/to/*`). +// ListFiles creates a pipe containing the files or directories specified by +// path, one per line. path can be a glob expression, as for [filepath.Match]. +// For example: +// +// ListFiles("/data/*").Stdout() // -// ListFiles does not recurse into subdirectories (use FindFiles for this). +// ListFiles does not recurse into subdirectories; use [FindFiles] instead. func ListFiles(path string) *Pipe { if strings.ContainsAny(path, "[]^*?\\{}!") { fileNames, err := filepath.Glob(path) @@ -285,33 +163,73 @@ func ListFiles(path string) *Pipe { return Slice(fileNames) } -// Slice creates a pipe containing each element of the supplied slice of -// strings, one per line. +// NewPipe creates a new pipe with an empty reader (use [Pipe.WithReader] to +// attach another reader to it). +func NewPipe() *Pipe { + return &Pipe{ + Reader: ReadAutoCloser{}, + mu: &sync.Mutex{}, + err: nil, + stdout: os.Stdout, + httpClient: http.DefaultClient, + } +} + +// Post creates a pipe that makes an HTTP POST request to URL, with an empty +// body, and produces the response. See [Pipe.Do] for how the HTTP response +// status is interpreted. +func Post(URL string) *Pipe { + return NewPipe().Post(URL) +} + +// Slice creates a pipe containing each element of s, one per line. func Slice(s []string) *Pipe { return Echo(strings.Join(s, "\n") + "\n") } -// Stdin creates a pipe that reads from os.Stdin. +// Stdin creates a pipe that reads from [os.Stdin]. func Stdin() *Pipe { return NewPipe().WithReader(os.Stdin) } -// Basename reads a list of filepaths from the pipe, one per line, and removes -// any leading directory components from each line. So, for example, -// `/usr/local/bin/foo` would become just `foo`. This is the complementary -// operation to Dirname. +// AppendFile appends the contents of the pipe to the file path, creating it if +// necessary, and returns the number of bytes successfully written, or an +// error. +func (p *Pipe) AppendFile(path string) (int64, error) { + return p.writeOrAppendFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY) +} + +var exitStatusPattern = regexp.MustCompile(`exit status (\d+)$`) + +// Basename reads paths from the pipe, one per line, and removes any leading +// directory components from each. So, for example, /usr/local/bin/foo would +// become just foo. This is the complementary operation to [Pipe.Dirname]. // -// If a line is empty, Basename will produce '.'. Trailing slashes are removed. -// The behaviour of Basename is the same as filepath.Base (not by coincidence). +// If any line is empty, Basename will transform it to a single dot. Trailing +// slashes are removed. The behaviour of Basename is the same as +// [filepath.Base] (not by coincidence). func (p *Pipe) Basename() *Pipe { return p.FilterLine(filepath.Base) } -// Column produces only the Nth column of each line of input, where '1' is the -// first column, and columns are delimited by whitespace. Specifically, whatever -// Unicode defines as whitespace ('WSpace=yes'). -// -// Lines containing less than N columns will be dropped altogether. +// Bytes returns the contents of the pipe as a []]byte, or an error. +func (p *Pipe) Bytes() ([]byte, error) { + res, err := io.ReadAll(p) + if err != nil { + p.SetError(err) + } + return res, err +} + +// Close closes the pipe's associated reader. This is a no-op if the reader is +// not an [io.Closer]. +func (p *Pipe) Close() error { + return p.Reader.Close() +} + +// Column produces column col of each line of input, where the first column is +// column 1, and columns are delimited by Unicode whitespace. Lines containing +// fewer than col columns will be skipped. func (p *Pipe) Column(col int) *Pipe { return p.FilterScan(func(line string, w io.Writer) { columns := strings.Fields(line) @@ -321,28 +239,28 @@ func (p *Pipe) Column(col int) *Pipe { }) } -// Concat reads a list of file paths from the pipe, one per line, and produces -// the contents of all those files in sequence. If there are any errors (for -// example, non-existent files), these will be ignored, execution will continue, -// and the pipe's error status will not be set. +// Concat reads paths from the pipe, one per line, and produces the contents of +// all the corresponding files in sequence. If there are any errors (for +// example, non-existent files), these will be ignored, execution will +// continue, and the pipe's error status will not be set. // -// This makes it convenient to write programs that take a list of input files on -// the command line. For example: +// This makes it convenient to write programs that take a list of paths on the +// command line. For example: // -// script.Args().Concat().Stdout() +// script.Args().Concat().Stdout() // -// The list of files could also come from a file: +// The list of paths could also come from a file: // -// script.File("filelist.txt").Concat() +// script.File("filelist.txt").Concat() // -// ...or from the output of a command: +// Or from the output of a command: // -// script.Exec("ls /var/app/config/").Concat().Stdout() +// script.Exec("ls /var/app/config/").Concat().Stdout() // // Each input file will be closed once it has been fully read. If any of the -// files can't be opened or read, `Concat` will simply skip these and carry on, +// files can't be opened or read, Concat will simply skip these and carry on, // without setting the pipe's error status. This mimics the behaviour of Unix -// `cat`. +// cat(1). func (p *Pipe) Concat() *Pipe { var readers []io.Reader p.FilterScan(func(line string, w io.Writer) { @@ -355,14 +273,23 @@ func (p *Pipe) Concat() *Pipe { return p.WithReader(io.MultiReader(readers...)) } -// Dirname reads a list of pathnames from the pipe, one per line, and produces -// only the parent directories of each pathname. For example, -// `/usr/local/bin/foo` would become just `/usr/local/bin`. This is the -// complementary operation to Basename. +// CountLines returns the number of lines of input, or an error. +func (p *Pipe) CountLines() (int, error) { + lines := 0 + p.FilterScan(func(line string, w io.Writer) { + lines++ + }).Wait() + return lines, p.Error() +} + +// Dirname reads paths from the pipe, one per line, and produces only the +// parent directories of each path. For example, /usr/local/bin/foo would +// become just /usr/local/bin. This is the complementary operation to +// [Pipe.Basename]. // -// If a line is empty, Dirname will produce a '.'. Trailing slashes are removed, -// unless Dirname returns the root folder. Otherwise, the behaviour of Dirname -// is the same as filepath.Dir (not by coincidence). +// If a line is empty, Dirname will transform it to a single dot. Trailing +// slashes are removed, unless Dirname returns the root folder. Otherwise, the +// behaviour of Dirname is the same as [filepath.Dir] (not by coincidence). func (p *Pipe) Dirname() *Pipe { return p.FilterLine(func(line string) string { // filepath.Dir() does not handle trailing slashes correctly @@ -378,11 +305,34 @@ func (p *Pipe) Dirname() *Pipe { }) } -// EachLine calls the specified function for each line of input, passing it the -// line as a string, and a *strings.Builder to write its output to. +// Do performs the HTTP request req using the pipe's configured HTTP client, as +// set by [Pipe.WithHTTPClient], or [http.DefaultClient] otherwise. The +// response body is streamed concurrently to the pipe's output. If the response +// status is anything other than HTTP 200-299, the pipe's error status is set. +func (p *Pipe) Do(req *http.Request) *Pipe { + return p.Filter(func(r io.Reader, w io.Writer) error { + resp, err := p.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + // Any HTTP 2xx status code is considered okay + if resp.StatusCode/100 != 2 { + return fmt.Errorf("unexpected HTTP response status: %s", resp.Status) + } + _, err = io.Copy(w, resp.Body) + if err != nil { + return err + } + return nil + }) +} + +// EachLine calls the function process on each line of input, passing it the +// line as a string, and a [*strings.Builder] to write its output to. // -// Deprecated: use FilterLine or FilterScan instead, which run concurrently and -// don't do unnecessary reads on the input. +// Deprecated: use [Pipe.FilterLine] or [Pipe.FilterScan] instead, which run +// concurrently and don't do unnecessary reads on the input. func (p *Pipe) EachLine(process func(string, *strings.Builder)) *Pipe { return p.Filter(func(r io.Reader, w io.Writer) error { scanner := bufio.NewScanner(r) @@ -395,7 +345,8 @@ func (p *Pipe) EachLine(process func(string, *strings.Builder)) *Pipe { }) } -// Echo produces the supplied string. +// Echo sets the pipe's reader to one that produces the string s, detaching any +// existing reader without draining or closing it. func (p *Pipe) Echo(s string) *Pipe { if p.Error() != nil { return p @@ -403,18 +354,34 @@ func (p *Pipe) Echo(s string) *Pipe { return p.WithReader(NewReadAutoCloser(strings.NewReader(s))) } -// Exec runs an external command, sending it the contents of the pipe as input, -// and produces the command's combined output (`stdout` and `stderr`). The -// effect of this is to filter the contents of the pipe through the external -// command. +// Error returns any error present on the pipe, or nil otherwise. +func (p *Pipe) Error() error { + if p.mu == nil { // uninitialised pipe + return nil + } + p.mu.Lock() + defer p.mu.Unlock() + return p.err +} + +// Exec runs cmdLine as an external command, sending it the contents of the +// pipe as input, and produces the command's combined output. The effect of +// this is to filter the contents of the pipe through the external command. +// +// # Error handling // // If the command had a non-zero exit status, the pipe's error status will also -// be set to the string "exit status X", where X is the integer exit status. -func (p *Pipe) Exec(command string) *Pipe { +// be set to the string “exit status X”, where X is the integer exit status. +// Even in the event of a non-zero exit status, the command's output will still +// be available in the pipe. This is often helpful for debugging. However, +// because [Pipe.String] is a no-op if the pipe's error status is set, if you +// want output you will need to reset the error status before calling +// [Pipe.String]. +func (p *Pipe) Exec(cmdLine string) *Pipe { return p.Filter(func(r io.Reader, w io.Writer) error { - args, ok := shell.Split(command) // strings.Fields doesn't handle quotes + args, ok := shell.Split(cmdLine) // strings.Fields doesn't handle quotes if !ok { - return fmt.Errorf("unbalanced quotes or backslashes in [%s]", command) + return fmt.Errorf("unbalanced quotes or backslashes in [%s]", cmdLine) } cmd := exec.Command(args[0], args[1:]...) cmd.Stdin = r @@ -429,18 +396,19 @@ func (p *Pipe) Exec(command string) *Pipe { }) } -// ExecForEach runs the supplied command once for each line of input, and -// produces its combined output. The command string is interpreted as a Go -// template, so `{{.}}` will be replaced with the input value, for example. +// ExecForEach renders cmdLine as a Go template for each line of input, running +// the resulting command, and produces the combined output of all these +// commands in sequence. See [Pipe.Exec] for error handling details. +// +// This is mostly useful for substituting data into commands using Go template +// syntax. For example: // -// If any command resulted in a non-zero exit status, the pipe's error status -// will also be set to the string "exit status X", where X is the integer exit -// status. -func (p *Pipe) ExecForEach(command string) *Pipe { +// ListFiles("*").ExecForEach("touch {{.}}").Wait() +func (p *Pipe) ExecForEach(cmdLine string) *Pipe { if p.Error() != nil { return p } - tpl, err := template.New("").Parse(command) + tpl, err := template.New("").Parse(cmdLine) if err != nil { return p.WithError(err) } @@ -452,7 +420,8 @@ func (p *Pipe) ExecForEach(command string) *Pipe { if err != nil { return err } - args, ok := shell.Split(cmdLine.String()) // strings.Fields doesn't handle quotes + // strings.Fields doesn't handle quotes + args, ok := shell.Split(cmdLine.String()) if !ok { return fmt.Errorf("unbalanced quotes or backslashes in [%s]", cmdLine.String()) } @@ -470,13 +439,33 @@ func (p *Pipe) ExecForEach(command string) *Pipe { }) } -// Filter filters the contents of the pipe through the supplied function, which -// takes an io.Reader (the filter input) and an io.Writer (the filter output), -// and returns an error, which will be set on the pipe. +// ExitStatus returns the integer exit status of a previous command (for +// example run by [Pipe.Exec]). This will be zero unless the pipe's error +// status is set and the error matches the pattern “exit status %d”. +func (p *Pipe) ExitStatus() int { + if p.Error() == nil { + return 0 + } + match := exitStatusPattern.FindStringSubmatch(p.Error().Error()) + if len(match) < 2 { + return 0 + } + status, err := strconv.Atoi(match[1]) + if err != nil { + // This seems unlikely, but... + return 0 + } + return status +} + +// Filter sends the contents of the pipe to the function filter and produces +// the result. filter takes an [io.Reader] to read its input from and an +// [io.Writer] to write its output to, and returns an error, which will be set +// on the pipe. // -// The filter function runs concurrently, so its goroutine will not complete -// until the pipe has been fully read. If you just need to make sure all -// concurrent filters have completed, call Wait on the end of the pipe. +// filter runs concurrently, so its goroutine will not exit until the pipe has +// been fully read. Use [Pipe.Wait] to wait for all concurrent filters to +// complete. func (p *Pipe) Filter(filter func(io.Reader, io.Writer) error) *Pipe { pr, pw := io.Pipe() q := NewPipe().WithReader(pr) @@ -488,18 +477,19 @@ func (p *Pipe) Filter(filter func(io.Reader, io.Writer) error) *Pipe { return q } -// FilterLine filters the contents of the pipe, a line at a time, through the -// supplied function, which takes the line as a string and returns a string (the -// filter output). The filter function runs concurrently. +// FilterLine sends the contents of the pipe to the function filter, a line at +// a time, and produces the result. filter takes each line as a string and +// returns a string as its output. See [Pipe.Filter] for concurrency handling. func (p *Pipe) FilterLine(filter func(string) string) *Pipe { return p.FilterScan(func(line string, w io.Writer) { fmt.Fprintln(w, filter(line)) }) } -// FilterScan filters the contents of the pipe, a line at a time, through the -// supplied function, which takes the line as a string and an io.Writer (the -// filtero output). The filter function runs concurrently. +// FilterScan sends the contents of the pipe to the function filter, a line at +// a time, and produces the result. filter takes each line as a string and an +// [io.Writer] to write its output to. See [Pipe.Filter] for concurrency +// handling. func (p *Pipe) FilterScan(filter func(string, io.Writer)) *Pipe { return p.Filter(func(r io.Reader, w io.Writer) error { scanner := bufio.NewScanner(r) @@ -510,8 +500,9 @@ func (p *Pipe) FilterScan(filter func(string, io.Writer)) *Pipe { }) } -// First produces only the first N lines of input, or the whole input if there -// are less than N lines. If N is zero or negative, there is no output at all. +// First produces only the first n lines of the pipe's contents, or all the +// lines if there are less than n. If n is zero or negative, there is no output +// at all. func (p *Pipe) First(n int) *Pipe { if n <= 0 { return NewPipe() @@ -526,28 +517,24 @@ func (p *Pipe) First(n int) *Pipe { }) } -// Freq produces only unique lines from the input, prefixed with a frequency -// count, in descending numerical order (most frequent lines first). Lines with -// equal frequency will be sorted alphabetically. +// Freq produces only the unique lines from the pipe's contents, each prefixed +// with a frequency count, in descending numerical order (most frequent lines +// first). Lines with equal frequency will be sorted alphabetically. // -// This is a common pattern in shell scripts to find the most -// frequently-occurring lines in a file: +// For example, we could take a common shell pipeline like this: // -// sort testdata/freq.input.txt |uniq -c |sort -rn +// sort input.txt |uniq -c |sort -rn // -// Freq's behaviour is like the combination of Unix `sort`, `uniq -c`, and `sort -// -rn` used here. You can use Freq in combination with First to get, for -// example, the ten most common lines in a file: +// and replace it with: // -// script.Stdin().Freq().First(10).Stdout() +// File("input.txt").Freq().Stdout() // -// Like `uniq -c`, Freq left-pads its count values if necessary to make them -// easier to read: +// Or to get only the ten most common lines: // -// 10 apple -// 4 banana -// 2 orange -// 1 kumquat +// File("input.txt").Freq().First(10).Stdout() +// +// Like Unix uniq(1), Freq right-justifies its count values in a column for +// readability, padding with spaces if necessary. func (p *Pipe) Freq() *Pipe { freq := map[string]int{} type frequency struct { @@ -582,8 +569,19 @@ func (p *Pipe) Freq() *Pipe { }) } -// Join produces its input as a single space-separated string, which will always -// end with a newline. +// Get makes an HTTP GET request to URL, sending the contents of the pipe as +// the request body, and produces the server's response. See [Pipe.Do] for how +// the HTTP response status is interpreted. +func (p *Pipe) Get(URL string) *Pipe { + req, err := http.NewRequest(http.MethodGet, URL, p.Reader) + if err != nil { + return p.WithError(err) + } + return p.Do(req) +} + +// Join joins all the lines in the pipe's contents into a single +// space-separated string, which will always end with a newline. func (p *Pipe) Join() *Pipe { return p.Filter(func(r io.Reader, w io.Writer) error { scanner := bufio.NewScanner(r) @@ -602,13 +600,12 @@ func (p *Pipe) Join() *Pipe { }) } -// JQ takes a query in the 'jq' language and applies it to the input (presumed -// to be JSON), producing the result. An invalid query will set the appropriate -// error on the pipe. +// JQ executes query on the pipe's contents (presumed to be JSON), producing +// the result. An invalid query will set the appropriate error on the pipe. // // The exact dialect of JQ supported is that provided by -// github.com/itchyny/gojq, whose documentation explains the differences between -// it and 'standard' JQ. +// [github.com/itchyny/gojq], whose documentation explains the differences +// between it and standard JQ. func (p *Pipe) JQ(query string) *Pipe { return p.Filter(func(r io.Reader, w io.Writer) error { q, err := gojq.Parse(query) @@ -638,8 +635,9 @@ func (p *Pipe) JQ(query string) *Pipe { }) } -// Last produces only the last N lines of input, or the whole input if there are -// less than N lines. If N is zero or negative, there is no output at all. +// Last produces only the last n lines of the pipe's contents, or all the lines +// if there are less than n. If n is zero or negative, there is no output at +// all. func (p *Pipe) Last(n int) *Pipe { if n <= 0 { return NewPipe() @@ -660,7 +658,7 @@ func (p *Pipe) Last(n int) *Pipe { }) } -// Match produces only lines that contain the specified string. +// Match produces only the input lines that contain the string s. func (p *Pipe) Match(s string) *Pipe { return p.FilterScan(func(line string, w io.Writer) { if strings.Contains(line, s) { @@ -669,8 +667,7 @@ func (p *Pipe) Match(s string) *Pipe { }) } -// MatchRegexp produces only lines that match the specified compiled regular -// expression. +// MatchRegexp produces only the input lines that match the compiled regexp re. func (p *Pipe) MatchRegexp(re *regexp.Regexp) *Pipe { return p.FilterScan(func(line string, w io.Writer) { if re.MatchString(line) { @@ -679,7 +676,25 @@ func (p *Pipe) MatchRegexp(re *regexp.Regexp) *Pipe { }) } -// Reject produces only lines that do not contain the specified string. +// Post makes an HTTP POST request to URL, using the contents of the pipe as +// the request body, and produces the server's response. See [Pipe.Do] for how +// the HTTP response status is interpreted. +func (p *Pipe) Post(URL string) *Pipe { + req, err := http.NewRequest(http.MethodPost, URL, p.Reader) + if err != nil { + return p.WithError(err) + } + return p.Do(req) +} + +// Read reads up to len(b) bytes from the pipe into b. It returns the number of +// bytes read and any error encountered. At end of file, or on a nil pipe, Read +// returns 0, [io.EOF]. +func (p *Pipe) Read(b []byte) (int, error) { + return p.Reader.Read(b) +} + +// Reject produces only lines that do not contain the string s. func (p *Pipe) Reject(s string) *Pipe { return p.FilterScan(func(line string, w io.Writer) { if !strings.Contains(line, s) { @@ -688,8 +703,7 @@ func (p *Pipe) Reject(s string) *Pipe { }) } -// RejectRegexp produces only lines that don't match the specified compiled -// regular expression. +// RejectRegexp produces only lines that don't match the compiled regexp re. func (p *Pipe) RejectRegexp(re *regexp.Regexp) *Pipe { return p.FilterScan(func(line string, w io.Writer) { if !re.MatchString(line) { @@ -698,27 +712,48 @@ func (p *Pipe) RejectRegexp(re *regexp.Regexp) *Pipe { }) } -// Replace replaces all occurrences of the 'search' string with the 'replace' -// string. +// Replace replaces all occurrences of the string search with the string +// replace. func (p *Pipe) Replace(search, replace string) *Pipe { return p.FilterLine(func(line string) string { return strings.ReplaceAll(line, search, replace) }) } -// ReplaceRegexp replaces all matches of the specified compiled regular -// expression with the 'replace' string. '$' characters in the replace string -// are interpreted as in regexp.Expand; for example, "$1" represents the text of -// the first submatch. +// ReplaceRegexp replaces all matches of the compiled regexp re with the string +// re. $x variables in the replace string are interpreted as by +// [regexp.Expand]; for example, $1 represents the text of the first submatch. func (p *Pipe) ReplaceRegexp(re *regexp.Regexp, replace string) *Pipe { return p.FilterLine(func(line string) string { return re.ReplaceAllString(line, replace) }) } -// SHA256Sums reads a list of file paths from the pipe, one per line, and -// produces the hex-encoded SHA-256 hash of each file. Any files that cannot be -// opened or read will be ignored. +// SetError sets the error err on the pipe. +func (p *Pipe) SetError(err error) { + if p.mu == nil { // uninitialised pipe + return + } + p.mu.Lock() + defer p.mu.Unlock() + p.err = err +} + +// SHA256Sum returns the hex-encoded SHA-256 hash of the entire contents of the +// pipe, or an error. +func (p *Pipe) SHA256Sum() (string, error) { + hasher := sha256.New() + _, err := io.Copy(hasher, p) + if err != nil { + p.SetError(err) + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), p.Error() +} + +// SHA256Sums reads paths from the pipe, one per line, and produces the +// hex-encoded SHA-256 hash of each corresponding file, one per line. Any files +// that cannot be opened or read will be ignored. func (p *Pipe) SHA256Sums() *Pipe { return p.FilterScan(func(line string, w io.Writer) { f, err := os.Open(line) @@ -735,48 +770,12 @@ func (p *Pipe) SHA256Sums() *Pipe { }) } -// AppendFile appends the contents of the pipe to the specified file, and -// returns the number of bytes successfully written, or an error. If the file -// does not exist, it is created. -func (p *Pipe) AppendFile(fileName string) (int64, error) { - return p.writeOrAppendFile(fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY) -} - -// Bytes returns the contents of the pipe as a []]byte, or an error. -func (p *Pipe) Bytes() ([]byte, error) { - res, err := io.ReadAll(p) - if err != nil { - p.SetError(err) - } - return res, err -} - -// CountLines returns the number of lines of input, or an error. -func (p *Pipe) CountLines() (int, error) { - lines := 0 - p.FilterScan(func(line string, w io.Writer) { - lines++ - }).Wait() - return lines, p.Error() -} - -// SHA256Sum returns the hex-encoded SHA-256 hash of its input, or an error. -func (p *Pipe) SHA256Sum() (string, error) { - hasher := sha256.New() - _, err := io.Copy(hasher, p) - if err != nil { - p.SetError(err) - return "", err - } - return hex.EncodeToString(hasher.Sum(nil)), p.Error() -} - -// Slice returns the input as a slice of strings, one element per line, or an -// error. +// Slice returns the pipe's contents as a slice of strings, one element per +// line, or an error. // // An empty pipe will produce an empty slice. A pipe containing a single empty -// line (that is, a single `\n` character) will produce a slice containing the -// empty string. +// line (that is, a single \n character) will produce a slice containing the +// empty string as its single element. func (p *Pipe) Slice() ([]string, error) { result := []string{} p.FilterScan(func(line string, w io.Writer) { @@ -785,8 +784,9 @@ func (p *Pipe) Slice() ([]string, error) { return result, p.Error() } -// Stdout writes the input to the pipe's configured standard output, and returns -// the number of bytes successfully written, or an error. +// Stdout copies the pipe's contents to its configured standard output (using +// [Pipe.WithStdout]), or to [os.Stdout] otherwise, and returns the number of +// bytes successfully written, together with any error. func (p *Pipe) Stdout() (int, error) { n64, err := io.Copy(p.stdout, p) if err != nil { @@ -799,7 +799,7 @@ func (p *Pipe) Stdout() (int, error) { return n, p.Error() } -// String returns the input as a string, or an error. +// String returns the pipe's contents as a string, together with any error. func (p *Pipe) String() (string, error) { data, err := p.Bytes() if err != nil { @@ -808,8 +808,9 @@ func (p *Pipe) String() (string, error) { return string(data), p.Error() } -// Wait reads the input to completion and discards it. This is mostly useful for -// waiting until all concurrent filter stages have finished. +// Wait reads the pipe to completion and discards the result. This is mostly +// useful for waiting until concurrent filters have completed (see +// [Pipe.Filter]). func (p *Pipe) Wait() { _, err := io.Copy(io.Discard, p) if err != nil { @@ -817,15 +818,46 @@ func (p *Pipe) Wait() { } } -// WriteFile writes the input to the specified file, and returns the number of -// bytes successfully written, or an error. If the file already exists, it is -// truncated and the new data will replace the old. -func (p *Pipe) WriteFile(fileName string) (int64, error) { - return p.writeOrAppendFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC) +// WithError sets the error err on the pipe. +func (p *Pipe) WithError(err error) *Pipe { + p.SetError(err) + return p +} + +// WithHTTPClient sets the HTTP client c for use with subsequent requests via +// [Pipe.Do], [Pipe.Get], or [Pipe.Post]. For example, to make a request using +// a client with a timeout: +// +// NewPipe().WithHTTPClient(&http.Client{ +// Timeout: 10 * time.Second, +// }).Get("https://example.com").Stdout() +func (p *Pipe) WithHTTPClient(c *http.Client) *Pipe { + p.httpClient = c + return p +} + +// WithReader sets the pipe's input reader to r. Once r has been completely +// read, it will be closed if necessary. +func (p *Pipe) WithReader(r io.Reader) *Pipe { + p.Reader = NewReadAutoCloser(r) + return p +} + +// WithStdout sets the pipe's standard output to the writer w, instead of the +// default [os.Stdout]. +func (p *Pipe) WithStdout(w io.Writer) *Pipe { + p.stdout = w + return p +} + +// WriteFile writes the pipe's contents to the file path, truncating it if it +// exists, and returns the number of bytes successfully written, or an error. +func (p *Pipe) WriteFile(path string) (int64, error) { + return p.writeOrAppendFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC) } -func (p *Pipe) writeOrAppendFile(fileName string, mode int) (int64, error) { - out, err := os.OpenFile(fileName, mode, 0666) +func (p *Pipe) writeOrAppendFile(path string, mode int) (int64, error) { + out, err := os.OpenFile(path, mode, 0666) if err != nil { p.SetError(err) return 0, err @@ -838,3 +870,44 @@ func (p *Pipe) writeOrAppendFile(fileName string, mode int) (int64, error) { } return wrote, nil } + +// ReadAutoCloser wraps an [io.ReadCloser] so that it will be automatically +// closed once it has been fully read. +type ReadAutoCloser struct { + r io.ReadCloser +} + +// NewReadAutoCloser returns a [ReadAutoCloser] wrapping the reader r. +func NewReadAutoCloser(r io.Reader) ReadAutoCloser { + if _, ok := r.(io.Closer); !ok { + return ReadAutoCloser{io.NopCloser(r)} + } + rc, ok := r.(io.ReadCloser) + if !ok { + // This can never happen, but just in case it does... + panic("internal error: type assertion to io.ReadCloser failed") + } + return ReadAutoCloser{rc} +} + +// Close closes ra's reader, returning any resulting error. +func (ra ReadAutoCloser) Close() error { + if ra.r == nil { + return nil + } + return ra.r.(io.Closer).Close() +} + +// Read reads up to len(b) bytes from ra's reader into b. It returns the number +// of bytes read and any error encountered. At end of file, Read returns 0, +// [io.EOF]. If end-of-file is reached, the reader will be closed. +func (ra ReadAutoCloser) Read(b []byte) (n int, err error) { + if ra.r == nil { + return 0, io.EOF + } + n, err = ra.r.Read(b) + if err == io.EOF { + ra.Close() + } + return n, err +} diff --git a/script_test.go b/script_test.go index a7e9430..4584867 100644 --- a/script_test.go +++ b/script_test.go @@ -6,11 +6,13 @@ import ( "errors" "fmt" "io" + "log" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "regexp" - "runtime" "strings" "testing" @@ -245,6 +247,26 @@ func TestDirname_RemovesFilenameComponentFromInputLines(t *testing.T) { } } +func TestDoPerformsSuppliedHTTPRequest(t *testing.T) { + t.Parallel() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "some data") + })) + defer ts.Close() + req, err := http.NewRequest(http.MethodGet, ts.URL, nil) + if err != nil { + t.Fatal(err) + } + want := "some data\n" + got, err := script.Do(req).String() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !cmp.Equal(want, got) { + t.Error(cmp.Diff(want, got)) + } +} + func TestEachLine_FiltersInputThroughSuppliedFunction(t *testing.T) { t.Parallel() p := script.Echo("Hello\nGoodbye") @@ -370,7 +392,6 @@ func TestFilterReadsNoMoreThanRequested(t *testing.T) { fmt.Fprintln(w, text) return nil }) - runtime.Gosched() // give filter goroutine a chance to run want := "firstline\n" got, err := p.String() if err != nil { @@ -531,6 +552,73 @@ func TestFreqProducesCorrectFrequencyTableForInput(t *testing.T) { } } +func TestGetMakesHTTPGetRequestToGivenURL(t *testing.T) { + t.Parallel() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Errorf("want HTTP method GET, got %q", r.Method) + } + fmt.Fprintln(w, "some data") + })) + defer ts.Close() + want := "some data\n" + got, err := script.Get(ts.URL).String() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !cmp.Equal(want, got) { + t.Error(cmp.Diff(want, got)) + } +} + +func TestGetSetsErrorStatusWhenHTTPResponseStatusIsNotOK(t *testing.T) { + t.Parallel() + // With no handler, all requests will get 404 + ts := httptest.NewServer(nil) + defer ts.Close() + p := script.Get(ts.URL) + p.Wait() + if p.Error() == nil { + t.Fatalf("want error for non-OK request, got nil") + } +} + +func TestGetConsidersHTTPStatus201CreatedToBeOK(t *testing.T) { + t.Parallel() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusCreated) + fmt.Fprintln(w, "some data") + })) + defer ts.Close() + want := "some data\n" + got, err := script.Get(ts.URL).String() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !cmp.Equal(want, got) { + t.Error(cmp.Diff(want, got)) + } +} + +func TestGetUsesPipeContentsAsRequestBody(t *testing.T) { + t.Parallel() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + want := []byte("request data") + got, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal("reading request body", err) + } + if !cmp.Equal(want, got) { + t.Error(cmp.Diff(want, string(got))) + } + })) + defer ts.Close() + _, err := script.Echo("request data").Get(ts.URL).String() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + func TestJoinJoinsInputLinesIntoSpaceSeparatedString(t *testing.T) { t.Parallel() input := "hello\nfrom\nthe\njoin\ntest" @@ -861,6 +949,33 @@ func TestRejectDropsMatchingLinesFromInput(t *testing.T) { } } +func TestPostPostsToGivenURLUsingPipeAsRequestBody(t *testing.T) { + t.Parallel() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("want HTTP method POST, got %q", r.Method) + } + want := []byte("request data") + got, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal("reading request body", err) + } + if !cmp.Equal(want, got) { + t.Error(cmp.Diff(want, string(got))) + } + fmt.Fprintln(w, "response data") + })) + defer ts.Close() + want := "response data\n" + got, err := script.Echo("request data").Post(ts.URL).String() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !cmp.Equal(want, got) { + t.Error(cmp.Diff(want, got)) + } +} + func TestRejectRegexp_DropsMatchingLinesFromInput(t *testing.T) { t.Parallel() input := "hello world" @@ -1384,6 +1499,28 @@ func TestWriteFile_TruncatesExistingFile(t *testing.T) { } } +func TestWithHTTPClient_SetsSuppliedClientOnPipe(t *testing.T) { + t.Parallel() + ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "some data") + })) + defer ts.Close() + req, err := http.NewRequest(http.MethodGet, ts.URL, nil) + if err != nil { + t.Fatal(err) + } + want := "some data\n" + // Unless the pipe uses the supplied ts.Client, we'll get a + // 'certificate is not trusted' error on making the request + got, err := script.NewPipe().WithHTTPClient(ts.Client()).Do(req).String() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !cmp.Equal(want, got) { + t.Error(cmp.Diff(want, got)) + } +} + func TestWithReader_SetsSuppliedReaderOnPipe(t *testing.T) { t.Parallel() want := "Hello, world." @@ -1497,6 +1634,20 @@ func ExampleArgs() { // prints command-line arguments } +func ExampleDo() { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "some data") + })) + defer ts.Close() + req, err := http.NewRequest(http.MethodGet, ts.URL, nil) + if err != nil { + log.Fatal(err) + } + script.Do(req).Stdout() + // Output: + // some data +} + func ExampleEcho() { script.Echo("Hello, world!").Stdout() // Output: @@ -1525,6 +1676,16 @@ func ExampleFile() { // hello world } +func ExampleGet() { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "some data") + })) + defer ts.Close() + script.Get(ts.URL).Stdout() + // Output: + // some data +} + func ExampleIfExists_true() { script.IfExists("./testdata/hello.txt").Echo("found it").Stdout() // Output: @@ -1586,6 +1747,24 @@ func ExamplePipe_CountLines() { // 3 } +func ExamplePipe_Do() { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, err := io.ReadAll(r.Body) + if err != nil { + log.Fatal(err) + } + fmt.Fprintf(w, "You said: %s", data) + })) + defer ts.Close() + req, err := http.NewRequest(http.MethodGet, ts.URL, strings.NewReader("hello")) + if err != nil { + log.Fatal(err) + } + script.NewPipe().Do(req).Stdout() + // Output: + // You said: hello +} + func ExamplePipe_EachLine() { script.File("testdata/test.txt").EachLine(func(line string, out *strings.Builder) { out.WriteString("> " + line + "\n") @@ -1685,6 +1864,20 @@ func ExamplePipe_Freq() { // 1 kumquat } +func ExamplePipe_Get() { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, err := io.ReadAll(r.Body) + if err != nil { + log.Fatal(err) + } + fmt.Fprintf(w, "You said: %s", data) + })) + defer ts.Close() + script.Echo("hello").Get(ts.URL).Stdout() + // Output: + // You said: hello +} + func ExamplePipe_Join() { script.Echo("hello\nworld\n").Join().Stdout() // Output: @@ -1720,6 +1913,20 @@ func ExamplePipe_MatchRegexp() { // world } +func ExamplePipe_Post() { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, err := io.ReadAll(r.Body) + if err != nil { + log.Fatal(err) + } + fmt.Fprintf(w, "You said: %s", data) + })) + defer ts.Close() + script.Echo("hello").Post(ts.URL).Stdout() + // Output: + // You said: hello +} + func ExamplePipe_Read() { buf := make([]byte, 12) n, err := script.Echo("hello world\n").Read(buf)