ex_test.go

// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"testing"

	"github.com/google/mtail/metrics"
	"github.com/google/mtail/mtail"
	"github.com/google/mtail/vm"
	"github.com/google/mtail/watcher"
	"github.com/kylelemons/godebug/pretty"
)

var (
	recordBenchmark = flag.Bool("record_benchmark", false, "Record the benchmark results to 'benchmark_results.csv'.")
)

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	jsonfile    string // Expected metrics after processing.
}{
	{
		"examples/rsyncd.mtail",
		"testdata/rsyncd.log",
		"testdata/rsyncd.json",
	},
	{
		"examples/sftp.mtail",
		"testdata/sftp_chroot.log",
		"testdata/sftp_chroot.json",
	},
	{
		"examples/dhcpd.mtail",
		"testdata/anonymised_dhcpd_log",
		"testdata/anonymised_dhcpd_log.json",
	},
}

func CompileAndLoad(programfile string, ms *metrics.Store, lines chan string) (*vm.Loader, error) {
	p, err := os.Open(programfile)
	if err != nil {
		return nil, fmt.Errorf("%s: could not open program file: %s", programfile, err)
	}
	defer p.Close()

	name := filepath.Base(programfile)
	w := watcher.NewFakeWatcher()
	o := vm.LoaderOptions{W: w, Store: ms, Lines: lines, SyslogUseCurrentYear: false}

	l, err := vm.NewLoader(o)
	if err != nil {
		return nil, fmt.Errorf("couldn't create program loader: %s", err)
	}
	if pErr := l.CompileAndRun(name, p); pErr != nil {
		return nil, fmt.Errorf("couldn't compile program: %s: %s", programfile, pErr)
	}
	return l, nil
}

func TestExamplePrograms(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	for _, tc := range exampleProgramTests {
		w := watcher.NewFakeWatcher()
		o := mtail.Options{Progs: tc.programfile, W: w}
		mtail, err := mtail.New(o)
		if err != nil {
			t.Fatalf("create mtail failed: %s", err)
		}

		if err := mtail.OneShot(tc.logfile); err != nil {
			t.Errorf("Oneshot failed for %s: %s", tc.logfile, err)
			continue
		}

		// Dirty hack to create json files :)
		if false {
			j, err := os.Create(tc.jsonfile)
			if err != nil {
				t.Errorf("%s: could not open json file: %s", tc.jsonfile, err)
				continue
			}
			defer j.Close()
			if err := mtail.WriteMetrics(j); err != nil {
				t.Errorf("couldn't marshall metrics: %q", err)
				continue
			}
		}

		j, err := os.Open(tc.jsonfile)
		if err != nil {
			t.Fatalf("%s: could not open json file: %s", tc.jsonfile, err)
		}
		defer j.Close()

		var m, ex bytes.Buffer
		mtail.Close()
		mtail.WriteMetrics(&m)
		m.WriteString("\n") // Golden data has trailing newline.

		if _, err := ex.ReadFrom(j); err != nil {
			t.Fatalf("Coldn't read from json: %s", err)
		}
		diff := pretty.Compare(m.String(), ex.String())
		if len(diff) > 0 {
			t.Errorf("%s: metrics don't match:\n%s", tc.programfile, diff)
		}
	}
}

func benchmarkProgram(b *testing.B, programfile string, logfile string) {
	w := watcher.NewFakeWatcher()
	o := mtail.Options{Progs: programfile, W: w}
	mtail, err := mtail.New(o)
	if err != nil {
		b.Fatalf("Failed to create mtail: %s", err)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err := mtail.OneShot(logfile); err != nil {
			b.Errorf("OneShot log parse failed: %s", err)
		}
	}
	b.StopTimer()
	mtail.Close()
	l, err := strconv.ParseInt(vm.LineCount.String(), 10, 64)
	if err != nil {
		b.Fatalf("strconv.ParseInt failed: %s", err)
		return
	}
	b.SetBytes(l)
}

func BenchmarkRsyncdProgram(b *testing.B) {
	benchmarkProgram(b, "examples/rsync.mtail", "testdata/rsyncd.log")
}
func BenchmarkSftpProgram(b *testing.B) {
	benchmarkProgram(b, "examples/sftp.mtail", "testdata/sftp_chroot.log")
}
func BenchmarkDhcpdProgram(b *testing.B) {
	benchmarkProgram(b, "examples/dhcpd.mtail", "testdata/anonymised_dhcpd_log")
}