Merge branch 'master' into log-watcher-coverage

.gitignore

@@ -14,3 +14,5 @@ emgen/emgen
 foo.log
 /watcher/watcher.test
 /vm/vm.test
+/coverage.html
+*.coverprofile

.travis.yml

@@ -0,0 +1,37 @@
+language: go
+
+# Must generate the parser before installing deps or go get will error out on
+# undefined lexer tokens.
+before_install:
+ - make vm/parser.go
+
+# Default dependency installation command, which is disabled when Makefile
+# detected.  Also install three tools for measuring coverage and sending to
+# coveralls.io.
+install:
+ - go get -t ./...
+ - go get golang.org/x/tools/cmd/cover
+ - go get github.com/modocache/gover
+ - go get github.com/mattn/goveralls
+
+# Run all tests, under race detector.
+script: make testrace
+
+# If the full suite passes, run again to collect coverage, merge all of the
+# packages reports, and send to coveralls.io.
+after_success:
+ - make coverage
+ - goveralls -coverprofile=gover.coverprofile -service=travis-ci
+
+# Run tests at these Go versions.
+go:
+  - tip
+  - 1.4
+
+# Allow tip to fail, and only wait for 1.4 to succeed before reporting build
+# status.
+matrix:
+  fast_finish: true
+  allow_failures:
+    - go: tip
+

Makefile

@@ -48,15 +48,15 @@ emgen/emgen: emgen/emgen.go
 
 .PHONY: test
 test: $(GOFILES) $(GOTESTFILES)
-	go test -v ./...
+	go test -v -timeout 60s ./...
 
 .PHONY: testrace
 testrace: $(GOFILES) $(GOTESTFILES)
-	go test -v -race ./...
+	go test -v -timeout 5m -race ./...
 
 .PHONY: smoke
 smoke: $(GOFILES) $(GOTESTFILES)
-	go test -v -test.short ./...
+	go test -v -timeout 10s -test.short ./...
 
 .PHONY: bench
 bench: $(GOFILES) $(GOTESTFILES)
@@ -66,5 +66,20 @@ bench: $(GOFILES) $(GOTESTFILES)
 recbench: $(GOFILES) $(GOTESTFILES)
 	go test -bench=. -run=XXX --record_benchmark ./...
 
+.PHONY: coverage
+coverage: gover.coverprofile
+gover.coverprofile: $(GOFILES) $(GOTESTFILES)
+	for package in exporter metrics tailer vm watcher; do\
+		go test -coverprofile=$$package.coverprofile ./$$package;\
+    done
+	go test -coverprofile=main.coverprofile .
+	gover
+
+.PHONY: covrep
+covrep: coverage.html
+coverage.html: gover.coverprofile
+	go tool cover -html=$< -o $@
+	xdg-open $@
+
 .PHONY: testall
 testall: testrace bench

README.md

@@ -2,6 +2,8 @@ mtail - extract whitebox monitoring data from application logs for collection in
 ========================================================================================================
 
 [![GoDoc](https://godoc.org/github.com/google/mtail?status.png)](http://godoc.org/github.com/google/mtail)
+[![Build Status](https://travis-ci.org/google/mtail.svg)](https://travis-ci.org/google/mtail)
+[![Coverage Status](https://coveralls.io/repos/google/mtail/badge.svg)](https://coveralls.io/r/google/mtail)
 
 mtail is a tool for extracting metrics from application logs to be exported into a timeseries database or timeseries calculator for alerting and dashboarding.
 

TODO

@@ -8,8 +8,6 @@ standard library, search path
 
 ntp bitfield parsing
 
-gocov
-
 refactor fs and notify into single interface
 
 close inofity then use it to force errors

ex_test.go

@@ -9,16 +9,16 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"path/filepath"
 	"runtime"
 	"sort"
 	"strconv"
-	"strings"
 	"testing"
 	"time"
 
-	"github.com/golang/glog"
 	"github.com/google/mtail/metrics"
 	"github.com/google/mtail/vm"
+	"github.com/google/mtail/watcher"
 	"github.com/kylelemons/godebug/pretty"
 )
 
@@ -48,22 +48,23 @@ var exampleProgramTests = []struct {
 	},
 }
 
-func CompileAndLoad(programfile string, ms *metrics.Store, lines chan string) error {
+func CompileAndLoad(programfile string, ms *metrics.Store, lines chan string) (*vm.Loader, error) {
 	p, err := os.Open(programfile)
 	if err != nil {
-		return fmt.Errorf("%s: could not open program file: %s", programfile, err)
+		return nil, fmt.Errorf("%s: could not open program file: %s", programfile, err)
 	}
 	defer p.Close()
 
-	// MtailDebug = 999 // All the debugging.
-
-	v, errs := vm.Compile(programfile, p, ms)
-	if errs != nil {
-		return fmt.Errorf("%s: compile failed: %s", programfile, strings.Join(errs, "\n"))
+	name := filepath.Base(programfile)
+	w := watcher.NewFakeWatcher()
+	l := vm.NewLoader(w, ms, lines)
+	if l == nil {
+		return nil, fmt.Errorf("couldn't create program loader")
 	}
-
-	go v.Run(lines)
-	return nil
+	if pErr := l.CompileAndRun(name, p); pErr != nil {
+		return nil, fmt.Errorf("couldn't compile program: %s: %s", programfile, pErr)
+	}
+	return l, nil
 }
 
 func TestExamplePrograms(t *testing.T) {
@@ -73,14 +74,13 @@ func TestExamplePrograms(t *testing.T) {
 	*vm.SyslogUseCurrentYear = false
 	for _, tc := range exampleProgramTests {
 		mtail := newMtail()
-		err := CompileAndLoad(tc.programfile, &mtail.store, mtail.lines)
+		l, err := CompileAndLoad(tc.programfile, &mtail.store, mtail.lines)
 		if err != nil {
-			t.Errorf("%s", err)
-			continue
+			t.Fatalf("Compile failed: %s", err)
 		}
+		mtail.l = l
 
-		err = mtail.OneShot(tc.logfile, mtail.lines)
-		if err != nil {
+		if err := mtail.OneShot(tc.logfile, mtail.lines); err != nil {
 			t.Errorf("Oneshot failed for %s: %s", tc.logfile, err)
 			continue
 		}
@@ -116,11 +116,12 @@ func TestExamplePrograms(t *testing.T) {
 			t.Errorf("%s: could not decode json: %s", tc.jsonfile, err)
 			continue
 		}
+		mtail.Close()
 		sort.Sort(metrics.Metrics(expectedMetrics))
-		glog.Infof("expected: %s", expectedMetrics)
+		mtail.store.Lock()
 		sort.Sort(metrics.Metrics(mtail.store.Metrics))
-		glog.Infof("received: %s", mtail.store.Metrics)
-		diff := pretty.Compare(expectedMetrics, mtail.store.Metrics)
+		diff := pretty.Compare(mtail.store.Metrics, expectedMetrics)
+		mtail.store.Unlock()
 		if len(diff) > 0 {
 			t.Errorf("%s: metrics don't match:\n%s\n", tc.programfile, diff)
 		}
@@ -136,11 +137,11 @@ func BenchmarkExamplePrograms(b *testing.B) {
 	for _, tc := range exampleProgramTests {
 		mtail := newMtail()
 		spareLines := make(chan string)
-		err := CompileAndLoad(tc.programfile, &metrics.Store{}, spareLines)
+		l, err := CompileAndLoad(tc.programfile, &metrics.Store{}, spareLines)
 		if err != nil {
-			b.Errorf("%s", err)
-			continue
+			b.Errorf("Compile failed: %s", err)
 		}
+		mtail.l = l
 		r := testing.Benchmark(func(b *testing.B) {
 			for i := 0; i < b.N; i++ {
 				b.StopTimer()

exporter/export.go

@@ -185,7 +185,8 @@ func metricToStatsd(m *metrics.Metric, l *metrics.LabelSet) string {
 
 // WriteMetrics writes metrics to each of the configured services.
 func (e *Exporter) WriteMetrics() {
-	if metrics.MetricUpdateTime.Sub(lastMetricPushTime) <= 0 {
+	lastUpdateTime := metrics.MetricUpdateTime.Load().(time.Time)
+	if lastUpdateTime.Sub(lastMetricPushTime) <= 0 {
 		return
 	}
 	if *collectdSocketPath != "" {
@@ -206,7 +207,7 @@ func (e *Exporter) WriteMetrics() {
 			glog.Infof("statsd error: %s", err)
 		}
 	}
-	lastMetricPushTime = time.Now()
+	lastMetricPushTime = time.Now().UTC()
 }
 
 // StartMetricPush pushes metrics to the configured services each interval.

exporter/export_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/google/mtail/metrics"
+	"github.com/kylelemons/godebug/pretty"
 )
 
 func FakeSocketWrite(f formatter, m *metrics.Metric) []string {
@@ -39,8 +40,9 @@ func TestMetricToCollectd(t *testing.T) {
 	d.Set(37, ts)
 	r := FakeSocketWrite(metricToCollectd, scalarMetric)
 	expected := []string{"PUTVAL \"" + hostname + "/mtail-prog/counter-foo\" interval=60 1343124840:37\n"}
-	if !reflect.DeepEqual(expected, r) {
-		t.Errorf("String didn't match:\n\texpected: %v\n\treceived: %v", expected, r)
+	diff := pretty.Compare(r, expected)
+	if len(diff) > 0 {
+		t.Errorf("String didn't match:\n%s", diff)
 	}
 
 	dimensionedMetric := metrics.NewMetric("bar", "prog", metrics.Gauge, "label")
@@ -52,8 +54,9 @@ func TestMetricToCollectd(t *testing.T) {
 	expected = []string{
 		"PUTVAL \"" + hostname + "/mtail-prog/gauge-bar-label-quux\" interval=60 1343124840:37\n",
 		"PUTVAL \"" + hostname + "/mtail-prog/gauge-bar-label-snuh\" interval=60 1343124840:37\n"}
-	if !reflect.DeepEqual(expected, r) {
-		t.Errorf("String didn't match:\n\texpected: %v\n\treceived: %v", expected, r)
+	diff = pretty.Compare(r, expected)
+	if len(diff) > 0 {
+		t.Errorf("String didn't match:\n%s", diff)
 	}
 }
 
@@ -68,8 +71,9 @@ func TestMetricToGraphite(t *testing.T) {
 	d.Set(37, ts)
 	r := FakeSocketWrite(metricToGraphite, scalarMetric)
 	expected := []string{"prog.foo 37 1343124840\n"}
-	if !reflect.DeepEqual(expected, r) {
-		t.Errorf("String didn't match:\n\texpected: %v\n\treceived: %v", expected, r)
+	diff := pretty.Compare(r, expected)
+	if len(diff) > 0 {
+		t.Errorf("String didn't match:\n%s", diff)
 	}
 
 	dimensionedMetric := metrics.NewMetric("bar", "prog", metrics.Gauge, "l")
@@ -81,8 +85,9 @@ func TestMetricToGraphite(t *testing.T) {
 	expected = []string{
 		"prog.bar.l.quux 37 1343124840\n",
 		"prog.bar.l.snuh 37 1343124840\n"}
-	if !reflect.DeepEqual(expected, r) {
-		t.Errorf("String didn't match:\n\texpected: %v\n\treceived: %v", expected, r)
+	diff = pretty.Compare(r, expected)
+	if len(diff) > 0 {
+		t.Errorf("String didn't match:\n%s", diff)
 	}
 }
 

metrics/metric.go

@@ -27,7 +27,7 @@ const (
 var (
 	// MetricUpdateTime contains the timestamp of the last update of a metric.
 	// TODO(jaq): move this global value to be a property of the Store.
-	MetricUpdateTime time.Time
+	MetricUpdateTime atomic.Value
 )
 
 func (m MetricType) String() string {
@@ -139,34 +139,41 @@ func (m *Metric) EmitLabelSets(c chan *LabelSet) {
 
 // Datum describes a LabelSet's or LabelValue's value at a given timestamp.
 type Datum struct {
+	sync.RWMutex
 	Value int64
 	Time  time.Time
 }
 
 func (d *Datum) stamp(timestamp time.Time) {
 	if timestamp.IsZero() {
-		d.Time = time.Now()
+		d.Time = time.Now().UTC()
 	} else {
 		d.Time = timestamp
 	}
-	MetricUpdateTime = time.Now()
+	MetricUpdateTime.Store(time.Now().UTC())
 }
 
 // Set implements the Settable interface for a Datum.
 func (d *Datum) Set(value int64, timestamp time.Time) {
-	atomic.StoreInt64(&d.Value, value)
+	d.Lock()
+	defer d.Unlock()
+	d.Value = value
 	d.stamp(timestamp)
 }
 
 // IncBy implements the Incrementable interface for a Datum.
 func (d *Datum) IncBy(delta int64, timestamp time.Time) {
-	atomic.AddInt64(&d.Value, delta)
+	d.Lock()
+	defer d.Unlock()
+	d.Value += delta
 	d.stamp(timestamp)
 }
 
 // Get returns the value of the Datum.
 func (d *Datum) Get() int64 {
-	return atomic.LoadInt64(&d.Value)
+	d.RLock()
+	defer d.RUnlock()
+	return d.Value
 }
 
 // Store contains Metrics.
@@ -190,6 +197,8 @@ func (ms *Store) ClearMetrics() {
 }
 
 func (d *Datum) String() string {
+	d.RLock()
+	defer d.RUnlock()
 	return fmt.Sprintf("%+#v", *d)
 }
 
@@ -198,6 +207,8 @@ func (lv *LabelValue) String() string {
 }
 
 func (m *Metric) String() string {
+	m.RLock()
+	defer m.RUnlock()
 	return fmt.Sprintf("%+#v", *m)
 }