From 6404b91b83a7043bb5ed6e81bc18b5822926a0c8 Mon Sep 17 00:00:00 2001 From: Katie Hockman Date: Wed, 10 Nov 2021 16:22:08 -0500 Subject: [PATCH] internal/fuzz: set timeout for each exec of fuzz target This change sets a timeout of 10 seconds on each execution of the fuzz target, both during fuzzing and during minimization. This is not currently customizable by the user, but issue #48157 tracks this work. Deadlocks will be considered non-recoverable errors, and as such, will not be minimizable. Fixes #48591 Change-Id: Ic86e8e9e9a0255e7860f7cbf5654e832785d1cbc Reviewed-on: https://go-review.googlesource.com/c/go/+/363134 Trust: Katie Hockman Run-TryBot: Katie Hockman TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills --- fuzz/minimize_test.go | 9 ++++++--- fuzz/worker.go | 36 +++++++++++++++++++++++------------- fuzz/worker_test.go | 3 ++- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/fuzz/minimize_test.go b/fuzz/minimize_test.go index dc153d0..04d785c 100644 --- a/fuzz/minimize_test.go +++ b/fuzz/minimize_test.go @@ -13,6 +13,7 @@ import ( "fmt" "reflect" "testing" + "time" "unicode" "unicode/utf8" ) @@ -279,7 +280,9 @@ func TestMinimizeInput(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() ws := &workerServer{ - fuzzFn: tc.fn, + fuzzFn: func(e CorpusEntry) (time.Duration, error) { + return time.Second, tc.fn(e) + }, } count := int64(0) vals := tc.input @@ -304,8 +307,8 @@ func TestMinimizeInput(t *testing.T) { // input and a flaky failure occurs, that minimization was not indicated // to be successful, and the error isn't returned (since it's flaky). func TestMinimizeFlaky(t *testing.T) { - ws := &workerServer{fuzzFn: func(e CorpusEntry) error { - return errors.New("ohno") + ws := &workerServer{fuzzFn: func(e CorpusEntry) (time.Duration, error) { + return time.Second, errors.New("ohno") }} keepCoverage := make([]byte, len(coverageSnapshot)) count := int64(0) diff --git a/fuzz/worker.go b/fuzz/worker.go index 02efa7f..48a3923 100644 --- a/fuzz/worker.go +++ b/fuzz/worker.go @@ -142,7 +142,7 @@ func (w *worker) coordinate(ctx context.Context) error { } // Worker exited non-zero or was terminated by a non-interrupt // signal (for example, SIGSEGV) while fuzzing. - return fmt.Errorf("fuzzing process terminated unexpectedly: %w", err) + return fmt.Errorf("fuzzing process hung or terminated unexpectedly: %w", err) // TODO(jayconrod,katiehockman): if -keepfuzzing, restart worker. case input := <-w.coordinator.inputC: @@ -183,7 +183,7 @@ func (w *worker) coordinate(ctx context.Context) error { // Unexpected termination. Set error message and fall through. // We'll restart the worker on the next iteration. // Don't attempt to minimize this since it crashed the worker. - resp.Err = fmt.Sprintf("fuzzing process terminated unexpectedly: %v", w.waitErr) + resp.Err = fmt.Sprintf("fuzzing process hung or terminated unexpectedly: %v", w.waitErr) canMinimize = false } result := fuzzResult{ @@ -255,7 +255,7 @@ func (w *worker) minimize(ctx context.Context, input fuzzMinimizeInput) (min fuz limit: input.limit, }, nil } - return fuzzResult{}, fmt.Errorf("fuzzing process terminated unexpectedly while minimizing: %w", w.waitErr) + return fuzzResult{}, fmt.Errorf("fuzzing process hung or terminated unexpectedly while minimizing: %w", w.waitErr) } if input.crasherMsg != "" && resp.Err == "" { @@ -471,8 +471,16 @@ func RunFuzzWorker(ctx context.Context, fn func(CorpusEntry) error) error { } srv := &workerServer{ workerComm: comm, - fuzzFn: fn, - m: newMutator(), + fuzzFn: func(e CorpusEntry) (time.Duration, error) { + timer := time.AfterFunc(10*time.Second, func() { + panic("deadlocked!") // this error message won't be printed + }) + defer timer.Stop() + start := time.Now() + err := fn(e) + return time.Since(start), err + }, + m: newMutator(), } return srv.serve(ctx) } @@ -604,9 +612,12 @@ type workerServer struct { // coverage is found. coverageMask []byte - // fuzzFn runs the worker's fuzz function on the given input and returns - // an error if it finds a crasher (the process may also exit or crash). - fuzzFn func(CorpusEntry) error + // fuzzFn runs the worker's fuzz target on the given input and returns an + // error if it finds a crasher (the process may also exit or crash), and the + // time it took to run the input. It sets a deadline of 10 seconds, at which + // point it will panic with the assumption that the process is hanging or + // deadlocked. + fuzzFn func(CorpusEntry) (time.Duration, error) } // serve reads serialized RPC messages on fuzzIn. When serve receives a message, @@ -699,9 +710,8 @@ func (ws *workerServer) fuzz(ctx context.Context, args fuzzArgs) (resp fuzzRespo } fuzzOnce := func(entry CorpusEntry) (dur time.Duration, cov []byte, errMsg string) { mem.header().count++ - start := time.Now() - err := ws.fuzzFn(entry) - dur = time.Since(start) + var err error + dur, err = ws.fuzzFn(entry) if err != nil { errMsg = err.Error() if errMsg == "" { @@ -803,7 +813,7 @@ func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, c // If not, then whatever caused us to think the value was interesting may // have been a flake, and we can't minimize it. *count++ - retErr = ws.fuzzFn(CorpusEntry{Values: vals}) + _, retErr = ws.fuzzFn(CorpusEntry{Values: vals}) if keepCoverage != nil { if !hasCoverageBit(keepCoverage, coverageSnapshot) || retErr != nil { return false, nil @@ -870,7 +880,7 @@ func (ws *workerServer) minimizeInput(ctx context.Context, vals []interface{}, c panic("impossible") } *count++ - err := ws.fuzzFn(CorpusEntry{Values: vals}) + _, err := ws.fuzzFn(CorpusEntry{Values: vals}) if err != nil { retErr = err if keepCoverage != nil { diff --git a/fuzz/worker_test.go b/fuzz/worker_test.go index c6f83fd..ed9722f 100644 --- a/fuzz/worker_test.go +++ b/fuzz/worker_test.go @@ -14,6 +14,7 @@ import ( "os/signal" "reflect" "testing" + "time" ) var benchmarkWorkerFlag = flag.Bool("benchmarkworker", false, "") @@ -36,7 +37,7 @@ func BenchmarkWorkerFuzzOverhead(b *testing.B) { os.Setenv("GODEBUG", fmt.Sprintf("%s,fuzzseed=123", origEnv)) ws := &workerServer{ - fuzzFn: func(_ CorpusEntry) error { return nil }, + fuzzFn: func(_ CorpusEntry) (time.Duration, error) { return time.Second, nil }, workerComm: workerComm{memMu: make(chan *sharedMem, 1)}, }