// Copyright 2018 syzkaller project authors. All rights reserved.
|
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
|
|
package bisect
|
|
import (
|
"fmt"
|
"io"
|
"path/filepath"
|
"time"
|
|
"github.com/google/syzkaller/pkg/build"
|
"github.com/google/syzkaller/pkg/instance"
|
"github.com/google/syzkaller/pkg/mgrconfig"
|
"github.com/google/syzkaller/pkg/osutil"
|
"github.com/google/syzkaller/pkg/vcs"
|
)
|
|
type Config struct {
|
Trace io.Writer
|
Fix bool
|
BinDir string
|
DebugDir string
|
Kernel KernelConfig
|
Syzkaller SyzkallerConfig
|
Repro ReproConfig
|
Manager mgrconfig.Config
|
}
|
|
type KernelConfig struct {
|
Repo string
|
Branch string
|
Commit string
|
Cmdline string
|
Sysctl string
|
Config []byte
|
Userspace string
|
}
|
|
type SyzkallerConfig struct {
|
Repo string
|
Commit string
|
Descriptions string
|
}
|
|
type ReproConfig struct {
|
Opts []byte
|
Syz []byte
|
C []byte
|
}
|
|
type env struct {
|
cfg *Config
|
repo vcs.Repo
|
head *vcs.Commit
|
inst *instance.Env
|
numTests int
|
buildTime time.Duration
|
testTime time.Duration
|
}
|
|
type buildEnv struct {
|
compiler string
|
}
|
|
func Run(cfg *Config) (*vcs.Commit, error) {
|
repo, err := vcs.NewRepo(cfg.Manager.TargetOS, cfg.Manager.Type, cfg.Manager.KernelSrc)
|
if err != nil {
|
return nil, err
|
}
|
env := &env{
|
cfg: cfg,
|
repo: repo,
|
}
|
if cfg.Fix {
|
env.log("searching for fixing commit since %v", cfg.Kernel.Commit)
|
} else {
|
env.log("searching for guilty commit starting from %v", cfg.Kernel.Commit)
|
}
|
start := time.Now()
|
res, err := env.bisect()
|
env.log("revisions tested: %v, total time: %v (build: %v, test: %v)",
|
env.numTests, time.Since(start), env.buildTime, env.testTime)
|
if err != nil {
|
env.log("error: %v", err)
|
return nil, err
|
}
|
if res == nil {
|
env.log("the crash is still unfixed")
|
return nil, nil
|
}
|
what := "bad"
|
if cfg.Fix {
|
what = "good"
|
}
|
env.log("first %v commit: %v %v", what, res.Hash, res.Title)
|
env.log("cc: %q", res.CC)
|
return res, nil
|
}
|
|
func (env *env) bisect() (*vcs.Commit, error) {
|
cfg := env.cfg
|
var err error
|
if env.inst, err = instance.NewEnv(&cfg.Manager); err != nil {
|
return nil, err
|
}
|
if env.head, err = env.repo.Poll(cfg.Kernel.Repo, cfg.Kernel.Branch); err != nil {
|
return nil, err
|
}
|
if err := build.Clean(cfg.Manager.TargetOS, cfg.Manager.TargetVMArch,
|
cfg.Manager.Type, cfg.Manager.KernelSrc); err != nil {
|
return nil, fmt.Errorf("kernel clean failed: %v", err)
|
}
|
env.log("building syzkaller on %v", cfg.Syzkaller.Commit)
|
if err := env.inst.BuildSyzkaller(cfg.Syzkaller.Repo, cfg.Syzkaller.Commit); err != nil {
|
return nil, err
|
}
|
if _, err := env.repo.SwitchCommit(cfg.Kernel.Commit); err != nil {
|
return nil, err
|
}
|
if res, err := env.test(); err != nil {
|
return nil, err
|
} else if res != vcs.BisectBad {
|
return nil, fmt.Errorf("the crash wasn't reproduced on the original commit")
|
}
|
res, bad, good, err := env.commitRange()
|
if err != nil {
|
return nil, err
|
}
|
if res != nil {
|
return res, nil // happens on the oldest release
|
}
|
if good == "" {
|
return nil, nil // still not fixed
|
}
|
return env.repo.Bisect(bad, good, cfg.Trace, func() (vcs.BisectResult, error) {
|
res, err := env.test()
|
if cfg.Fix {
|
if res == vcs.BisectBad {
|
res = vcs.BisectGood
|
} else if res == vcs.BisectGood {
|
res = vcs.BisectBad
|
}
|
}
|
return res, err
|
})
|
}
|
|
func (env *env) commitRange() (*vcs.Commit, string, string, error) {
|
if env.cfg.Fix {
|
return env.commitRangeForFix()
|
}
|
return env.commitRangeForBug()
|
}
|
|
func (env *env) commitRangeForFix() (*vcs.Commit, string, string, error) {
|
env.log("testing current HEAD %v", env.head.Hash)
|
if _, err := env.repo.SwitchCommit(env.head.Hash); err != nil {
|
return nil, "", "", err
|
}
|
res, err := env.test()
|
if err != nil {
|
return nil, "", "", err
|
}
|
if res != vcs.BisectGood {
|
return nil, "", "", nil
|
}
|
return nil, env.head.Hash, env.cfg.Kernel.Commit, nil
|
}
|
|
func (env *env) commitRangeForBug() (*vcs.Commit, string, string, error) {
|
cfg := env.cfg
|
tags, err := env.repo.PreviousReleaseTags(cfg.Kernel.Commit)
|
if err != nil {
|
return nil, "", "", err
|
}
|
for i, tag := range tags {
|
if tag == "v3.8" {
|
// v3.8 does not work with modern perl, and as we go further in history
|
// make stops to work, then binutils, glibc, etc. So we stop at v3.8.
|
// Up to that point we only need an ancient gcc.
|
tags = tags[:i]
|
break
|
}
|
}
|
if len(tags) == 0 {
|
return nil, "", "", fmt.Errorf("no release tags before this commit")
|
}
|
lastBad := cfg.Kernel.Commit
|
for i, tag := range tags {
|
env.log("testing release %v", tag)
|
commit, err := env.repo.SwitchCommit(tag)
|
if err != nil {
|
return nil, "", "", err
|
}
|
res, err := env.test()
|
if err != nil {
|
return nil, "", "", err
|
}
|
if res == vcs.BisectGood {
|
return nil, lastBad, tag, nil
|
}
|
if res == vcs.BisectBad {
|
lastBad = tag
|
}
|
if i == len(tags)-1 {
|
return commit, "", "", nil
|
}
|
}
|
panic("unreachable")
|
}
|
|
func (env *env) test() (vcs.BisectResult, error) {
|
cfg := env.cfg
|
env.numTests++
|
current, err := env.repo.HeadCommit()
|
if err != nil {
|
return 0, err
|
}
|
be, err := env.buildEnvForCommit(current.Hash)
|
if err != nil {
|
return 0, err
|
}
|
compilerID, err := build.CompilerIdentity(be.compiler)
|
if err != nil {
|
return 0, err
|
}
|
env.log("testing commit %v with %v", current.Hash, compilerID)
|
buildStart := time.Now()
|
if err := build.Clean(cfg.Manager.TargetOS, cfg.Manager.TargetVMArch,
|
cfg.Manager.Type, cfg.Manager.KernelSrc); err != nil {
|
return 0, fmt.Errorf("kernel clean failed: %v", err)
|
}
|
err = env.inst.BuildKernel(be.compiler, cfg.Kernel.Userspace,
|
cfg.Kernel.Cmdline, cfg.Kernel.Sysctl, cfg.Kernel.Config)
|
env.buildTime += time.Since(buildStart)
|
if err != nil {
|
if verr, ok := err.(*osutil.VerboseError); ok {
|
env.log("%v", verr.Title)
|
env.saveDebugFile(current.Hash, 0, verr.Output)
|
} else {
|
env.log("%v", err)
|
}
|
return vcs.BisectSkip, nil
|
}
|
testStart := time.Now()
|
results, err := env.inst.Test(8, cfg.Repro.Syz, cfg.Repro.Opts, cfg.Repro.C)
|
env.testTime += time.Since(testStart)
|
if err != nil {
|
env.log("failed: %v", err)
|
return vcs.BisectSkip, nil
|
}
|
bad, good := env.processResults(current, results)
|
res := vcs.BisectSkip
|
if bad != 0 {
|
res = vcs.BisectBad
|
} else if good != 0 {
|
res = vcs.BisectGood
|
}
|
return res, nil
|
}
|
|
func (env *env) processResults(current *vcs.Commit, results []error) (bad, good int) {
|
var verdicts []string
|
for i, res := range results {
|
if res == nil {
|
good++
|
verdicts = append(verdicts, "OK")
|
continue
|
}
|
switch err := res.(type) {
|
case *instance.TestError:
|
if err.Boot {
|
verdicts = append(verdicts, fmt.Sprintf("boot failed: %v", err))
|
} else {
|
verdicts = append(verdicts, fmt.Sprintf("basic kernel testing failed: %v", err))
|
}
|
output := err.Output
|
if err.Report != nil {
|
output = err.Report.Output
|
}
|
env.saveDebugFile(current.Hash, i, output)
|
case *instance.CrashError:
|
bad++
|
verdicts = append(verdicts, fmt.Sprintf("crashed: %v", err))
|
output := err.Report.Report
|
if len(output) == 0 {
|
output = err.Report.Output
|
}
|
env.saveDebugFile(current.Hash, i, output)
|
default:
|
verdicts = append(verdicts, fmt.Sprintf("failed: %v", err))
|
}
|
}
|
unique := make(map[string]bool)
|
for _, verdict := range verdicts {
|
unique[verdict] = true
|
}
|
if len(unique) == 1 {
|
env.log("all runs: %v", verdicts[0])
|
} else {
|
for i, verdict := range verdicts {
|
env.log("run #%v: %v", i, verdict)
|
}
|
}
|
return
|
}
|
|
// Note: linux-specific.
|
func (env *env) buildEnvForCommit(commit string) (*buildEnv, error) {
|
cfg := env.cfg
|
tags, err := env.repo.PreviousReleaseTags(commit)
|
if err != nil {
|
return nil, err
|
}
|
be := &buildEnv{
|
compiler: filepath.Join(cfg.BinDir, "gcc-"+linuxCompilerVersion(tags), "bin", "gcc"),
|
}
|
return be, nil
|
}
|
|
func linuxCompilerVersion(tags []string) string {
|
for _, tag := range tags {
|
switch tag {
|
case "v4.12":
|
return "8.1.0"
|
case "v4.11":
|
return "7.3.0"
|
case "v3.19":
|
return "5.5.0"
|
}
|
}
|
return "4.9.4"
|
}
|
|
func (env *env) saveDebugFile(hash string, idx int, data []byte) {
|
if env.cfg.DebugDir == "" || len(data) == 0 {
|
return
|
}
|
osutil.WriteFile(filepath.Join(env.cfg.DebugDir, fmt.Sprintf("%v.%v", hash, idx)), data)
|
}
|
|
func (env *env) log(msg string, args ...interface{}) {
|
fmt.Fprintf(env.cfg.Trace, msg+"\n", args...)
|
}
|
