Add benchmarking
This commit is contained in:
parent
de714d4ed8
commit
3f59977ba5
4 changed files with 233 additions and 10 deletions
13
go.mod
13
go.mod
|
|
@ -10,9 +10,14 @@ require (
|
|||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
|
||||
github.com/mattn/go-colorable v0.1.2 // indirect
|
||||
github.com/mattn/go-isatty v0.0.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.13 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect
|
||||
golang.org/x/term v0.0.0-20210503060354-a79de5458b56 // indirect
|
||||
golang.org/x/text v0.3.3 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/rivo/uniseg v0.2.0 // indirect
|
||||
github.com/schollz/progressbar/v3 v3.8.3 // indirect
|
||||
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
|
||||
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||
golang.org/x/text v0.3.6 // indirect
|
||||
)
|
||||
|
|
|
|||
29
go.sum
29
go.sum
|
|
@ -6,10 +6,12 @@ github.com/alexflint/go-arg v1.4.2 h1:lDWZAXxpAnZUq4qwb86p/3rIJJ2Li81EoMbTMujhVa
|
|||
github.com/alexflint/go-arg v1.4.2/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM=
|
||||
github.com/alexflint/go-scalar v1.0.0 h1:NGupf1XV/Xb04wXskDFzS0KWOLH632W/EO4fAFi+A70=
|
||||
github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ=
|
||||
github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A=
|
||||
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
|
||||
github.com/kr/pty v1.1.4 h1:5Myjjh3JY/NaAi4IsUbHADytDyl1VE1Y9PXDlL+P/VQ=
|
||||
|
|
@ -21,25 +23,52 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
|
|||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
|
||||
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8=
|
||||
github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U=
|
||||
github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI=
|
||||
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc=
|
||||
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210503060354-a79de5458b56 h1:b8jxX3zqjpqb2LklXPzKSGJhzyxCOZSz8ncv8Nv+y7w=
|
||||
golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
|
|
|||
182
runtime/benchmark/benchmark.go
Normal file
182
runtime/benchmark/benchmark.go
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
package benchmark
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/codemicro/adventOfCode/runtime/challenge"
|
||||
"github.com/codemicro/adventOfCode/runtime/runners"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
"io/ioutil"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func makeBenchmarkID(part runners.Part, number int) string {
|
||||
if number == -1 {
|
||||
return fmt.Sprintf("benchmark.part.%d", part)
|
||||
}
|
||||
return fmt.Sprintf("benchmark.part.%d.%d", part, number)
|
||||
}
|
||||
|
||||
func meanFloatSlice(arr []float64) float64 {
|
||||
var sum float64
|
||||
for _, v := range arr {
|
||||
sum += v
|
||||
}
|
||||
return sum / float64(len(arr))
|
||||
}
|
||||
|
||||
func minFloatSlice(arr []float64) float64 {
|
||||
min := arr[0]
|
||||
for _, v := range arr {
|
||||
if v < min {
|
||||
min = v
|
||||
}
|
||||
}
|
||||
return min
|
||||
}
|
||||
|
||||
func maxFloatSlice(arr []float64) float64 {
|
||||
max := arr[0]
|
||||
for _, v := range arr {
|
||||
if v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
func Run(selectedChallenge *challenge.Challenge, input string, numberRuns int) error {
|
||||
|
||||
implementations, err := selectedChallenge.GetImplementations()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var valueSets []*values
|
||||
|
||||
for _, implementation := range implementations {
|
||||
v, err := benchmarkImplementation(implementation, selectedChallenge.Dir, input, numberRuns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
valueSets = append(valueSets, v)
|
||||
}
|
||||
|
||||
// make file
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("Day %d (%s) benchmark\n\n", selectedChallenge.Number, selectedChallenge.Name))
|
||||
sb.WriteString(fmt.Sprintf("Dir: %s\n", selectedChallenge.Dir))
|
||||
sb.WriteString(fmt.Sprintf("Runs per part: %d\n", numberRuns))
|
||||
|
||||
for _, vs := range valueSets {
|
||||
sb.WriteString("--------------------------------------------------------------------------------\n")
|
||||
sb.WriteString(vs.implementation)
|
||||
sb.WriteString("\n\n")
|
||||
for _, v := range vs.values {
|
||||
sb.WriteString(fmt.Sprintf("%s: %f seconds\n", v.key, v.value))
|
||||
}
|
||||
}
|
||||
sb.WriteString("--------------------------------------------------------------------------------\n")
|
||||
|
||||
fpath := filepath.Join(selectedChallenge.Dir, "benchmark.txt")
|
||||
|
||||
fmt.Println("Writing results to", fpath)
|
||||
|
||||
return ioutil.WriteFile(
|
||||
fpath,
|
||||
[]byte(sb.String()),
|
||||
0644,
|
||||
)
|
||||
}
|
||||
|
||||
type values struct {
|
||||
implementation string
|
||||
values []kv
|
||||
}
|
||||
|
||||
type kv struct {
|
||||
key string
|
||||
value float64
|
||||
}
|
||||
|
||||
func benchmarkImplementation(implementation string, dir string, inputString string, numberRuns int) (*values, error) {
|
||||
|
||||
var results []*runners.Result
|
||||
|
||||
runner := runners.Available[implementation](dir)
|
||||
for i := 0; i < numberRuns; i++ {
|
||||
|
||||
runner.Queue(&runners.Task{
|
||||
TaskID: makeBenchmarkID(runners.PartOne, i),
|
||||
Part: runners.PartOne,
|
||||
Input: inputString,
|
||||
})
|
||||
|
||||
runner.Queue(&runners.Task{
|
||||
TaskID: makeBenchmarkID(runners.PartTwo, i),
|
||||
Part: runners.PartTwo,
|
||||
Input: inputString,
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
pb := progressbar.NewOptions(
|
||||
numberRuns * 2, // two parts means 2x the number of runs
|
||||
progressbar.OptionSetDescription(
|
||||
fmt.Sprintf("Running %s benchmarks", runners.RunnerNames[implementation]),
|
||||
),
|
||||
)
|
||||
|
||||
res, cleanup := runner.Run()
|
||||
|
||||
for roe := range res {
|
||||
if roe.Error != nil {
|
||||
_ = pb.Close()
|
||||
return nil, roe.Error
|
||||
}
|
||||
results = append(results, roe.Result)
|
||||
_ = pb.Add(1)
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
pb = progressbar.NewOptions(
|
||||
len(results) + 1, // two parts means 2x the number of runs
|
||||
progressbar.OptionSetDescription(
|
||||
fmt.Sprintf("Processing %s results", runners.RunnerNames[implementation]),
|
||||
),
|
||||
)
|
||||
|
||||
var (
|
||||
p1, p2 []float64
|
||||
p1id = makeBenchmarkID(runners.PartOne, -1)
|
||||
p2id = makeBenchmarkID(runners.PartTwo, -1)
|
||||
)
|
||||
|
||||
for _, result := range results {
|
||||
if strings.HasPrefix(result.TaskID, p1id) {
|
||||
p1 = append(p1, result.Duration)
|
||||
} else if strings.HasPrefix(result.TaskID, p2id) {
|
||||
p2 = append(p2, result.Duration)
|
||||
}
|
||||
_ = pb.Add(1)
|
||||
}
|
||||
|
||||
_ = pb.Finish()
|
||||
fmt.Println()
|
||||
|
||||
if cleanup != nil {
|
||||
cleanup()
|
||||
}
|
||||
|
||||
return &values{
|
||||
implementation: runners.RunnerNames[implementation],
|
||||
values: []kv{
|
||||
{"benchmark.part.1.avg", meanFloatSlice(p1)},
|
||||
{"benchmark.part.1.min", minFloatSlice(p1)},
|
||||
{"benchmark.part.1.max", maxFloatSlice(p1)},
|
||||
{"benchmark.part.2.avg", meanFloatSlice(p2)},
|
||||
{"benchmark.part.2.min", minFloatSlice(p2)},
|
||||
{"benchmark.part.2.max", maxFloatSlice(p2)},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -3,6 +3,7 @@ package main
|
|||
import (
|
||||
"fmt"
|
||||
"github.com/alexflint/go-arg"
|
||||
"github.com/codemicro/adventOfCode/runtime/benchmark"
|
||||
"github.com/codemicro/adventOfCode/runtime/challenge"
|
||||
"github.com/codemicro/adventOfCode/runtime/runners"
|
||||
"io/ioutil"
|
||||
|
|
@ -18,6 +19,8 @@ var args struct {
|
|||
Year string `arg:"-y,--year" help:"AoC year to use"`
|
||||
ChallengeDay *int `arg:"-d,--day" help:"challenge day number to run"`
|
||||
Implementation string `arg:"-i,--implementation" help:"implementation to use"`
|
||||
Benchmark bool `arg:"-b,--benchmark" help:"benchmark a day's implementations'"`
|
||||
BenchmarkN int `arg:"-n,--benchmark-n" help:"Number of iterations to run for benchmarking" default:"50"`
|
||||
}
|
||||
|
||||
func run() error {
|
||||
|
|
@ -36,12 +39,6 @@ func run() error {
|
|||
return err
|
||||
}
|
||||
|
||||
// List and select implementations
|
||||
selectedImplementation, err := selectImplementation(selectedChallenge)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Load info.json file
|
||||
challengeInfo, err := challenge.LoadChallengeInfo(filepath.Join(selectedChallenge.Dir, challengeInfoFile))
|
||||
if err != nil {
|
||||
|
|
@ -55,6 +52,16 @@ func run() error {
|
|||
}
|
||||
challengeInputString := string(challengeInput)
|
||||
|
||||
if args.Benchmark {
|
||||
return benchmark.Run(selectedChallenge, challengeInputString, args.BenchmarkN)
|
||||
}
|
||||
|
||||
// List and select implementations
|
||||
selectedImplementation, err := selectImplementation(selectedChallenge)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
runner := runners.Available[selectedImplementation](selectedChallenge.Dir)
|
||||
|
||||
lookupTable := make(taskLookupTable)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue