From b045410134c6f429aab54f0c0d1beb4a684144c7 Mon Sep 17 00:00:00 2001 From: Lukas Anliker Date: Mon, 6 Nov 2023 16:10:38 +0100 Subject: [PATCH] improve sliceutil.RemoveMatch, additional tests for ctxutil & traceutil --- util/ctxutil/stack_private_test.go | 58 +++++++ util/sliceutil/sliceutil.go | 20 ++- util/sliceutil/sliceutil_benchmark_test.go | 73 +++++++++ util/sliceutil/sliceutil_examples_test.go | 21 +++ util/sliceutil/sliceutil_test.go | 22 ++- .../traceutil_multi_goroutine_test.go | 146 ++++++++++++++++++ 6 files changed, 327 insertions(+), 13 deletions(-) create mode 100644 util/ctxutil/stack_private_test.go create mode 100644 util/sliceutil/sliceutil_benchmark_test.go create mode 100644 util/sliceutil/sliceutil_examples_test.go create mode 100644 util/traceutil/traceutil_multi_goroutine_test.go diff --git a/util/ctxutil/stack_private_test.go b/util/ctxutil/stack_private_test.go new file mode 100644 index 0000000..693189e --- /dev/null +++ b/util/ctxutil/stack_private_test.go @@ -0,0 +1,58 @@ +package ctxutil + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/eluv-io/common-go/util/traceutil/trace" +) + +func TestCleanup(t *testing.T) { + // make sure cleanup works with wrapped spans + root := newLogSpan(Current().InitTracing("root")) + + wg := sync.WaitGroup{} + for i := 0; i < 100; i++ { + wg.Add(1) + ctx := Current().Ctx() + go func() { + defer Current().Push(ctx)() + span := Current().StartSpan("s1") + span.Attribute("attr", "blub") + + span2 := Current().StartSpan("s2") + span2.End() + + span.End() + wg.Done() + }() + } + + wg.Wait() + + root.End() + require.Empty(t, Current().(*contextStack).stacks) + + root.Log() +} + +func newLogSpan(span trace.Span) *logSpan { + return &logSpan{ + Span: span, + } +} + +type logSpan struct { + trace.Span + once sync.Once +} + +func (s *logSpan) Log() { + s.once.Do(func() { + log.Info("collected trace", "trace", s.Span.Json()) + // would be more efficient if logging disabled: + // log.Info("collected trace", "trace", jsonutil.Stringer(s.Span)) + }) +} diff --git a/util/sliceutil/sliceutil.go b/util/sliceutil/sliceutil.go index 85db037..4b6ca9c 100644 --- a/util/sliceutil/sliceutil.go +++ b/util/sliceutil/sliceutil.go @@ -159,18 +159,24 @@ func RemoveFn[T any](slice []T, element T, equal func(e1, e2 T) bool) ([]T, int) }) } -// RemoveMatch removes all occurrences of an element from the given slice that match according to the provided match -// function. Returns the new slice and the number of removed elements. +// RemoveMatch removes all elements that match according to the provided match function from the given slice. Removal is +// performed inline, freed up slots at the end of the slice are zeroed out. Returns the updated slice and the number of +// removed elements. func RemoveMatch[T any](slice []T, match func(e T) bool) ([]T, int) { - orgLen := len(slice) - for i := 0; i < len(slice); { + var zero T + removed := 0 + for i := 0; i < len(slice); i++ { if match(slice[i]) { - slice = RemoveIndex(slice, i) + removed++ + slice[i] = zero } else { - i++ + if removed > 0 { + slice[i-removed] = slice[i] + slice[i] = zero + } } } - return slice, orgLen - len(slice) + return slice[:len(slice)-removed], removed } // RemoveIndex removes the element at the given index from the provided slice. Removes nothing if the index is diff --git a/util/sliceutil/sliceutil_benchmark_test.go b/util/sliceutil/sliceutil_benchmark_test.go new file mode 100644 index 0000000..7bedc71 --- /dev/null +++ b/util/sliceutil/sliceutil_benchmark_test.go @@ -0,0 +1,73 @@ +package sliceutil_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/eluv-io/common-go/util/sliceutil" +) + +// baseline current version: +// +// goos: darwin +// goarch: amd64 +// pkg: github.com/eluv-io/common-go/util/sliceutil +// cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz +// BenchmarkRemoveMatch +// BenchmarkRemoveMatch/slice_len_10 +// BenchmarkRemoveMatch/slice_len_10-16 487131 2367 ns/op +// BenchmarkRemoveMatch/slice_len_100 +// BenchmarkRemoveMatch/slice_len_100-16 387724 2637 ns/op +// BenchmarkRemoveMatch/slice_len_1000 +// BenchmarkRemoveMatch/slice_len_1000-16 232868 5655 ns/op +// BenchmarkRemoveMatch/slice_len_10000 +// BenchmarkRemoveMatch/slice_len_10000-16 40525 30455 ns/op +// BenchmarkRemoveMatch/slice_len_100000 +// BenchmarkRemoveMatch/slice_len_100000-16 4180 307099 ns/op +// PASS +// +// original version: +// +// goos: darwin +// goarch: amd64 +// pkg: github.com/eluv-io/common-go/util/sliceutil +// cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz +// BenchmarkRemoveMatch +// BenchmarkRemoveMatch/slice_len_10 +// BenchmarkRemoveMatch/slice_len_10-16 491574 2329 ns/op +// BenchmarkRemoveMatch/slice_len_100 +// BenchmarkRemoveMatch/slice_len_100-16 381914 3376 ns/op +// BenchmarkRemoveMatch/slice_len_1000 +// BenchmarkRemoveMatch/slice_len_1000-16 37308 29885 ns/op +// BenchmarkRemoveMatch/slice_len_10000 +// BenchmarkRemoveMatch/slice_len_10000-16 312 3780946 ns/op +// BenchmarkRemoveMatch/slice_len_100000 +// BenchmarkRemoveMatch/slice_len_100000-16 2 611002579 ns/op +// PASS +func BenchmarkRemoveMatch(b *testing.B) { + sliceLens := []int{10, 100, 1000, 10000, 100_000} + for _, sliceLen := range sliceLens { + b.Run(fmt.Sprintf("slice len %d", sliceLen), func(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + slice := generateSlice(sliceLen) + b.StartTimer() + res, removed := sliceutil.RemoveMatch(slice, func(e int) bool { + return e >= 5 + }) + require.Equal(b, sliceLen/2, len(res)) + require.Equal(b, sliceLen/2, removed) + } + }) + } +} + +func generateSlice(len int) []int { + slice := make([]int, len) + for i, _ := range slice { + slice[i] = i % 10 + } + return slice +} diff --git a/util/sliceutil/sliceutil_examples_test.go b/util/sliceutil/sliceutil_examples_test.go new file mode 100644 index 0000000..faf474f --- /dev/null +++ b/util/sliceutil/sliceutil_examples_test.go @@ -0,0 +1,21 @@ +package sliceutil_test + +import ( + "fmt" + + "github.com/eluv-io/common-go/util/sliceutil" +) + +func ExampleRemoveMatch() { + slice := []int{1, 2, 3, 1, 2, 3, 6, 6, 4, 2, 4} + res, i := sliceutil.RemoveMatch(slice, func(e int) bool { + return e >= 3 + }) + fmt.Printf("removed %d elements: %v\n", i, res) + fmt.Printf("removed slots in original slice are zeroed: %v\n", slice) + + // Output: + // + // removed 6 elements: [1 2 1 2 2] + // removed slots in original slice are zeroed: [1 2 1 2 2 0 0 0 0 0 0] +} diff --git a/util/sliceutil/sliceutil_test.go b/util/sliceutil/sliceutil_test.go index 4cadd35..457bfe4 100644 --- a/util/sliceutil/sliceutil_test.go +++ b/util/sliceutil/sliceutil_test.go @@ -339,22 +339,32 @@ func TestRemove(t *testing.T) { remove: 3, want: []int{}, }, - { + { // no match + slice: []int{1, 2, 3, 4, 5}, + remove: 6, + want: []int{1, 2, 3, 4, 5}, + }, + { // one match at beginning + slice: []int{1, 2, 3, 4, 5}, + remove: 1, + want: []int{2, 3, 4, 5}, + }, + { // one match in middle slice: []int{1, 2, 3, 4, 5}, remove: 3, want: []int{1, 2, 4, 5}, }, - { + { // one match at end slice: []int{1, 2, 3, 4, 5}, - remove: 6, - want: []int{1, 2, 3, 4, 5}, + remove: 5, + want: []int{1, 2, 3, 4}, }, - { + { // multiple matches slice: []int{1, 2, 1, 2, 1, 2, 3}, remove: 2, want: []int{1, 1, 1, 3}, }, - { + { // all match slice: []int{1, 1, 1, 1, 1}, remove: 1, want: []int{}, diff --git a/util/traceutil/traceutil_multi_goroutine_test.go b/util/traceutil/traceutil_multi_goroutine_test.go new file mode 100644 index 0000000..f431a1b --- /dev/null +++ b/util/traceutil/traceutil_multi_goroutine_test.go @@ -0,0 +1,146 @@ +package traceutil_test + +import ( + "fmt" + "sync" + "time" + + "github.com/eluv-io/common-go/format/duration" + "github.com/eluv-io/common-go/util/ctxutil" + "github.com/eluv-io/common-go/util/jsonutil" + "github.com/eluv-io/common-go/util/traceutil" + "github.com/eluv-io/common-go/util/traceutil/trace" +) + +func A() { + span := traceutil.StartSpan("func A") + time.Sleep(time.Second) + defer span.End() +} + +func B() { + span := traceutil.StartSpan("func B") + time.Sleep(500 * time.Millisecond) + defer span.End() +} + +func Sequential() { + span := traceutil.StartSpan("calling A() and B() sequentially") + defer span.End() + A() + B() +} + +func Parallel() { + span := traceutil.StartSpan("calling A() and B() concurrently") + defer span.End() + + wg := sync.WaitGroup{} + wg.Add(2) + + ctx := ctxutil.Ctx() + go func() { + defer ctxutil.Current().Push(ctx)() + A() + wg.Done() + }() + time.Sleep(50 * time.Millisecond) + go func() { + defer ctxutil.Current().Push(ctx)() + B() + wg.Done() + }() + + wg.Wait() +} + +func Parallel2() { + span := traceutil.StartSpan("calling A() and B() concurrently") + defer span.End() + + wg := sync.WaitGroup{} + wg.Add(2) + + ctxutil.Current().Go(func() { + A() + wg.Done() + }) + time.Sleep(50 * time.Millisecond) + ctxutil.Current().Go(func() { + B() + wg.Done() + }) + + wg.Wait() +} + +func ExampleMultiGoRoutine() { + rootSpan := traceutil.InitTracing("root span") + + Sequential() + Parallel() + Parallel2() + + rootSpan.End() + + // truncate span durations to have reproducible output + spans := []trace.Span{rootSpan.FindByName("root span")} + for len(spans) > 0 { + span := spans[len(spans)-1].(*trace.RecordingSpan) + span.Data.Duration = duration.Spec(span.Data.Duration.Duration().Truncate(500 * time.Millisecond)) + spans = append(spans[:len(spans)-1], span.Data.Subs...) + } + fmt.Println(jsonutil.MustPretty(rootSpan.Json())) + + // Output: + // + // { + // "name": "root span", + // "time": "3.5s", + // "subs": [ + // { + // "name": "calling A() and B() sequentially", + // "time": "1.5s", + // "subs": [ + // { + // "name": "func A", + // "time": "1s" + // }, + // { + // "name": "func B", + // "time": "500ms" + // } + // ] + // }, + // { + // "name": "calling A() and B() concurrently", + // "time": "1s", + // "subs": [ + // { + // "name": "func A", + // "time": "1s" + // }, + // { + // "name": "func B", + // "time": "500ms" + // } + // ] + // }, + // { + // "name": "calling A() and B() concurrently", + // "time": "1s", + // "subs": [ + // { + // "name": "func A", + // "time": "1s" + // }, + // { + // "name": "func B", + // "time": "500ms" + // } + // ] + // } + // ] + // } + +}