improve sliceutil.RemoveMatch, additional tests for ctxutil & traceutil

util/ctxutil/stack_private_test.go

@@ -0,0 +1,58 @@
+package ctxutil
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/eluv-io/common-go/util/traceutil/trace"
+)
+
+func TestCleanup(t *testing.T) {
+	// make sure cleanup works with wrapped spans
+	root := newLogSpan(Current().InitTracing("root"))
+
+	wg := sync.WaitGroup{}
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		ctx := Current().Ctx()
+		go func() {
+			defer Current().Push(ctx)()
+			span := Current().StartSpan("s1")
+			span.Attribute("attr", "blub")
+
+			span2 := Current().StartSpan("s2")
+			span2.End()
+
+			span.End()
+			wg.Done()
+		}()
+	}
+
+	wg.Wait()
+
+	root.End()
+	require.Empty(t, Current().(*contextStack).stacks)
+
+	root.Log()
+}
+
+func newLogSpan(span trace.Span) *logSpan {
+	return &logSpan{
+		Span: span,
+	}
+}
+
+type logSpan struct {
+	trace.Span
+	once sync.Once
+}
+
+func (s *logSpan) Log() {
+	s.once.Do(func() {
+		log.Info("collected trace", "trace", s.Span.Json())
+		// would be more efficient if logging disabled:
+		// log.Info("collected trace", "trace", jsonutil.Stringer(s.Span))
+	})
+}

util/sliceutil/sliceutil.go

@@ -159,18 +159,24 @@ func RemoveFn[T any](slice []T, element T, equal func(e1, e2 T) bool) ([]T, int)
 	})
 }
 
-// RemoveMatch removes all occurrences of an element from the given slice that match according to the provided match
-// function. Returns the new slice and the number of removed elements.
+// RemoveMatch removes all elements that match according to the provided match function from the given slice. Removal is
+// performed inline, freed up slots at the end of the slice are zeroed out. Returns the updated slice and the number of
+// removed elements.
 func RemoveMatch[T any](slice []T, match func(e T) bool) ([]T, int) {
-	orgLen := len(slice)
-	for i := 0; i < len(slice); {
+	var zero T
+	removed := 0
+	for i := 0; i < len(slice); i++ {
 		if match(slice[i]) {
-			slice = RemoveIndex(slice, i)
+			removed++
+			slice[i] = zero
 		} else {
-			i++
+			if removed > 0 {
+				slice[i-removed] = slice[i]
+				slice[i] = zero
+			}
 		}
 	}
-	return slice, orgLen - len(slice)
+	return slice[:len(slice)-removed], removed
 }
 
 // RemoveIndex removes the element at the given index from the provided slice. Removes nothing if the index is

util/sliceutil/sliceutil_benchmark_test.go

@@ -0,0 +1,73 @@
+package sliceutil_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/eluv-io/common-go/util/sliceutil"
+)
+
+// baseline current version:
+//
+//	goos: darwin
+//	goarch: amd64
+//	pkg: github.com/eluv-io/common-go/util/sliceutil
+//	cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+//	BenchmarkRemoveMatch
+//	BenchmarkRemoveMatch/slice_len_10
+//	BenchmarkRemoveMatch/slice_len_10-16         	  487131	      2367 ns/op
+//	BenchmarkRemoveMatch/slice_len_100
+//	BenchmarkRemoveMatch/slice_len_100-16        	  387724	      2637 ns/op
+//	BenchmarkRemoveMatch/slice_len_1000
+//	BenchmarkRemoveMatch/slice_len_1000-16       	  232868	      5655 ns/op
+//	BenchmarkRemoveMatch/slice_len_10000
+//	BenchmarkRemoveMatch/slice_len_10000-16      	   40525	     30455 ns/op
+//	BenchmarkRemoveMatch/slice_len_100000
+//	BenchmarkRemoveMatch/slice_len_100000-16     	    4180	    307099 ns/op
+//	PASS
+//
+// original version:
+//
+//	goos: darwin
+//	goarch: amd64
+//	pkg: github.com/eluv-io/common-go/util/sliceutil
+//	cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+//	BenchmarkRemoveMatch
+//	BenchmarkRemoveMatch/slice_len_10
+//	BenchmarkRemoveMatch/slice_len_10-16         	  491574	      2329 ns/op
+//	BenchmarkRemoveMatch/slice_len_100
+//	BenchmarkRemoveMatch/slice_len_100-16        	  381914	      3376 ns/op
+//	BenchmarkRemoveMatch/slice_len_1000
+//	BenchmarkRemoveMatch/slice_len_1000-16       	   37308	     29885 ns/op
+//	BenchmarkRemoveMatch/slice_len_10000
+//	BenchmarkRemoveMatch/slice_len_10000-16      	     312	   3780946 ns/op
+//	BenchmarkRemoveMatch/slice_len_100000
+//	BenchmarkRemoveMatch/slice_len_100000-16     	       2	 611002579 ns/op
+//	PASS
+func BenchmarkRemoveMatch(b *testing.B) {
+	sliceLens := []int{10, 100, 1000, 10000, 100_000}
+	for _, sliceLen := range sliceLens {
+		b.Run(fmt.Sprintf("slice len %d", sliceLen), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				b.StopTimer()
+				slice := generateSlice(sliceLen)
+				b.StartTimer()
+				res, removed := sliceutil.RemoveMatch(slice, func(e int) bool {
+					return e >= 5
+				})
+				require.Equal(b, sliceLen/2, len(res))
+				require.Equal(b, sliceLen/2, removed)
+			}
+		})
+	}
+}
+
+func generateSlice(len int) []int {
+	slice := make([]int, len)
+	for i, _ := range slice {
+		slice[i] = i % 10
+	}
+	return slice
+}

util/sliceutil/sliceutil_examples_test.go

@@ -0,0 +1,21 @@
+package sliceutil_test
+
+import (
+	"fmt"
+
+	"github.com/eluv-io/common-go/util/sliceutil"
+)
+
+func ExampleRemoveMatch() {
+	slice := []int{1, 2, 3, 1, 2, 3, 6, 6, 4, 2, 4}
+	res, i := sliceutil.RemoveMatch(slice, func(e int) bool {
+		return e >= 3
+	})
+	fmt.Printf("removed %d elements: %v\n", i, res)
+	fmt.Printf("removed slots in original slice are zeroed: %v\n", slice)
+
+	// Output:
+	//
+	// removed 6 elements: [1 2 1 2 2]
+	// removed slots in original slice are zeroed: [1 2 1 2 2 0 0 0 0 0 0]
+}

util/sliceutil/sliceutil_test.go

@@ -339,22 +339,32 @@ func TestRemove(t *testing.T) {
 			remove: 3,
 			want:   []int{},
 		},
-		{
+		{ // no match
+			slice:  []int{1, 2, 3, 4, 5},
+			remove: 6,
+			want:   []int{1, 2, 3, 4, 5},
+		},
+		{ // one match at beginning
+			slice:  []int{1, 2, 3, 4, 5},
+			remove: 1,
+			want:   []int{2, 3, 4, 5},
+		},
+		{ // one match in middle
 			slice:  []int{1, 2, 3, 4, 5},
 			remove: 3,
 			want:   []int{1, 2, 4, 5},
 		},
-		{
+		{ // one match at end
 			slice:  []int{1, 2, 3, 4, 5},
-			remove: 6,
-			want:   []int{1, 2, 3, 4, 5},
+			remove: 5,
+			want:   []int{1, 2, 3, 4},
 		},
-		{
+		{ // multiple matches
 			slice:  []int{1, 2, 1, 2, 1, 2, 3},
 			remove: 2,
 			want:   []int{1, 1, 1, 3},
 		},
-		{
+		{ // all match
 			slice:  []int{1, 1, 1, 1, 1},
 			remove: 1,
 			want:   []int{},

util/traceutil/traceutil_multi_goroutine_test.go

@@ -0,0 +1,146 @@
+package traceutil_test
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/eluv-io/common-go/format/duration"
+	"github.com/eluv-io/common-go/util/ctxutil"
+	"github.com/eluv-io/common-go/util/jsonutil"
+	"github.com/eluv-io/common-go/util/traceutil"
+	"github.com/eluv-io/common-go/util/traceutil/trace"
+)
+
+func A() {
+	span := traceutil.StartSpan("func A")
+	time.Sleep(time.Second)
+	defer span.End()
+}
+
+func B() {
+	span := traceutil.StartSpan("func B")
+	time.Sleep(500 * time.Millisecond)
+	defer span.End()
+}
+
+func Sequential() {
+	span := traceutil.StartSpan("calling A() and B() sequentially")
+	defer span.End()
+	A()
+	B()
+}
+
+func Parallel() {
+	span := traceutil.StartSpan("calling A() and B() concurrently")
+	defer span.End()
+
+	wg := sync.WaitGroup{}
+	wg.Add(2)
+
+	ctx := ctxutil.Ctx()
+	go func() {
+		defer ctxutil.Current().Push(ctx)()
+		A()
+		wg.Done()
+	}()
+	time.Sleep(50 * time.Millisecond)
+	go func() {
+		defer ctxutil.Current().Push(ctx)()
+		B()
+		wg.Done()
+	}()
+
+	wg.Wait()
+}
+
+func Parallel2() {
+	span := traceutil.StartSpan("calling A() and B() concurrently")
+	defer span.End()
+
+	wg := sync.WaitGroup{}
+	wg.Add(2)
+
+	ctxutil.Current().Go(func() {
+		A()
+		wg.Done()
+	})
+	time.Sleep(50 * time.Millisecond)
+	ctxutil.Current().Go(func() {
+		B()
+		wg.Done()
+	})
+
+	wg.Wait()
+}
+
+func ExampleMultiGoRoutine() {
+	rootSpan := traceutil.InitTracing("root span")
+
+	Sequential()
+	Parallel()
+	Parallel2()
+
+	rootSpan.End()
+
+	// truncate span durations to have reproducible output
+	spans := []trace.Span{rootSpan.FindByName("root span")}
+	for len(spans) > 0 {
+		span := spans[len(spans)-1].(*trace.RecordingSpan)
+		span.Data.Duration = duration.Spec(span.Data.Duration.Duration().Truncate(500 * time.Millisecond))
+		spans = append(spans[:len(spans)-1], span.Data.Subs...)
+	}
+	fmt.Println(jsonutil.MustPretty(rootSpan.Json()))
+
+	// Output:
+	//
+	// {
+	//   "name": "root span",
+	//   "time": "3.5s",
+	//   "subs": [
+	//     {
+	//       "name": "calling A() and B() sequentially",
+	//       "time": "1.5s",
+	//       "subs": [
+	//         {
+	//           "name": "func A",
+	//           "time": "1s"
+	//         },
+	//         {
+	//           "name": "func B",
+	//           "time": "500ms"
+	//         }
+	//       ]
+	//     },
+	//     {
+	//       "name": "calling A() and B() concurrently",
+	//       "time": "1s",
+	//       "subs": [
+	//         {
+	//           "name": "func A",
+	//           "time": "1s"
+	//         },
+	//         {
+	//           "name": "func B",
+	//           "time": "500ms"
+	//         }
+	//       ]
+	//     },
+	//     {
+	//       "name": "calling A() and B() concurrently",
+	//       "time": "1s",
+	//       "subs": [
+	//         {
+	//           "name": "func A",
+	//           "time": "1s"
+	//         },
+	//         {
+	//           "name": "func B",
+	//           "time": "500ms"
+	//         }
+	//       ]
+	//     }
+	//   ]
+	// }
+
+}