show stacktrace when panic

golangid · Aug 22, 2024 · cd94184 · cd94184
1 parent c67e92a
commit cd94184
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 11 deletions.
diff --git a/codebase/app/grpc_server/interceptor.go b/codebase/app/grpc_server/interceptor.go
@@ -69,6 +69,7 @@ func (i *interceptor) unaryTracerInterceptor(ctx context.Context, req interface{
 	trace, ctx := tracer.StartTraceFromHeader(ctx, "GRPC-Server", header)
 	defer func() {
 		if rec := recover(); rec != nil {
+			trace.SetTag("panic", true)
 			err = status.Errorf(codes.Aborted, "%v", rec)
 		}
 		i.logInterceptor(start, err, info.FullMethod, "GRPC")
@@ -84,7 +85,6 @@ func (i *interceptor) unaryTracerInterceptor(ctx context.Context, req interface{
 	}
 
 	resp, err = handler(ctx, req)
-	trace.SetError(err)
 	if respBody := candihelper.ToBytes(resp); len(respBody) < i.opt.jaegerMaxPacketSize { // limit response body size to 65000 bytes (if higher tracer cannot show root span)
 		trace.Log("response.body", respBody)
 	} else {
@@ -152,6 +152,7 @@ func (i *interceptor) streamTracerInterceptor(srv interface{}, stream grpc.Serve
 	trace, ctx := tracer.StartTraceFromHeader(ctx, "GRPC-STREAM", header)
 	defer func() {
 		if rec := recover(); rec != nil {
+			trace.SetTag("panic", true)
 			err = status.Errorf(codes.Aborted, "%v", rec)
 		}
 		i.logInterceptor(start, err, info.FullMethod, "GRPC-STREAM")
@@ -161,7 +162,6 @@ func (i *interceptor) streamTracerInterceptor(srv interface{}, stream grpc.Serve
 	trace.SetTag("method", info.FullMethod)
 	trace.Log("metadata", meta)
 	err = handler(srv, &wrappedServerStream{ServerStream: stream, wrappedContext: ctx})
-	trace.SetError(err)
 	return
 }
 

diff --git a/codebase/app/rest_server/middleware.go b/codebase/app/rest_server/middleware.go
@@ -127,6 +127,8 @@ func HTTPMiddlewareTracer() func(http.Handler) http.Handler {
 			trace, ctx := tracer.StartTraceFromHeader(req.Context(), "REST-Server", header)
 			defer func() {
 				if rec := recover(); rec != nil {
+					trace.SetTag("panic", true)
+					trace.SetError(fmt.Errorf("%v", rec))
 					wrapper.NewHTTPResponse(http.StatusInternalServerError, "Something error").JSON(rw)
 				}
 				trace.Finish()

diff --git a/config/env/environment.go b/config/env/environment.go
@@ -155,7 +155,7 @@ func Load(serviceName string) {
 		}
 		env.TaskQueueDashboardPort = uint16(port)
 		env.TaskQueueDashboardMaxClientSubscribers, _ = strconv.Atoi(os.Getenv("TASK_QUEUE_DASHBOARD_MAX_CLIENT"))
-		if env.TaskQueueDashboardPort <= 0 || env.TaskQueueDashboardMaxClientSubscribers > 10 {
+		if env.TaskQueueDashboardMaxClientSubscribers <= 0 || env.TaskQueueDashboardMaxClientSubscribers > 10 {
 			env.TaskQueueDashboardMaxClientSubscribers = 10 // default
 		}
 	}

diff --git a/tracer/jaeger.go b/tracer/jaeger.go
@@ -186,11 +186,11 @@ func (j *jaegerPlatform) Disconnect(ctx context.Context) error { return j.closer
 
 // jaeger span tracer implementation
 type jaegerTraceImpl struct {
-	ctx           context.Context
-	span          opentracing.Span
-	operationName string
-	isRoot        bool
-	errWhitelist  []error
+	ctx             context.Context
+	span            opentracing.Span
+	operationName   string
+	isRoot, isPanic bool
+	errWhitelist    []error
 }
 
 // Context get active context
@@ -204,6 +204,8 @@ func (t *jaegerTraceImpl) SetTag(key string, value interface{}) {
 		return
 	}
 
+	v, _ := value.(bool)
+	t.isPanic = key == "panic" && v
 	t.span.SetTag(key, toValue(value))
 }
 
@@ -242,7 +244,7 @@ func (t *jaegerTraceImpl) SetError(err error) {
 	t.span.LogKV("error.message", err.Error())
 
 	var stackTraces []string
-	for i := 1; i < 10 && len(stackTraces) <= 5 && !t.isRoot; i++ {
+	for i := 1; i < 10 && len(stackTraces) <= 5 && (!t.isRoot || t.isPanic); i++ {
 		if caller := parseCaller(runtime.Caller(i)); caller != "" {
 			stackTraces = append(stackTraces, caller)
 		}
@@ -291,7 +293,7 @@ func (t *jaegerTraceImpl) Finish(opts ...FinishOptionFunc) {
 				stackTraces = append(stackTraces, caller)
 			}
 		}
-		t.logStackTrace(32, t.operationName, stackTraces)
+		t.logStackTrace(0, t.operationName, stackTraces)
 	}
 
 	if finishOpt.OnFinish != nil {
@@ -304,7 +306,11 @@ func (t *jaegerTraceImpl) Finish(opts ...FinishOptionFunc) {
 }
 
 func (t *jaegerTraceImpl) logStackTrace(color int, header string, stackTraces []string) {
-	log.Printf("\x1b[%d;5m%s\x1b[0m", color, strings.Join(append([]string{header}, stackTraces...), "\n"))
+	format := "%s"
+	if color > 0 {
+		format = "\x1b[" + strconv.Itoa(color) + ";5m%s\x1b[0m"
+	}
+	log.Printf(format, strings.Join(append([]string{header}, stackTraces...), "\n"))
 	if len(stackTraces) > 0 {
 		t.span.LogKV("stacktrace", strings.Join(stackTraces, "\n"))
 	}