Improve performance of depth-limited type printing in stack traces

[charleskawczynski]

Can we improve the performance of depth-limited type printing in stack traces? Here is an example where printing truncated types takes quite a while with a highly nested struct:

Base.@kwdef struct Nested{A,B}
    num::Int = 1
end
nest_val(na, nb, ::Val{1}) = Nested{na, nb}()
nest_val(na, nb, ::Val{n}) where {n} = nest_val(Nested{na, nb}, Nested{na, nb}, Val(n-1))
nest_val(na, nb, n::Int) = nest_val(na, nb, Val(n))
nest_val(n) = nest_val(1, 1, n)
foo1(t::Nested) = error("oops")
foo2(t::Nested) = foo1(t)
foo3(t::Nested) = foo2(t)
foo4(t::Nested) = foo3(t)
foo5(t::Nested) = foo4(t)
foo6(t::Nested) = foo5(t)
foo7(t::Nested) = foo6(t)
foo8(t::Nested) = foo7(t)
foo9(t::Nested) = foo8(t)
foo10(t::Nested) = foo9(t)
NV = nest_val(20); # be careful with changing 20 to a larger number
foo10(NV) # errors, takes ~45 seconds on Julia 1.10

I haven't looked at the implementation in a while, but I imagine that this could be related to #55807.

[nsajko]

Try the latest nightly, xref #55575.

[charleskawczynski]

Try the latest nightly, xref #55575.

I just tried it. It's actually slower on nightly. ~ 148 seconds

[KristofferC]

I think this is the same as #37990.

The code tries to check if this type could potentially have a type alias somewhere and in that case print the type alias. This check is very slow and it is done over and over and over.

julia/base/show.jl

Lines 729 to 754 in 1cfda3f

	function show_typealias(io::IO, name::GlobalRef, x::Type, env::SimpleVector, wheres::Vector)
	if !(get(io, :compact, false)::Bool)
	# Print module prefix unless alias is visible from module passed to
	# IOContext. If :module is not set, default to Main.
	# nothing can be used to force printing prefix.
	from = get(io, :module, Main)
	if (from === nothing || !isvisible(name.name, name.mod, from))
	show(io, name.mod)
	print(io, ".")
	end
	end
	print(io, name.name)
	isempty(env) && return
	io = IOContext(io)
	for p in wheres
	io = IOContext(io, :unionall_env => p)
	end
	orig = getfield(name.mod, name.name)
	vars = TypeVar[]
	while orig isa UnionAll
	push!(vars, orig.var)
	orig = orig.body
	end
	show_typeparams(io, env, Core.svec(vars...), wheres)
	nothing
	end

[KristofferC]

The solution here is to not form the whole type string because it is ridiculously large. Instead the printing should stop going down the nesting once we know we have gone far enough down to not make a difference to the stacktrace type printing.