LuaTeX nodelist visualization

viznodelist.lua

-- usage example:

-- \setbox0\hbox{\vbox{\hbox{abc}}\vbox{x}}
-- \directlua{
--   require("viznodelist")
--   viznodelist.nodelist_visualize(0,"mybox.gv")
-- }
-- 
-- \bye

-- and then open "mybox.gv" with graphviz

-- 2010-08-29, Patrick Gundlach
-- Status: experimental/usable, including debug info


local io,string,table = io,string,table
local assert,tostring = assert,tostring
local tex,texio,node,unicode=tex,texio,node,unicode

module(...)

local function w( ... )
  texio.write_nl(string.format(...))
end

local glue_type = {
[1]   = "lineskip",
[2]   = "baselineskip",
[3]   = "parskip",
[4]   = "abovedisplayskip",
[5]   = "belowdisplayskip",
[6]   = "abovedisplayshortskip",
[7]   = "belowdisplayshortskip",
[8]   = "leftskip",
[9]   = "rightskip",
[10]  = "topskip",
[11]  = "splittopskip",
[12]  = "tabskip",
[13]  = "spaceskip",
[14]  = "xspaceskip",
[15]  = "parfillskip",
[16]  = "thinmuskip",
[17]  = "medmuskip",
[18]  = "thickmuskip",
[100] = "leaders",
[101] = "cleaders",
[102] = "xleaders",
[103] = "gleaders"
}


-- tostring(a_node) looks like "<node    nil <    172 >    nil : hlist 2>", so we can
-- grab the number in the middle (172 here) as a unique id. So the node
-- is named "node172"
local function get_nodename(n)
  return "\"n" .. string.gsub(tostring(n), "^<node%s+%S+%s+<%s+(%d+).*","%1") .. "\""
end



local function link_to( n,nodename,label )
  if n then
    local t = node.type(n.id)
    local nodename_n = get_nodename(n)
    if t=="temp" or t=="nested_list" then return end
    local ret
    -- texio.write_nl(string.format("unknown: %s",t))
    -- if  nodename_n == "\"n30\"" or nodename_n == "\"n155\"" or nodename_n == "\"n65\"" or nodename_n == "\"n187" then
    --   w("n30 gefunden!")
    --   w(tostring(n))
    -- end
    if label=="prev" then
      ret = string.format("%s:%s:w -> %s:title\n",nodename,label,get_nodename(n))
    else
      ret = string.format("%s:%s -> %s:title\n",nodename,label,get_nodename(n)) 
    end
    return ret
  end
end

local function label(n,tab )
  local typ = node.type(n.id)
  local nodename = get_nodename(n)
  local ret = string.format("%s [ label = \"<title> %s | { <prev> prev |<next> next }",nodename,typ)
  if tab then
    for i=1,#tab do
      ret = ret .. string.format("|<%s> %s",tab[i][1],tab[i][2])
    end
  end
  return ret .. "\"]\n"
end

local function draw_node( n,tab )
  local ret = {}
  local nodename = get_nodename(n)
  ret[#ret + 1] = label(n,tab)
  ret[#ret + 1] = link_to(n.next,nodename,"next")
  ret[#ret + 1] = link_to(n.prev,nodename,"prev")
  return table.concat(ret)
end

local function dot_analyze_nodelist( head )
  local ret = {}
  local typ,nodename
	while head do
	  typ = node.type(head.id)
	  nodename = get_nodename(head)
    
  	if typ == "hlist" then
      ret[#ret + 1] = draw_node(head,{ {"list", "list"}, {"shift", string.format("shift: %d", head.shift)} })
  	  if head.list then
	      ret[#ret + 1] = link_to(head.list,nodename,"list")
  	    ret[#ret + 1] = dot_analyze_nodelist(head.list)
  	  end
  	elseif typ == "vlist" then
      ret[#ret + 1] = draw_node(head,{ {"list", "list"} })
  	  if head.list then
	      ret[#ret + 1] = link_to(head.list,nodename,"list")
  	    ret[#ret + 1] = dot_analyze_nodelist(head.list)
  	  end
  	elseif typ == "glue" then
  	  local subtype = string.format("%s",glue_type[head.subtype])
  	  local spec = string.format("%gpt", head.spec.width / 2^16)
  	  if head.spec.stretch ~= 0 then
  	    spec = spec .. string.format(" plus %g fi%s", head.spec.stretch  / 2^16, string.rep("l",head.spec.stretch_order))
  	  end
  	  if head.spec.shrink ~= 0 then
  	    spec = spec .. string.format(" minus %g fi%s",head.spec.shrink  / 2^16, string.rep("l",head.spec.shrink_order))
  	  end
      ret[#ret + 1] = draw_node(head,{ {"subtype", subtype},{"spec",spec} })
  	elseif typ == "kern" then
      ret[#ret + 1] = draw_node(head,{ {"kern", string.format("kern: %gpt",head.kern / 2^16) } })
  	elseif typ == "whatsit" then
      ret[#ret + 1] = draw_node(head,{ {"type", node.whatsits()[head.subtype] } })
  	elseif typ == "glyph" then
      ret[#ret + 1] = draw_node(head,{ {"char", string.format("char: %s",unicode.utf8.char(head.char))},{"lang",string.format("lang: %d",head.lang)} })
	  else
      ret[#ret + 1] = draw_node(head)
    end
    
    head = head.next
	end
  return table.concat(ret)
end


function nodelist_visualize( box,filename )
  assert(box,"No box given")
  assert(filename,"No filename given")

  local gv = dot_analyze_nodelist(tex.box[box])
  local outfile = io.open(filename,"w")
  outfile:write([[
digraph g {
graph [
rankdir = "LR"
];
node [ shape = "record"]
]])
  outfile:write(gv)
  outfile:write("}\n")
  outfile:close()
end

An issue with hlist and vlist nodes. Edges departing from the field labelled list always head south, while in most cases they should head east, like, e.g., the edges departing from next fields do.

An issue with hlist and vlist nodes. Field list should be renamed to head. To quote from the LuaTeX manual:

Note: the new field name head was introduced in 0.65 to replace the old name list. Use of the name list is now deprecated, but it will stay available until at least version 0.80.

I missed your comment, sorry. I will rename head/list and have a look at the edges departing from list/head field. Thank you very much for your comment!

An RFE this time: In user-defined whatsit nodes, would it be possible to show fields user_id, type and value?

Thanks @hennigs, I've finally implemented your suggestions.

To limit output for quickly checking a node's neighbours max_prev, max_next and max_head options would be useful, i.e. called on node n with max_head=1 would draw n and n.head but not n.head.head, but instead a dangling arrow.

I have also witten a package which visualizes the node lists. My package nodetree tries to solve the challenge to visualize the node structure in a different approach. https://www.ctan.org/pkg/nodetree

Here is an example of the output of the file:

I tried the code on the AtBeginShipoutBox with luatex 1.07 from texlive 2018 pretest:

\documentclass{article}
\usepackage{atbegshi,lipsum}
\AtBeginShipout {%
 \directlua{require("viznodelist")
  viznodelist.nodelist_visualize(tex.box["AtBeginShipoutBox"],"mybox.gv")}}
  \pagestyle{empty}
\begin{document}
abc

\end{document}

When I try to convert this to a pdf with dot.exe I get a warning

Warning: node n80, port title unrecognized
Warning: node n80, port title unrecognized

and there is a mysterious n80 node in the middle:

I can confirm this (Linux, TeXLive 2017, luatex 1.0.4). I get

Warning: node n78, port title unrecognized

...Rolf

Another issue with current TeXLive (2018, luatex 1.09.0). If I compile ulrikes (u-fischer) example above I see the following error:

./viznodelist.lua:124: attempt to index a nil value (field 'spe
c')
stack traceback:
        ./viznodelist.lua:124: in upvalue 'dot_analyze_nodelist'
        ./viznodelist.lua:120: in upvalue 'dot_analyze_nodelist'
        ./viznodelist.lua:152: in function 'viznodelist.nodelist_visualize'
        [\directlua]:1: in main chunk.
<argument> ...list_visualize("AtBeginShipoutBox","mybox.gv")}

l.10 \end{document}

It would be nice to see a graphic of the ShipoutBox. Thanks in advance.

@u-fischer I cannot reproduce this problem with my installation of tl2018.
@rolfn I don't have LuaTeX 1.09 at the moment (& I don't know how to upgrade)

Both: sorry for the late reply, I somehow don't get a notice when there are new comments here.

@pgundlach: I still get the node message:

\dot.exe -Tpdf mybox.gv -o dot.pdf
Warning: node n80, port title unrecognized
Warning: node n80, port title unrecognized

Regarding the luatex1.09 error: As it uses lua5.3 you should probably replace glue_set %d by glue_set %.0f (twice).

@pgundlach: Fails with a lua error when HarfBuzz renderer is used instead of Node renderer for the following short test-case (Noto Sans Devanagari is available from google fonts GitHub here, though I guess any Devanagari font should do). I encountered this same error in my code while playing with char fields in glyph nodes, someone posted a solution (which I don't really understand) on stack exchange. Please have a look: HarfBuzz UTF-8 issue as it causes visnodelist.lua to produce error. Btw, thanks for the script, graphical visualization helps debug otherwise inhumane tex debug experience. @Josef-Friedrich I filed a bug on your package nodetree too, it seems we all are affected by the same issue :)

\documentclass{article}
\usepackage[lmargin=0.5in,tmargin=0.5in,rmargin=0.5in,bmargin=0.5in]{geometry}
\usepackage{fontspec}

%\newfontscript{Devanagari}{deva,dev2}
\newfontfamily{\devanagarifam}{Noto Sans Devanagari}[Script=Devanagari, Scale=1, Renderer=HarfBuzz]

\begin{document}


\setbox0=\hbox{Hey \devanagarifam एक गांव -- में मोहन} % The error happens on compound glyph: 'में'. As a slightly longer test case, replace contents of box 0 with: Příliš žluťoučký \textit{kůň} úpěl \hbox{ďábelské} ódy difference diffierence. \devanagarifam एक गांव -- में मोहन नाम का लड़का रहता था। उसके पिताजी एक मामूली मजदूर थे।

\directlua{
	require("viznodelist")
	viznodelist.nodelist_visualize(0,"mybox.gv")
}

\box0

\end{document}

Error:

./viznodelist.lua:332: bad argument #1 to 'char' (invalid value)
stack traceback:
	[C]: in field 'char'
	./viznodelist.lua:332: in upvalue 'dot_analyze_nodelist'
	./viznodelist.lua:226: in upvalue 'dot_analyze_nodelist'
	./viznodelist.lua:421: in function 'viznodelist.nodelist_visualize'
	[\directlua]:1: in main chunk.
l.16 }

I have also witten a package which visualizes the node lists. My package nodetree tries to solve the challenge to visualize the node structure in a different approach. https://www.ctan.org/pkg/nodetree

@Josef-Friedrich I really like and use nodetree a lot! Thank you for sharing this (I know this is an old message)

@reportaman Thank you for the bug report. I now present everything I can't display as U+FFFD REPLACEMENT CHARACTER (�). Not sure if this is the best way to do, but better than an error.

@pgundlach: Its not the best way to do it as the characters used in the test case do actually have unicode code-points. They could be hidden in the way LuaHBTeX uses 'char' field of glyph nodes. I think they are using it as a 'string' now? I have dissected the problem further in my comment on same bug filed on package nodetree here. Please review, and let us know what you think.

@reportaman I'll have a look if I can use get_glyph_info. Since viznodelist is used outside LaTeX/Luaotfload, I am not sure if I implement this.

@pgundlach: Thanks! You might also be interested in this question posted on TeX.SE, given you have a tool (Speedata publisher) with LuaTeX backbone: LuaTeX glyph reordering issue.