Draft nif handling code. Parses nif.xml and some Morrowing nif files

nif.ijs

NB. roughly patterened after http://rpmfind.net/linux/RPM/sourceforge/p/py/pyffi/OldFiles/PyFFI-0.0-1.noarch.html
NB. you should not expect to read this any faster than you read the whole of PyFFI
NB. and if you are new to J you should also expect to spend some time learning the language
NB. recommendation: take breaks occasionally, play with this, try to make it fun

require '~user/nifxml.ijs'
coinsert 'nifxml' NB. for cond expressions
NB. FIXME: do not inherit from nifxml - factor out the expr support
NB. FIXME: most of this nif.ijs should be factored out into readnif.ijs which inherits from nif.ijs

NB. extract crude definition from nif.xml 
readdef=: extract_nifxml_&'~user/nif.xml'

NB. cached variant, also boxes crue definition
getdef=:3 :0
  nams=. ;:^:(0=L.) y
  mask=. -. nams e. defNAMES
  vals=. readdef&.> mask#nams
  defNAMES=: defNAMES, mask#nams
  defVALUES=: defVALUES, vals
  (defNAMES i. nams) { defVALUES
)
defNAMES=: ''
defVALUES=: ''


NB. ----------------- scratch -------------------------------------------
NB. get name of every definition to complete the named crude definition
depends=: ''1 :(0 :0-.LF)
  ;:inv @~. @:(
     ,   [: ; (
         e.& (;:'inherit type niflibtype storage')@[ # ]
     )/ @|: @; @(
         1&{ , 1 {::each 2&{::
     ) @; ^:(1 < L.) @getdef&.>
   ) @;: ^:_
)

NB. get the leaf node definitions
NB. we must generate manual definitions for all of these
primitives=: (/: toupper)@: >@ ~.@ (#~ (1 = #@;:@depends&>))@ ;:@ depends
primitives=: (/: toupper)@: >@ ~.@ (;: #~ (<<'basic') = [: {.&.> getdef)@ depends

NB. clear out dev junk
erase 'read_'names''

lr=:3 :'5!:5 <''y'''
readinherit=:''1 :(0 :0-.LF)
  [ (] 1:@".@, '=.', lr@[)&>/
)

NB. from nif file name get J array
read_nif=:3 :0
  READNIF=: fread y
  assert. _1 ~: READNIF [ 'does file exist?'
  POS=: 0
  headerstring=. 40 read_char ''
  version=. read_int ''
  assert. version = Version_nifxml_
  numblocks=. read_int ''
  DATANIF=: (<(".&.> ,: ]) ;:'headerstring version numblocks'),:<'Header'
  for_block.i. numblocks do.
    DATANIF=: DATANIF,.(read_block block);block
  end.
)

read_block=: 3 :0
  type=. read_string''
  type build_reader
  ".'read_',type,''''''
)

build_reader=: 1 :0
  type=. m
  reader=. 'read_',type
  if. 3 = nc <reader do. reader return. end.
  'base overview detail'=. ,>getdef m
  select. base
    case. 'basic'    do. assert. 0-:'fail' [smoutput m
    case. 'compound' do. m build_compound
    case. 'enum'     do. m build_enum
    case. 'niobject' do. m build_compound
  end.
)

fixexpr=: fixcondexp@;
build_compound=:1 :0
  'base overview detail'=. ,>getdef m
  def=. '  r=. i.2 0',LF
  if. 'enum'-: base do.
  end.
  if. #inherit=. ;(#~ e.&(<'inherit'))~/|:overview do.
    def=. '  readinherit r=. read_',(inherit),'''''',LF
    inherit build_reader
  end.
  for_add. detail do.
    recipe=. |:1 {::,>add
    type=. ;(#~ e.&(<'type'))~/recipe
    type build_reader
    label=. ' '-.~;(#~ e.&(<'name'))~/recipe
    arr2=. (,'*'#~0<#) fixexpr (#~ e.&(<'arr2'))~/recipe
    arr1=. ,&arr2 (,'*'#~0<#) fixexpr (#~ e.&(<'arr1'))~/recipe
    if. #cond=. fixexpr (#~ e.&(<'cond'))~/recipe do.
      def=. def,'  r=. r,.(',label,'=. (',arr1,'{.',cond,') read_',type,''''');''',label,'''',LF
    else.
      arr=. ('(',')',~])^:('*'e.]) _1}.arr1
      def=. def,'  r=. r,.(',label,'=. ',arr,' read_',type,''''');''',label,'''',LF
    end.
  end.
  def=. def,':',LF,'  r=. x read_',m,' Repeated ''''',LF
  smoutput '''',m,''' defRD (3 :0)',LF,def,')',LF
  m defRD (3 :def)
)

Repeated=:1 :0
  u y
:
  select. {.x
    case. 0 do. i.0
    case. 1 do. u y
    case.   do. u&> x#<y
  end.
)

build_enum=:1 :0
  'base overview detail'=. ,>getdef m
  fetcher=. 'read_',;(#~ e.&(<'storage'))~/@|: overview
  enum=. >1{&>,detail
  values=. 0".>, (#~ e.&(<'value'))~/@|:"2 enum
  names=. , (#~ e.&(<'name'))~/@|:"2 enum
  data=. names values} a:#~1+>./values
  smoutput '''',m,''' defRD ((<;._2]0 :0) {~ ',fetcher,')',LF,(;data,each LF),')',LF
  m defRD ((names values} a:#~1+>./values) {~ fetcher~) 
)
    

NB. ------------- primitive defs ----------------
NB. assumed globals:
READNIF=: '' NB. is read or mapped from file
POS=: 0      NB. marks next unread byte

defRD=: 2 :0
  ('read_',m)=: v
)

NB. readers - left argument of dyad is vector length to read

'char' defRD (3 :0)
  (POS=:POS+1) ] READNIF{~POS
:
  (POS=:POS+x) ] READNIF{~POS+i.x
)
'byte' defRD (a.i.read_char)

ic=: 3!:4
'Flags' defRD (3 :0)
  (POS=:POS+2) ] {. 0 ic READNIF {~ POS+i.2
:
  (POS=:POS+2*x) ] 0 ic READNIF {~ POS+i.2*x
)


fc=: 3!:5
'float' defRD  (3 :0)
  (POS=:POS+4) ] {. _1 fc READNIF {~ POS+i.4
:
  (POS=:POS+4*x) ] _1 fc READNIF {~ POS+i.4*x
)

'IndexString' defRD [: NB. don't ask

'int' defRD (3 :0)
  (POS=:POS+4) ] {. _2 ic READNIF {~ POS+i.4
:
  (POS=:POS+4*x) ] _2 ic READNIF {~ POS+i.4*x
)

NB. bools have a dual existence:
NB. a true/false value (a count - 0 or 1 times)
NB. a literal value (an arbitrary number)
'bool' defRD ((,"0~*)@read_int)

'NiObject' defRD ((i.2 0)"_)

'Ptr' defRD read_int

'Ref' defRD read_int

'short' defRD (3 :0)
  (POS=:POS+2) ] {. _1 ic READNIF {~ POS+i.2
:
  (POS=:POS+2*x) ] _1 ic READNIF {~ POS+i.2*x
)

'ushort' defRD (3 :0)
  (POS=:POS+2) ] {. 0 ic READNIF {~ POS+i.2
:
  (POS=:POS+2*x) ] 0 ic READNIF {~ POS+i.2*x
)

'unsigned' defRD ((2^32) | read_int) 
'uint' defRD read_unsigned

NB. special compound (or other) defs
'SizedString' defRD (3 :0)
  length=. read_int ''
  value=. length read_char ''
:
  NB. boxing needed to preserve length of strings
  x <@read_SizedString Repeated ''
)
'string' defRD read_SizedString

'NiGeometryData' defRD (3 :0)
  readinherit r=. read_NiObject''
  r=. r,.(NumVertices=. read_ushort'');'NumVertices' NB. avoid NiPSysData bogosity
  r=. r,.(HasVertices=.  read_bool'');'HasVertices'
  r=. r,.(Vertices=. (NumVertices*{.HasVertices) read_Vector3'');'Vertices'
  r=. r,.(HasNormals=.  read_bool'');'HasNormals'
  r=. r,.(Normals=. (NumVertices*{.HasNormals) read_Vector3'');'Normals'
  r=. r,.(Center=.  read_Vector3'');'Center'
  r=. r,.(Radius=.  read_float'');'Radius'
  r=. r,.(HasVertexColors=.  read_bool'');'HasVertexColors'
  r=. r,.(VertexColors=. (NumVertices*{.HasVertexColors) read_Color4'');'VertexColors'
  r=. r,.(NumUVSets=.  read_ushort'');'NumUVSets'
  r=. r,.(HasUV=.  read_bool'');'HasUV'
  r=. r,.(UVSets=. (((NumUVSets bAnd 63) bOr (BSNumUVSets bAnd 1))*NumVertices) read_TexCoord'');'UVSets'
:
  r=. x read_NiGeometryData Repeated ''
)

NB. 'NiTriShapeData' defRD (3 :0)
NB.   readinherit r=. read_NiGeometryData''
NB.   r=. r,.(NumTriangles=.  read_ushort'');'NumTriangles'
NB.   r=. r,.(NumTrianglePoints=. read_uint'');'NumTrianglePoints'
NB.   r=. r,.(Triangles=. NumTriangles read_Triangle'');'Triangles'
NB.   r=. r,.(NumMatchGroups=. read_ushort'');'NumMatchGroups'
NB.   r=. r,.(MatchGroups=. NumMatchGroups read_MatchGroup'');'MatchGroups'
NB. :
NB.   r=. x read_NiTriShapeData Repeated ''
NB. )

NB. names would be: u v
'TexCoord' defRD ((2 read_float ]) Repeated)

NB. autogenerated version from nif.xml
NB. fails because of name conflict on 'r'
'Color3' defRD (3 :0)
  r=. i.2 0
  r=. r,.(r=. read_float'');'r'
  r=. r,.(g=. read_float'');'g'
  r=. r,.(b=. read_float'');'b'
:
  r=. x read_Color3 Repeated ''
)
'Color3' defRD ((3 read_float ]) Repeated)

NB. similar name conflict with r g b a
'Color4' defRD ((4 read_float ]) Repeated)

NB. efficient version
NB. labels would have been v1 v2 v3
'Triangle' defRD ((3 read_ushort ]) Repeated)


NB. efficient version
NB. labels would have been: x y z
'Vector3' defRD ((3 read_float ]) Repeated)

NB. efficient version
NB. labels would have been:
NB.    m11 m12 m13
NB.    m21 m22 m23
NB.    m31 m32 m33
'Matrix33' defRD ((3 3 |:@$ 9 read_float ]) Repeated)

NB. autogenerated version from nif.xml
'Matrix22' defRD (3 :0)
  r=. i.2 0
  r=. r,.(m11=. read_float'');'m11'
  r=. r,.(m21=. read_float'');'m21'
  r=. r,.(m12=. read_float'');'m12'
  r=. r,.(m22=. read_float'');'m22'
:
  r=. x read_Matrix22 Repeated ''
)
NB. efficient version
NB. labels would have been:
NB.    m11 m12
NB.    m21 m22
'Matrix22' defRD ((2 2 |:@$ 4 read_float ]) Repeated)

NB. work around pythonesque warts
HasUnknown2Texture=: 0
BSNumUVSets=: 0

NB. ------------------- cached copies of autogenerated code -----------
NB. included here for speed and/or illustration purposes

'NiObjectNET' defRD (3 :0)
  readinherit r=. read_NiObject''
  r=. r,.(Name=.  read_string'');'Name'
  r=. r,.(ExtraData=.  read_Ref'');'ExtraData'
  r=. r,.(Controller=.  read_Ref'');'Controller'
:
  r=. x read_NiObjectNET Repeated ''
)

'BoundingBox' defRD (3 :0)
  r=. i.2 0
  r=. r,.(UnknownInt=.  read_uint'');'UnknownInt'
  r=. r,.(Translation=.  read_Vector3'');'Translation'
  r=. r,.(Rotation=.  read_Matrix33'');'Rotation'
  r=. r,.(Radius=.  read_Vector3'');'Radius'
:
  r=. x read_BoundingBox Repeated ''
)

'NiAVObject' defRD (3 :0)
  readinherit r=. read_NiObjectNET''
  r=. r,.(Flags=.  read_Flags'');'Flags'
  r=. r,.(Translation=.  read_Vector3'');'Translation'
  r=. r,.(Rotation=.  read_Matrix33'');'Rotation'
  r=. r,.(Scale=.  read_float'');'Scale'
  r=. r,.(Velocity=.  read_Vector3'');'Velocity'
  r=. r,.(NumProperties=.  read_uint'');'NumProperties'
  r=. r,.(Properties=. (NumProperties) read_Ref'');'Properties'
  r=. r,.(HasBoundingBox=.  read_bool'');'HasBoundingBox'
  r=. r,.(BoundingBox=. ({.(HasBoundingBox)) read_BoundingBox'');'BoundingBox'
:
  r=. x read_NiAVObject Repeated ''
)

'NiNode' defRD (3 :0)
  readinherit r=. read_NiAVObject''
  r=. r,.(NumChildren=.  read_uint'');'NumChildren'
  r=. r,.(Children=. (NumChildren) read_Ref'');'Children'
  r=. r,.(NumEffects=.  read_uint'');'NumEffects'
  r=. r,.(Effects=. (NumEffects) read_Ref'');'Effects'
:
  r=. x read_NiNode Repeated ''
)

'NiGeometry' defRD (3 :0)
  readinherit r=. read_NiAVObject''
  r=. r,.(Data=.  read_Ref'');'Data'
  r=. r,.(SkinInstance=.  read_Ref'');'SkinInstance'
:
  r=. x read_NiGeometry Repeated ''
)

'NiTriBasedGeom' defRD (3 :0)
  readinherit r=. read_NiGeometry''
:
  r=. x read_NiTriBasedGeom Repeated ''
)

'NiTriShape' defRD (3 :0)
  readinherit r=. read_NiTriBasedGeom''
:
  r=. x read_NiTriShape Repeated ''
)

'NiProperty' defRD (3 :0)
  readinherit r=. read_NiObjectNET''
:
  r=. x read_NiProperty Repeated ''
)

'ApplyMode' defRD ((<;._2]0 :0) {~ read_uint)
APPLY_REPLACE
APPLY_DECAL
APPLY_MODULATE
APPLY_HILIGHT
APPLY_HILIGHT2
)

'TexClampMode' defRD ((<;._2]0 :0) {~ read_uint)
CLAMP_S_CLAMP_T
CLAMP_S_WRAP_T
WRAP_S_CLAMP_T
WRAP_S_WRAP_T
)


NB. ------------------ test on load

0 0 $ read_nif '~user\furniture\unpack\Furn_OrcLC_Table01.nif'
NB. needs file from http://www.nexusmods.com/morrowind/mods/42513/

nifxml.ijs

require 'xml/sax/x2j regex'

x2jclass 'nifxml'

extract=:4 :0
  Name=: x
  process fread y
)

NB. nif versioning:
vton=: 256 #. '.' 0&".;._1@, ]
Version=: vton '4.0.0.2' NB. Morrowind
UserVersion=: 0
UserVersion2=: 0

'Items' x2jDefn NB. dispatch xml event handlers
  /        := Result : Result=: ''
  compound := defEnd y : x defStart y
  basic    := defEnd y : x defStart y
  bitflags := defEnd y : x defStart y
  enum     := defEnd y : x defStart y
  niobject := defEnd y : x defStart y
  add      := aEnd y : x aStart y
  add      := aChr y
  option   := aEnd y : x aStart y
  option   := aChr y
)

NB. --------- "low level" implementation --------
Interesting=: 0

ver1=: (>: vton)`1:@.(-:&_1@]) atr bind 'ver1' 
ver2=: (<: vton)`1:@.(-:&_1@]) atr bind 'ver2'

NB. ------- "glue" ------------------------------
defStart=:4 :0 NB. event handlers for <compound>
  if. Name -: atr 'name' do.
    Interesting=: 1
  end.
  if. Interesting do.
    Element=:    y;<attributes x
    Attributes=: i.0 1
  end.
)
defEnd=:3 :0
  if. Interesting do.
    Result=: Result,Element,<Attributes
  end.
  Interesting=: 0
)

NB. translate a python expression to a J expression
NB. this is necessarily a heuristic with limited power
NB. FIXME: this really belongs in its own locale so it can be inherited without dragging along everything in nifxml
bAnd=: 17 b.
bOr=:  23 b.
verpat=: rxcomp '\d+\.\d+\.\d+\.\d+'
fixcondexp=:3 :0
  expr=. ('<=';'<:';'>=';'>:';'==';'=';'||';'+.';'&&';'*.';'!';'-.';'&';' bAnd ';'|';' bOr ') stringreplace y -.' '
  ('(',')',~])^:(0<#) verpat ('(vton ''',''')',~]) rxapply expr
)

validVer=:3 :0
  if. _1-: y do. 1 return.end.
  expr=. fixcondexp y
  ".expr
)

aStart=:4 :0  NB. event handlers for <add>
  if. WasInteresting=: Interesting do.
    if. Interesting=: (ver1 * ver2) Version do.
      if. Interesting=: validVer atr 'vercond' do.
        if. Interesting=: _1=atr 'userver' do.
          Attribute=:    y;<attributes x
          Comment=: ''
        end.
      end.
    end.
  end.
)
aEnd=:3 :0
  if. Interesting do.
    Attributes=: Attributes,<Attribute,<Comment
  end.
  Interesting=: WasInteresting
)
aChr=:3 :0
  if. Interesting do.
    Comment=: Comment,y
  end.
)

This generates a file named DATANIF which represents the data from the file. In the furniture example (loaded at the bottom of the first .ijs file), I get:

$DATANIF
2 12

$":DATANIF
440 2675

So this is a small example. The code a bit crude in spots, but I much prefer refining working code than I do refining non-working code.

Next, I need to do some refactoring and also write the file back out. If I do everything right (according to my concept of "right") the generated file will identically match the original file. If I do it wrong and have a lossy transform I will consider that an error.

And I have a variety of further stages also planned. This file handling stuff is just preparation for some later tools.