A Scalariform based Lexer for RSyntaxTextArea

ScalariformTokenMaker.scala

/*
 * Copyright (C) 2012 Lalit Pant <pant.lalit@gmail.com>
 *
 * The contents of this file are subject to the GNU General Public License
 * Version 3 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of
 * the License at http://www.gnu.org/copyleft/gpl.html
 *
 * Software distributed under the License is distributed on an "AS
 * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * rights and limitations under the License.
 *
 */

package net.kogics.kojo.lexer

import org.fife.ui.rsyntaxtextarea.AbstractTokenMaker
import org.fife.ui.rsyntaxtextarea.Token
import org.fife.ui.rsyntaxtextarea.TokenMap
import org.fife.ui.rsyntaxtextarea.TokenTypes

import javax.swing.text.Segment
import scalariform.lexer.ScalaLexer
import scalariform.lexer.{ Token => SfToken }
import scalariform.lexer.TokenType
import scalariform.lexer.Tokens

class ScalariformTokenMaker extends AbstractTokenMaker {

  override def getCurlyBracesDenoteCodeBlocks = true

  override def getMarkOccurrencesOfTokenType(tpe: Int) =
    tpe == TokenTypes.IDENTIFIER || tpe == TokenTypes.FUNCTION

  override def getShouldIndentNextLineAfter(t: Token) = {
    if (t != null && t.textCount == 1) {
      val ch = t.text(t.textOffset)
      ch == '{' || ch == '('
    }
    else {
      false
    }
  }

  override def getTokenList(pSegment: Segment, pInitialTokenType: Int, pSegmentOffset: Int): Token = {
    throw new UnsupportedOperationException
  }

  override def getLastTokenTypeOnLine(text: Segment, initialTokenType: Int) = {
    TokenTypes.NULL
  }

  def addTokenHook(array: Array[Char], start: Int, end: Int, tokenType: Int, startOffset: Int) {
    addToken(array, start, end, tokenType, startOffset)
  }

  override def getTokenList2(segment: Segment, initialTokenType: Int, segmentOffset: Int, doc: String): Token = {

    def isLastMultiline(ts: List[SfToken]) = ts match {
      case Nil => false
      case ts2 =>
        val t = ts2.last
        t.tokenType match {
          case Tokens.WS =>
            if (t.rawText(0) == '\n' && t.rawText.size > 1) true else false
          case Tokens.STRING_LITERAL =>
            if (t.rawText.startsWith("\"\"\"") && t.rawText.contains("\n")) true else false
          case _ =>
            if (t.rawText.contains("\n")) true else false
        }
    }

    def splitLast(t: SfToken) = {
      val delta = segmentOffset - t.offset
      SfToken(t.tokenType, t.rawText.slice(delta, t.rawText.length), t.offset + delta, t.rawText.slice(delta, t.rawText.length))
    }

    def addRstaToken(t: SfToken) {
      val tRSTATokenStart = t.offset + segment.offset - segmentOffset
      val tRSTATokenEnd = tRSTATokenStart + t.length - 1
      val tRSTATokenOffset = t.offset
      addTokenHook(segment.array, tRSTATokenStart, tRSTATokenEnd, convertTokenType(t.tokenType), tRSTATokenOffset)
    }

    val tokens = ScalaLexer.rawTokenise(doc, true)
    val (active, inactive) = tokens.partition { t => t.offset >= segmentOffset }

    resetTokenList()
    
    if (isLastMultiline(inactive)) {
      val t2 = splitLast(inactive.last)
      addRstaToken(t2)
    }

    val active2 = if (active.size > 1) active.take(active.size - 1) else active
    active2.foreach { addRstaToken }
    firstToken
  }

  override def getWordsToHighlight: TokenMap = wordsToHighlight

  def convertTokenType(sfType: TokenType): Int = {
    if (Tokens.KEYWORDS.contains(sfType)) {
      TokenTypes.RESERVED_WORD
    }
    else if (Tokens.COMMENTS.contains(sfType)) {
      TokenTypes.COMMENT_MULTILINE
    }
    else {
      sfType match {
        case Tokens.WS                         => TokenTypes.WHITESPACE
        case Tokens.CHARACTER_LITERAL          => TokenTypes.LITERAL_CHAR
        case Tokens.INTEGER_LITERAL            => TokenTypes.LITERAL_NUMBER_DECIMAL_INT
        case Tokens.FLOATING_POINT_LITERAL     => TokenTypes.LITERAL_NUMBER_FLOAT
        case Tokens.STRING_LITERAL             => TokenTypes.LITERAL_STRING_DOUBLE_QUOTE
        case Tokens.STRING_PART                => TokenTypes.LITERAL_CHAR
        case Tokens.SYMBOL_LITERAL             => TokenTypes.LITERAL_STRING_DOUBLE_QUOTE
        case Tokens.TRUE                       => TokenTypes.LITERAL_BOOLEAN
        case Tokens.FALSE                      => TokenTypes.LITERAL_BOOLEAN
        case Tokens.NULL                       => TokenTypes.LITERAL_CHAR
        case Tokens.EOF                        => TokenTypes.WHITESPACE
        case Tokens.LBRACE                     => TokenTypes.SEPARATOR
        case Tokens.RBRACE                     => TokenTypes.SEPARATOR
        case Tokens.LBRACKET                   => TokenTypes.SEPARATOR
        case Tokens.RBRACKET                   => TokenTypes.SEPARATOR
        case Tokens.LPAREN                     => TokenTypes.SEPARATOR
        case Tokens.RPAREN                     => TokenTypes.SEPARATOR

        case Tokens.XML_START_OPEN             => TokenTypes.MARKUP_TAG_DELIMITER
        case Tokens.XML_EMPTY_CLOSE            => TokenTypes.MARKUP_TAG_DELIMITER
        case Tokens.XML_TAG_CLOSE              => TokenTypes.MARKUP_TAG_DELIMITER
        case Tokens.XML_END_OPEN               => TokenTypes.MARKUP_TAG_DELIMITER
        case Tokens.XML_WHITESPACE             => TokenTypes.WHITESPACE
        case Tokens.XML_ATTR_EQ                => TokenTypes.MARKUP_TAG_ATTRIBUTE
        case Tokens.XML_ATTR_VALUE             => TokenTypes.MARKUP_TAG_ATTRIBUTE_VALUE
        case Tokens.XML_NAME                   => TokenTypes.MARKUP_TAG_NAME
        case Tokens.XML_PCDATA                 => TokenTypes.IDENTIFIER
        case Tokens.XML_COMMENT                => TokenTypes.COMMENT_MARKUP
        case Tokens.XML_CDATA                  => TokenTypes.MARKUP_CDATA
        case Tokens.XML_UNPARSED               => TokenTypes.MARKUP_CDATA
        case Tokens.XML_PROCESSING_INSTRUCTION => TokenTypes.MARKUP_PROCESSING_INSTRUCTION

        case _                                 => TokenTypes.IDENTIFIER
      }
    }
  }
}