Source highlighter for programming or markup languages. Currently only few languages are supported, other languages may be added. The interface supports one language nested in another.

Note: Import packages/docutils/highlite to use this module

You can use this to build your own syntax highlighting, check this example:

let code = """for x in $int.high: echo x.ord mod 2 == 0"""
var toknizr: GeneralTokenizer
initGeneralTokenizer(toknizr, code)
while true:
  getNextToken(toknizr, langNim)
  case toknizr.kind
  of gtEof: break  # End Of File (or string)
  of gtWhitespace:
    echo gtWhitespace # Maybe you want "visible" whitespaces?.
    echo substr(code, toknizr.start, toknizr.length + toknizr.start - 1)
  of gtOperator:
    echo gtOperator # Maybe you want Operators to use a specific color?.
    echo substr(code, toknizr.start, toknizr.length + toknizr.start - 1)
  # of gtSomeSymbol: syntaxHighlight("Comic Sans", "bold", "99px", "pink")
  else:
    echo toknizr.kind # All the kinds of tokens can be processed here.
    echo substr(code, toknizr.start, toknizr.length + toknizr.start - 1)

The proc getSourceLanguage can get the language enum from a string:

for l in ["C", "c++", "jAvA", "Nim", "c#"]: echo getSourceLanguage(l)

Types

TokenClass = enum gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber, gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit, gtLongStringLit, gtCharLit, gtEscapeSequence, gtOperator, gtPunctuation, gtComment, gtLongComment, gtRegularExpression, gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler, gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel, gtReference, gtOther: Source Edit
GeneralTokenizer = object of RootObj kind*: TokenClass start*, length*: int buf: cstring pos: int state: TokenClass: Source Edit
SourceLanguage = enum langNone, langNim, langCpp, langCsharp, langC, langJava, langYaml: Source Edit

Consts

sourceLanguageToStr: array[SourceLanguage, string] = ["none", "Nim", "C++", "C#", "C", "Java", "Yaml"]: Source Edit
tokenClassToStr: array[TokenClass, string] = ["Eof", "None", "Whitespace", "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", "EscapeSequence", "Operator", "Punctuation", "Comment", "LongComment", "RegularExpression", "TagStart", "TagEnd", "Key", "Value", "RawData", "Assembler", "Preprocessor", "Directive", "Command", "Rule", "Hyperlink", "Label", "Reference", "Other"]: Source Edit

Procs

proc getSourceLanguage(name: string): SourceLanguage {...}{.raises: [], tags: [].}: Source Edit
proc initGeneralTokenizer(g: var GeneralTokenizer; buf: cstring) {...}{.raises: [], tags: [].}: Source Edit
proc initGeneralTokenizer(g: var GeneralTokenizer; buf: string) {...}{.raises: [], tags: [].}: Source Edit
proc deinitGeneralTokenizer(g: var GeneralTokenizer) {...}{.raises: [], tags: [].}: Source Edit
proc getNextToken(g: var GeneralTokenizer; lang: SourceLanguage) {...}{.raises: [], tags: [].}: Source Edit