TokenClass = enum
  gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber, gtOctNumber,
  gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit, gtLongStringLit, gtCharLit,
  gtEscapeSequence, gtOperator, gtPunctuation, gtComment, gtLongComment,
  gtRegularExpression, gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler,
  gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel, gtReference,
  gtOther

GeneralTokenizer = object of RootObj
  kind*: TokenClass
  start*, length*: int
  buf: cstring
  pos: int
  state: TokenClass

SourceLanguage = enum
  langNone, langNim, langCpp, langCsharp, langC, langJava, langYaml

sourceLanguageToStr: array[SourceLanguage, string] = ["none", "Nim", "C++", "C#", "C",
    "Java", "Yaml"]

tokenClassToStr: array[TokenClass, string] = ["Eof", "None", "Whitespace", "DecNumber",
    "BinNumber", "HexNumber", "OctNumber", "FloatNumber", "Identifier", "Keyword",
    "StringLit", "LongStringLit", "CharLit", "EscapeSequence", "Operator",
    "Punctuation", "Comment", "LongComment", "RegularExpression", "TagStart",
    "TagEnd", "Key", "Value", "RawData", "Assembler", "Preprocessor", "Directive",
    "Command", "Rule", "Hyperlink", "Label", "Reference", "Other"]

proc getSourceLanguage(name: string): SourceLanguage {...}{.raises: [], tags: [].}

proc initGeneralTokenizer(g: var GeneralTokenizer; buf: cstring) {...}{.raises: [], tags: [].}

proc initGeneralTokenizer(g: var GeneralTokenizer; buf: string) {...}{.raises: [], tags: [].}

proc deinitGeneralTokenizer(g: var GeneralTokenizer) {...}{.raises: [], tags: [].}

proc getNextToken(g: var GeneralTokenizer; lang: SourceLanguage) {...}{.raises: [], tags: [].}