ErrorHandler = proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind;
                     arg: string)

Lexer = object of TBaseLexer
  fileIdx*: FileIndex
  indentAhead*: int
  currLineIndent*: int
  strongSpaces*, allowTabs*: bool
  errorHandler*: ErrorHandler
  cache*: IdentCache
  when defined(nimsuggest):
      tokenEnd*: TLineInfo
      previousTokenEnd*: TLineInfo

  config*: ConfigRef

NumericalBase = enum
  base10, base2, base8, base16

Token = object
  tokType*: TokType
  base*: NumericalBase
  spacing*: set[TokenSpacing]
  indent*: int
  ident*: PIdent
  iNumber*: BiggestInt
  fNumber*: BiggestFloat
  literal*: string
  line*, col*: int
  when defined(nimpretty):
      offsetA*, offsetB*: int
      commentOffsetA*, commentOffsetB*: int

  

TokenSpacing = enum
  tsLeading, tsTrailing, tsEof

TokType = enum
  tkInvalid = "tkInvalid", tkEof = "[EOF]", tkSymbol = "tkSymbol",
  tkAddr = "addr", tkAnd = "and", tkAs = "as", tkAsm = "asm", tkBind = "bind",
  tkBlock = "block", tkBreak = "break", tkCase = "case", tkCast = "cast",
  tkConcept = "concept", tkConst = "const", tkContinue = "continue",
  tkConverter = "converter", tkDefer = "defer", tkDiscard = "discard",
  tkDistinct = "distinct", tkDiv = "div", tkDo = "do", tkElif = "elif",
  tkElse = "else", tkEnd = "end", tkEnum = "enum", tkExcept = "except",
  tkExport = "export", tkFinally = "finally", tkFor = "for", tkFrom = "from",
  tkFunc = "func", tkIf = "if", tkImport = "import", tkIn = "in",
  tkInclude = "include", tkInterface = "interface", tkIs = "is",
  tkIsnot = "isnot", tkIterator = "iterator", tkLet = "let", tkMacro = "macro",
  tkMethod = "method", tkMixin = "mixin", tkMod = "mod", tkNil = "nil",
  tkNot = "not", tkNotin = "notin", tkObject = "object", tkOf = "of",
  tkOr = "or", tkOut = "out", tkProc = "proc", tkPtr = "ptr", tkRaise = "raise",
  tkRef = "ref", tkReturn = "return", tkShl = "shl", tkShr = "shr",
  tkStatic = "static", tkTemplate = "template", tkTry = "try",
  tkTuple = "tuple", tkType = "type", tkUsing = "using", tkVar = "var",
  tkWhen = "when", tkWhile = "while", tkXor = "xor", tkYield = "yield",
  tkIntLit = "tkIntLit", tkInt8Lit = "tkInt8Lit", tkInt16Lit = "tkInt16Lit",
  tkInt32Lit = "tkInt32Lit", tkInt64Lit = "tkInt64Lit", tkUIntLit = "tkUIntLit",
  tkUInt8Lit = "tkUInt8Lit", tkUInt16Lit = "tkUInt16Lit",
  tkUInt32Lit = "tkUInt32Lit", tkUInt64Lit = "tkUInt64Lit",
  tkFloatLit = "tkFloatLit", tkFloat32Lit = "tkFloat32Lit",
  tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit",
  tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit",
  tkTripleStrLit = "tkTripleStrLit", tkGStrLit = "tkGStrLit",
  tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit",
  tkCustomLit = "tkCustomLit", tkParLe = "(", tkParRi = ")", tkBracketLe = "[",
  tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}", tkBracketDotLe = "[.",
  tkBracketDotRi = ".]", tkCurlyDotLe = "{.", tkCurlyDotRi = ".}",
  tkParDotLe = "(.", tkParDotRi = ".)", tkComma = ",", tkSemiColon = ";",
  tkColon = ":", tkColonColon = "::", tkEquals = "=", tkDot = ".",
  tkDotDot = "..", tkBracketLeColon = "[:", tkOpr, tkComment, tkAccent = "`",
  tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr, tkHideableStart,
  tkHideableEnd

TokTypes = set[TokType]

numChars: set[char] = {'0'..'9', 'a'..'z', 'A'..'Z'}

OpChars: set[char] = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
                      '|', '=', '%', '&', '$', '@', '~', ':'}

SymChars: set[char] = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'}

SymStartChars: set[char] = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}

tokKeywordHigh = tkYield

tokKeywordLow = tkAddr