CodeSnip Old Code (SVN)

{
 * UHilitePasLexer.pas
 *
 * Defines class that analyses and tokenises Pascal source code.
 *
 * $Rev$
 * $Date$
 *
 * ***** BEGIN LICENSE BLOCK *****
 *
 * Version: MPL 1.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with the
 * License. You may obtain a copy of the License at http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
 * the specific language governing rights and limitations under the License.
 *
 * The Original Code is UHilitePasLexer.pas
 *
 * The Initial Developer of the Original Code is Peter Johnson
 * (http://www.delphidabbler.com/).
 *
 * Portions created by the Initial Developer are Copyright (C) 2005-2010 Peter
 * Johnson. All Rights Reserved.
 *
 * Contributor(s)
 *   NONE
 *
 * ***** END LICENSE BLOCK *****
}


unit UHilitePasLexer;


interface


uses
  // Delphi
  Classes,
  // Project
  UTextStreamReader;


type

  {
  THilitePasToken:
    Tokens describing the different components of Pascal source code returned by
    the lexical analyser.
  }
  THilitePasToken = (
    tkKeyword,        // Pascal keyword
    tkComment,        // comment including opening and closing symbols
    tkCompilerDir,    // compiler directive including include comment symbols
    tkDirective,      // Pascal directive (Delphi 7)
    tkIdentifier,     // identifier: identifier that is not keyword or directive
    tkString,         // string literal including quotes
    tkChar,           // literal character "#" [Hex | Whole number ]
    tkNumber,         // integral whole number
    tkFloat,          // floating point number (may use 'E' notation)
    tkHex,            // hex digit "$" + { 0..9 | A..F }+
    tkSymbol,         // symbol (single or double character eg '=' and ':=')
    tkWhitespace,     // white space (spaces, tabs etc, excluding CR and LF)
    tkEOL,            // end of line (usually CRLF but CR and LF on own valid)
    tkEOF,            // end of file
    tkError           // error condition: shouldn't occur
  );

  {
  THilitePasLexer:
    Class that analyses and tokenises Pascal code.
  }
  THilitePasLexer = class(TObject)
  strict private
    fTokenStr: string;            // Text of last token read from input
    fToken: THilitePasToken;      // Kind of last token read from input
    fCommentState: record         // Records state of comment being processed
      InComment: Boolean;           // whether currently processing comment
      CommentType: THilitePasToken; // indicates comment or compiler directive
      CommentCloser: string;        // closing comment symbol ( *), } or EOL )
    end;
    fReader: TTextStreamReader;   // Object that reads characters from input
    procedure UpdateTokenStr; overload;
      {Appends current character in input to token string. Ignores EOF.
      }
    procedure UpdateTokenStr(const Ch: Char); overload;
      {Appends a character to token string. Ignores EOF.
        @param Ch [in] Character to append.
      }
    function ParseChar: THilitePasToken;
      {Analyses a literal character (made from # followed by number) from input
      and stores in token string.
        @return Token indicating literal char (tkChar).
      }
    function ParseCommentFromStart: THilitePasToken;
      {Begins parsing of a new comment or compiler directive.
        @return Token telling whether this is a comment or compiler directive
          (tkComment, tkCompilerDir).
      }
    function ParseCommentInterior: THilitePasToken;
      {Analyses body of comment after start or after resuming processing multi-
      line comments.
        @return Token telling whether this is a comment or compiler directive
          (tkComment, tkCompilerDir).
      }
    function ParseEOL: THilitePasToken;
      {Analyses end of line from input and stores in token string.
        @return End of line token (tkEOL).
      }
    function ParseHex: THilitePasToken;
      {Analyses a hexadecimal integer from input and stores in token string.
        @return Token indicating hexadecimal value (tkHex).
      }
    function ParseIdent: THilitePasToken;
      {Analyses an alphanumeric identifier from input and stores in token
      string. Checks if identifier is keyword or directive.
        @return Token representing identifier: tkKeyword, tkDirective or
          tkIdentifier.
      }
    function ParseNumber: THilitePasToken;
      {Analyses a number from input and stores in token string. Number can be
      integer or real.
        @return Appropriate token for number (tkNumber or tkFloat).
      }
    function ParseString: THilitePasToken;
      {Analyses a string literal from input and stores in token string.
        @return String token (tkString).
      }
    function ParseSymbol: THilitePasToken;
      {Determines whether the current symbol character on input represents a
      symbol or introduces some other syntactic entity (i.e. comment, string,
      character literal or a hex number). Analyses the input accordingly and
      stores the whole token in the token string.
        @return Token describing entity parsed.
      }
    function ParseUnknown: THilitePasToken;
      {Analyses an unrecognised entity from input and adds it to token string.
        @return Error token (tkError).
      }
    function ParseWhiteSpace: THilitePasToken;
      {Analyses a sequence of white space from input and appends space for each
      white space character read to token string.
        @return White space token (tkWhiteSpace).
      }
    function ParseWholeNumber: THilitePasToken;
      {Analyses a whole number from input and appends to token string.
        @return Whole number token (tkNumber).
      }
  public
    constructor Create(const Stm: TStream);
      {Class constructor. Sets up object to analyse code on a stream.
        @param Stm [in] Stream containing Pascal source.
      }
    destructor Destroy; override;
      {Class destructor. Tears down object.
      }
    function NextToken: THilitePasToken;
      {Gets and analyses next pascal token from input and stores details in
      token string.
        @return Token identifier for type of token read.
      }
    property TokenStr: string read fTokenStr;
      {Text that makes up the token last read from input}
    property Token: THilitePasToken read fToken;
      {Kind of token last read from input}
  end;


implementation


uses
  // Delphi
  SysUtils, Windows {for inlining},
  // Project
  UConsts, UExceptions, UUnicodeHelper;


const

  // Character constants
  cDecimalPoint     = '.';
  cCompilerDirChar  = '$';
  cStringDelim      = '''';
  cCloseParen       = ')';
  cEOL = TTextStreamReader.EOL;
  cEOF = TTextStreamReader.EOF;

  // String tables
  cDoubleSyms: array[0..9] of string = (         // list of valid double symbols
    '(*', '(.', '*)', '.)', '..', '//', ':=', '<=', '>=', '<>'
  );
  cCommentOpeners: array[0..2] of string = (       // symbols that open comments
    '{', '(*', '//'
  );
  cCommentClosers: array[0..2] of string = (      // symbols that close comments
    // item at given index matches openers at same index
    '}', '*)', cEOL
  );
  cCompilerDirOpeners: array[0..1] of string = (    // comment symbols that open
    '{', '(*'                                             // compiler directives
  );
  cKeywords: array[0..68] of string = (     // table of keywords per Delphi 2006
    'and', 'array', 'as', 'asm',
    'begin',
    'case', 'class', 'const', 'constructor',
    'destructor', 'dispinterface', 'div', 'do', 'downto',
    'else', 'end', 'except', 'exports',
    'file', 'final', 'finalization', 'finally', 'for', 'function',
    'goto',
    'if', 'implementation', 'in', 'inherited',
    'initialization', 'inline', 'interface', 'is',
    'label', 'library',
    'mod',
    'nil', 'not',
    'object', 'of', 'or', 'out',
    'packed', 'procedure', 'program', 'property',
    'raise', 'record', 'repeat', 'resourcestring',
    'sealed', 'set', 'shl', 'shr', 'static', 'string',
    'then', 'threadvar', 'to', 'try', 'type',
    'unit', 'unsafe', 'until', 'uses',
    'var',
    'while', 'with',
    'xor'
  );

  cDirectives: array[0..45] of string = (                 // table of directives
    'absolute',     // used in variable declaration
    'abstract',     // method directive
    'assembly',     // flags routine as containing assembler
    'at',           // only occurs in raise statement
    'automated',    // used in class declarations
    'cdecl',        // calling convention
    'contains',     // package clause
    'default',      // used in property declarations
    'deprecated',   // portability directive
    'dispid',       // used in automated properties
    'dynamic',      // method directive
    'export',       // calling convention (ignored)
    'external',     // routine directive
    'far',          // calling convention (ignored)
    'forward',      // routine directive
    'implements',   // used in property declarations
    'index',        // used in property declarations and re DLLs
    'inline',       // flags a routine as inlinable
    'local',        // routine directive
    'message',      // method directive
    'name',         // used re DLLs
    'near',         // calling convention (ignored)
    'nodefault',    // used in property declarations
    'on',           // used in exception handlers
    'overload',     // method / routine directive
    'override',     // method directive
    'package',      // introduces a package
    'pascal',       // calling convention
    'platform',     // portability directive
    'private',      // used in class declarations
    'protected',    // used in class declarations
    'public',       // used in class declarations
    'published',    // used in class declarations
    'read',         // used in property declarations
    'readonly',     // property directive in dispinterfaces
    'register',     // calling convention
    'reintroduce',  // method directive
    'requires',     // package clause
    'resident',     // directive used in exports clauses (ignored)
    'safecall',     // calling convention
    'stdcall',      // calling convention
    'stored',       // used in property declarations
    'varargs',      // method / routine directive
    'virtual',      // method directive
    'write',        // used in property declarations
    'writeonly'     // property directive in dispinterfaces
  );

  // Maps symbols onto likely tokens or error if token shouldn't occur (eg
  // close comments).
  cSymToTokenMap: array[0..32] of record
    Symbol: string;           // symbol strings
    Token: THilitePasToken;   // related token
  end = (
    ( Symbol: '$';  Token: tkHex;     ),
    ( Symbol: '#';  Token: tkChar;    ),
    ( Symbol: '&';  Token: tkSymbol;  ),
    ( Symbol: ''''; Token: tkString;  ),
    ( Symbol: '(';  Token: tkSymbol;  ),
    ( Symbol: ')';  Token: tkSymbol;  ),
    ( Symbol: '*';  Token: tkSymbol;  ),
    ( Symbol: '+';  Token: tkSymbol;  ),
    ( Symbol: ',';  Token: tkSymbol;  ),
    ( Symbol: '-';  Token: tkSymbol;  ),
    ( Symbol: '.';  Token: tkSymbol;  ),
    ( Symbol: '/';  Token: tkSymbol;  ),
    ( Symbol: ':';  Token: tkSymbol;  ),
    ( Symbol: ';';  Token: tkSymbol;  ),
    ( Symbol: '<';  Token: tkSymbol;  ),
    ( Symbol: '=';  Token: tkSymbol;  ),
    ( Symbol: '>';  Token: tkSymbol;  ),
    ( Symbol: '@';  Token: tkSymbol;  ),
    ( Symbol: '[';  Token: tkSymbol;  ),
    ( Symbol: ']';  Token: tkSymbol;  ),
    ( Symbol: '^';  Token: tkSymbol;  ),
    ( Symbol: '{';  Token: tkComment; ),
    ( Symbol: '}';  Token: tkError;   ),
    ( Symbol: '(*'; Token: tkComment; ),
    ( Symbol: '*)'; Token: tkError;   ),
    ( Symbol: '(.'; Token: tkSymbol;  ),
    ( Symbol: '.)'; Token: tkSymbol;  ),
    ( Symbol: '..'; Token: tkSymbol;  ),
    ( Symbol: '//'; Token: tkComment; ),
    ( Symbol: ':='; Token: tkSymbol;  ),
    ( Symbol: '<='; Token: tkSymbol;  ),
    ( Symbol: '>='; Token: tkSymbol;  ),
    ( Symbol: '<>'; Token: tkSymbol;  )
  );


var
  // Private objects used to store and search lists of symbols and keywords
  pvtKeywords: TStringList = nil;   // keywords list
  pvtDirectives: TStringList = nil; // directives list
  pvtDoubleSyms: TStringList = nil; // list of double symbols
  pvtSymMap: TStringList = nil;     // map of symbols to tokens


{ Helper routines }

function IsValidIdentBodyChar(const C: Char): Boolean; inline;
  {Checks if a character is valid for inclusion in the body of a Delphi
  identifier, after the first character.
    @param C [in] Character to be tested.
    @return True if C is valid, False otherwise.
  }
begin
  Result := IsAlphaNumeric(C) or (C = '_');
end;

function IsValidIdentStartChar(const C: Char): Boolean; inline;
  {Checks if a character is a valid first character of a Delphi identifier.
    @param C [in] Character to be tested.
    @return True if C is valid, False otherwise.
  }
begin
  Result := IsLetter(C) or (C = '_');
end;

function IsWhiteSpaceChar(const C: Char): Boolean; inline;
  {Checks if a character is a whitespace character but not end of line or end
  of file character}
begin
  Result := IsWhiteSpace(C) and not IsCharInSet(C, [CR, LF, cEOF]);
end;

function IsSymbolChar(const C: Char): Boolean; inline;
  {Checks if a character is a symbol.
    @param C [in] Character to check.
    @return True if C is a symbol, False if not.
  }
const
  // valid symbols
  cSymbols = [
    '#', '$', '&', '''', '(', ')', '*', '+', ',', '-', '.',
    '/', ':', ';', '<', '=', '>', '@', '[', ']', '^', '{', '}'
  ];
begin
  Result := IsCharInSet(C, cSymbols);
end;

function IsExponentChar(const C: Char): Boolean; inline;
  {Checks if a character is an exponent.
    @param C [in] Character to check.
    @return True if C is an exponent, False if not.
  }
begin
  Result := IsCharInSet(C, ['E', 'e']);
end;

function IsUnaryPlusMinusChar(const C: Char): Boolean; inline;
  {Checks if a character is a unary plus or minus operator.
    @param C [in] Character to check.
    @return True if C is a unary plus or minus, False if not.
  }
begin
  Result := IsCharInSet(C, ['+', '-']);
end;

function IsSeparatorChar(const C: Char): Boolean; inline;
  {Checks if a character is a separator character.
    @param C [in] Character to check.
    @return True if C is a separator, False if not.
  }
begin
  Result := IsWhiteSpaceChar(C) or IsSymbolChar(C) or (C = cEOL);
end;

function IndexInTable(const Str: string; const Table: array of string): Integer;
  {Gets the index of a string in a table.
    @param Str [in] String to search for.
    @param Table [in] Table of strings to search.
    @return Index of string in table or -1 if string not in table.
  }
var
  I: Integer;   // loops thru table
begin
  // Note: calling code assumes Table is zero based
  Result := -1;
  for I := Low(Table) to High(Table) do
    if AnsiSameText(Table[I], Str) then
    begin
      Result := I;
      Break;
    end;
end;

procedure InitStringList(out Strings: TStringList;
  const Table: array of string);
  {Creates and initialises a sorted string list from a table of values.
    @param Strings [out] String list we create and initialise.
    @param Table [in] Table of strings to place in string list.
  }
var
  Idx: Integer; // loops thru rows of table
begin
  Strings := TStringList.Create;
  for Idx := Low(Table) to High(Table) do
    Strings.Add(Table[Idx]);
  Strings.Sorted := True;
  Strings.CaseSensitive := False;
end;

procedure InitSymbolMap(out Strings: TStringList);
  {Initialises string list object used to map valid symbols to tokens.
    @param Strings [out] String list we create and initialises.
  }
var
  I: Integer; // loops thru entries in symbol map constant table.
begin
  Strings := TStringList.Create;
  // Use Objects[] property to store tokens with related string
  for I := Low(cSymToTokenMap) to High(cSymToTokenMap) do
    Strings.AddObject(
      cSymToTokenMap[I].Symbol, TObject(cSymToTokenMap[I].Token)
    );
end;

function IsDoubleSym(const Symbol: string): Boolean;
  {Checks if a symbol is a valid double character symbol.
    @param Symbol [in] Symbol to check.
    @return True if symbol is valid double character symbol.
  }
begin
  if not Assigned(pvtDoubleSyms) then
    InitStringList(pvtDoubleSyms, cDoubleSyms);
  Result := pvtDoubleSyms.IndexOf(Symbol) >= 0;
end;

function IsDirective(const Ident: string): Boolean;
  {Checks if an identifier is a directive.
    @param Ident [in] Identifier to check.
    @return True if Ident is a directive, false otherwise.
  }
begin
  if not Assigned(pvtDirectives) then
    InitStringList(pvtDirectives, cDirectives);
  Result := pvtDirectives.IndexOf(Ident) >= 0;
end;

function IsKeyword(const Ident: string): Boolean;
  {Checks if an identifier is a keyword.
    @param Ident [in] Identifier to check.
    @return True if Ident is a keyword, false otherwise.
  }
begin
  if not Assigned(pvtKeywords) then
    InitStringList(pvtKeywords, cKeywords);
  Result := pvtKeywords.IndexOf(Ident) >= 0;
end;

function SymbolToToken(const Symbol: string): THilitePasToken;
  {Gets the likely token associated with a symbol.
    @param Symbol [in] Symbol to check
    @return Token associated with symbol.
  }
var
  Idx: Integer;   // index of symbol in map table
begin
  // Assumes Str is a symbol (single or double)
  if not Assigned(pvtSymMap) then
    InitSymbolMap(pvtSymMap);
  Idx := pvtSymMap.IndexOf(Symbol);
  if Idx >= 0 then
    Result := THilitePasToken(pvtSymMap.Objects[Idx])
  else
    Result := tkError;
end;

function MatchingCommentCloser(const CommentOpener: string): string;
  {Given a comment opening symbol gets the matching closing comment symbol.
    @param CommentOpener [in] Opening comment we need to match.
    @return Closing comment symbol.
  }
var
  Idx: Integer; // index of opening / closing symbols in table
begin
  // Note: this code assumes cCommentXXX arrays are zero based
  Idx := IndexInTable(CommentOpener, cCommentOpeners);
  Assert(Idx >= 0, 'MatchingCommentCloser: invalid comment opener');
  Result := cCommentClosers[Idx];
end;

function IsCompilerDirOpener(const Str: string): Boolean;
  {Checks if text is a comment opening symbol that is valid for a compiler
  directive.
    @param Str [in] String we are check is a compiler directive opening symbol.
    @return True if is a compiler directive opening symbol.
  }
begin
  Result := IndexInTable(Str, cCompilerDirOpeners) >= 0;
end;


{ THilitePasLexer }

constructor THilitePasLexer.Create(const Stm: TStream);
  {Class constructor. Sets up object to analyse code on a stream.
    @param Stm [in] Stream containing Pascal source.
  }
begin
  inherited Create;
  fReader := TTextStreamReader.Create(Stm);
end;

destructor THilitePasLexer.Destroy;
  {Class destructor. Tears down object.
  }
begin
  fReader.Free;
  inherited;
end;

function THilitePasLexer.NextToken: THilitePasToken;
  {Gets and analyses next Pascal token from input and stores details in token
  string.
    @return Token identifiing type of token read.
  }
begin
  // Reset token string
  fTokenStr := '';
  // Decide on method used to parse the token
  if not fCommentState.InComment then
  begin
    // We are not in a multi-line comment: process normally
    if IsWhiteSpaceChar(fReader.Ch) then
      Result := ParseWhiteSpace
    else if IsValidIdentStartChar(fReader.Ch) then
      Result := ParseIdent
    else if IsSymbolChar(fReader.Ch) then
      Result := ParseSymbol
    else if IsDigit(fReader.Ch) then
      Result := ParseNumber
    else if fReader.Ch = cEOL then
      Result := ParseEOL
    else if fReader.Ch = cEOF then
      Result := tkEOF
    else
      Result := ParseUnknown;
  end
  else
  begin
    // We're in a multiline comment: char is either from inside comment or EOL
    if fReader.Ch <> cEOL then
      Result := ParseCommentInterior
    else
      Result := ParseEOL;
  end;
  // Record the token
  fToken := Result;
end;

function THilitePasLexer.ParseChar: THilitePasToken;
  {Analyses a literal character (made from # followed by number) from input and
  stores in token string.
    @return Token indicating literal char (tkChar).
  }
begin
  // This method called with token string already containing '#' and current
  // char is char after '#'
  // Numeric part can either by whole number or hex number
  Result := tkChar;
  if SymbolToToken(fReader.Ch) = tkHex then
  begin
    // Hex number ('$' detected)
    // store '$' and skip to next
    UpdateTokenStr;
    fReader.NextChar;
    // now read hex digits
    ParseHex;
  end
  else if IsDigit(fReader.Ch) then
    // This is whole number: parse it
    ParseWholeNumber
  else
    // Not valid character: error token
    Result := tkError;
end;

function THilitePasLexer.ParseCommentFromStart: THilitePasToken;
  {Begins parsing of a new comment or compiler directive.
    @return Token telling whether this is a comment or compiler directive
      (tkComment, tkCompilerDir).
  }
begin
  // Token string contains comment opening symbol and current char is that which
  // follows opening symbol

  // Record information about the comment
  fCommentState.InComment := True;
  fCommentState.CommentCloser := MatchingCommentCloser(fTokenStr);
  // if char following opener is '$' we have compiler directive
  // (but only if comment opener is '{' or '(*' )
  if (fReader.Ch = cCompilerDirChar) and
    IsCompilerDirOpener(fTokenStr) then
    fCommentState.CommentType := tkCompilerDir
  else
    fCommentState.CommentType := tkComment;

  // Parse body of comment
  Result := ParseCommentInterior;
end;

function THilitePasLexer.ParseCommentInterior: THilitePasToken;
  {Analyses body of comment after start or after resuming processing multi-line
  comments.
    @return Token telling whether this is a comment or compiler directive
      (tkComment, tkCompilerDir).
  }
var
  Done: Boolean;  // flag true when we have finished comment
begin
  Assert(fCommentState.InComment,
    ClassName + '.ParseCommentInterior: called when not in comment');
  Assert(fCommentState.CommentType in [tkComment, tkCompilerDir],
    ClassName + '.ParseCommentInterior: invalid comment type');
  Assert(Length(fCommentState.CommentCloser) > 0,
    ClassName + '.ParseCommentInterior: invalid comment closer');

  Result := fCommentState.CommentType;

  // Loop thru all comment, looking for closing comment symbol
  Done := False;
  while (fReader.Ch <> cEOF) and not Done do
  begin
    if fReader.Ch = fCommentState.CommentCloser[1] then
    begin
      // We have encountered 1st char of a comment "closer"
      if Length(fCommentState.CommentCloser) = 1 then
      begin
        // Our closer is a single char: comment is closed
        Done := True;
        fCommentState.InComment := False;
        if fCommentState.CommentCloser[1] = cEOL then
          // closer is EOL: put it back to be read later
          fReader.PutBackChar
        else
          // closer not EOL: add it to token string
          UpdateTokenStr;
      end
      else
      begin
        // Our possible closer has two chars
        // Record first char in token string
        UpdateTokenStr;
        // Peek ahead at next char
        fReader.NextChar;
        if fReader.Ch = fCommentState.CommentCloser[2] then
        begin
          // This is the expected closer: comment is closed
          Done := True;
          fCommentState.InComment := False;
          UpdateTokenStr;
        end
        else
          // False alarm: put back the char we peeked at
          fReader.PutBackChar;
      end;
    end
    else
    begin
      // Ordinary comment text
      if fReader.Ch = cEOL then
      begin
        // EOL: put it back and stop parsing
        // the comment stays open: we will continue processing after EOL handled
        Done := True;
        fReader.PutBackChar;
      end
      else
        // Not EOL: add char to token string
        UpdateTokenStr;
    end;
    fReader.NextChar;
  end;
  // If at EOF ensure that comment is closed
  if fReader.Ch = cEOF then
    fCommentState.InComment := False;
end;

function THilitePasLexer.ParseEOL: THilitePasToken;
  {Analyses end of line from input and stores in token string.
    @return End of line token (tkEOL).
  }
begin
  UpdateTokenStr(cEOL);
  Result := tkEOL;
  fReader.NextChar;
end;

function THilitePasLexer.ParseHex: THilitePasToken;
  {Analyses a hexadecimal integer from input and stores in token string.
    @return Token indicating hexadecimal value (tkHex).
  }
begin
  // Called with fTokenStr = '$' and fReader.Ch with char after '$'
  // Build string of hex digits
  while IsHexDigit(fReader.Ch) do
  begin
    UpdateTokenStr;
    fReader.NextChar;
  end;
  // Check that we ended in a valid way: error if not
  if not IsSeparatorChar(fReader.Ch) then
    Result := tkError
  else
    Result := tkHex;
end;

function THilitePasLexer.ParseIdent: THilitePasToken;
  {Analyses an alphanumeric identifier from input and stores in token string.
  Checks if identifier is keyword or directive.
    @return Token representing identifier: tkKeyword, tkDirective or
      tkIdentifier.
  }
begin
  Assert(IsValidIdentStartChar(fReader.Ch),
    ClassName + '.ParseIdent: identifier starting character expected');
  // Build identifier in token string
  while IsValidIdentBodyChar(fReader.Ch) do
  begin
    UpdateTokenStr;
    fReader.NextChar;
  end;
  // Check if token is keyword or directive or is plain identifier
  if IsKeyword(fTokenStr) then
    Result := tkKeyword
  else if IsDirective(fTokenStr) then
    Result := tkDirective
  else
    Result := tkIdentifier;
end;

function THilitePasLexer.ParseNumber: THilitePasToken;
  {Analyses a number from input and stores in token string. Number can be
  integer or real.
    @return Appropriate token for number (tkNumber or tkFloat).
  }
var
  TempCh: Char; // temporary storage for a character read from input
begin
  Assert(IsDigit(fReader.Ch), ClassName + '.ParseNumber: digit expected');
  // All numbers start with a whole number: read it
  ParseWholeNumber; // leaves current char as one immediately after number
  // Assume we have whole number and see if we can disprove it
  Result := tkNumber;
  if fReader.Ch = cDecimalPoint then
  begin
    // Char after whole number is a decimal point: this *may* indicate a float,
    // but may not since there are other symbols that start with '.'
    // Store the decimal point then read ahead to see what next char is
    TempCh := fReader.Ch;
    fReader.NextChar;
    if IsCharInSet(fReader.Ch, [cDecimalPoint, cCloseParen]) then
    begin
      // decimal point was followed by '.' or ')' making valid two char symbols
      // .. and .) => we put back the read character and get out, leaving first
      // decimal point as current character and returning whole number
      fReader.PutBackChar;
      Exit;
    end;
    // Decimal point was valid: record in token string
    UpdateTokenStr(TempCh);
    // If we have digits after decimal point read them into token str
    // Note: there may not necessarily be digits after '.' (e.g. 2. is a valid
    // Delphi float)
    if IsDigit(fReader.Ch) then
      ParseWholeNumber;
    Result := tkFloat;
  end;
  if IsExponentChar(fReader.Ch) then
  begin
    // Next char is an exponent (e or E) that is present in numbers in
    // "scientific" notation. This can either follow whole number, follow
    // decimal point or follow digits after decimal point. I.e. 2e4, 2.e3 and
    // 2.0e4 are all valid, as is 2.0e-4 etc.
    // Record exponent in token string
    UpdateTokenStr;
    // Read chars after exponent (first may be unary + or -)
    fReader.NextChar;
    if IsUnaryPlusMinusChar(fReader.Ch) then
    begin
      UpdateTokenStr;
      fReader.NextChar;
    end;
    // Next comes whole number: get it
    ParseWholeNumber;
    Result := tkFloat;
  end;
end;

function THilitePasLexer.ParseString: THilitePasToken;
  {Analyses a string literal from input and stores in token string.
    @return String token (tkString).
  }
var
  Done: Boolean;  // flag true when done parsing string
begin
  // Note: token string already contains opening quote - current char is first
  // character of the string after the quote
  Done := False;
  // Loop thru characters until end of string found
  while (fReader.Ch <> cEOF) and not Done do
  begin
    UpdateTokenStr;
    if fReader.Ch = cStringDelim then
    begin
      // Could be closing quote or pair of quotes used to embed quote in string
      // we need to read ahead to check this
      fReader.NextChar;
      if fReader.Ch = cStringDelim then
        // this is a pair of quotes ('') => embeds quote in string => not done
        UpdateTokenStr
      else
      begin
        // not a pair of quotes => string completed
        Done := True;
        // put back char we read ahead
        fReader.PutBackChar;
      end;
    end;
    fReader.NextChar;
  end;
  Result := tkString;
end;

function THilitePasLexer.ParseSymbol: THilitePasToken;
  {Determines whether the current symbol character on input represents a symbol
  or introduces some other syntactic entity (i.e. comment, string, character
  literal or a hex number). Analyses the input accordingly and stores the whole
  token in the token string.
    @return Token describing entity parsed.
  }
var
  AToken: THilitePasToken; // token represented by the symbol
begin
  Assert(IsSymbolChar(fReader.Ch), ClassName + '.ParseSymbol: symbol expected');
  // Add character that starts symbol to token string and read next char
  UpdateTokenStr;
  fReader.NextChar;
  // Check if char read is second char of a two char symbol and process if so
  if IsSymbolChar(fReader.Ch) then
  begin
    if IsDoubleSym(fTokenStr + fReader.Ch) then
    begin
      // this is 2 char symbol: store in token string and skip over
      UpdateTokenStr;
      fReader.NextChar;
    end
  end;
  // Token string now holds symbol: check which kind of token it represents
  // and parse accordingly
  AToken := SymbolToToken(TokenStr);
  case AToken of
    tkComment:
      Result := ParseCommentFromStart;
    tkString:
      Result := ParseString;
    tkChar:
      Result := ParseChar;
    tkHex:
      Result := ParseHex;
    else
      Result := AToken;       // no special processing: return token
  end;
end;

function THilitePasLexer.ParseUnknown: THilitePasToken;
  {Analyses an unrecognised entity from input and adds it to token string.
    @return Error token (tkError).
  }
begin
  Result := tkError;
  UpdateTokenStr;
  fReader.NextChar;
end;

function THilitePasLexer.ParseWhiteSpace: THilitePasToken;
  {Analyses a sequence of white space from input and appends space for each
  white space character read to token string.
    @return White space token (tkWhiteSpace).
  }
begin
  Assert(IsWhiteSpaceChar(fReader.Ch),
    ClassName + '.ParseWhiteSpace: current char not white space');
  while IsWhiteSpaceChar(fReader.Ch) do
  begin
    UpdateTokenStr;
    fReader.NextChar;
  end;
  Result := tkWhiteSpace;
end;

function THilitePasLexer.ParseWholeNumber: THilitePasToken;
  {Analyses a whole number from input and appends to token string.
    @return Whole number token (tkNumber).
  }
begin
  Assert(IsDigit(fReader.Ch),
    ClassName + '.ParseWholeNumber: current char not a digit');
  while IsDigit(fReader.Ch) do
  begin
    UpdateTokenStr;
    fReader.NextChar;
  end;
  Result := tkNumber;
end;

procedure THilitePasLexer.UpdateTokenStr;
  {Appends current character in input to token string. Ignores EOF.
  }
begin
  UpdateTokenStr(fReader.Ch);
end;

procedure THilitePasLexer.UpdateTokenStr(const Ch: Char);
  {Appends a character to token string. Ignores EOF.
    @param Ch [in] Character to append.
  }
begin
  if Ch <> cEOF then
    fTokenStr := fTokenStr + Ch;
end;


initialization


finalization

pvtKeywords.Free;
pvtDirectives.Free;
pvtDoubleSyms.Free;
pvtSymMap.Free;

end.