program test_words;

uses words, lexer_interfaces, classes, sysutils;

var
  sourceText : string;
  source : TStringStream;
  lexer : ILexer;
begin
  sourceText := 'this  is  a test';
  //             0123456789012345
  
  source := TStringStream.Create(sourceText);
  
  lexer := TDefaultWordBreakLexer.Create;
  lexer.SourceStream := source; // ....
  
  assert(lexer.Token = words.TokenWord);
  assert(lexer.MatchBeginning = 0);
  assert(lexer.MatchEnd = 4);
  
  lexer.Consume;
  
  assert(lexer.Token = words.TokenWhitespace);
  assert(lexer.MatchBeginning = 4);
  assert(lexer.MatchEnd = 6);
  lexer.Consume;
  
  assert(lexer.Token = words.TokenWord);
  assert(lexer.MatchBeginning = 6);
  assert(lexer.MatchEnd = 8);
  lexer.Consume;
  
  assert(lexer.Token = words.TokenWhitespace);
  assert(lexer.MatchBeginning = 8);
  assert(lexer.MatchEnd = 10);
  lexer.Consume;
  
  assert(lexer.Token = words.TokenWord);
  assert(lexer.MatchBeginning = 10);
  assert(lexer.MatchEnd = 11);
  lexer.Consume;
  
  assert(lexer.Token = words.TokenWhitespace);
  assert(lexer.MatchBeginning = 11);
  assert(lexer.MatchEnd = 12);
  lexer.Consume;
  
  assert(lexer.Token = words.TokenWord);
  assert(lexer.MatchBeginning = 12);
  assert(lexer.MatchEnd = 16);
  lexer.Consume;
  
  // TODO check proper EOF handling.
  
  FreeAndNil(source);
end.