# File lib/rdoc/tom_doc.rb, line 195
  def tokenize text
    text.sub!(/\A(Public|Internal|Deprecated):\s+/, '')

    s = StringScanner.new text

    @line = 0
    @line_pos = 0

    until s.eos? do
      pos = s.pos

      # leading spaces will be reflected by the column of the next token
      # the only thing we loose are trailing spaces at the end of the file
      next if s.scan(/ +/)

      @tokens << case
                 when s.scan(/\r?\n/) then
                   token = [:NEWLINE, s.matched, *token_pos(pos)]
                   @line_pos = s.pos
                   @line += 1
                   token
                 when s.scan(/(Examples|Signature)$/) then
                   @tokens << [:HEADER, 3, *token_pos(pos)]

                   [:TEXT, s[1], *token_pos(pos)]
                 when s.scan(/([:\w]\w*)[ ]+- /) then
                   [:NOTE, s[1], *token_pos(pos)]
                 else
                   s.scan(/.*/)
                   [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
                 end
    end

    self
  end