def tokenize text
text.sub!(/\A(Public|Internal|Deprecated):\s+/, '')
s = StringScanner.new text
@line = 0
@line_pos = 0
until s.eos? do
pos = s.pos
next if s.scan(/ +/)
@tokens << case
when s.scan(/\r?\n/) then
token = [:NEWLINE, s.matched, *token_pos(pos)]
@line_pos = s.pos
@line += 1
token
when s.scan(/(Examples|Signature)$/) then
@tokens << [:HEADER, 3, *token_pos(pos)]
[:TEXT, s[1], *token_pos(pos)]
when s.scan(/([:\w]\w*)[ ]+- /) then
[:NOTE, s[1], *token_pos(pos)]
else
s.scan(/.*/)
[:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
end
end
self
end