diff --git a/lib/oga/lexer.rl b/lib/oga/lexer.rl index 0739681..bb037e6 100644 --- a/lib/oga/lexer.rl +++ b/lib/oga/lexer.rl @@ -120,6 +120,10 @@ module Oga @text_buffer << text } + action emit_text_buffer { + emit_text_buffer + } + action buffer_string { @string_buffer << text } @@ -326,24 +330,30 @@ module Oga *|; main := |* - newline => emit_newline; + newline @emit_text_buffer => emit_newline; - doctype_start => { + doctype_start @emit_text_buffer => { t(:T_DOCTYPE_START) fcall doctype; }; - cdata_start => { + cdata_start @emit_text_buffer => { t(:T_CDATA_START) fcall cdata; }; - comment_start => { + comment_start @emit_text_buffer => { t(:T_COMMENT_START) fcall comment; }; - element_start => open_element; + element_start @emit_text_buffer => open_element; + + any => { + @text_buffer << text + + emit_text_buffer if @te == eof + }; *|; }%% end # Lexer diff --git a/spec/oga/lexer/elements_spec.rb b/spec/oga/lexer/elements_spec.rb index a7deee5..893a86b 100644 --- a/spec/oga/lexer/elements_spec.rb +++ b/spec/oga/lexer/elements_spec.rb @@ -25,6 +25,14 @@ describe Oga::Lexer do [:T_ELEM_CLOSE, nil, 1, 9] ] end + + example 'lex text followed by a paragraph element' do + lex('Foo
').should == [ + [:T_TEXT, 'Foo', 1, 1], + [:T_ELEM_OPEN, nil, 1, 4], + [:T_ELEM_NAME, 'p', 1, 5] + ] + end end context 'elements with attributes' do diff --git a/spec/oga/lexer/general_spec.rb b/spec/oga/lexer/general_spec.rb index f65ca01..ba9792e 100644 --- a/spec/oga/lexer/general_spec.rb +++ b/spec/oga/lexer/general_spec.rb @@ -5,9 +5,7 @@ describe Oga::Lexer do example 'lex regular text' do lex('hello').should == [[:T_TEXT, 'hello', 1, 1]] end - end - context 'whitespace' do example 'lex regular whitespace' do lex(' ').should == [[:T_TEXT, ' ', 1, 1]] end @@ -22,5 +20,12 @@ describe Oga::Lexer do [:T_TEXT, ' ', 2, 1] ] end + + example 'lex text followed by a newline' do + lex("foo\n").should == [ + [:T_TEXT, 'foo', 1, 1], + [:T_TEXT, "\n", 1, 4] + ] + end end end