Lexing of bare regular text.

This is currently a bit of a hack but at least we're slowly getting there.

lib/oga/lexer.rl

@@ -120,6 +120,10 @@ module Oga
         @text_buffer << text
       }
 
+      action emit_text_buffer {
+        emit_text_buffer
+      }
+
       action buffer_string {
         @string_buffer << text
       }
@@ -326,24 +330,30 @@ module Oga
       *|;
 
       main := |*
-        newline => emit_newline;
+        newline @emit_text_buffer => emit_newline;
 
-        doctype_start => {
+        doctype_start @emit_text_buffer => {
           t(:T_DOCTYPE_START)
           fcall doctype;
         };
 
-        cdata_start => {
+        cdata_start @emit_text_buffer => {
           t(:T_CDATA_START)
           fcall cdata;
         };
 
-        comment_start => {
+        comment_start @emit_text_buffer => {
           t(:T_COMMENT_START)
           fcall comment;
         };
 
-        element_start => open_element;
+        element_start @emit_text_buffer => open_element;
+
+        any => {
+          @text_buffer << text
+
+          emit_text_buffer if @te == eof
+        };
       *|;
     }%%
   end # Lexer

spec/oga/lexer/elements_spec.rb

@@ -25,6 +25,14 @@ describe Oga::Lexer do
         [:T_ELEM_CLOSE, nil, 1, 9]
       ]
     end
+
+    example 'lex text followed by a paragraph element' do
+      lex('Foo<p>').should == [
+        [:T_TEXT, 'Foo', 1, 1],
+        [:T_ELEM_OPEN, nil, 1, 4],
+        [:T_ELEM_NAME, 'p', 1, 5]
+      ]
+    end
   end
 
   context 'elements with attributes' do

spec/oga/lexer/general_spec.rb

@@ -5,9 +5,7 @@ describe Oga::Lexer do
     example 'lex regular text' do
       lex('hello').should == [[:T_TEXT, 'hello', 1, 1]]
     end
-  end
 
-  context 'whitespace' do
     example 'lex regular whitespace' do
       lex(' ').should == [[:T_TEXT, ' ', 1, 1]]
     end
@@ -22,5 +20,12 @@ describe Oga::Lexer do
         [:T_TEXT, ' ', 2, 1]
       ]
     end
+
+    example 'lex text followed by a newline' do
+      lex("foo\n").should == [
+        [:T_TEXT, 'foo', 1, 1],
+        [:T_TEXT, "\n", 1, 4]
+      ]
+    end
   end
 end