%%machine base_lexer; %%{ ## # Base grammar for the XML lexer. # # This grammar is shared between the C and Java extensions. As a result of # this you should **not** include language specific code in Ragel # actions/callbacks. # # To call back in to Ruby you can use one of the following two functions: # # * callback # * callback_simple # # The first function takes 5 arguments: # # * The name of the Ruby method to call. # * The input data. # * The encoding of the input data. # * The start of the current buffer. # * The end of the current buffer. # # The function callback_simple only takes one argument: the name of the # method to call. This function should be used for callbacks that don't # require any values. # # When you call a method in Ruby make sure that said method is defined as # an instance method in the `Oga::XML::Lexer` class. # # The name of the callback to invoke should be an identifier starting with # "id_". The identifier should be defined in the associated C and Java code. # In case of C code its value should be a Symbol as a ID object, for Java # it should be a String. For example: # # ID id_foo = rb_intern("foo"); # # And for Java: # # String id_foo = "foo"; # # ## Machine Transitions # # To transition from one machine to another always use `fnext` instead of # `fcall` and `fret`. This removes the need for the code to keep track of a # stack. # newline = '\r\n' | '\n' | '\r'; whitespace = [ \t]; unicode = any - ascii; ident_char = unicode | [a-zA-Z0-9\-_\.]; identifier = ident_char+; whitespace_or_newline = whitespace | newline; action count_newlines { if ( fc == '\n' ) lines++; } action advance_newline { advance_line(1); } action hold_and_return { fhold; fret; } # Comments # # http://www.w3.org/TR/html/syntax.html#comments # # Unlike the W3C specification these rules *do* allow character sequences # such as `--` and `->`. Putting extra checks in for these sequences would # actually make the rules/actions more complex. # comment_start = ''; # Everything except "-" OR a single "-" comment_allowed = (^'-'+ | '-') $count_newlines; action start_comment { callback_simple(id_on_comment_start); fnext comment_body; } comment_body := |* comment_allowed => { callback(id_on_comment_body, data, encoding, ts, te); if ( lines > 0 ) { advance_line(lines); lines = 0; } }; comment_end => { callback_simple(id_on_comment_end); fnext main; }; *|; # CDATA # # http://www.w3.org/TR/html/syntax.html#cdata-sections # # In HTML CDATA tags have no meaning/are not supported. Oga does # support them but treats their contents as plain text. # cdata_start = ''; # Everything except "]" OR a single "]" cdata_allowed = (^']'+ | ']') $count_newlines; action start_cdata { callback_simple(id_on_cdata_start); fnext cdata_body; } cdata_body := |* cdata_allowed => { callback(id_on_cdata_body, data, encoding, ts, te); if ( lines > 0 ) { advance_line(lines); lines = 0; } }; cdata_end => { callback_simple(id_on_cdata_end); fnext main; }; *|; # Processing Instructions # # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes # http://en.wikipedia.org/wiki/Processing_Instruction # # These are tags meant to be used by parsers/libraries for custom behaviour. # One example are the tags used by PHP: . Note that the XML # declaration tags () are not considered to be a processing # instruction. # proc_ins_start = ''; # Everything except "?" OR a single "?" proc_ins_allowed = (^'?'+ | '?') $count_newlines; action start_proc_ins { callback_simple(id_on_proc_ins_start); callback(id_on_proc_ins_name, data, encoding, ts + 2, te); fnext proc_ins_body; } proc_ins_body := |* proc_ins_allowed => { callback(id_on_proc_ins_body, data, encoding, ts, te); if ( lines > 0 ) { advance_line(lines); lines = 0; } }; proc_ins_end => { callback_simple(id_on_proc_ins_end); fnext main; }; *|; # Strings # # Strings in HTML can either be single or double quoted. If a string # starts with one of these quotes it must be closed with the same type # of quote. # dquote = '"'; squote = "'"; action emit_string { callback(id_on_string_body, data, encoding, ts, te); if ( lines > 0 ) { advance_line(lines); lines = 0; } } action start_string_squote { callback_simple(id_on_string_squote); fcall string_squote; } action start_string_dquote { callback_simple(id_on_string_dquote); fcall string_dquote; } string_squote := |* ^squote* $count_newlines => emit_string; squote => { callback_simple(id_on_string_squote); fret; }; *|; string_dquote := |* ^dquote* $count_newlines => emit_string; dquote => { callback_simple(id_on_string_dquote); fret; }; *|; # DOCTYPES # # http://www.w3.org/TR/html/syntax.html#the-doctype # # These rules support the 3 flavours of doctypes: # # 1. Normal doctypes, as introduced in the HTML5 specification. # 2. Deprecated doctypes, the more verbose ones used prior to HTML5. # 3. Legacy doctypes # doctype_start = ' 0 ) { advance_line(lines); lines = 0; } fnext doctype; } # Machine for processing inline rules of a doctype. doctype_inline := |* ^']'* $count_newlines => { callback(id_on_doctype_inline, data, encoding, ts, te); if ( lines > 0 ) { advance_line(lines); lines = 0; } }; ']' => { fnext doctype; }; *|; # Machine for processing doctypes. Doctype values such as the public # and system IDs are treated as T_STRING tokens. doctype := |* 'PUBLIC' | 'SYSTEM' => { callback(id_on_doctype_type, data, encoding, ts, te); }; # Starts a set of inline doctype rules. '[' => { fnext doctype_inline; }; # Lex the public/system IDs as regular strings. squote => start_string_squote; dquote => start_string_dquote; identifier => { callback(id_on_doctype_name, data, encoding, ts, te); }; '>' => { callback_simple(id_on_doctype_end); fnext main; }; newline => advance_newline; whitespace; *|; # XML declaration tags # # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd # xml_decl_start = ''; action start_xml_decl { callback_simple(id_on_xml_decl_start); fnext xml_decl; } # Machine that processes the contents of an XML declaration tag. xml_decl := |* xml_decl_end => { if ( lines > 0 ) { advance_line(lines); lines = 0; } callback_simple(id_on_xml_decl_end); fnext main; }; # Attributes and their values (e.g. version="1.0"). identifier => { if ( lines > 0 ) { advance_line(lines); lines = 0; } callback(id_on_attribute, data, encoding, ts, te); }; squote => start_string_squote; dquote => start_string_dquote; any $count_newlines; *|; # Elements # # http://www.w3.org/TR/html/syntax.html#syntax-elements # # Lexing of elements is broken up into different machines that handle the # name/namespace, contents of the open tag and the body of an element. The # body of an element is lexed using the `main` machine. # action start_element { fhold; fnext element_name; } action start_close_element { fnext element_close; } action close_element { callback(id_on_element_end, data, encoding, ts, te); } action close_element_fnext_main { callback_simple(id_on_element_end); fnext main; } element_start = '<' ident_char; element_end = ' { callback(id_on_element_ns, data, encoding, ts, te - 1); }; identifier => { callback(id_on_element_name, data, encoding, ts, te); fnext element_head; }; *|; # Machine used for lexing the closing tag of an element element_close := |* # namespace prefixes, currently not used but allows the rule below it # to be used for the actual element name. identifier ':'; identifier => close_element; '>' => { if ( lines > 0 ) { advance_line(lines); lines = 0; } fnext main; }; any $count_newlines; *|; # Machine used after matching the "=" of an attribute and just before moving # into the actual attribute value. attribute_pre := |* whitespace_or_newline $count_newlines; squote | dquote => { fhold; if ( lines > 0 ) { advance_line(lines); lines = 0; } fnext quoted_attribute_value; }; any => { fhold; if ( lines > 0 ) { advance_line(lines); lines = 0; } if ( html_p ) { fnext unquoted_attribute_value; } /* XML doesn't support unquoted attribute values */ else { fret; } }; *|; # Machine for processing unquoted HTML attribute values. # # The HTML specification describes a set of characters that can be allowed # in an unquoted value at https://html.spec.whatwg.org/multipage/introduction.html#intro-early-example. # # As is always the case with HTML everybody completely ignores this # specification and thus every library and browser out these is expected to # support input such as `. # # Oga too has to support this, thus the only characters it disallows in # unquoted attribute values are: # # * > (used for terminating open tags) # * whitespace # unquoted_attribute_value := |* ^('>' | whitespace_or_newline)+ => { callback_simple(id_on_string_squote); callback(id_on_string_body, data, encoding, ts, te); callback_simple(id_on_string_squote); }; any => hold_and_return; *|; # Machine used for processing quoted XML/HTML attribute values. quoted_attribute_value := |* # The following two actions use "fnext" instead of "fcall". Combined # with "element_head" using "fcall" to jump to this machine this means # we can return back to "element_head" after processing a single string. squote => { callback_simple(id_on_string_squote); fnext string_squote; }; dquote => { callback_simple(id_on_string_dquote); fnext string_dquote; }; any => hold_and_return; *|; # Machine used for processing the contents of an element's starting tag. # This includes the name, namespace and attributes. element_head := |* newline => advance_newline; # Attribute names and namespaces. identifier ':' => { callback(id_on_attribute_ns, data, encoding, ts, te - 1); }; identifier => { callback(id_on_attribute, data, encoding, ts, te); }; # Attribute values. '=' => { fcall attribute_pre; }; # We're done with the open tag of the element. '>' => { callback_simple(id_on_element_open_end); if ( html_script_p() ) { fnext html_script; } else if ( html_style_p() ) { fnext html_style; } else { fnext main; } }; # Self closing tags. '/>' => { callback_simple(id_on_element_end); fnext main; }; any; *|; # Text # # http://www.w3.org/TR/xml/#syntax # http://www.w3.org/TR/html/syntax.html#text # # Text content is everything leading up to certain special tags such as " 0 ) { advance_line(lines); lines = 0; } } text := |* terminate_text | allowed_text => { callback(id_on_text, data, encoding, ts, te); if ( lines > 0 ) { advance_line(lines); lines = 0; } fnext main; }; # Text followed by a special tag, such as "foo