php-doc-en/reference/tokenizer/functions/token-get-all.xml

<?xml version="1.0" encoding="iso-8859-1"?>
<!-- $Revision: 1.7 $ -->
  <refentry id="function.token-get-all">
   <refnamediv>
    <refname>token_get_all</refname>
    <refpurpose>Split given source into PHP tokens</refpurpose>
   </refnamediv>
   <refsect1>
    <title>Description</title>
    <methodsynopsis>
     <type>array</type><methodname>token_get_all</methodname>
     <methodparam><type>string</type><parameter>source</parameter></methodparam>
    </methodsynopsis>
    <simpara>
     <function>token_get_all</function> parses the given <parameter>source</parameter> 
     string into PHP language tokens using the Zend engine&apos;s lexical scanner.
     The function returns an array of token identifiers.  Each individual token
     identifier is either a single character (i.e.: <constant>;</constant>,
     <constant>.</constant>, <constant>&gt;</constant>, <constant>!</constant>, etc...),
     or a two element array containing the token index in element 0, and the string
     content of the original token in element 1.
    </simpara>
    <simpara>
     For a list of parser tokens, see <xref linkend="tokens"/>, or use 
     <function>token_name</function> to translate a token value into its string
     representation.
    </simpara>
    <example>
     <title><function>token_get_all</function> examples</title>
     <programlisting role="php">
<![CDATA[
<?php
  $tokens = token_get_all('<?php'); // => array(array(T_OPEN_TAG, '<?'));
  $tokens = token_get_all('<?php echo; ?>'); /* => array(
                                                    array(T_OPEN_TAG, '<?php'), 
                                                    array(T_ECHO, 'echo'),
                                                    ';',
                                                    array(T_CLOSE_TAG, '?>') ); */

/* Note in the following example that the string is parsed as T_INLINE_HTML
   rather than the otherwise expected T_COMMENT (T_ML_COMMENT in PHP <5).
   This is because no open/close tags were used in the "code" provided.
   This would be equivalent to putting a comment outside of <?php ?> tags in a normal file. */
  $tokens = token_get_all('/* comment */'); // => array(array(T_INLINE_HTML, '/* comment */'));
?>
]]>
     </programlisting>
    </example>
   </refsect1>
  </refentry>

<!-- Keep this comment at the end of the file
Local variables:
mode: sgml
sgml-omittag:t
sgml-shorttag:t
sgml-minimize-attributes:nil
sgml-always-quote-attributes:t
sgml-indent-step:1
sgml-indent-data:t
indent-tabs-mode:nil
sgml-parent-document:nil
sgml-default-dtd-file:"../../../../manual.ced"
sgml-exposed-tags:nil
sgml-local-catalogs:nil
sgml-local-ecat-files:nil
End:
vim600: syn=xml fen fdm=syntax fdl=2 si
vim: et tw=78 syn=sgml
vi: ts=1 sw=1
-->