php-doc-en/reference/tokenizer/reference.xml

<?xml version="1.0" encoding="iso-8859-1"?>
<!-- $Revision: 1.20 $ -->
<!-- Purpose: basic.other -->
<!-- Membership: core -->

 <reference id="ref.tokenizer">
  <title>Tokenizer Functions</title>
  <titleabbrev>Tokenizer</titleabbrev>

  <partintro>
   <section id="tokenizer.intro">
    &reftitle.intro;
    <para>
     The tokenizer functions provide an interface to the
     PHP tokenizer embedded in the Zend Engine. Using these
     functions you may write your own PHP source analyzing
     or modification tools without having to deal with the
     language specification at the lexical level.
    </para>
    <para>
     See also the <link linkend="tokens">appendix about tokens</link>.
    </para>
   </section>

   <section id="tokenizer.requirements">
    &reftitle.required; 
    &no.requirement; 
   </section> 

   &reference.tokenizer.configure;

   &reference.tokenizer.constants;

   <section id="tokenizer.examples">
    &reftitle.examples;
    <para>
     Here is a simple example PHP scripts using the tokenizer that 
     will read in a PHP file, strip all comments from the source
     and print the pure code only.
    </para>
    <example>
     <title>Strip comments with the tokenizer</title>
     <programlisting role="php">
<![CDATA[
<?php
/*
 * T_ML_COMMENT does not exist in PHP 5.
 * The following three lines define it in order to
 * preserve backwards compatibility.
 *
 * The next two lines define the PHP 5 only T_DOC_COMMENT,
 * which we will mask as T_ML_COMMENT for PHP 4.
 */
if (!defined('T_ML_COMMENT')) {
    define('T_ML_COMMENT', T_COMMENT);
} else {
    define('T_DOC_COMMENT', T_ML_COMMENT);
}

$source = file_get_contents('example.php');
$tokens = token_get_all($source);

foreach ($tokens as $token) {
    if (is_string($token)) {
        // simple 1-character token
        echo $token;
    } else {
        // token array
        list($id, $text) = $token;
 
        switch ($id) { 
            case T_COMMENT: 
            case T_ML_COMMENT: // we've defined this
            case T_DOC_COMMENT: // and this
                // no action on comments
                break;

            default:
                // anything else -> output "as is"
                echo $text;
                break;
        }
    }
}
?>
]]>
     </programlisting>
    </example>    
   </section>


  </partintro>

&reference.tokenizer.functions;

 </reference>
<!-- Keep this comment at the end of the file
Local variables:
mode: sgml
sgml-omittag:t
sgml-shorttag:t
sgml-minimize-attributes:nil
sgml-always-quote-attributes:t
sgml-indent-step:1
sgml-indent-data:t
indent-tabs-mode:nil
sgml-parent-document:nil
sgml-default-dtd-file:"../../../manual.ced"
sgml-exposed-tags:nil
sgml-local-catalogs:nil
sgml-local-ecat-files:nil
End:
-->