php-doc-en/reference/tokenizer/reference.xml

<?xml version="1.0" encoding="iso-8859-1"?>
<!-- $Revision: 1.11 $ -->
 <reference id="ref.tokenizer">
  <title>Tokenizer functions</title>
  <titleabbrev>Tokenizer functions</titleabbrev>

  <partintro>
   <section id="tokenizer.intro">
    &reftitle.intro;
    <para>
     The tokenizer functions provide an interface to the
     PHP tokenizer embedded in the Zend Engine. Using these
     functions you may write your own PHP source analyzation
     or modification tools without having to deal with the
     language specification at the lexical level.
    </para>
    <para>
     See also the <link linkend="tokens">appendix about tokens</link>.
    </para>
   </section>

   <section id="tokenizer.requirements">
    &reftitle.required; 
    &no.requirement; 
   </section> 

   &reference.tokenizer.configure;

   &reference.tokenizer.constants;

   <section id="tokenizer.examples">
    &reftitle.examples;
    <para>
     Here is a simple example PHP scripts using the tokenizer that 
     will read in a PHP file, strip all comments from the source
     and print the pure code only.
    </para>
    <example>
     <title>Strip comments with the tokenizer</title>
     <programlisting role="php">
<![CDATA[
<?php
  $source = file_get_contents("somefile.php");
  $tokens = token_get_all($source);
  foreach ($tokens as $token) {
    if (is_string($token)) {
      // simple 1-character token
      echo $token;
    } else {
      // token array
      list($id, $text) = $token;
      switch($id) { 
        case T_COMMENT: 
        case T_ML_COMMENT:
          // no action on comments
          break;
        default:
          // anything else -> output "as is"
          echo $text;
          break;
      }
    }
  }
?>
]]>
     </programlisting>
    </example>    
   </section>


  </partintro>

&reference.tokenizer.functions;

 </reference>
<!-- Keep this comment at the end of the file
Local variables:
mode: sgml
sgml-omittag:t
sgml-shorttag:t
sgml-minimize-attributes:nil
sgml-always-quote-attributes:t
sgml-indent-step:1
sgml-indent-data:t
indent-tabs-mode:nil
sgml-parent-document:nil
sgml-default-dtd-file:"../../../manual.ced"
sgml-exposed-tags:nil
sgml-local-catalogs:nil
sgml-local-ecat-files:nil
End:
-->