From 1de2f41d925600d597129fb5861a53f9d3481dee Mon Sep 17 00:00:00 2001
From: Hartmut Holzgraefe <hholzgra@php.net>
Date: Thu, 6 Feb 2003 09:38:26 +0000
Subject: [PATCH] tokenizer no longer undocumented :)

git-svn-id: https://svn.php.net/repository/phpdoc/en/trunk@114924 c90b9560-bf6c-de11-be94-00142212c4b1
---
 .../tokenizer/functions/token-get-all.xml     | 32 +++++++++---
 reference/tokenizer/functions/token-name.xml  | 32 +++++++++---
 reference/tokenizer/reference.xml             | 50 ++++++++++++++++++-
 3 files changed, 96 insertions(+), 18 deletions(-)

diff --git a/reference/tokenizer/functions/token-get-all.xml b/reference/tokenizer/functions/token-get-all.xml
index c0f4a3d69c..69a73aa021 100644
--- a/reference/tokenizer/functions/token-get-all.xml
+++ b/reference/tokenizer/functions/token-get-all.xml
@@ -1,18 +1,34 @@
 <?xml version="1.0" encoding="iso-8859-1"?>
-<!-- $Revision: 1.1 $ -->
+<!-- $Revision: 1.2 $ -->
   <refentry id="function.token-get-all">
    <refnamediv>
     <refname>token_get_all</refname>
-    <refpurpose>Split given source in tokens</refpurpose>
+    <refpurpose>Split given source into PHP tokens</refpurpose>
    </refnamediv>
    <refsect1>
     <title>Description</title>
-     <methodsynopsis>
-      <type>array</type><methodname>token_get_all</methodname>
-      <methodparam><type>string</type><parameter>source</parameter></methodparam>
-     </methodsynopsis>
-     &warn.experimental.func;
-     &warn.undocumented.func;
+    <methodsynopsis>
+     <type>array</type><methodname>token_get_all</methodname>
+     <methodparam><type>string</type><parameter>source</parameter></methodparam>
+    </methodsynopsis>
+    <para>
+     <function>token_get_all</function> parses the given <parameter>source</parameter> 
+     string into PHP language tokens using the Zend engines lexical scanner.
+     The function returns an array of token descriptions. Each array element itself is
+     either a one character string or itself an array containing a token id and the 
+     string representation of that token in the source code.
+    </para>
+    <informalexample>
+     <programlisting role="php">
+<![CDATA[
+<?php
+  $tokens = token_get_all(";"); // => array(";");
+  $tokens = token_get_all("foreach") // => array(T_FOREACH => "foreach");
+  $tokens = token_get_all("/* comment */") // => array(T_ML_COMMENT => "/* comment */");
+?>
+]]>
+     </programlisting>
+    </informalexample>
    </refsect1>
   </refentry>
 
diff --git a/reference/tokenizer/functions/token-name.xml b/reference/tokenizer/functions/token-name.xml
index b8489b48c6..3e491233d1 100644
--- a/reference/tokenizer/functions/token-name.xml
+++ b/reference/tokenizer/functions/token-name.xml
@@ -1,18 +1,34 @@
 <?xml version="1.0" encoding="iso-8859-1"?>
-<!-- $Revision: 1.1 $ -->
+<!-- $Revision: 1.2 $ -->
   <refentry id="function.token-name">
    <refnamediv>
     <refname>token_name</refname>
-    <refpurpose>Get the name of a given token</refpurpose>
+    <refpurpose>Get the symbolic name of a given PHP token</refpurpose>
    </refnamediv>
    <refsect1>
     <title>Description</title>
-     <methodsynopsis>
-      <type>string</type><methodname>token_name</methodname>
-      <methodparam><type>int</type><parameter>type</parameter></methodparam>
-     </methodsynopsis>
-     &warn.experimental.func;
-     &warn.undocumented.func;
+    <methodsynopsis>
+     <type>string</type><methodname>token_name</methodname>
+     <methodparam><type>int</type><parameter>token</parameter></methodparam>
+    </methodsynopsis>
+    <para>
+     This function will returen the symbolic name for a PHP 
+     <parameter>token</parameter> value. The symbolic name 
+     returned matches the name of the matching token constant.
+    </para> 
+    <informalexample>
+     <programlisting role="php">
+<![CDATA[
+<?php
+  // 260 is the token value for the T_REQUIRE token
+  echo token_name(260);        // -> "T_REQUIRE"
+
+  // a token constant maps to its own name
+  echo token_name(T_FUNCTION); // -> "T_FUNCTION"
+?>
+]]>
+     </programlisting>
+    </informalexample>
    </refsect1>
   </refentry>
 
diff --git a/reference/tokenizer/reference.xml b/reference/tokenizer/reference.xml
index 8663f0a97e..afcfa54db2 100644
--- a/reference/tokenizer/reference.xml
+++ b/reference/tokenizer/reference.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="iso-8859-1"?>
-<!-- $Revision: 1.7 $ -->
+<!-- $Revision: 1.8 $ -->
  <reference id="ref.tokenizer">
   <title>Tokenizer functions</title>
   <titleabbrev>Tokenizer functions</titleabbrev>
@@ -7,7 +7,13 @@
   <partintro>
    <section id="tokenizer.intro">
     &reftitle.intro;
-    &warn.experimental;
+    <para>
+		 The tokenizer functions provide an interface to the
+		 PHP tokenizer embedded in the Zend Engine. Using these
+		 functions you may write your own PHP source analyzation
+		 or modification tools without having to deal with the
+		 language specification at the lexical level.
+    </para>
     <para>
      See also the <link linkend="tokens">appendix about tokens</link>.
     </para>
@@ -26,6 +32,46 @@
 
    &reference.tokenizer.constants;
 
+   <section id="tokenizer.examples">
+    &reftitle.examples;
+    <para>
+     Here is a simple example PHP scripts using the tokenizer that 
+     will read in a PHP file, strip all comments from the source
+     and print the pure code only.
+    </para>
+    <example>
+     <title>Strip comments</title>
+     <programlisting role="php">
+<![CDATA[
+<?php
+  $source = file_get_contents("somefile.php");
+  $tokens = token_get_all($source);
+  foreach ($tokens as $token) {
+    if (is_string($token)) {
+      // simple 1-character token
+      echo $token;
+    } else {
+      // token array
+      list($id, $text) = $token;
+      switch($id) { 
+        case T_COMMENT: 
+        case T_ML_COMMENT:
+          // no action on comments
+          break;
+        default:
+          // anything else -> output "as is"
+          echo $text;
+          break;
+      }
+    }
+  }
+?>
+]]>
+     </programlisting>
+    </example>    
+   </section>
+
+
   </partintro>
 
 &reference.tokenizer.functions;