Enable control for tokenizer buffer size

This change adds an optional bufferSize parameter to the tokenize(), parse(),
and parseFragment() methods of sax.HtmlParser instances. That bufferSize
parameter controls the size of the buffer which gets fed to the tokenizer.

The control provided by that parameter allows the tokenizer buffer to be
set, for example, to "1" — and that is particularly useful for emulating
the behavior of the Firefox HTML parser, which feeds the tokenizer one
single code unit at a time.

Otherwise, without this change, the tokenizer buffer size for HtmlParser
instances is hardcoded to 2048.

Author: sideshowbarker

src/nu/validator/htmlparser/io/Driver.java

@@ -173,6 +173,24 @@ public boolean isCheckingNormalization() {
      *             if the stream threw
      */
     public void tokenize(InputSource is) throws SAXException, IOException {
+        int bufferSize = 2048;
+        tokenize(is, bufferSize);
+    }
+    /**
+     * Runs the tokenization. This is the main entry point.
+     *
+     * @param is
+     *            the input source
+     * @param bufferSize
+     *            the size of the buffer to feed to the tokenizer
+     * @throws SAXException
+     *             on fatal error (if configured to treat XML violations as
+     *             fatal) or if the token handler threw
+     * @throws IOException
+     *             if the stream threw
+     */
+    public void tokenize(InputSource is, int bufferSize)
+            throws SAXException, IOException {
         if (is == null) {
             throw new IllegalArgumentException("InputSource was null.");
         }
@@ -216,7 +234,7 @@ public void tokenize(InputSource is) throws SAXException, IOException {
                         CharacterHandler ch = characterHandlers[i];
                         ch.start();
                     }
-                    runStates();
+                    runStates(bufferSize);
                     break;
                 } catch (ReparseException e) {
                     if (rewindableInputStream == null) {
@@ -270,8 +288,8 @@ void dontSwallowBom() {
         swallowBom = false;
     }
 
-    private void runStates() throws SAXException, IOException {
-        char[] buffer = new char[2048];
+    private void runStates(int bufferSize) throws SAXException, IOException {
+        char[] buffer = new char[bufferSize];
         UTF16Buffer bufr = new UTF16Buffer(buffer, 0, 0);
         boolean lastWasCR = false;
         int len = -1;

src/nu/validator/htmlparser/sax/HtmlParser.java

@@ -402,10 +402,17 @@ public Object getProperty(String name) throws SAXNotRecognizedException,
      * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource)
      */
     public void parse(InputSource input) throws IOException, SAXException {
+        parse(input, -1);
+    }
+
+    /**
+     * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource)
+     */
+    public void parse(InputSource input, int bufferSize) throws IOException, SAXException {
         lazyInit();
         try {
             treeBuilder.setFragmentContext(null);
-            tokenize(input);
+            tokenize(input, bufferSize);
         } finally {
             if (saxTreeBuilder != null) {
                 Document document = saxTreeBuilder.getDocument();
@@ -426,10 +433,27 @@ public void parse(InputSource input) throws IOException, SAXException {
      */
     public void parseFragment(InputSource input, String context)
             throws IOException, SAXException {
+            parseFragment(input, context, -1);
+    }
+    /**
+     * Parses a fragment with HTML context.
+     *
+     * @param input the input to parse
+     * @param context the name of the context element (HTML namespace assumed)
+     * @param bufferSize the size of the buffer to feed to the tokenizer
+     * @throws IOException
+     * @throws SAXException
+     */
+    public void parseFragment(InputSource input, String context, int bufferSize)
+            throws IOException, SAXException {
         lazyInit();
         try {
             treeBuilder.setFragmentContext(context.intern());
-            tokenize(input);
+            if (bufferSize == -1) {
+                tokenize(input);
+            } else {
+                tokenize(input, bufferSize);
+            }
         } finally {
             if (saxTreeBuilder != null) {
                 DocumentFragment fragment = saxTreeBuilder.getDocumentFragment();
@@ -449,10 +473,29 @@ public void parseFragment(InputSource input, String context)
      */
     public void parseFragment(InputSource input, String contextLocal, String contextNamespace)
             throws IOException, SAXException {
+            parseFragment(input, contextLocal, contextNamespace, -1);
+    }
+    /**
+     * Parses a fragment.
+     *
+     * @param input the input to parse
+     * @param contextLocal the local name of the context element
+     * @param contextNamespace the namespace of the context element
+     * @param bufferSize the size of the buffer to feed to the tokenizer
+     * @throws IOException
+     * @throws SAXException
+     */
+    public void parseFragment(InputSource input, String contextLocal,
+            String contextNamespace, int bufferSize)
+            throws IOException, SAXException {
         lazyInit();
         try {
             treeBuilder.setFragmentContext(contextLocal.intern(), contextNamespace.intern(), null, false);
-            tokenize(input);
+            if (bufferSize == -1) {
+                tokenize(input);
+            } else {
+                tokenize(input, bufferSize);
+            }
         } finally {
             if (saxTreeBuilder != null) {
                 DocumentFragment fragment = saxTreeBuilder.getDocumentFragment();
@@ -468,6 +511,10 @@ public void parseFragment(InputSource input, String contextLocal, String context
      * @throws MalformedURLException
      */
     private void tokenize(InputSource is) throws SAXException, IOException, MalformedURLException {
+        tokenize(is, -1);
+    }
+    private void tokenize(InputSource is, int bufferSize) throws SAXException,
+            IOException, MalformedURLException {
         if (is == null) {
             throw new IllegalArgumentException("Null input.");            
         }
@@ -485,7 +532,11 @@ private void tokenize(InputSource is) throws SAXException, IOException, Malforme
                 is.setByteStream(new URL(systemId).openStream());
             }
         }
-        driver.tokenize(is);
+        if (bufferSize == -1) {
+            driver.tokenize(is);
+        } else {
+            driver.tokenize(is, bufferSize);
+        }
     }
 
     /**