My problem is
even with threading it is running very slow, 45 seconds to load 15,000
lines. Is this accurate or is my code off?
For super speed read the entire file into ram as an array of bytes,
then convert to string, then parse it yourself. Here is the code to do
it. This way, I process 200 files a second.
The other way to do it, is to crank your BufferedReader buffer up to
say 64K.
package com.mindprod.hunkio;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.i
utputStreamWriter;
import java.io.UnsupportedEncodingException;
/**
* Read and write entire files as Strings. Presumes the default
encoding. No
* main method.
*
* @author Roedy Green <br>
* version 1.5 - 2006-01-25 - readFile/writeFile with
non-default
* encoding, raw read/write entire file. <br>
* version 1.4 - 2006-01-14 - readEntire file now works with
multi-byte
* encodings. <br>
* version 1.3 - 2005-06-18 add more consistent bat files.<br>
* @since 2002 July 12
*/
public class HunkIO {
/**
* Create a temporary file, Slightly smarter version of
File.createTempFile
*
* @param prefix
* beginning letters of filename
* @param suffix
* ending letters of filename.
* @param near
* directory where to put file, or file to place this temp
file near
* in the same directory. null means put the temp file in
the current
* directory.
* @return A temporary file. It will not automatically delete on
program
* completion, however.
* @exception IOException
*/
public static File createTempFile ( String prefix, String suffix,
File near )
throws IOException
{
if ( near != null )
{
if ( near.isDirectory() )
{
return File.createTempFile( prefix, suffix, near );
}
else if ( near.isFile() )
{
String parent = near.getParent();
if ( parent != null )
{
File dir = new File( parent );
if ( dir.isDirectory() )
{
return File.createTempFile( prefix, suffix,
dir );
}
}
}
}
// anything else, just create in the current directory.
return File.createTempFile( prefix, suffix );
}
/**
* read file of bytes without conversion
*
* @param fromFile
* file to read
* @return byte array representing whole file
* @throws IOException
*/
public static byte[] rawReadEntireFile ( File fromFile ) throws
IOException
{
int size = (int)fromFile.length();
FileInputStream fis = new FileInputStream( fromFile );
// R E A D
byte[] rawContents = new byte[ size ];
int bytesRead = fis.read( rawContents );
if ( bytesRead != size )
{
throw new IOException( "error: problems reading file " +
fromFile );
}
// C L O S E
fis.close();
return rawContents;
}
/**
* read file of bytes without conversion
*
* @param fromFile
* file to read
* @return byte array representing whole file
* @throws IOException
*/
public static byte[] rawReadEntireFile ( String fromFile )
throws IOException
{
return rawReadEntireFile( new File( fromFile ) );
}
/**
* Get all text in a file.
*
* @param fromFile
* file where to get the text.
* @return all the text in the File.
* @exception IOException
*/
public static String readEntireFile ( File fromFile ) throws
IOException
{
// decode with default encoding and return entire file as one
big string
return new String( rawReadEntireFile( fromFile ) );
} // end readEntireFile
/**
* Get all text in a file.
*
* @param fromFile
* file where to get the text.
* @param encoding
* name of the encoding to use to translate the bytes in
the file
* into chars e.g. "windows-1252" "UTF-8" "UTF-16"
* @return all the text in the File.
* @exception IOException
* @exception UnsupportedEncodingException
* If the named encoding is not supported
*/
public static String readEntireFile ( File fromFile, String
encoding )
throws IOException, UnsupportedEncodingException
{
// decode with encoding and return entire file as one big
string
return new String( rawReadEntireFile( fromFile ), encoding );
} // end readEntireFile
/**
* Get all text in a file.
*
* @param fromFile
* Name of file where to get the text.
* @return all the text in the File.
* @exception IOException
*/
public static String readEntireFile ( String fromFile ) throws
IOException
{
return readEntireFile( new File( fromFile ) );
} // end readEntireFile
/**
* Get all text in a file.
*
* @param fromFile
* Name of file where to get the text.
* @param encoding
* name of the encoding to use to translate the bytes in
the file
* into chars e.g. "windows-1252" "UTF-8" "UTF-16"
* @return all the text in the File.
* @exception IOException
* @exception UnsupportedEncodingException
* If the named encoding is not supported
*/
public static String readEntireFile ( String fromFile, String
encoding )
throws IOException, UnsupportedEncodingException
{
return readEntireFile( new File( fromFile ), encoding );
} // end readEntireFile
/**
* Write all the text in a file
*
* @param toFile
* file where to write the text
* @param text
* Text to write
* @exception IOException
*/
public static void writeEntireFile ( File toFile, String text )
throws IOException
{
// default encoding
FileWriter fw = new FileWriter( toFile );
// W R I T E
fw.write( text );
// C L O S E
fw.close();
} // end writeEntireFile
/**
* Write all the text in a file
*
* @param toFile
* file where to write the text
* @param text
* Text to write
* @param encoding
* name of the encoding to use to translate chars to bytes
e.g.
* "windows-1252" "UTF-8" "UTF-16"
* @exception IOException
* @exception UnsupportedEncodingException
* If the named encoding is not supported
*/
public static void writeEntireFile ( File toFile, String text,
String encoding ) throws IOException
{
// supplied encoding
FileOutputStream fos = new FileOutputStream( toFile );
OutputStreamWriter osw = new OutputStreamWriter( fos, encoding
);
// W R I T E
osw.write( text );
// C L O S E
osw.close();
} // end writeEntireFile
/**
* Write all the text in a file
*
* @param toFile
* Name of file where to write the text
* @param text
* Text to write
* @exception IOException
*/
public static void writeEntireFile ( String toFile, String text )
throws IOException
{
writeEntireFile( new File( toFile ), text );
} // end writeEntireFile
/**
* Write all the text in a file
*
* @param toFile
* file where to write the text
* @param text
* Text to write
* @param encoding
* name of the encoding to use to translate chars to bytes
e.g.
* "windows-1252" "UTF-8" "UTF-16"
* @exception IOException
* @exception UnsupportedEncodingException
* If the named encoding is not supported
*/
public static void writeEntireFile ( String toFile, String text,
String encoding ) throws IOException
{
writeEntireFile( new File( toFile ), text, encoding );
} // end writeEntireFile
} // end HunkIO
