support reading from streams (fb48d2d9) · Commits · Matthew Dawson / JMatIO

README.markdown

+2 −1

Original line number	Diff line number	Diff line
		@@ -11,4 +11,5 @@ For example: in MatFileWriter.java I have cut down the use of needless ByteArray

		These changes are completely irrelevant with small matlab files (sub 100Mb) but when the data being written is in the gigabytes these changes resulted in substantial memory usage improvements

		In addition, support for reading from streams has been added.

pom.xml

+1 −1

Original line number	Diff line number	Diff line
		@@ -3,7 +3,7 @@
		<modelVersion>4.0.0</modelVersion>
		<groupId>jmatio</groupId>
		<artifactId>jmatio</artifactId>
		<version>0.2.3</version>
		<version>0.2.3.1</version>
		<name>The OpenIMAJ branch of JMATIO</name>
		<url>http://www.openimaj.org/thirdparty/jmatio</url>
		<inceptionYear>2011</inceptionYear>

src/main/java/com/jmatio/io/MatFileReader.java

+1221 −1072

File changed.

Preview size limit exceeded, changes collapsed.

src/main/java/com/jmatio/io/MatFileWriter.java

+528 −490

Original line number	Diff line number	Diff line
		package com.jmatio.io;


		import java.io.ByteArrayOutputStream;
		import java.io.DataOutputStream;
		import java.io.File;
		@@ -27,7 +26,9 @@ import com.jmatio.types.MLStructure;
		* MAT-file writer.
		*
		* Usage:
		* <pre><code>
		*
		* <pre>
		* <code>
		* //1. First create example arrays
		* double[] src = new double[] { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 };
		* MLDouble mlDouble = new MLDouble( "double_arr", src, 3 );
		@@ -39,21 +40,28 @@ import com.jmatio.types.MLStructure;
		* list.add( mlChar );
		*
		* new MatFileWriter( "mat_file.mat", list );
		* </code></pre>
		* </code>
		* </pre>
		*
		* this is "equal" to Matlab commands:
		* <pre><code>
		*
		* <pre>
		* <code>
		* >> double_arr = [ 1 2; 3 4; 5 6];
		* >> char_arr = 'I am dummy';
		* >>
		* >> save('mat_file.mat', 'double_arr', 'char_arr');
		* </pre></code>
		* </pre>
		*
		* </code>
		*
		* @author Wojciech Gradkowski (<a href="mailto:wgradkowski@gmail.com">wgradkowski@gmail.com</a>)
		* @author Wojciech Gradkowski (<a
		* href="mailto:wgradkowski@gmail.com">wgradkowski@gmail.com</a>)
		*/
		public class MatFileWriter
		{
		// private static final Logger logger = Logger.getLogger(MatFileWriter.class);
		// private static final Logger logger =
		// Logger.getLogger(MatFileWriter.class);

		/**
		* Creates the new <code>{@link MatFileWriter}</code> instance
		@@ -62,11 +70,14 @@ public class MatFileWriter
		{
		super();
		}

		/**
		* Writes MLArrays into file given by <code>fileName</code>.
		*
		* @param fileName - name of ouput file
		* @param data - <code>Collection</code> of <code>MLArray</code> elements
		* @param fileName
		* - name of ouput file
		* @param data
		* - <code>Collection</code> of <code>MLArray</code> elements
		* @throws IOException
		* @throws DataFormatException
		*/
		@@ -74,11 +85,14 @@ public class MatFileWriter
		{
		this(new File(fileName), data);
		}

		/**
		* Writes MLArrays into <code>File</code>.
		*
		* @param file - an output <code>File</code>
		* @param data - <code>Collection</code> of <code>MLArray</code> elements
		* @param file
		* - an output <code>File</code>
		* @param data
		* - <code>Collection</code> of <code>MLArray</code> elements
		* @throws IOException
		* @throws DataFormatException
		*/
		@@ -86,13 +100,16 @@ public class MatFileWriter
		{
		this((new FileOutputStream(file)).getChannel(), data);
		}

		/**
		* Writes MLArrays into <code>OuputSteram</code>.
		*
		* Writes MAT-file header and compressed data (<code>miCOMPRESSED</code>).
		*
		* @param output - <code>OutputStream</code>
		* @param data - <code>Collection</code> of <code>MLArray</code> elements
		* @param output
		* - <code>OutputStream</code>
		* @param data
		* - <code>Collection</code> of <code>MLArray</code> elements
		* @throws IOException
		*/
		public MatFileWriter(WritableByteChannel channel, Collection<MLArray> data) throws IOException
		@@ -105,8 +122,10 @@ public class MatFileWriter
		*
		* Writes MAT-file header and compressed data (<code>miCOMPRESSED</code>).
		*
		* @param stream - <code>OutputStream</code>
		* @param data - <code>Collection</code> of <code>MLArray</code> elements
		* @param stream
		* - <code>OutputStream</code>
		* @param data
		* - <code>Collection</code> of <code>MLArray</code> elements
		* @throws IOException
		*/
		public MatFileWriter(OutputStream stream, Collection<MLArray> data) throws IOException
		@@ -115,8 +134,8 @@ public class MatFileWriter
		}

		/**
		* Writes <code>MLArrays</code> into file created from
		* <code>filepath</code>.
		* Writes <code>MLArrays</code> into file created from <code>filepath</code>
		* .
		*
		* @param filepath
		* the absolute file path of a MAT-file to which data is written
		@@ -144,34 +163,41 @@ public class MatFileWriter
		public synchronized void write(File file, Collection<MLArray> data)
		throws IOException
		{
		FileOutputStream fos = new FileOutputStream(file);
		final FileOutputStream fos = new FileOutputStream(file);

		try
		{
		write(fos.getChannel(), data);
		}
		catch ( IOException e )
		} catch (final IOException e)
		{
		throw e;
		}
		finally
		} finally
		{
		fos.close();
		}
		}

		/**
		* A hack stolen from Greg Wilkins of Mortbay.
		* A {@link ByteArrayOutputStream} with revealed internals so there is no more wasteful
		* copying when calling {@link ByteArrayOutputStream#toByteArray()}
		* A hack stolen from Greg Wilkins of Mortbay. A
		* {@link ByteArrayOutputStream} with revealed internals so there is no more
		* wasteful copying when calling {@link ByteArrayOutputStream#toByteArray()}
		*
		* @author ss
		*
		*/
		private static class ByteArrayOutputStream2 extends ByteArrayOutputStream
		static class ByteArrayOutputStream2 extends ByteArrayOutputStream
		{
		public ByteArrayOutputStream2(){super();}
		public byte[] getBuf(){return buf;}
		public int getCount(){return count;}
		public ByteArrayOutputStream2() {
		super();
		}

		public byte[] getBuf() {
		return buf;
		}

		public int getCount() {
		return count;
		}
		}

		/**
		@@ -193,13 +219,13 @@ public class MatFileWriter
		writeHeader(channel);

		// write data
		for ( MLArray matrix : data )
		for (final MLArray matrix : data)
		{
		// compress data to save storage
		Deflater compresser = new Deflater();
		final Deflater compresser = new Deflater();
		// prepare buffer for MATRIX data
		ByteArrayOutputStream2 compressed = new ByteArrayOutputStream2();
		DataOutputStream dout = new DataOutputStream(new DeflaterOutputStream(compressed, compresser));
		final ByteArrayOutputStream2 compressed = new ByteArrayOutputStream2();
		final DataOutputStream dout = new DataOutputStream(new DeflaterOutputStream(compressed, compresser));
		// write MATRIX bytes into compressed buffer buffer
		writeMatrix(dout, matrix);
		dout.flush();
		@@ -207,8 +233,11 @@ public class MatFileWriter

		// write COMPRESSED tag and compressed data into output channel
		// byte[] compressedBytes = compressed.toByteArray();
		int compressedSize = compressed.getCount();
		ByteBuffer buf = ByteBuffer.allocateDirect(2 * 4 /* Int size */ + compressedSize);
		final int compressedSize = compressed.getCount();
		final ByteBuffer buf = ByteBuffer.allocateDirect(2 * 4 /*
		* Int
		* size
		*/+ compressedSize);
		buf.putInt(MatDataTypes.miCOMPRESSED);
		buf.putInt(compressedSize);
		buf.put(compressed.getBuf(), 0, compressedSize);
		@@ -216,12 +245,10 @@ public class MatFileWriter
		buf.flip();
		channel.write(buf);
		}
		}
		catch ( IOException e )
		} catch (final IOException e)
		{
		throw e;
		}
		finally
		} finally
		{
		channel.close();
		}
		@@ -229,20 +256,25 @@ public class MatFileWriter

		/**
		* Writes MAT-file header into <code>OutputStream</code>
		* @param os <code>OutputStream</code>
		*
		* @param os
		* <code>OutputStream</code>
		* @throws IOException
		*/
		private void writeHeader(WritableByteChannel channel) throws IOException
		{
		// write descriptive text
		MatFileHeader header = MatFileHeader.createHeader();
		char[] dest = new char[116];
		char[] src = header.getDescription().toCharArray();
		final MatFileHeader header = MatFileHeader.createHeader();
		final char[] dest = new char[116];
		final char[] src = header.getDescription().toCharArray();
		System.arraycopy(src, 0, dest, 0, src.length);

		byte[] endianIndicator = header.getEndianIndicator();
		final byte[] endianIndicator = header.getEndianIndicator();

		ByteBuffer buf = ByteBuffer.allocateDirect(dest.length * 2 /* Char size */ + 2 + endianIndicator.length);
		final ByteBuffer buf = ByteBuffer.allocateDirect(dest.length * 2 /*
		* Char
		* size
		*/+ 2 + endianIndicator.length);

		for (int i = 0; i < dest.length; i++)
		{
		@@ -252,7 +284,7 @@ public class MatFileWriter
		buf.position(buf.position() + 8);

		// write version
		int version = header.getVersion();
		final int version = header.getVersion();
		buf.put((byte) (version >> 8));
		buf.put((byte) version);

		@@ -265,8 +297,10 @@ public class MatFileWriter
		/**
		* Writes MATRIX into <code>OutputStream</code>.
		*
		* @param os - <code>OutputStream</code>
		* @param array - a <code>MLArray</code>
		* @param os
		* - <code>OutputStream</code>
		* @param array
		* - a <code>MLArray</code>
		* @throws IOException
		*/
		private void writeMatrix(DataOutputStream output, MLArray array) throws IOException
		@@ -274,8 +308,8 @@ public class MatFileWriter
		OSArrayTag tag;
		ByteArrayOutputStream2 buffer;
		DataOutputStream bufferDOS;
		ByteArrayOutputStream2 baos = new ByteArrayOutputStream2();
		DataOutputStream dos = new DataOutputStream(baos);
		final ByteArrayOutputStream2 baos = new ByteArrayOutputStream2();
		final DataOutputStream dos = new DataOutputStream(baos);

		// flags
		writeFlags(dos, array);
		@@ -292,7 +326,7 @@ public class MatFileWriter
		// write char data
		buffer = new ByteArrayOutputStream2();
		bufferDOS = new DataOutputStream(buffer);
		Character[] ac = ((MLChar)array).exportChar();
		final Character[] ac = ((MLChar) array).exportChar();
		for (int i = 0; i < ac.length; i++)
		{
		bufferDOS.writeByte((byte) ac[i].charValue());
		@@ -387,7 +421,7 @@ public class MatFileWriter
		break;
		case MLArray.mxSTRUCT_CLASS:
		// field name length
		int itag = 4 << 16 \| MatDataTypes.miINT32 & 0xffff;
		final int itag = 4 << 16 \| MatDataTypes.miINT32 & 0xffff;
		dos.writeInt(itag);
		dos.writeInt(((MLStructure) array).getMaxFieldLenth());

		@@ -395,13 +429,13 @@ public class MatFileWriter
		tag = new OSArrayTag(MatDataTypes.miINT8, ((MLStructure) array).getKeySetToByteArray());
		tag.writeTo(dos);

		for ( MLArray a : ((MLStructure)array).getAllFields() )
		for (final MLArray a : ((MLStructure) array).getAllFields())
		{
		writeMatrix(dos, a);
		}
		break;
		case MLArray.mxCELL_CLASS:
		for ( MLArray a : ((MLCell)array).cells() )
		for (final MLArray a : ((MLCell) array).cells())
		{
		writeMatrix(dos, a);
		}
		@@ -412,7 +446,7 @@ public class MatFileWriter
		buffer = new ByteArrayOutputStream2();
		bufferDOS = new DataOutputStream(buffer);
		ai = ((MLSparse) array).getIR();
		for ( int i : ai )
		for (final int i : ai)
		{
		bufferDOS.writeInt(i);
		}
		@@ -422,7 +456,7 @@ public class MatFileWriter
		buffer = new ByteArrayOutputStream2();
		bufferDOS = new DataOutputStream(buffer);
		ai = ((MLSparse) array).getJC();
		for ( int i : ai )
		for (final int i : ai)
		{
		bufferDOS.writeInt(i);
		}
		@@ -460,7 +494,6 @@ public class MatFileWriter

		}


		// write matrix
		output.writeInt(MatDataTypes.miMATRIX); // matrix tag
		output.writeInt(baos.size()); // size of matrix
		@@ -470,14 +503,16 @@ public class MatFileWriter
		/**
		* Writes MATRIX flags into <code>OutputStream</code>.
		*
		* @param os - <code>OutputStream</code>
		* @param array - a <code>MLArray</code>
		* @param os
		* - <code>OutputStream</code>
		* @param array
		* - a <code>MLArray</code>
		* @throws IOException
		*/
		private void writeFlags(DataOutputStream os, MLArray array) throws IOException
		{
		ByteArrayOutputStream buffer = new ByteArrayOutputStream();
		DataOutputStream bufferDOS = new DataOutputStream(buffer);
		final ByteArrayOutputStream buffer = new ByteArrayOutputStream();
		final DataOutputStream bufferDOS = new DataOutputStream(buffer);

		bufferDOS.writeInt(array.getFlags());

		@@ -489,7 +524,7 @@ public class MatFileWriter
		{
		bufferDOS.writeInt(0);
		}
		OSArrayTag tag = new OSArrayTag(MatDataTypes.miUINT32, buffer.toByteArray() );
		final OSArrayTag tag = new OSArrayTag(MatDataTypes.miUINT32, buffer.toByteArray());
		tag.writeTo(os);

		}
		@@ -497,21 +532,23 @@ public class MatFileWriter
		/**
		* Writes MATRIX dimensions into <code>OutputStream</code>.
		*
		* @param os - <code>OutputStream</code>
		* @param array - a <code>MLArray</code>
		* @param os
		* - <code>OutputStream</code>
		* @param array
		* - a <code>MLArray</code>
		* @throws IOException
		*/
		private void writeDimensions(DataOutputStream os, MLArray array) throws IOException
		{
		ByteArrayOutputStream buffer = new ByteArrayOutputStream();
		DataOutputStream bufferDOS = new DataOutputStream(buffer);
		final ByteArrayOutputStream buffer = new ByteArrayOutputStream();
		final DataOutputStream bufferDOS = new DataOutputStream(buffer);

		int[] dims = array.getDimensions();
		final int[] dims = array.getDimensions();
		for (int i = 0; i < dims.length; i++)
		{
		bufferDOS.writeInt(dims[i]);
		}
		OSArrayTag tag = new OSArrayTag(MatDataTypes.miUINT32, buffer.toByteArray() );
		final OSArrayTag tag = new OSArrayTag(MatDataTypes.miUINT32, buffer.toByteArray());
		tag.writeTo(os);

		}
		@@ -519,8 +556,10 @@ public class MatFileWriter
		/**
		* Writes MATRIX name into <code>OutputStream</code>.
		*
		* @param os - <code>OutputStream</code>
		* @param array - a <code>MLArray</code>
		* @param os
		* - <code>OutputStream</code>
		* @param array
		* - a <code>MLArray</code>
		* @throws IOException
		*/
		private void writeName(DataOutputStream os, MLArray array) throws IOException
		@@ -528,13 +567,12 @@ public class MatFileWriter
		ByteArrayOutputStream buffer = new ByteArrayOutputStream();
		DataOutputStream bufferDOS = new DataOutputStream(buffer);

		byte[] nameByteArray = array.getNameToByteArray();
		final byte[] nameByteArray = array.getNameToByteArray();
		buffer = new ByteArrayOutputStream();
		bufferDOS = new DataOutputStream(buffer);
		bufferDOS.write(nameByteArray);
		OSArrayTag tag = new OSArrayTag(16, buffer.toByteArray() );
		final OSArrayTag tag = new OSArrayTag(16, buffer.toByteArray());
		tag.writeTo(os);
		}


		}