How to: chunk serialized data

Two issues that occur when sending large data sets in Web service messages are:

1. A large working set (memory) due to buffering by the serialization engine.

2. Inordinate bandwidth consumption due to 33 percent inflation after Base64 encoding.

To solve these problems, implement the IXmlSerializable interface to control the serialization and deserialization. Specifically, implement the WriteXml and ReadXml methods to chunk the data.

To implement server-side chunking

1. On the server machine, the Web method must turn off ASP.NET buffering and return a type that implements IXmlSerializable.

2. The type that implements IXmlSerializable chunks the data in the WriteXml method.

To implement client-side processing

1. Alter the Web method on the client proxy to return the type that implements IXmlSerializable. You can use a SchemaImporterExtension to do this automatically, but this isn't shown here.

2. Implement the ReadXml method to read the chunked data stream and write the bytes to disk. This implementation also raises progress events that can be used by a graphic control, such as a progress bar.

Example

The following code example shows the Web method on the client that turns off ASP.NET buffering. It also shows the client-side implementation of the IXmlSerializable interface that chunks the data in the WriteXml method.

[WebMethod]
[SoapDocumentMethod(ParameterStyle = SoapParameterStyle.Bare)]
public SongStream DownloadSong(DownloadAuthorization Authorization, string filePath)
{
 // Turn off response buffering.
 System.Web.HttpContext.Current.Response.Buffer = false;
 // Return a song.
 SongStream song = new SongStream(filePath);
 return song;
}

 <WebMethod(), SoapDocumentMethodAttribute(ParameterStyle:=SoapParameterStyle.Bare)>
 Public Function DownloadSong(ByVal Authorization As DownloadAuthorization, ByVal filePath As String) As SongStream

 ' Turn off response buffering.
 System.Web.HttpContext.Current.Response.Buffer = False
 ' Return a song.
 Dim song As New SongStream(filePath)
 Return song

 End Function
End Class

[XmlSchemaProvider("MySchema")]
public class SongStream : IXmlSerializable
{
 private const string ns = "http://demos.Contoso.com/webservices";
 private string filePath;

 public SongStream() { }

 public SongStream(string filePath)
 {
 this.filePath = filePath;
 }

 // This is the method named by the XmlSchemaProviderAttribute applied to the type.
 public static XmlQualifiedName MySchema(XmlSchemaSet xs)
 {
 // This method is called by the framework to get the schema for this type.
 // We return an existing schema from disk.

 XmlSerializer schemaSerializer = new XmlSerializer(typeof(XmlSchema));
 string xsdPath = null;
 // NOTE: replace the string with your own path.
 xsdPath = System.Web.HttpContext.Current.Server.MapPath("SongStream.xsd");
 XmlSchema s = (XmlSchema)schemaSerializer.Deserialize(
 new XmlTextReader(xsdPath), null);
 xs.XmlResolver = new XmlUrlResolver();
 xs.Add(s);

 return new XmlQualifiedName("songStream", ns);
 }

 void IXmlSerializable.WriteXml(System.Xml.XmlWriter writer)
 {
 // This is the chunking code.
 // ASP.NET buffering must be turned off for this to work.

 int bufferSize = 4096;
 char[] songBytes = new char[bufferSize];
 FileStream inFile = File.Open(this.filePath, FileMode.Open, FileAccess.Read);

 long length = inFile.Length;

 // Write the file name.
 writer.WriteElementString("fileName", ns, Path.GetFileNameWithoutExtension(this.filePath));

 // Write the size.
 writer.WriteElementString("size", ns, length.ToString());

 // Write the song bytes.
 writer.WriteStartElement("song", ns);

 StreamReader sr = new StreamReader(inFile, true);
 int readLen = sr.Read(songBytes, 0, bufferSize);

 while (readLen > 0)
 {
 writer.WriteStartElement("chunk", ns);
 writer.WriteChars(songBytes, 0, readLen);
 writer.WriteEndElement();

 writer.Flush();
 readLen = sr.Read(songBytes, 0, bufferSize);
 }

 writer.WriteEndElement();
 inFile.Close();
 }

 XmlSchema IXmlSerializable.GetSchema()
 {
 throw new NotImplementedException();
 }

 void IXmlSerializable.ReadXml(System.Xml.XmlReader reader)
 {
 throw new NotImplementedException();
 }
}

<XmlSchemaProvider("MySchema")>
Public Class SongStream
 Implements IXmlSerializable

 Private Const ns As String = "http://demos.Contoso.com/webservices"
 Private filePath As String

 Public Sub New()

 End Sub

 Public Sub New(ByVal filePath As String)
 Me.filePath = filePath
 End Sub

 ' This is the method named by the XmlSchemaProviderAttribute applied to the type.
 Public Shared Function MySchema(ByVal xs As XmlSchemaSet) As XmlQualifiedName
 ' This method is called by the framework to get the schema for this type.
 ' We return an existing schema from disk.
 Dim schemaSerializer As New XmlSerializer(GetType(XmlSchema))
 Dim xsdPath As String = Nothing
 ' NOTE: replace SongStream.xsd with your own schema file.
 xsdPath = System.Web.HttpContext.Current.Server.MapPath("SongStream.xsd")
 Dim s As XmlSchema = CType(schemaSerializer.Deserialize(New XmlTextReader(xsdPath)), XmlSchema)
 xs.XmlResolver = New XmlUrlResolver()
 xs.Add(s)

 Return New XmlQualifiedName("songStream", ns)

 End Function

 Sub WriteXml(ByVal writer As System.Xml.XmlWriter) Implements IXmlSerializable.WriteXml
 ' This is the chunking code.
 ' ASP.NET buffering must be turned off for this to work.

 Dim bufferSize As Integer = 4096
 Dim songBytes(bufferSize) As Char
 Dim inFile As FileStream = File.Open(Me.filePath, FileMode.Open, FileAccess.Read)

 Dim length As Long = inFile.Length

 ' Write the file name.
 writer.WriteElementString("fileName", ns, Path.GetFileNameWithoutExtension(Me.filePath))

 ' Write the size.
 writer.WriteElementString("size", ns, length.ToString())

 ' Write the song bytes.
 writer.WriteStartElement("song", ns)

 Dim sr As New StreamReader(inFile, True)
 Dim readLen As Integer = sr.Read(songBytes, 0, bufferSize)

 While readLen > 0
 writer.WriteStartElement("chunk", ns)
 writer.WriteChars(songBytes, 0, readLen)
 writer.WriteEndElement()

 writer.Flush()
 readLen = sr.Read(songBytes, 0, bufferSize)
 End While

 writer.WriteEndElement()
 inFile.Close()
 End Sub

 Function GetSchema() As System.Xml.Schema.XmlSchema Implements IXmlSerializable.GetSchema
 Throw New System.NotImplementedException()
 End Function

 Sub ReadXml(ByVal reader As System.Xml.XmlReader) Implements IXmlSerializable.ReadXml
 Throw New System.NotImplementedException()
 End Sub
End Class

public class SongFile : IXmlSerializable
{
 public static event ProgressMade OnProgress;

 public SongFile()
 { }

 private const string ns = "http://demos.teched2004.com/webservices";
 public static string MusicPath;
 private string filePath;
 private double size;

 void IXmlSerializable.ReadXml(System.Xml.XmlReader reader)
 {
 reader.ReadStartElement("DownloadSongResult", ns);
 ReadFileName(reader);
 ReadSongSize(reader);
 ReadAndSaveSong(reader);
 reader.ReadEndElement();
 }

 void ReadFileName(XmlReader reader)
 {
 string fileName = reader.ReadElementString("fileName", ns);
 this.filePath =
 Path.Combine(MusicPath, Path.ChangeExtension(fileName, ".mp3"));
 }

 void ReadSongSize(XmlReader reader)
 {
 this.size = Convert.ToDouble(reader.ReadElementString("size", ns));
 }

 void ReadAndSaveSong(XmlReader reader)
 {
 FileStream outFile = File.Open(
 this.filePath, FileMode.Create, FileAccess.Write);

 string songBase64;
 byte[] songBytes;
 reader.ReadStartElement("song", ns);
 double totalRead = 0;
 while (true)
 {
 if (reader.IsStartElement("chunk", ns))
 {
 songBase64 = reader.ReadElementString();
 totalRead += songBase64.Length;
 songBytes = Convert.FromBase64String(songBase64);
 outFile.Write(songBytes, 0, songBytes.Length);
 outFile.Flush();

 if (OnProgress != null)
 {
 OnProgress(100 * (totalRead / size));
 }
 }

 else
 {
 break;
 }
 }

 outFile.Close();
 reader.ReadEndElement();
 }

 public void Play()
 {
 System.Diagnostics.Process.Start(this.filePath);
 }

 XmlSchema IXmlSerializable.GetSchema()
 {
 throw new NotImplementedException();
 }

 public void WriteXml(XmlWriter writer)
 {
 throw new NotImplementedException();
 }
}

Public Class SongFile
 Implements IXmlSerializable
 Public Shared Event OnProgress As ProgressMade

 Public Sub New()

 End Sub

 Private Const ns As String = "http://demos.teched2004.com/webservices"
 Public Shared MusicPath As String
 Private filePath As String
 Private size As Double

 Sub ReadXml(ByVal reader As System.Xml.XmlReader) Implements IXmlSerializable.ReadXml
 reader.ReadStartElement("DownloadSongResult", ns)
 ReadFileName(reader)
 ReadSongSize(reader)
 ReadAndSaveSong(reader)
 reader.ReadEndElement()
 End Sub

 Sub ReadFileName(ByVal reader As XmlReader)
 Dim fileName As String = reader.ReadElementString("fileName", ns)
 Me.filePath = Path.Combine(MusicPath, Path.ChangeExtension(fileName, ".mp3"))

 End Sub

 Sub ReadSongSize(ByVal reader As XmlReader)
 Me.size = Convert.ToDouble(reader.ReadElementString("size", ns))

 End Sub

 Sub ReadAndSaveSong(ByVal reader As XmlReader)
 Dim outFile As FileStream = File.Open(Me.filePath, FileMode.Create, FileAccess.Write)

 Dim songBase64 As String
 Dim songBytes() As Byte
 reader.ReadStartElement("song", ns)
 Dim totalRead As Double = 0
 While True
 If reader.IsStartElement("chunk", ns) Then
 songBase64 = reader.ReadElementString()
 totalRead += songBase64.Length
 songBytes = Convert.FromBase64String(songBase64)
 outFile.Write(songBytes, 0, songBytes.Length)
 outFile.Flush()
 RaiseEvent OnProgress((100 * (totalRead / size)))
 Else
 Exit While
 End If
 End While

 outFile.Close()
 reader.ReadEndElement()
 End Sub

 Public Sub Play()
 System.Diagnostics.Process.Start(Me.filePath)
 End Sub

 Function GetSchema() As System.Xml.Schema.XmlSchema Implements IXmlSerializable.GetSchema
 Throw New System.NotImplementedException()
 End Function

 Public Sub WriteXml(ByVal writer As XmlWriter) Implements IXmlSerializable.WriteXml
 Throw New System.NotImplementedException()
 End Sub
End Class

Compiling the code

• The code uses the following namespaces: System, System.Runtime.Serialization, System.Web.Services, System.Web.Services.Protocols, System.Xml, System.Xml.Serialization, and System.Xml.Schema.

See also

• Custom Serialization