System.Net.ScatterGatherBuffers.MemoryChuck allocates large byte[]

G

Guest

Summary: System.Net.ScatterGatherBuffers.MemoryChuck allocates inordinately
large byte[]s when sending large post data.

The following application consumes inordinate quantities of memory. My code
does not explicitly allocate memory in a loop nor does it explicitly allocate
large blocks of memory. Yet, the application’s memory footprint will grow as
large as 370 MB. Rarely will it run to completion; usually, it throws an out
of memory exception or locks up just before sending the packet that would
total 512 MBs.

SciTech .NET Memory Profiler, reports that
System.Net.ScatterGatherBuffers.MemoryChuck is holding onto very large blocks
of memory, specifically byte[]s, which it appears to have allocated in a
doubling fashion. For example, the sizes of these blocks are 226MB, 113MB,
56.5MB, 28MB, 14MB, and so on down. It’s as if
System.Net.ScatterGatherBuffers is allocating a byte of memory for every byte
I send on the socket, but is allocating increasingly large buffers to store
this.

If I send a smaller amount of data, the send completes, but the
System.Net.ScatterGatherBuffers.MemoryChuck byte[]s are not released when the
socket stream is closed, all references to the WebRequest and its stream are
released, and a garbage collection is explicitly requested.

I’ve also tried performing the socket communication on the UI message pump
thread without seeing a different behavior.

I’m running Windows 2000 on a box with 512 MB of memory. I ran this test on
..NET Fx 1.1 SP1. I haven’t tried any other framework versions.

Am I doing something wrong? Is this a known issue? Do you have any
suggestions or explanations?

Thanks in advance for any help you can provide.


using System;
using System.Drawing;
using System.Collections;
using System.ComponentModel;
using System.Windows.Forms;
using System.Data;
using System.Net;
using System.IO;
using System.Threading;

namespace SocketMemoryLeak {
public class Form1 : System.Windows.Forms.Form {
private const int KB = 1024;
private const int MB = KB * KB;
private System.Windows.Forms.Button btnSend;
private System.ComponentModel.Container components = null;
private System.Windows.Forms.Label lblMBsSent;
private System.Windows.Forms.Label lblGCed;
private System.Windows.Forms.Label lblMemoryAllocated;

public Form1() {
InitializeComponent();
}

protected override void Dispose(bool disposing) {
if (disposing) {
if (components != null) {
components.Dispose();
}
}
base.Dispose(disposing);
}

#region Windows Form Designer generated code
private void InitializeComponent() {
this.btnSend = new System.Windows.Forms.Button();
this.lblMBsSent = new System.Windows.Forms.Label();
this.lblGCed = new System.Windows.Forms.Label();
this.lblMemoryAllocated = new System.Windows.Forms.Label();
this.SuspendLayout();
//
// btnSend
//
this.btnSend.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.btnSend.Location = new System.Drawing.Point(16, 8);
this.btnSend.Name = "btnSend";
this.btnSend.Size = new System.Drawing.Size(184, 23);
this.btnSend.TabIndex = 0;
this.btnSend.Text = "Start";
this.btnSend.Click += new System.EventHandler(this.btnSend_Click);
//
// lblMBsSent
//
this.lblMBsSent.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblMBsSent.Location = new System.Drawing.Point(16, 40);
this.lblMBsSent.Name = "lblMBsSent";
this.lblMBsSent.Size = new System.Drawing.Size(184, 23);
this.lblMBsSent.TabIndex = 1;
this.lblMBsSent.Text = "Sent: 0 MBs";
this.lblMBsSent.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// lblGCed
//
this.lblGCed.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblGCed.Location = new System.Drawing.Point(16, 88);
this.lblGCed.Name = "lblGCed";
this.lblGCed.Size = new System.Drawing.Size(184, 23);
this.lblGCed.TabIndex = 2;
this.lblGCed.Text = "No previous garbage collection";
this.lblGCed.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// lblMemoryAllocated
//
this.lblMemoryAllocated.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblMemoryAllocated.Location = new System.Drawing.Point(16,
64);
this.lblMemoryAllocated.Name = "lblMemoryAllocated";
this.lblMemoryAllocated.Size = new System.Drawing.Size(184, 23);
this.lblMemoryAllocated.TabIndex = 3;
this.lblMemoryAllocated.Text = "Memory Allocated: 0 MB";
this.lblMemoryAllocated.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// Form1
//
this.AutoScaleBaseSize = new System.Drawing.Size(5, 13);
this.ClientSize = new System.Drawing.Size(208, 117);
this.Controls.Add(this.lblMemoryAllocated);
this.Controls.Add(this.lblGCed);
this.Controls.Add(this.lblMBsSent);
this.Controls.Add(this.btnSend);
this.Name = "Form1";
this.Text = "Form1";
this.ResumeLayout(false);

}
#endregion

[STAThread]
static void Main() {
Application.Run(new Form1());
}

private void btnSend_Click(object sender, System.EventArgs e) {
this.btnSend.Enabled = false;
Thread t = new Thread(new ThreadStart(this.Send));
t.IsBackground = true;
t.Start();
}

private void DoGC(int bytesSent) {
this.lblGCed.Text = "Starting GC...";
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
this.lblGCed.Text = "GC'd at " + (bytesSent / MB) + " MB";
}

private void SetSent(int bytesSent) {
this.lblMBsSent.Text = "Sent: " + (bytesSent / MB) + " MB";
this.lblMemoryAllocated.Text = "Memory Allocated: " +
(Environment.WorkingSet / MB) + " MB";
}

private void Send() {
byte[] buffer = new byte[16 * KB];
int totalBytesToSend = 514 * MB;

HttpWebRequest request = (HttpWebRequest)
WebRequest.Create("http://marias/cgi-bin/leif.cfg/php/enduser/fileUpload.php");
request.Method = "POST";
request.ContentLength = totalBytesToSend;
Stream requestStream = request.GetRequestStream();

int bytesSent = 0;
int gcInterval = 64 * MB;
int displayInterval = 1 * MB;
while (bytesSent < totalBytesToSend) {
requestStream.Write(buffer, 0, buffer.Length);
bytesSent += buffer.Length;
if (bytesSent % displayInterval == 0) {
this.SetSent(bytesSent);
}
if (bytesSent % gcInterval == 0) {
this.DoGC(bytesSent);
}
}

requestStream.Flush();
requestStream.Close();
requestStream = null;
request = null;

this.DoGC(bytesSent);
MessageBox.Show("Send thread ending.");
this.btnSend.Enabled = true;
}
}
}
 
F

Feroze [MSFT]

this is because httpwebrequest buffers data, as it migt need to repost
it in so9me cases.

you can disable this by setting AllowWriteStreamBuffering=false

-----
feroze

This posting is provided as-is

http://weblogs.asp.net/feroze_daud


lwickland said:
Summary: System.Net.ScatterGatherBuffers.MemoryChuck allocates inordinately
large byte[]s when sending large post data.

The following application consumes inordinate quantities of memory. My code
does not explicitly allocate memory in a loop nor does it explicitly allocate
large blocks of memory. Yet, the application’s memory footprint will grow as
large as 370 MB. Rarely will it run to completion; usually, it throws an out
of memory exception or locks up just before sending the packet that would
total 512 MBs.

SciTech .NET Memory Profiler, reports that
System.Net.ScatterGatherBuffers.MemoryChuck is holding onto very large blocks
of memory, specifically byte[]s, which it appears to have allocated in a
doubling fashion. For example, the sizes of these blocks are 226MB, 113MB,
56.5MB, 28MB, 14MB, and so on down. It’s as if
System.Net.ScatterGatherBuffers is allocating a byte of memory for every byte
I send on the socket, but is allocating increasingly large buffers to store
this.

If I send a smaller amount of data, the send completes, but the
System.Net.ScatterGatherBuffers.MemoryChuck byte[]s are not released when the
socket stream is closed, all references to the WebRequest and its stream are
released, and a garbage collection is explicitly requested.

I’ve also tried performing the socket communication on the UI message pump
thread without seeing a different behavior.

I’m running Windows 2000 on a box with 512 MB of memory. I ran this test on
.NET Fx 1.1 SP1. I haven’t tried any other framework versions.

Am I doing something wrong? Is this a known issue? Do you have any
suggestions or explanations?

Thanks in advance for any help you can provide.


using System;
using System.Drawing;
using System.Collections;
using System.ComponentModel;
using System.Windows.Forms;
using System.Data;
using System.Net;
using System.IO;
using System.Threading;

namespace SocketMemoryLeak {
public class Form1 : System.Windows.Forms.Form {
private const int KB = 1024;
private const int MB = KB * KB;
private System.Windows.Forms.Button btnSend;
private System.ComponentModel.Container components = null;
private System.Windows.Forms.Label lblMBsSent;
private System.Windows.Forms.Label lblGCed;
private System.Windows.Forms.Label lblMemoryAllocated;

public Form1() {
InitializeComponent();
}

protected override void Dispose(bool disposing) {
if (disposing) {
if (components != null) {
components.Dispose();
}
}
base.Dispose(disposing);
}

#region Windows Form Designer generated code
private void InitializeComponent() {
this.btnSend = new System.Windows.Forms.Button();
this.lblMBsSent = new System.Windows.Forms.Label();
this.lblGCed = new System.Windows.Forms.Label();
this.lblMemoryAllocated = new System.Windows.Forms.Label();
this.SuspendLayout();
//
// btnSend
//
this.btnSend.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.btnSend.Location = new System.Drawing.Point(16, 8);
this.btnSend.Name = "btnSend";
this.btnSend.Size = new System.Drawing.Size(184, 23);
this.btnSend.TabIndex = 0;
this.btnSend.Text = "Start";
this.btnSend.Click += new System.EventHandler(this.btnSend_Click);
//
// lblMBsSent
//
this.lblMBsSent.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblMBsSent.Location = new System.Drawing.Point(16, 40);
this.lblMBsSent.Name = "lblMBsSent";
this.lblMBsSent.Size = new System.Drawing.Size(184, 23);
this.lblMBsSent.TabIndex = 1;
this.lblMBsSent.Text = "Sent: 0 MBs";
this.lblMBsSent.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// lblGCed
//
this.lblGCed.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblGCed.Location = new System.Drawing.Point(16, 88);
this.lblGCed.Name = "lblGCed";
this.lblGCed.Size = new System.Drawing.Size(184, 23);
this.lblGCed.TabIndex = 2;
this.lblGCed.Text = "No previous garbage collection";
this.lblGCed.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// lblMemoryAllocated
//
this.lblMemoryAllocated.Anchor =
((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.lblMemoryAllocated.Location = new System.Drawing.Point(16,
64);
this.lblMemoryAllocated.Name = "lblMemoryAllocated";
this.lblMemoryAllocated.Size = new System.Drawing.Size(184, 23);
this.lblMemoryAllocated.TabIndex = 3;
this.lblMemoryAllocated.Text = "Memory Allocated: 0 MB";
this.lblMemoryAllocated.TextAlign =
System.Drawing.ContentAlignment.MiddleLeft;
//
// Form1
//
this.AutoScaleBaseSize = new System.Drawing.Size(5, 13);
this.ClientSize = new System.Drawing.Size(208, 117);
this.Controls.Add(this.lblMemoryAllocated);
this.Controls.Add(this.lblGCed);
this.Controls.Add(this.lblMBsSent);
this.Controls.Add(this.btnSend);
this.Name = "Form1";
this.Text = "Form1";
this.ResumeLayout(false);

}
#endregion

[STAThread]
static void Main() {
Application.Run(new Form1());
}

private void btnSend_Click(object sender, System.EventArgs e) {
this.btnSend.Enabled = false;
Thread t = new Thread(new ThreadStart(this.Send));
t.IsBackground = true;
t.Start();
}

private void DoGC(int bytesSent) {
this.lblGCed.Text = "Starting GC...";
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
this.lblGCed.Text = "GC'd at " + (bytesSent / MB) + " MB";
}

private void SetSent(int bytesSent) {
this.lblMBsSent.Text = "Sent: " + (bytesSent / MB) + " MB";
this.lblMemoryAllocated.Text = "Memory Allocated: " +
(Environment.WorkingSet / MB) + " MB";
}

private void Send() {
byte[] buffer = new byte[16 * KB];
int totalBytesToSend = 514 * MB;

HttpWebRequest request = (HttpWebRequest)
WebRequest.Create("http://marias/cgi-bin/leif.cfg/php/enduser/fileUpload.php");
request.Method = "POST";
request.ContentLength = totalBytesToSend;
Stream requestStream = request.GetRequestStream();

int bytesSent = 0;
int gcInterval = 64 * MB;
int displayInterval = 1 * MB;
while (bytesSent < totalBytesToSend) {
requestStream.Write(buffer, 0, buffer.Length);
bytesSent += buffer.Length;
if (bytesSent % displayInterval == 0) {
this.SetSent(bytesSent);
}
if (bytesSent % gcInterval == 0) {
this.DoGC(bytesSent);
}
}

requestStream.Flush();
requestStream.Close();
requestStream = null;
request = null;

this.DoGC(bytesSent);
MessageBox.Show("Send thread ending.");
this.btnSend.Enabled = true;
}
}
}
Click to expand...
 

Ask a Question

Want to reply to this thread or ask your own question?

You'll need to choose a username for the site, which only take a couple of moments. After that, you can post your question and our members will help you out.

Ask a Question

Similar Threads

FlowLayoutPanel and anchoring child controls - ahh! 1
UserControl and Fliker - Take 2 4
Complete program that give exception ObjectDisposed was unhandled by user 1
problems with arrays 2
threading and delegates 4
TreeView doesn't show last item 6
A complete program using Threads. Can this program be shorter 1
ComboBox OnPropertyChanged problem 3

Top