I have a C# program which converts Ebcdic to Ascii. The input files for the program are around 250MB to 300 MB in size, When I am processing by placing single file,in the share path the file processing is happening without any issues, but when there are more than 1 files in the input file location, when the second file is processed to 50% ,I am getting System.OutOfMemoryException , I have pasted sample code where I have used garbage collection (after single files are processed) for freeing up the memory. but its not working as expected, at the same time I also made parser object to null and then tried to garbage collection but still getting the same error.
using System;
using System.Text;
using System.IO;
using System.Linq;
namespace Ebcdic2Ascii
{
class Program
{
static void Main(string[] args)
{
DirectoryInfo d = new DirectoryInfo(@"C:\SampleFiles\input\");
FileInfo[] Files = d.GetFiles("*.dat");
StreamWriter[] writer = new StreamWriter[Files.Count()];
LineTemplate lineTemplate = new LineTemplate(73, "ReservationsData");
lineTemplate.AddFieldTemplate(new FieldTemplate("RESERVATION-NUMBER", FieldType.String, 0, 11));
lineTemplate.AddFieldTemplate(new FieldTemplate("CHECKIN-DATE", FieldType.DateString, 11, 6));
lineTemplate.AddFieldTemplate(new FieldTemplate("CALC-NET-AMOUNT", FieldType.BinaryNum, 17, 4, 2));
lineTemplate.AddFieldTemplate(new FieldTemplate("CUSTOMER-NAME", FieldType.String, 21, 30));
lineTemplate.AddFieldTemplate(new FieldTemplate("RUNDATE", FieldType.DateStringMMDDYY, 51, 6));
lineTemplate.AddFieldTemplate(new FieldTemplate("CURRENCY-CONV-RATE", FieldType.Packed, 57, 6, 6));
lineTemplate.AddFieldTemplate(new FieldTemplate("US-DOLLAR-AMOUNT-DUE", FieldType.Packed, 63, 6, 2));
lineTemplate.AddFieldTemplate(new FieldTemplate("DATE-OF-BIRTH", FieldType.PackedDate, 69, 4));
int i = 0;
foreach (FileInfo File in Files)
{
EbcdicParser parser = new EbcdicParser(File.FullName, lineTemplate);
string Outputpath = "C:\\output\\" File.Name ".txt";
writer[i] = new StreamWriter(Outputpath);
foreach (ParsedLine line in parser.ParsedLines)
{
writer[i].WriteLine(line["RESERVATION-NUMBER"].ToString() "\t" line["CHECKIN-DATE"].ToString() "\t" line["CALC-NET-AMOUNT"].ToString() "\t" line["CALC-NET-AMOUNT"].ToString() "\t" line["CUSTOMER-NAME"].ToString() "\t" line["RUNDATE"].ToString() "\t" line["CURRENCY-CONV-RATE"].ToString() "\t" line["US-DOLLAR-AMOUNT-DUE"].ToString() "\t" line["DATE - OF - BIRTH"].ToString());
}
i = i 1;
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
GC.Collect();
GC.WaitForPendingFinalizers();
}
}
}
}
Code for the parser this also has garbage collection method
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace Ebcdic2Ascii
{
public class EbcdicParser
{
public ParsedLine[] Lines { get; private set; }
public EbcdicParser()
{
//Empty constructor
}
public EbcdicParser(byte[] allBytes, LineTemplate lineTemplate)
{
double expectedRows = (double)allBytes.Length / lineTemplate.LineSize;
Console.WriteLine("{0}: Parsing started", DateTime.Now);
Console.WriteLine("{1}: Line count est {0:#,###.00}", expectedRows, DateTime.Now);
this.Lines = this.ParseAllLines(lineTemplate, allBytes);
//Collect garbage
GC.Collect();
GC.WaitForPendingFinalizers();
Console.WriteLine("{1}: {0} line(s) have been parsed", this.Lines.Count(), DateTime.Now);
}
public EbcdicParser(string sourceFilePath, LineTemplate lineTemplate)
: this(File.ReadAllBytes(sourceFilePath), lineTemplate)
{
//Constructor with the file path
}
public ParsedLine[] ParseAllLines(LineTemplate lineTemplate, byte[] allBytes)
{
bool isSingleLine = false;
this.ValidateInputParameters(lineTemplate, allBytes, isSingleLine);
List<ParsedLine> parsedLines = new List<ParsedLine>();
byte[] lineBytes = new byte[lineTemplate.LineSize];
ParsedLine parsedLine;
for (int i = 0; i < allBytes.Length; i = lineTemplate.LineSize)
{
if (i % 1000 == 0)
{
//Print progress
Console.Write(i "\r");
}
Array.Copy(allBytes, i, lineBytes, 0, lineTemplate.LineSize);
parsedLine = this.ParseSingleLine(lineTemplate, lineBytes);
parsedLines.Add(parsedLine);
}
return parsedLines.ToArray();
}
public ParsedLine[] ParseAllLines(LineTemplate lineTemplate, string sourceFilePath)
{
return this.ParseAllLines(lineTemplate, File.ReadAllBytes(sourceFilePath));
}
public ParsedLine ParseSingleLine(LineTemplate lineTemplate, byte[] lineBytes)
{
bool isSingleLine = true;
this.ValidateInputParameters(lineTemplate, lineBytes, isSingleLine);
ParsedLine parsedLine = new ParsedLine(lineTemplate, lineBytes);
return parsedLine;
}
private bool ValidateInputParameters(LineTemplate lineTemplate, byte[] allBytes, bool isSingleLine)
{
if (allBytes == null)
{
throw new ArgumentNullException("Ebcdic data is not provided");
}
if (lineTemplate == null)
{
throw new ArgumentNullException("Line template is not provided");
}
if (lineTemplate.FieldsCount == 0)
{
throw new Exception("Line template must contain at least one field");
}
if (allBytes.Length < lineTemplate.LineSize)
{
throw new Exception("Data length is shorter than the line size");
}
if (isSingleLine && allBytes.Length != lineTemplate.LineSize)
{
throw new Exception("Bytes count doesn't equal to line size");
}
double expectedRows = (double)allBytes.Length / lineTemplate.LineSize;
if (expectedRows % 1 != 0) //Expected number of rows is not a whole number
{
throw new Exception("Expected number of rows is not a whole number. Check line template.");
}
return true;
}
public void CreateCsvFile(string outputFilePath, bool includeColumnNames, bool addQuotes)
{
if (this.Lines == null || this.Lines.Length == 0)
{
throw new Exception("No lines have been parsed");
}
ParserUtilities.ConvertLineArrayToCsv(this.Lines, outputFilePath, includeColumnNames, addQuotes);
}
}
}
ParsedLine class file
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace Ebcdic2Ascii
{
public class ParsedLine
{
public LineTemplate Line_Template { get; private set; }
public Dictionary<string, ParsedField> FieldDictionary
{ get; private set; } //= new Dictionary<string, ParsedField>();
public string this[string fieldName]
{
get
{
return this.FieldDictionary[fieldName].Value.Trim();
}
}
//Constructor
public ParsedLine(LineTemplate lineTemplate, byte[] lineBytes)
{
this.Line_Template = lineTemplate;
this.ParseLine(lineBytes, lineTemplate);
}
private void ParseLine(byte[] lineBytes, LineTemplate lineTemplate)
{
this.ValidateInputParameters(lineBytes, lineTemplate);
foreach (var fieldTemplate in lineTemplate.FieldTemplateDictionary)
{
FieldDictionary.Add(fieldTemplate.Key,
new ParsedField(lineBytes, lineTemplate.FieldTemplateDictionary[fieldTemplate.Key]));
}
}
private void ValidateInputParameters(byte[] lineBytes, LineTemplate template)
{
if (lineBytes == null)
{
throw new ArgumentNullException("Line bytes required");
}
if (lineBytes.Length < template.LineSize)
{
throw new Exception(String.Format(
"Bytes provided: {0}, line size: {1}", lineBytes.Length, template.LineSize));
}
if (template == null)
{
throw new ArgumentNullException("line template is required");
}
if (template.FieldsCount == 0)
{
throw new Exception("Field templates have not been defined in the line template");
}
}
public string GetParsedFieldValuesCSV(bool addQuotes)
{
StringBuilder sb = new StringBuilder();
int count = 0;
foreach (ParsedField parsedField in this.FieldDictionary.Values)
{
sb.Append(addQuotes ? "\"" : "");
sb.Append(parsedField.Value);
sb.Append(addQuotes ? "\"" : "");
sb.Append(this.FieldDictionary.Count < count ? "," : "");
count ;
}
return sb.ToString();
}
}
}
class ParsedField
using System;
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
namespace Ebcdic2Ascii
{
public class ParsedField
{
public FieldTemplate Field_Template { get; private set; }
public string Value { get; private set; }
public byte[] OriginalBytes { get; private set; }
public string OriginalBytesInHex
{
get
{
return BitConverter.ToString(this.OriginalBytes);
}
}
public string OriginalBytesInDec
{
get
{
return ParserUtilities.ConvertBytesToDec(this.OriginalBytes);
}
}
public bool ParsedSuccessfully { get; private set; }
//Constructor
public ParsedField(byte[] lineBytes, FieldTemplate fieldTemplate)
{
this.ParsedSuccessfully = true;
this.Field_Template = fieldTemplate;
this.Value = ParseField(lineBytes, fieldTemplate);
}
private string ParseField(byte[] lineBytes, FieldTemplate template)
{
if (lineBytes == null || lineBytes.Length == 0)
{
ParserUtilities.PrintError("Line bytes is null or empty");
this.ParsedSuccessfully = false;
return null;
}
if (lineBytes.Length < (template.StartPosition template.FieldSize))
{
this.ParsedSuccessfully = false;
throw new Exception(String.Format(
"Field \"{0}\" length falls outside the line length", template.FieldName));
}
byte[] fieldBytes = new byte[template.FieldSize];
Array.Copy(lineBytes, template.StartPosition, fieldBytes, 0, template.FieldSize);
this.OriginalBytes = fieldBytes;
if (this.Field_Template.Type == FieldType.AlphaNum)
{
return this.ConvertAlphaNumEbcdic(fieldBytes);
}
else if (this.Field_Template.Type == FieldType.Numeric)
{
return this.ConvertNumericEbcdic(fieldBytes, template.DecimalPlaces);
}
else if (this.Field_Template.Type == FieldType.Packed)
{
return this.Unpack(fieldBytes, template.DecimalPlaces);
}
else if (this.Field_Template.Type == FieldType.Binary)
{
return ConvertBinaryEbcdic(fieldBytes, template.DecimalPlaces);
}
else if (this.Field_Template.Type == FieldType.Date)
{
return ConvertDateStrEbcdic(fieldBytes);
}
else if (this.Field_Template.Type == FieldType.PackedDate)
{
return ConvertPackedDateStrEbcdic(fieldBytes);
}
else if (this.Field_Template.Type == FieldType.SourceBytesInHex)
{
return this.OriginalBytesInHex;
}
else if (this.Field_Template.Type == FieldType.SourceBytesInDec)
{
return this.OriginalBytesInDec;
}
else
{
this.ParsedSuccessfully = false;
throw new Exception(String.Format(
"Unable to parse field \"{0}\". Unknown field type: {1}",
template.FieldName, template.Type.ToString()));
}
}
private string ConvertAlphaNumEbcdic(byte[] ebcdicBytes)
{
if (this.ByteArrayIsFullOf_0xFF(ebcdicBytes))
{
return "";
}
//Encoding asciiEnc = Encoding.ASCII;
//Encoding ebcdicEnc = Encoding.GetEncoding("IBM037");
//string result = Encoding.ASCII.GetString(Encoding.Convert(ebcdicEnc, asciiEnc, ebcdicBytes));
//Thank you sx2008
Encoding ebcdicEnc = Encoding.GetEncoding("IBM037");
string result = ebcdicEnc.GetString(ebcdicBytes); // convert EBCDIC Bytes -> Unicode string
return result;
}
private string ConvertNumericEbcdic(byte[] ebcdicBytes, int decimalPlaces)
{
string tempNumStr = this.ConvertAlphaNumEbcdic(ebcdicBytes).Trim();
if (tempNumStr == null || tempNumStr.Length == 0)
{
return "";
}
if (Regex.IsMatch(tempNumStr, @"^\d $")) //Unsigned integer
{
string result = this.AdjustDecimalValues(Int64.Parse(tempNumStr), decimalPlaces);
return result;
}
else if (Regex.IsMatch(tempNumStr, @"^\d [A-R{}]$")) //Signed integer
{
string lastChar = tempNumStr.Substring(tempNumStr.Length - 1);
switch (lastChar)
{
case "{":
tempNumStr = tempNumStr.Replace("{", "0");
break;
case "A":
tempNumStr = tempNumStr.Replace("A", "1");
break;
case "B":
tempNumStr = tempNumStr.Replace("B", "2");
break;
case "C":
tempNumStr = tempNumStr.Replace("C", "3");
break;
case "D":
tempNumStr = tempNumStr.Replace("D", "4");
break;
case "E":
tempNumStr = tempNumStr.Replace("E", "5");
break;
case "F":
tempNumStr = tempNumStr.Replace("F", "6");
break;
case "G":
tempNumStr = tempNumStr.Replace("G", "7");
break;
case "H":
tempNumStr = tempNumStr.Replace("H", "8");
break;
case "I":
tempNumStr = tempNumStr.Replace("I", "9");
break;
case "}":
tempNumStr = "-" tempNumStr.Replace("}", "0");//Fixed
break;
case "J":
tempNumStr = "-" tempNumStr.Replace("J", "1");
break;
case "K":
tempNumStr = "-" tempNumStr.Replace("K", "2");
break;
case "L":
tempNumStr = "-" tempNumStr.Replace("L", "3");
break;
case "M":
tempNumStr = "-" tempNumStr.Replace("M", "4");
break;
case "N":
tempNumStr = "-" tempNumStr.Replace("N", "5");
break;
case "O":
tempNumStr = "-" tempNumStr.Replace("O", "6");
break;
case "P":
tempNumStr = "-" tempNumStr.Replace("P", "7");
break;
case "Q":
tempNumStr = "-" tempNumStr.Replace("Q", "8");
break;
case "R":
tempNumStr = "-" tempNumStr.Replace("R", "9");
break;
}
string result = this.AdjustDecimalValues(Int64.Parse(tempNumStr), decimalPlaces);
return result;
}
else
{
this.ParsedSuccessfully = false;
return tempNumStr;
}
}
private string ConvertBinaryEbcdic(byte[] ebcdicBytes, int decimalPlaces)
{
if (this.ByteArrayIsFullOf_0xFF(ebcdicBytes))
{
return "";
}
//BitConverter requires low order bytes goes first, followed by the higher order bytes.
//Bytes are stored in the file in the opposite order, thus need to reverse bytes
Array.Reverse(ebcdicBytes);
long tempNum;
if (ebcdicBytes.Length == 2)
{
//If 2 bytes are provided -- assume it's a short
tempNum = BitConverter.ToUInt16(ebcdicBytes, 0);
}
else if (ebcdicBytes.Length == 4)
{
//If 4 bytes are provided -- assume it's an int
tempNum = BitConverter.ToInt32(ebcdicBytes, 0);
}
else
{
//Just in case
throw new Exception(String.Format(
"Incorrect number of bytes provided for a binary field: {1}", decimalPlaces));
}
string result = this.AdjustDecimalValues(tempNum, decimalPlaces);
return result;
}
private string AdjustDecimalValues(long numericValue, int decimalPlaces)
{
if (decimalPlaces == 0)
{
return numericValue.ToString();
}
double result = numericValue / Math.Pow(10, decimalPlaces);
return result.ToString();
}
private string ConvertDateStrEbcdic(byte[] ebcdicBytes)
{
string dateStr = this.ConvertAlphaNumEbcdic(ebcdicBytes).Trim();
string result = this.ConvertDateStr(dateStr);
return result;
}
private string ConvertPackedDateStrEbcdic(byte[] ebcdicBytes)
{
string dateStr = this.Unpack(ebcdicBytes, 0);
string result = this.ConvertDateStr(dateStr);
return result;
}
private string ConvertDateStr(string dateStr)
{
dateStr = dateStr.Trim();
if (dateStr.Trim() == "" || dateStr == "0" ||
dateStr == "0000000" || dateStr == "9999999")
{
return "";
}
if (Regex.IsMatch(dateStr, @"^\d{3,5}$"))
{
dateStr = dateStr.PadLeft(6, '0');
}
Match match = Regex.Match(dateStr, @"^(?<Year>\d{3})(?<Month>\d{2})
(?<Day>\d{2})$"); //E.g.: 0801232 = 1980-12-31; 1811231 = 2080-12-31
if (match.Success)
{
int year = Int32.Parse(match.Groups["Year"].Value) 1900; //013 => 1913, 113 => 2013...
int month = Int32.Parse(match.Groups["Month"].Value);
int day = Int32.Parse(match.Groups["Day"].Value);
try
{
DateTime tempDate = new DateTime(year, month, day);
return tempDate.ToString("yyyy-MM-dd");
}
catch { }
}
if (Regex.IsMatch(dateStr, @"^\d{6}$"))
{
DateTime tempDate;
if (DateTime.TryParseExact(dateStr, "yyMMdd",
CultureInfo.InvariantCulture, DateTimeStyles.None, out tempDate))
{
return tempDate.ToString("yyyy-MM-dd");
}
}
this.ParsedSuccessfully = false;
return dateStr;
}
private string Unpack(byte[] ebcdicBytes, int decimalPlaces)
{
if (ByteArrayIsFullOf_0xFF(ebcdicBytes))
{
return "";
}
long lo = 0;
long mid = 0;
long hi = 0;
bool isNegative;
// this nybble stores only the sign, not a digit.
// "C" hex is positive, "D" hex is negative, and "F" hex is unsigned.
switch (Nibble(ebcdicBytes, 0))
{
case 0x0D:
isNegative = true;
break;
case 0x0F:
case 0x0C:
isNegative = false;
break;
default:
//throw new Exception("Bad sign nibble");
this.ParsedSuccessfully = false;
return this.ConvertAlphaNumEbcdic(ebcdicBytes);
}
long intermediate;
long carry;
long digit;
for (int j = ebcdicBytes.Length * 2 - 1; j > 0; j--)
{
// multiply by 10
intermediate = lo * 10;
lo = intermediate & 0xffffffff;
carry = intermediate >> 32;
intermediate = mid * 10 carry;
mid = intermediate & 0xffffffff;
carry = intermediate >> 32;
intermediate = hi * 10 carry;
hi = intermediate & 0xffffffff;
carry = intermediate >> 32;
// By limiting input length to 14, we ensure overflow will never occur
digit = Nibble(ebcdicBytes, j);
if (digit > 9)
{
//throw new Exception("Bad digit");
this.ParsedSuccessfully = false;
return this.ConvertAlphaNumEbcdic(ebcdicBytes);
}
intermediate = lo digit;
lo = intermediate & 0xffffffff;
carry = intermediate >> 32;
if (carry > 0)
{
intermediate = mid carry;
mid = intermediate & 0xffffffff;
carry = intermediate >> 32;
if (carry > 0)
{
intermediate = hi carry;
hi = intermediate & 0xffffffff;
carry = intermediate >> 32;
// carry should never be non-zero. Back up with validation
}
}
}
decimal result = new Decimal((int)lo, (int)mid, (int)hi, isNegative, (byte)decimalPlaces);
return result.ToString();
}
private int Nibble(byte[] ebcdicBytes, int nibbleNo)
{
int b = ebcdicBytes[ebcdicBytes.Length - 1 - nibbleNo / 2];
return (nibbleNo % 2 == 0) ? (b & 0x0000000F) : (b >> 4);
}
private bool ByteArrayIsFullOf_0xFF(byte[] ebcdicBytes)
{
if (ebcdicBytes == null || ebcdicBytes.Length == 0)
{
return false;
}
foreach (byte b in ebcdicBytes)
{
if (b != 0xFF)
{
return false;
}
}
return true;
}
}
}
LineTemplate class,FieldTemplate class,ParserUtilities class are present in page
Click 
CodePudding user response：
It is likely that the System.OutOfMemoryException is occurring because you have run out of file handles, not actual memory. You are creating a new StreamWriter for every file you process without disposing it.
See the documentation for a System.OutOfMemoryException. It specifically says:
Although the garbage collector is able to free memory allocated to managed types, it does not manage memory allocated to unmanaged resources such as operating system handles (including handles to files, memory-mapped files, pipes, registry keys, and wait handles) and memory blocks allocated directly by Windows API calls or by calls to memory allocation functions such as malloc. Types that consume unmanaged resources implement the IDisposable interface.
If you are consuming a type that uses unmanaged resources, you should be sure to call its IDisposable.Dispose method when you have finished using it. (Some types also implement a Close method that is identical in function to a Dispose method.) For more information, see the Using Objects That Implement IDisposable topic.
It's important to note that the Garbage Collector does not call Dispose() for you.
You must call writer[i].Dispose() when you have finished writing each stream.
CodePudding user response：
I agree with John Glenn, try put your parsing works in a method, and Enigmativity's advice, using a single StreamWriter.
static void Parse(.....)
{
EbcdicParser parser = new EbcdicParser(File.FullName, lineTemplate);
string Outputpath = "C:\\output\\" File.Name ".txt";
using(var writer = new StreamWriter(Outputpath))
{
foreach (ParsedLine line in parser.ParsedLines)
{
.....
}
}
}
Run GC outside the scope.
int i = 0;
foreach (FileInfo File in Files)
{
Parse(......);
GC.Collect();
}
CodePudding user response：
Your garbage collection calls do not have the effect that you intend. Within your foreach loop, the EbcdicParser parser is still accessible because it is in scope of the currently executing function, so it is not considered for garbage collection.
Similarly, within your EbcdicParser, the byte array is still accessible, so it isn't cleared by your garbage collection call either.