i have the following c# script on ssis project in visual studio that generates files based on a column value of a sql table and i would like to add a part that also splits the files when the record count reaches 200..Example of an extraction would be nameoffile_columnvalue_datetime_1.txt for the first 200,nameoffile_columnvalue_datetime_2.txt for the next 200 etc... thank you in advance!
public void Main()
{
// TODO: Add your code here
string datetime_1 = DateTime.Now.ToString("dd/MM/yyyy");
string datetime = datetime_1.Replace("/", String.Empty);
try
{
//Declare Variables
string FileNamePart = Dts.Variables["User::FlatFileNamePart"].Value.ToString();
string DestinationFolder = Dts.Variables["User::DestinationFolder"].Value.ToString();
string TableName = Dts.Variables["User::TableName"].Value.ToString();
string FileDelimiter = Dts.Variables["User::FileDelimiter"].Value.ToString();
string FileExtension = Dts.Variables["User::FileExtension"].Value.ToString();
string ColumnNameForGrouping = Dts.Variables["User::ColumnNameForGrouping"].Value.ToString();
string SubFolder = Dts.Variables["User::SubFolder"].Value.ToString();
Int32 RecordCntPerFile = (Int32)Dts.Variables["User::RecordsPerFile"].Value;
string RecordCntPerFileDecimal = RecordCntPerFile ".0";
//USE ADO.NET Connection from SSIS Package to get data from table
SqlConnection myADONETConnection = new SqlConnection();
myADONETConnection = (SqlConnection)(Dts.Connections["AR_GSLO_OLTP"].AcquireConnection(Dts.Transaction) as SqlConnection);
//Read distinct Group Values for each flat file
string query = "Select distinct " ColumnNameForGrouping " from " TableName;
//MessageBox.Show(query.ToString());
SqlCommand cmd = new SqlCommand(query, myADONETConnection);
//myADONETConnection.Open();
DataTable dt = new DataTable();
dt.Load(cmd.ExecuteReader());
myADONETConnection.Close();
//Loop through values for ColumnNameForGroup
foreach (DataRow dt_row in dt.Rows)
{
string ColumnValue = "";
object[] array = dt_row.ItemArray;
ColumnValue = array[0].ToString();
//Load Data into DataTable from SQL ServerTable
string queryString =
"SELECT * from " TableName " Where " ColumnNameForGrouping "='" ColumnValue "'";
SqlDataAdapter adapter = new SqlDataAdapter(queryString, myADONETConnection);
DataSet ds = new DataSet();
adapter.Fill(ds);
foreach (DataTable d_table in ds.Tables)
{
string FileFullPath = SubFolder "\\" FileNamePart "_" ColumnValue "_" datetime FileExtension;
StreamWriter sw = null;
sw = new StreamWriter(FileFullPath, false);
// Write the Header Row to File an thelw na exei kai header ta kanw uncomment apo 152 mexri 160 grammi
int ColumnCount = d_table.Columns.Count;
// for (int ic = 0; ic < ColumnCount; ic )
//{
// sw.Write(d_table.Columns[ic]);
// if (ic < ColumnCount - 1)
// {
// sw.Write(FileDelimiter);
// }
//}
// sw.Write(sw.NewLine);
// Write All Rows to the File
foreach (DataRow dr in d_table.Rows)
{
for (int ir = 0; ir < ColumnCount; ir )
{
if (!Convert.IsDBNull(dr[ir]))
{
sw.Write(dr[ir].ToString());
}
if (ir < ColumnCount - 1)
{
sw.Write(FileDelimiter);
}
}
sw.Write(sw.NewLine);
}
sw.Close();
}
}
}
catch (Exception exception)
{
// Create Log File for Errors
using (StreamWriter sw = File.CreateText(Dts.Variables["User::LogFolder"].Value.ToString() "\\"
"ErrorLog_" datetime ".log"))
{
sw.WriteLine(exception.ToString());
Dts.TaskResult = (int)ScriptResults.Failure;
}
}
}
CodePudding user response：
So assuming/understanding you have a script task which is the source of your rows?
You can add a Rowcount Task to capture the amount of rows in a variable for later validation.
after that, you might need a new script task where you can do the validation of how many records there is. something like a case statement/if statement or a type of loop for each 200.. odd records to change a value/filename.
Using that loop, you can change the filename/filenamevariable.
So you would need your ConnectionManager File Destination to have an expression with this variable filename. Thus as soon as it hits 200.. records, the script should change the filename and the output should create a new file.
I am not sure if it will like the in flight name change. but think this might be a way to point your thoughts.
CodePudding user response：
i did it first i used a foreach loop container to take the name of the txts as input variables then inside the loop i created a script with the following c# code
public void Main()
{
// TODO: Add your code here
string datetime_1 = DateTime.Now.ToString("dd/MM/yyyy");
string datetime = datetime_1.Replace("/", String.Empty);
StreamWriter writer = null;
try
{
//Declare Variables
string filename= Dts.Variables["User::FileName"].Value.ToString();
//MessageBox.Show(filename);
using (StreamReader inputfile = new System.IO.StreamReader(filename))
{
int count = 0;
int filecount = 0;
string line;
while ((line = inputfile.ReadLine()) != null)
{
if (writer == null || count > 199)
{
count = 0;
if (writer != null)
{
writer.Close();
writer = null;
}
filecount;
writer = new System.IO.StreamWriter(filename.Replace(".txt", "_") filecount.ToString() ".txt", true);
}
writer.WriteLine(line);
count;
}
}
}
catch (Exception exception)
{
// Create Log File for Errors
using (StreamWriter sw = File.CreateText(Dts.Variables["User::LogFolder"].Value.ToString() "\\"
"ErrorLog_" datetime ".log"))
{
sw.WriteLine(exception.ToString());
Dts.TaskResult = (int)ScriptResults.Failure;
}
}
finally
{
if (writer != null)
writer.Close();
}
Dts.TaskResult = (int)ScriptResults.Success;
}
and that was it!