Firstly I'm new at programming. I want to load CSV files, i have coded below code but its took to load 100mb file for 3-5 mins(multi threaded, single thread 5+ mins). In excel its just few secs and bulkinserting to cells. My CSV file format is;
"title1","title2","title3","title4"...
"1","1_werwq","","","myname","this a sentence with comma,,,,,",""...
i couldn't use OLEDBCONNECTION it not loading properly so i coded this;
private: System::Void backgroundWorker1_DoWork(System::Object^ sender, System::ComponentModel::DoWorkEventArgs^ e) {
StreamReader^ file = gcnew StreamReader(openFileDialog2->FileName, Encoding::Default, true);
String^ columnNames = file->ReadLine();
std::wstring ws = marshal_as<std::wstring>(columnNames);
std::vector<std::wstring> vws;
seperate(ws, vws);
for (auto & s : vws) {
DataColumn ^clm = gcnew DataColumn(gcnew String(s.c_str()));
clm->DefaultValue = String::Empty;
alldata->Columns->Add(clm);
}
setColumnNames();
int ilines = 0;
while (!file->EndOfStream) {
iList->Add(file->ReadLine());
ilines++;
}
ParallelOptions^ ops = gcnew ParallelOptions();
ops->MaxDegreeOfParallelism = Environment::ProcessorCount * 2;
CancellationTokenSource ^cts = gcnew CancellationTokenSource();
CancellationToken ^ct = cts->Token;
ops->CancellationToken = cts->Token;
//Parallel::ForEach(iList, gcnew Action<iStringList^>(this, &MyForm::readStr));
System::Diagnostics::Stopwatch ^sw = gcnew System::Diagnostics::Stopwatch();
sw->Start();
Parallel::For(0, iList->Count, ops, gcnew Action<int>(this, &MyForm::readStr));
sw->Stop();
}
private: System::Void readStr(int index) {
try {
String^ str = iList->getLine(index);
std::wstring aline = marshal_as<std::wstring>(str);
std::vector<std::wstring> wlines;
seperate(aline, wlines);
DataRow^ oDataRow = getnewRow();
int i = 0;
for (auto &c : wlines) {
String^lan = gcnew String(c.c_str());
oDataRow[i] = lan;
i++;
}
backgroundWorker1->ReportProgress(1);
setRow(oDataRow);
iList->performStep();
}
catch (...) {
toolStripStatusLabel2->Text = "HATAAAAAAAAAAAA";
}
}
How excel loading files so fast and bulkinsert to cells without memory leak? i tried native code but its takes alot ram...my Data files can reach 2000 columns and 20k rows. Thank you for help.
Aucun commentaire:
Enregistrer un commentaire