samedi 22 avril 2017

How to speed up loading text file to multi vector

I have to load large files (several GB) with data and I want to load them to two dimensional vector. Code below does the job, but it is insanely slow. To be more specific, the goal is to get all lines where values in 2nd column are equal to index(_lh,_sh). And then exclude the lines where 4th column value is same as line+1 and line-1. Now, I'am new to c++ and I usualy code in Python (have working code for this problem already). But I need it to be as fast as posible so I tried to rewrite my python code to C++. But it rus slower than Python now (and only getting the data to vector is implemented)... so before I proceed, I want to improve that. From what I have found in similar questions, the problem would be dynamic vectors, .push_back() and getline().

I am rather confused about maping and chunk loading mentioned in similar questions so I am not able to change the code acording to these.

Could you help me to optimize this code?

Thank you.

#include <iostream>
#include <sstream>
#include <fstream>
#include <array>
#include <string>
#include <vector>

using namespace std;

int pixel(int radek, int sloupec, int rozmer = 256) {
    int index = (radek - 1) * rozmer + sloupec;
    int index_lh = (index - rozmer - 1);
    int index_sh = (index - rozmer);
    int index_ph = (index - rozmer + 1);
    int index_l = (index - 1);
    int index_p = (index + 1);
    int index_ld = (index + rozmer - 1);
    int index_sd = (index + rozmer);
    int index_pd = (index + rozmer + 1);
    array<int, 9> index_all = { {index, index_lh, index_sh, index_ph, index_l, index_p, index_ld, index_sd, index_pd } };
    vector<vector<string>> Data;
    vector<string> Line;
    string line;

    for (int m = 2; m < 3; m++) {
        string url = ("e:/TPX3 - kalibrace - 170420/ToT_ToA_calib_Zn_" + to_string(m) + string(".t3pa"));
        cout << url << endl;
        ifstream infile(url);
        if (!infile)
        {
            cout << "Error opening output file" << endl;
            system("pause");
            return -1;
        }
        while (getline(infile, line))
        {
            Line.push_back(line);
            istringstream txtStream(line);
            string txtElement;
            vector<string> Element;
            while (getline(txtStream, txtElement, '\t')){
                Element.push_back(txtElement);
            }
            Data.push_back(Element);
        }
    }
    cout << Data[1][0] << ' ' << Data[1][1] << ' ' << Data[1][2] << endl;
    return 0; 
}

int main()
{   
    int x = pixel(120, 120);
    cout << x << endl;
    system("pause");
    return 0;
}

Aucun commentaire:

Enregistrer un commentaire