The function call is slow in C++

I'm calling a function in C++ main function, for some reason it's taking long to call the function.

#include <iostream>
#include <opencv2/opencv.hpp>
#include "Eigen/Dense"


int inRo(float val){
    return (int)round(val);}

std::pair<int,int> inRo(std::pair<float,float> pt){
    return std::make_pair(inRo(pt.first),inRo(pt.second));}


// Gives the boundary points of Obstacles (in this case, the boundary points of white region in img)
std::vector<std::pair<int,int>> getBoundPts(cv::Mat img){
    std::vector<std::pair<int,int>> bound_pts;
    cv::Mat img_dilate;

    cv::dilate(img.clone(),img_dilate,cv::Mat(),cv::Point(-1,-1),1,1,1);
    cv::Mat img_bound = img_dilate - img;
    std::vector<cv::Point> bound_pts_cv;
    cv::findNonZero(img_bound,bound_pts_cv);

    for(int i = 0;i < bound_pts_cv.size();i++){
        std::pair<int,int> pt = std::make_pair(bound_pts_cv[i].x,bound_pts_cv[i].y);
        bound_pts.push_back(pt);
    }

    return bound_pts;
}

void getNeighPts(std::pair<float,float> pt,Eigen::MatrixXf potentials_map,std::vector<std::pair<int,int>>& list_pts_bound,std::vector<float>& list_vals_bound,std::vector<std::pair<int,int>>& list_pts_empty){
//this function tries divides the neighbourhood points of a point 'pt'
//into two categories, empty (or list_pts_empty, are the ones with zero
//potential value, which can be obtained from map_potential) and non-zero ones
//(list_pts_bound, which are the ones with a non-zero potential value which
//can also be obtained from map_potential and list_val_bound is the corresponding 
//potential values of the list_pts_bound).

//I'M SURE THE FUNCTION CAN BE WRITTEN IN SIMPLER WAYS BUT THE ONE I
//NEED IS MORE GENERIC AND COULD ABLE TO MODIFIED FOR VARIOUS TYPES OF
//NEIGHBOURHOODS AROUND A POINT.


    clock_t ct1,ct2;

    ct1 = clock();

    list_pts_empty.clear();
    list_vals_bound.clear();
    list_pts_bound.clear();
    std::pair<int,int> ind;
    float pot_val_ind;

    std::vector<std::pair<int,int>> neighs_perp;
    neighs_perp.push_back(std::make_pair(0,1));
    neighs_perp.push_back(std::make_pair(0,-1));
    neighs_perp.push_back(std::make_pair(1,0));
    neighs_perp.push_back(std::make_pair(-1,0));

    int i,j;

    for(int k = 0;k < neighs_perp.size();k++)
    {
        i = neighs_perp[k].first;
        j = neighs_perp[k].second;

        ind = std::make_pair(pt.first+i,pt.second+j);
        if(ind.first >= 0 && ind.second >= 0 && ind.first < potentials_map.cols() && ind.second < potentials_map.rows())
        {
            pot_val_ind = potentials_map(ind.second,ind.first);
            if(pot_val_ind > 0)
            {
                list_pts_bound.push_back(ind);
                list_vals_bound.push_back(potentials_map(ind.second,ind.first) + 1);
            }

            else if(inRo(pot_val_ind) == 0)
            {
                list_pts_empty.push_back(ind);
            }
        }

    }

    ct2 = clock();

    std::cout<<"Inside : "<<float(ct2-ct0)/CLOCKS_PER_SEC<<std::endl;
}


int main(){
    clock_t t1,t2;

    cv::Mat img = cv::Mat::zeros(500,500,CV_8UC1);
    cv::Mat img_win = img(cv::Rect(125,125,250,250));
    img_win.setTo(255);

    std::vector<std::pair<int,int>> brush_que = getBoundPts(img);
    Eigen::MatrixXf mat_potential(500,500);
    Eigen::MatrixXf mat_block = Eigen::MatrixXf::Ones(250,250);
    mat_potential.block(125,125,250,250) = -255*mat_block;

    for(int i = 0;i < brush_que.size();i++){
        std::pair<float,float> pt = brush_que[i];
        mat_potential(pt.second,pt.first) = 1;
    }

    std::vector<std::pair<int,int>> neighs_empty;
    std::vector<std::pair<int,int>> neighs_pts;
    std::vector<float> neighs_val;

    std::pair<int,int> pt_ex = brush_que[0];

    t1 = clock();
    getNeighPts(pt_ex,mat_potential,neighs_pts,neighs_val,neighs_empty);
    t2 = clock();

    std::cout<<"Outside : "<<float(t2-t1)/CLOCKS_PER_SEC<<std::endl;

}

the t2-t1 >>> ct2-ct1. This is the trimmed version of my main code, and here I'm getting,

Inside : 1e-05
Outside : 0.002574

and in the main code, the difference is going way higher depending on the type of 'img'. Any suggesions on the problem would be appreciated. Thanks in advance :)