Accessing a small subset of a large numpy array quickly

[iwishiwasaneagle]

I'm trying to access the surrounding 8 squares from a given x,y of a image. This image is around 5000x4000 pixels large. For smaller images (5x5) my pybind code is much much faster than anything I could do with python. However, for the file that I'm intending on using it's as slow, if not slower than the python. This is leading me to believe that the array is being passed by value rather than reference, or I'm forcing the entire array to be loaded. Any advice? Code below.

std::vector<double>
get_vicinity(float x, float y, py::array_t<double> map, float north, float east, float south, float west){
    float width_m  = east-west, height_m = north-south;
    int width_px = map.shape(1), height_px = map.shape(0);

    std::vector<int> rows = {1,1,1,0,0,0,-1,-1,-1};
    std::vector<int> cols  = {-1, 0, 1, -1, 0, 1, -1, 0, 1};

    for(int& c : cols)
        c += x;

    for(int& r : rows)
        r += y;

    std::vector<bool> mask_rows_above_0(rows.size());
    for(std::size_t i = 0; i < rows.size(); i++)
        mask_rows_above_0[i] = rows[i] > 0;

    for(int& r : rows)
        if (r<0){
            r = -1;
        } else {
            r = height_px - 1 - r;
        }

    std::vector<double> vicinity(9);
    auto buf = map.request();
    double* buf_ptr  =  (double*) buf.ptr;
    double c;
    int row,col;
    for(std::size_t i = 0; i<rows.size();i++){
        row = rows[i];
        col = cols[i];
        c = 0;
        if ((col>=0) && (col <= width_px-1) && (row  >= 0) && (row <= height_px)){
            c = (double)buf_ptr[row*width_px + col];
        }
        vicinity[i] = c;
    }
    return vicinity;
}