CUDA设备指针丢失

Lost in CUDA device pointers

本文关键字:指针 CUDA      更新时间:2023-10-16

作为论文工作的一部分,我正在CUDA项目中工作(修改他人的代码、添加功能等)。作为CUDA的新手,这对我来说是一个真正的挑战。我正在使用计算能力1.3卡,4 x特斯拉C1060。遗憾的是,我遇到了平台的一些限制。

我需要将几个新结构传递给设备,我相信这些结构是正确复制的。但是,当我在内核调用中尝试将指针传递到设备上的结构时,我达到了256字节的限制(如本问题所述)。

我的代码是这样的:

// main.cu
static void RunGPU(HostThreadState *hstate)
{
SimState *HostMem = &(hstate->host_sim_state);
SimState DeviceMem;
TetrahedronStructGPU *h_root = &(hstate->root);
TetrahedronStructGPU *d_root;
TriangleFacesGPU *h_faces = &(hstate->faces);
TriangleFacesGPU *d_faces;
GPUThreadStates tstates;
unsigned int n_threads = hstate->n_tblks * NUM_THREADS_PER_BLOCK;
unsigned int n_tetras  = hstate->n_tetras; // 9600
unsigned int n_faces   = hstate->n_faces;  // 38400
InitGPUStates(HostMem, h_root, h_faces, &DeviceMem, &tstates, hstate->sim, 
d_root, d_faces, n_threads, n_tetras, n_faces );
cudaThreadSynchronize();
...
kernel<<<dimGrid, dimBlock, k_smem_sz>>>(DeviceMem, tstates, /*OK, these 2*/
d_root, d_faces);
// Limit of 256 bytes adding d_root and/or d_faces
cudaThreadSynchronize();
...
}

InitGPUStates函数在另一个源文件中:

// kernel.cu
int InitGPUStates(SimState* HostMem, TetrahedronStructGPU* h_root,
TriangleFacesGPU* h_faces,
SimState* DeviceMem, GPUThreadStates *tstates,
SimulationStruct* sim, 
TetrahedronStructGPU* d_root, TriangleFacesGPU* d_faces,
int n_threads, int n_tetras, int n_faces)
{
unsigned int size;
// Allocate and copy RootTetrahedron (d_root) on device
size = n_tetras * sizeof(TetrahedronStructGPU); // Too big
checkCudaErrors(cudaMalloc((void**)&d_root, size));
checkCudaErrors(cudaMemcpy(d_root, h_root, size, cudaMemcpyHostToDevice));
// Allocate and copy Faces (d_faces) on device
size = n_faces * sizeof(TriangleFacesGPU); // Too big
checkCudaErrors(cudaMalloc((void**)&d_faces, size));
checkCudaErrors(cudaMemcpy(d_faces, h_faces, size, cudaMemcpyHostToDevice));     
...
}

我知道我只需要将指针传递到设备内存上的位置。如何在设备中获取地址?指针的传递是否正确?

这两个新结构是:

// header.h
typedef struct {
int idx;
int vertices[4];
float Nx, Ny, Nz, d;
} TriangleFacesGPU;
typedef struct {
int idx, region;
int vertices[4], faces[4], adjTetras[4];
float n, mua, mus, g;
} TetrahedronStructGPU;
// other structures
typedef struct {
BOOLEAN *is_active;
BOOLEAN *dead;
BOOLEAN *FstBackReflectionFlag;
int *NextTetrahedron;
UINT32 *NumForwardScatters;
UINT32 *NumBackwardScatters;
UINT32 *NumBackwardsSpecularReflections;
UINT32 *NumBiases;
UINT32 *p_layer;
GFLOAT *p_x, *p_y, *p_z;
GFLOAT *p_ux, *p_uy, *p_uz;
GFLOAT *p_w;
GFLOAT *Rspecular;
GFLOAT *LocationFstBias;
GFLOAT *OpticalPath;
GFLOAT *MaxDepth;
GFLOAT *MaxLikelihoodRatioIncrease;
GFLOAT *LikelihoodRatioIncreaseFstBias;
GFLOAT *LikelihoodRatio;
GFLOAT *LikelihoodRatioAfterFstBias;
GFLOAT *s, *sleft;
TetrahedronStructGPU *tetrahedron;
TriangleFacesGPU *faces;
} GPUThreadStates;
typedef struct {
UINT32 *n_p_left;
UINT64 *x;
UINT32 *a;
UINT64 *Rd_ra;
UINT64 *A_rz;
UINT64 *Tt_ra;
} SimState;

kernel的定义是

__global__ void kernel(SimState d_state, GPUThreadStates tstates,
TetrahedronStructGPU *d_root,
TriangleFacesGPU *d_faces);

我将把SimState d_state改为指针传递SimState *d_state。以及CCD_ 5至CCD_。

您似乎还没有初始化DeviceMem结构,它应该包含稍后应该用cudaMalloc初始化的指针。

你应该做一些类似的事情:

SimState* DeviceMem;
cudaMalloc(&DeviceMem, sizeof(SimState)) 

也可以(或为该指针分配内存的任何其他方式)。

最终解决了256字节的问题。但是,真的仍然迷失在指针

我修改后的代码是这样的:

// main.cu
static void RunGPU(HostThreadState *hstate)
{
SimState *HostMem = &(hstate->host_sim_state);
// new pointers to pass
SimState *DeviceMem = (SimState*)malloc(sizeof(SimState));
GPUThreadStates *tstates = (GPUThreadStates*)malloc(sizeof(GPUThreadStates));
TetrahedronStructGPU *h_root = hstate->root; //root, pointer in HostThreadState
TetrahedronStructGPU *d_root;
TriangleFacesGPU *h_faces = hstate->faces; //faces, pointer in HostThreadState
TriangleFacesGPU *d_faces;
unsigned int n_threads = hstate->n_tblks * NUM_THREADS_PER_BLOCK;
unsigned int n_tetras  = hstate->n_tetras; // 9600
unsigned int n_faces   = hstate->n_faces;  // 38400
InitGPUStates(HostMem, h_root, h_faces, DeviceMem, tstates, hstate->sim, 
d_root, d_faces, n_threads, n_tetras, n_faces );
cudaThreadSynchronize();
...
kernel<<<dimGrid, dimBlock, k_smem_sz>>>(DeviceMem, tstates,
d_root, d_faces);
// No limit reached!
cudaThreadSynchronize();
...      
}

InitGPUStates函数中,更改如下。特别注意DeviceMem的副本(我尝试了许多形式,但都没有成功)。有些表单(带括号,比如这个cudaMalloc((void **)&(*DeviceMem).n_p_left, size))不会给我任何错误。我假设没有错误意味着没有数据复制到设备。在当前形式中,错误为code=11(cudaErrorInvalidValue) "cudaMalloc((void**)&DeviceMem->n_photons_left, size)"

// kernel.cu
int InitGPUStates(SimState* HostMem, TetrahedronStructGPU* h_root,
TriangleFacesGPU* h_faces,
SimState* DeviceMem, GPUThreadStates *tstates,
SimulationStruct* sim, 
TetrahedronStructGPU* d_root, TriangleFacesGPU* d_faces,
int n_threads, int n_tetras, int n_faces)
{
unsigned int size;
// Allocate and copy RootTetrahedron (d_root) on device
size = n_tetras * sizeof(TetrahedronStructGPU); // Too big
checkCudaErrors(cudaMalloc((void**)&d_root, size));
checkCudaErrors(cudaMemcpy(d_root, h_root, size, cudaMemcpyHostToDevice));
// Allocate and copy Faces (d_faces) on device
size = n_faces * sizeof(TriangleFacesGPU); // Too big
checkCudaErrors(cudaMalloc((void**)&d_faces, size));
checkCudaErrors(cudaMemcpy(d_faces, h_faces, size, cudaMemcpyHostToDevice));     
// HELP NEEDED MAINLY FROM HERE REGARDING POINTER VALUE COPY!
checkCudaErrors( cudaMalloc((void**)&DeviceMem, sizeof(SimState) ); //Needed?
size = sizeof(UINT32);
checkCudaErrors( cudaMalloc(&DeviceMem->n_p_left, size) );
checkCudaErrors( cudaMemcpy(DeviceMem->n_p_left,
HostMem->n_p_left, size, cudaMemcpyHostToDevice) );
size = n_threads * sizeof(UINT32);
checkCudaErrors( cudaMalloc(&DeviceMem->a, size) );
checkCudaErrors( cudaMemcpy(DeviceMem->a, HostMem->a, size,
cudaMemcpyHostToDevice) );
size = n_threads * sizeof(UINT64);
checkCudaErrors( cudaMalloc(&DeviceMem->x, size) );
checkCudaErrors( cudaMemcpy(DeviceMem->x, HostMem->x, size,
cudaMemcpyHostToDevice) );
...
}

我知道我只需要将指针传递到设备内存上的位置。如何在设备中获取地址?指针的传递是否正确?

这两个新结构是:

// header.h
typedef struct {
int idx;
int vertices[4];
float Nx, Ny, Nz, d;
} TriangleFacesGPU;
typedef struct {
int idx, region;
int vertices[4], faces[4], adjTetras[4];
float n, mua, mus, g;
} TetrahedronStructGPU;
// other structures
typedef struct {
BOOLEAN *is_active;
BOOLEAN *dead;
BOOLEAN *FstBackReflectionFlag;
int *NextTetrahedron;
UINT32 *NumForwardScatters;
UINT32 *NumBackwardScatters;
UINT32 *NumBackwardsSpecularReflections;
UINT32 *NumBiases;
UINT32 *p_layer;
GFLOAT *p_x, *p_y, *p_z;
GFLOAT *p_ux, *p_uy, *p_uz;
GFLOAT *p_w;
GFLOAT *Rspecular;
GFLOAT *LocationFstBias;
GFLOAT *OpticalPath;
GFLOAT *MaxDepth;
GFLOAT *MaxLikelihoodRatioIncrease;
GFLOAT *LikelihoodRatioIncreaseFstBias;
GFLOAT *LikelihoodRatio;
GFLOAT *LikelihoodRatioAfterFstBias;
GFLOAT *s, *sleft;
TetrahedronStructGPU *tetrahedron;
TriangleFacesGPU *faces;
} GPUThreadStates;
typedef struct {
UINT32 *n_p_left;
UINT64 *x;
UINT32 *a;
UINT64 *Rd_ra;
UINT64 *A_rz;
UINT64 *Tt_ra;
} SimState;

kernel的定义更改为:

__global__ void kernel(SimState *d_state, GPUThreadStates *tstates,
TetrahedronStructGPU *d_root,
TriangleFacesGPU *d_faces);