发生错误后，cuFFT无法恢复

我找不到在之前失败的启动后启动cuFFT处理的方法。

这里是一个最小的例子。主要思想如下:我们创建一个简单的cuFFT处理器，它可以管理它的资源(设备内存和cuFFT计划)。我们检查这个处理器是否做FFT。然后我们要求创建太多的计划，因此我们强制执行cuFFT错误。然后我们释放所有资源，尝试重复成功的发布。但是，处理器在失败后什么也做不了。

首先，这里有一个相当长的序言:

#include <iostream>
using std::cout;
using std::cerr;
using std::endl;
#include <vector>
using std::vector;
#include "cuda_runtime.h"
#include "cufft.h"
// cuFFT API errors
static char* _cufftGetErrorEnum( cufftResult_t error )
{
    switch ( error )
    {
        case CUFFT_SUCCESS:
        return "CUFFT_SUCCESS";
        case CUFFT_INVALID_PLAN:
        return "cuFFT was passed an invalid plan handle";
        case CUFFT_ALLOC_FAILED:
        return "cuFFT failed to allocate GPU or CPU memory";
        // No longer used
        case CUFFT_INVALID_TYPE:
        return "CUFFT_INVALID_TYPE";
        case CUFFT_INVALID_VALUE:
        return "User specified an invalid pointer or parameter";
        case CUFFT_INTERNAL_ERROR:
        return "Driver or internal cuFFT library error";
        case CUFFT_EXEC_FAILED:
        return "Failed to execute an FFT on the GPU";
        case CUFFT_SETUP_FAILED:
        return "The cuFFT library failed to initialize";
        case CUFFT_INVALID_SIZE:
        return "User specified an invalid transform size";
        // No longer used
        case CUFFT_UNALIGNED_DATA:
        return "CUFFT_UNALIGNED_DATA";
        case CUFFT_INCOMPLETE_PARAMETER_LIST:
        return "Missing parameters in call";
        case CUFFT_INVALID_DEVICE:
        return "Execution of a plan was on different GPU than plan creation";
        case CUFFT_PARSE_ERROR:
        return "Internal plan database error";
        case CUFFT_NO_WORKSPACE:
        return "No workspace has been provided prior to plan execution";
        case CUFFT_NOT_IMPLEMENTED:
        return "CUFFT_NOT_IMPLEMENTED";
        case CUFFT_LICENSE_ERROR:
        return "CUFFT_LICENSE_ERROR";
    }
    return "<unknown>";
}
// check cuda runtime calls
bool cudaCheck( cudaError_t err )
{
    if ( err != cudaSuccess )
    {
        cudaDeviceSynchronize();
        cerr << cudaGetErrorString( cudaGetLastError() ) << endl;
        return false;
    }
    return true;
}
// check cuFFT calls
bool cufftCheck( cufftResult_t err )
{
    if ( err != CUFFT_SUCCESS )
    {
        cerr << _cufftGetErrorEnum( err ) << endl;
        return false;
    }
    return true;
}

接下来，我们定义一个简单的cuFFT处理器，它可以管理它的资源(设备内存和cuFFT计划)

class CCuFFT_Processor
{
    vector<cufftHandle> _plans;
    cufftComplex *_data;
    size_t _data_bytes;
    // Release resouces
    bool ReleaseAll();
    bool ReleaseMemory();
    bool ReleasePlans();
public:
    CCuFFT_Processor() :
    _data( NULL ),
    _data_bytes( 0 )
    {
        _plans.reserve( 32 );
        _plans.clear();
    }
    ~CCuFFT_Processor()
    {
        ReleaseAll();
    }
    bool Run();
    bool Alloc( size_t data_len, size_t batch_len );
};

下面是我们释放资源的方法:

bool     CCuFFT_Processor::ReleaseMemory()
{
    bool chk = true;
    if ( _data != NULL )
    {
        chk         = cudaCheck( cudaFree( _data ) );
        _data       = NULL;
        _data_bytes = 0;
    }
    return chk;
}
bool CCuFFT_Processor::ReleasePlans()
{
    bool chk = true;
    for ( auto & p : _plans )
        chk = chk && cufftCheck( cufftDestroy( p ) );
    _plans.clear();
    return chk;
}
bool CCuFFT_Processor::ReleaseAll()
{
    bool chk = true;
    chk = chk && cudaCheck( cudaDeviceSynchronize() );
    chk = chk && ReleaseMemory();
    chk = chk && ReleasePlans();
    chk = chk && cudaCheck( cudaDeviceReset() );
    return chk;
}

bool CCuFFT_Processor::Alloc( size_t data_len, size_t batch_len )
{
    bool   chk   = true;
    size_t bytes = sizeof( cufftComplex ) * data_len * batch_len;
    // CUDA resources
    if ( _data_bytes < bytes )
        chk = chk && ReleaseMemory();
    if ( _data == NULL )
    {
        chk         = chk && cudaCheck( cudaMalloc( (void **)&_data, bytes ) );
        _data_bytes = bytes;
    }
    // cuFFT resources
    chk = chk && ReleasePlans();
    for ( size_t b = 1; chk && ( b <= batch_len ); b *= 2 )
    {
        cufftHandle new_plan;
        chk = cufftCheck(
            cufftPlan1d( &new_plan, int(data_len), CUFFT_C2C, int(b) ) );
        if ( chk )
            _plans.push_back( new_plan );
    }
    if ( !chk )
        ReleaseAll();
    return chk;
}
bool CCuFFT_Processor::Run()
{
    bool chk = true;
    chk = cufftCheck(
        cufftExecC2C( *_plans.rbegin(), _data, _data, CUFFT_FORWARD ) );
    if ( !chk )
        ReleaseAll();
    chk = chk && cudaCheck( cudaDeviceSynchronize() );
    return chk;
}

最后，程序

int main()
{
    size_t batch  = 1 << 5;
    size_t length = 1 << 21;
    CCuFFT_Processor proc;
    // Normal run
    if ( proc.Alloc( length, batch ) )
        proc.Run();
    // Run with error
    length *= 4;
    if ( proc.Alloc( length, batch ) )
        proc.Run();
    // Normal run : check recovery
    length /= 4;
    if ( proc.Alloc( length, batch ) )
        proc.Run();
    return EXIT_SUCCESS;
}

如果我使用较小的length = 1 << 18，则不会发生错误。然而，对于较大的length = 1 << 21，出现两个错误:

cuFFT failed to allocate GPU or CPU memory
Failed to execute an FFT on the GPU

第一个错误是预期的，我们是有意这样做的。但第二个不是。虽然设备被重置并且成功分配了新的资源，但是cuFFT执行FFT失败。

我使用GTX 970。我尝试了所有的组合:cuda 6.5, cuda 7.5, 32位平台，64位平台等，但没有成功。