cudamemcpy error:"the launch timed out and was terminated"

Question

My code is a parallel implmentation that calculates the nth digit of pi. When I finish the kernel and try to copy the memory back to the host I get a "the launch timed out and was terminated" error. I used this code for error checking for each cudamalloc, cudamemcpy, and kernal launch.

std::string error = cudaGetErrorString(cudaGetLastError());
printf("%s\n", error);

These calls were saying everything was fine until the first cudamemcpy call after returning from the kernel. the error happens in the line "cudaMemcpy(avhost, avdev, size, cudaMemcpyDeviceToHost);" in main. Any help is appreciated.

#include <stdlib.h>
#include <stdio.h>
#include <math.h>

#define mul_mod(a,b,m) fmod( (double) a * (double) b, m)
///////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////
/* return the inverse of x mod y */
__device__ int inv_mod(int x,int y) {
  int q,u,v,a,c,t;

  u=x;
  v=y;
  c=1;
  a=0;
  do {
    q=v/u;

    t=c;
    c=a-q*c;
    a=t;

    t=u;
    u=v-q*u;
    v=t;
  } while (u!=0);
  a=a%y;
  if (a<0) a=y+a;
  return a;
}
///////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////
/* return the inverse of u mod v, if v is odd */
__device__ int inv_mod2(int u,int v) {
  int u1,u3,v1,v3,t1,t3;

  u1=1;
  u3=u;

  v1=v;
  v3=v;

  if ((u&1)!=0) {
    t1=0;
    t3=-v;
    goto Y4;
  } else {
    t1=1;
    t3=u;
  }

  do {

    do {
      if ((t1&1)==0) {
    t1=t1>>1;
    t3=t3>>1;
      } else {
    t1=(t1+v)>>1;
    t3=t3>>1;
      }
      Y4:;
    } while ((t3&1)==0);

    if (t3>=0) {
      u1=t1;
      u3=t3;
    } else {
      v1=v-t1;
      v3=-t3;
    }
    t1=u1-v1;
    t3=u3-v3;
    if (t1<0) {
      t1=t1+v;
    }
  } while (t3 != 0);
  return u1;
}


/* return (a^b) mod m */
__device__ int pow_mod(int a,int b,int m)
{
  int r,aa;

  r=1;
  aa=a;
  while (1) {
    if (b&1) r=mul_mod(r,aa,m);
    b=b>>1;
    if (b == 0) break;
    aa=mul_mod(aa,aa,m);
  }
  return r;
}
///////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////
/* return true if n is prime */
int is_prime(int n)
{
   int r,i;
   if ((n % 2) == 0) return 0;

   r=(int)(sqrtf(n));
   for(i=3;i<=r;i+=2) if ((n % i) == 0) return 0;
   return 1;
}
///////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////
/* return the prime number immediatly after n */
int next_prime(int n)
{
   do {
      n++;
   } while (!is_prime(n));
   return n;
}
///////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////
#define DIVN(t,a,v,vinc,kq,kqinc)       \
{                       \
  kq+=kqinc;                    \
  if (kq >= a) {                \
    do { kq-=a; } while (kq>=a);        \
    if (kq == 0) {              \
      do {                  \
    t=t/a;                  \
    v+=vinc;                \
      } while ((t % a) == 0);           \
    }                       \
  }                     \
}

///////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////

__global__ void digi_calc(int *s, int *av, int *primes, int N, int n, int nthreads){
    int a,vmax,num,den,k,kq1,kq2,kq3,kq4,t,v,i,t1, h;
    unsigned int tid = blockIdx.x*blockDim.x + threadIdx.x;
// GIANT LOOP
    for (h = 0; h<1; h++){
    if(tid > nthreads) continue;
    a = primes[tid];
    vmax=(int)(logf(3*N)/logf(a));
    if (a==2) {
      vmax=vmax+(N-n);
      if (vmax<=0) continue;
    }
    av[tid]=1;
    for(i=0;i<vmax;i++) av[tid]*= a;

    s[tid]=0;
    den=1;
    kq1=0;
    kq2=-1;
    kq3=-3;
    kq4=-2;
    if (a==2) {
      num=1;
      v=-n; 
    } else {
      num=pow_mod(2,n,av[tid]);
      v=0;
    }

    for(k=1;k<=N;k++) {

      t=2*k;
      DIVN(t,a,v,-1,kq1,2);
      num=mul_mod(num,t,av[tid]);

      t=2*k-1;
      DIVN(t,a,v,-1,kq2,2);
      num=mul_mod(num,t,av[tid]);

      t=3*(3*k-1);
      DIVN(t,a,v,1,kq3,9);
      den=mul_mod(den,t,av[tid]);

      t=(3*k-2);
      DIVN(t,a,v,1,kq4,3);
      if (a!=2) t=t*2; else v++;
      den=mul_mod(den,t,av[tid]);

      if (v > 0) {
    if (a!=2) t=inv_mod2(den,av[tid]);
    else t=inv_mod(den,av[tid]);
    t=mul_mod(t,num,av[tid]);
    for(i=v;i<vmax;i++) t=mul_mod(t,a,av[tid]);
    t1=(25*k-3);                                                                                                                                                                                                                                                                                                                                                                       
    t=mul_mod(t,t1,av[tid]);
    s[tid]+=t;
    if (s[tid]>=av[tid]) s-=av[tid];
      }
    }

    t=pow_mod(5,n-1,av[tid]);
    s[tid]=mul_mod(s[tid],t,av[tid]);
    }
    __syncthreads();
}
///////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////
int main(int argc,char *argv[])
{
  int N,n,i,totalp, h;
  double sum;
  const char *error;
  int *sdev, *avdev, *shost, *avhost, *adev, *ahost;
    argc = 2;
    argv[1] = "2";
  if (argc<2 || (n=atoi(argv[1])) <= 0) {
    printf("This program computes the n'th decimal digit of pi\n"
       "usage: pi n , where n is the digit you want\n"
       );
    exit(1);
  }
    sum = 0;
    N=(int)((n+20)*logf(10)/logf(13.5));
    totalp=(N/logf(N))+10;
    ahost = (int *)calloc(totalp, sizeof(int));
    i = 0;
    ahost[0]=2;
    for(i=1; ahost[i-1]<=(3*N); ahost[i+1]=next_prime(ahost[i])){
        i++;
    }
    // allocate host memory
    size_t size = i*sizeof(int);
    shost = (int *)malloc(size);
    avhost = (int *)malloc(size);

  //allocate memory on device
    cudaMalloc((void **) &sdev, size);
    cudaMalloc((void **) &avdev, size);
    cudaMalloc((void **) &adev, size);
    cudaMemcpy(adev, ahost, size, cudaMemcpyHostToDevice);

    if (i >= 512){
        h = 512;
    }
    else h = i;
    dim3 dimGrid(((i+512)/512),1,1);                   
    dim3 dimBlock(h,1,1);

    // launch kernel
    digi_calc <<<dimGrid, dimBlock >>> (sdev, avdev, adev, N, n, i);

    //copy memory back to host
    cudaMemcpy(avhost, avdev, size, cudaMemcpyDeviceToHost);
    cudaMemcpy(shost, sdev, size, cudaMemcpyDeviceToHost);

  // end malloc's, memcpy's, kernel calls
    for(h = 0; h <=i; h++){
    sum=fmod(sum+(double) shost[h]/ (double) avhost[h],1.0);
    }
  printf("Decimal digits of pi at position %d: %09d\n",n,(int)(sum*1e9));
    //free memory
    cudaFree(sdev);
    cudaFree(avdev);
    cudaFree(adev);
    free(shost);
    free(avhost);
    free(ahost);
  return 0;
}

score 8 · Accepted Answer · edited May 23 '17 at 10:34

8

This is exactly the same problem you asked about in this question. The kernel is getting terminated early by the driver because it is taking too long to finish. If you read the documentation for any of these runtime API functions you will see the following note:

Note: Note that this function may also return error codes from previous, asynchronous launches.

All that is happening is that the first API call after the kernel launch is returning the error incurred while the kernel was running - in this case the cudaMemcpy call. The way you can confirm this for yourself is to do something like this directly after the kernel launch:

// launch kernel
digi_calc <<<dimGrid, dimBlock >>> (sdev, avdev, adev, N, n, i);
std::string error = cudaGetErrorString(cudaPeekAtLastError());
printf("%s\n", error);
error = cudaGetErrorString(cudaThreadSynchronize());
printf("%s\n", error);

The cudaPeekAtLastError() call will show you if there are any errors in the kernel launch, and the error code returned by the cudaThreadSynchronize() call will show whether any errors were generated while the kernel was executing.

The solution is exactly as outlined in the previous question: probably the simplest way is redesign the code so it is "re-entrant" so you can split the work over several kernel launches, with each kernel launch safely under the display driver watchdog timer limit.

edited May 23 '17 at 10:34

Community

1
1

answered May 31 '11 at 09:06

talonmies

70,661
34
192
269

1

Ah I thought it was at least slightly different since I had done a cudagetlasterror right after the kernel finished executing and it said there was no error. In the other question the kernel actually ran for 5 seconds before being shut down by the watchdog but this kernel finishes in less than a second. – zetatr May 31 '11 at 09:15
I added the code you suggested and received no error for the cudaPeekAtLastError but the cudaThreadSynchronize timed out and was terminated since it lasted over 5 seconds. – zetatr May 31 '11 at 09:17
That is expected. the cudaPeekAtLastError would return an error if you used invalid kernel arguments, for example. The cudaThreadSynchronize blocks the host until the kernel is finished or terminated and gives any errors which occurred between the cudaPeekAtLastError call and the end of the kernel. – talonmies May 31 '11 at 09:28
1

I realized how often I was reading the same variables from global memory and decided it was much smarter to just read once and use a local variable to store it. The problem now is that writing the end result back to global memory causes the same error as before. I tried commenting the two global writes out and there is no error at all. I find it hard to believe that 2 writes to global memory per thread would kill my kernel execution time. – zetatr May 31 '11 at 10:15
It is compiler optimization. If you don't do the writes, the compiler is smart enough to work out that all the code which produces the writes is redundant and will remove that code as being "dead". So your kernel without the writes probably is empty. You can confirm this if you compile to PTX and have a look at the assembler. There should be a lot fewer instructions when the writes are commented out. – talonmies May 31 '11 at 10:20

score 0 · Answer 2 · edited Feb 03 '12 at 02:52

Cuda somehow buffers all the read/write operations on global memory. So you can batch the operations in some loop with some kernel, and it will take actually NO TIME. Then, when you call memcpy, all the buffered operations are done, and it can timeout. Method to go with, is to call cudaThreadSynchronize procedure between iterations.

So remember: if a kernel run takes only nanoseconds to calculate - it doesn't mean that it is so fast - some of the writes to the global memory, are done when memcpy or threadsynchronize is called.

cudamemcpy error:"the launch timed out and was terminated"

2 Answers2

Linked

Related