CUDA __global__ function not called

Question

I'm trying to compile simple helloworld example copied from here. I'm using CentOS 6.4 environment.

// This is the REAL "hello world" for CUDA!
// It takes the string "Hello ", prints it, then passes it to CUDA with an array
// of offsets. Then the offsets are added in parallel to produce the string "World!"
// By Ingemar Ragnemalm 2010

#include <stdio.h>

const int N = 16; 
const int blocksize = 16; 

__global__ 
void hello(char *a, int *b) 
{
    a[threadIdx.x] += b[threadIdx.x];
}

int main()
{
    char a[N] = "Hello \0\0\0\0\0\0";
    int b[N] = {15, 10, 6, 0, -11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};

    char *ad;
    int *bd;
    const int csize = N*sizeof(char);
    const int isize = N*sizeof(int);

    printf("%s", a);

    cudaMalloc( (void**)&ad, csize ); 
    cudaMalloc( (void**)&bd, isize ); 
    cudaMemcpy( ad, a, csize, cudaMemcpyHostToDevice ); 
    cudaMemcpy( bd, b, isize, cudaMemcpyHostToDevice ); 

    dim3 dimBlock( blocksize, 1 );
    dim3 dimGrid( 1, 1 );
    hello<<<dimGrid, dimBlock>>>(ad, bd);
    cudaMemcpy( a, ad, csize, cudaMemcpyDeviceToHost ); 
    cudaFree( ad );
    cudaFree( bd );

    printf("%s\n", a);
    return EXIT_SUCCESS;
}

Trying to compile it works fine:

$ nvcc hello_world.cu -o hello_world.bin

But when I run it:

$ ./hello_world.bin
Hello Hello

It doesn't print the expected 'Hello World', but instead 'Hello Hello'. If I comment some code out from the __global__ function there is no impact at all, or even adding printf into the hello() function does not result in anything. It seems the function isn't called. What am I missing? What can I check?

I have also tried some other example source codes, which work on another box. The problem seems to be the same, so something isn't right on this computer.

---

Edit:

$ nvcc --version
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2013 NVIDIA Corporation
Built on Wed_Jul_17_18:36:13_PDT_2013
Cuda compilation tools, release 5.5, V5.5.0
$ nvidia-smi -a
-bash: nvidia-smi: command not found
$ cat /proc/driver/nvidia/version
NVRM version: NVIDIA UNIX x86_64 Kernel Module  319.60  Wed Sep 25 14:28:26 PDT 2013
GCC version:  gcc version 4.4.7 20120313 (Red Hat 4.4.7-3) (GCC)
$ dmesg | grep NVRM
NVRM: loading NVIDIA UNIX x86_64 Kernel Module  319.60  Wed Sep 25 14:28:26 PDT 2013
NVRM: loading NVIDIA UNIX x86_64 Kernel Module  319.60  Wed Sep 25 14:28:26 PDT 2013

Answer 1

Thanks to advise from @RobertCrovella, I added return value checks all over my code:

#include <stdio.h>

const int N = 16;
const int blocksize = 16;

#define gpuErrchk(ans) { gpuAssert((ans), __FILE__, __LINE__); }
inline void gpuAssert(cudaError_t code, char *file, int line, bool abort=true)
{
   if (code != cudaSuccess)
   {
      fprintf(stderr,"GPUassert: %s %s %d\n", cudaGetErrorString(code), file, line);
      if (abort) exit(code);
   }
}
__global__
void hello(char *a, int *b)
{
    a[threadIdx.x] += b[threadIdx.x];
}

int main()
{
        char a[N] = "Hello \0\0\0\0\0\0";
        int b[N] = {15, 10, 6, 0, -11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};

        char *ad;
        int *bd;
        const int csize = N*sizeof(char);
        const int isize = N*sizeof(int);

        printf("%s", a);

        gpuErrchk(cudaMalloc( (void**)&ad, csize ));
        gpuErrchk(cudaMalloc( (void**)&bd, isize ));
        gpuErrchk(cudaMemcpy( ad, a, csize, cudaMemcpyHostToDevice ));
        gpuErrchk(cudaMemcpy( bd, b, isize, cudaMemcpyHostToDevice ));

        dim3 dimBlock( blocksize, 1 );
        dim3 dimGrid( 1, 1 );
        hello<<<dimGrid, dimBlock>>>(ad, bd);
        gpuErrchk( cudaPeekAtLastError() );
        gpuErrchk( cudaDeviceSynchronize() );
        gpuErrchk(cudaMemcpy( a, ad, csize, cudaMemcpyDeviceToHost ));
        gpuErrchk(cudaFree( ad ));
        gpuErrchk(cudaFree( bd ));

        printf("%s\n", a);
        return EXIT_SUCCESS;
}

This lead to discovery of this error when running the code:

$ nvcc hello_world.cu -o hello_world.bin
$ ./hello_world.bin
GPUassert: CUDA driver version is insufficient for CUDA runtime version hello_world.cu 39

I was running this on a cloud provider which did the setup of CUDA environment, so I suspected something was wrong in env I had done after that. In my environment, cuda env is set up by using

module load cuda55/toolkit/5.5.22

which should set up the environment fully. This was something I did not know at first, so before using that, I had tried to set up some paths myself. Due to that this was in my .bash_profile:

export CUDA_INSTALL_PATH=/cm/shared/apps/cuda55/toolkit/current
export PATH=$PATH:$CUDA_INSTALL_PATH/bin
export LD_LIBRARY_PATH=$CUDA_INSTALL_PATH/lib64
export PATH=$PATH:$CUDA_INSTALL_PATH/lib

Once I removed the stuff I had added to my .bash_profile and did a logout/login, everything started to work without issues.