/*
  1. Login to a cluster that has cuda/GPU installed:
      ssh rye01.stanford.edu
      module add cuda
      cd ~/Codes/MD++.svn

  2. Compile the code:
      make build=R SYS=rye CUDA=yes  phasefield

  3. Run the code as
       cuda is enabled by setting (single phase model) "MD++ model_type = 10"
          bin/phasefield_rye scripts/work/phasefield/pf3d.tcl 10

       CPU version is enabled by setting (single phase model) "MD++ model_type = 0"
          bin/phasefield_rye scripts/work/phasefield/pf3d.tcl  0


Performance:

----
GPU version (cuda)  "MD++ model_type = 10"   NX=NY=NZ=50    
curstep =  19500 F =   8.315100787117e+01 rel_vol =  11.2% 
curstep =  20000 F =   8.314954756421e+01 rel_vol =  11.2% 
CPU time spent:  64.604000 s

GPU version (cuda)  "MD++ model_type = 10"   NX=NY=NZ=50     -D_USEFLOAT
curstep =  19500 F =   8.312005295594e+01 rel_vol =  11.2% 
curstep =  20000 F =   8.311554071061e+01 rel_vol =  11.2% 
CPU time spent:  47.996000 s

GPU version (cuda)  "MD++ model_type = 10"   NX=NY=NZ=50     -D_USEFLOAT -D_USETEXTURE
curstep =  19500 F =   8.312005295594e+01 rel_vol =  11.2% 
curstep =  20000 F =   8.311554071061e+01 rel_vol =  11.2% 
CPU time spent:  46.632000 s

----
CPU version         "MD++ model_type =  0"   NX=NY=NZ=50     
curstep =  19500 F =   8.314971421228e+01 rel_vol =  11.2% 
curstep =  20000 F =   8.314825515021e+01 rel_vol =  11.2% 
CPU time spent:  631.172000 s

======

----
GPU version (cuda)  "MD++ model_type = 10"   NX=NY=NZ=64     -D_USEFLOAT 
curstep =  19500 F =   1.469005080300e+02 rel_vol =  11.2% 
curstep =  20000 F =   1.469029097643e+02 rel_vol =  11.2% 
CPU time spent:  91.520000 s

----
GPU version (cuda)  "MD++ model_type = 10"   NX=NY=NZ=64     -D_USEFLOAT -D_USETEXTURE
curstep =  19500 F =   1.469005080300e+02 rel_vol =  11.2% 
curstep =  20000 F =   1.469029097643e+02 rel_vol =  11.2% 
CPU time spent:  93.064000 s

----
CPU version         "MD++ model_type =  0"   NX=NY=NZ=64     (running on corn)
curstep =  19500 F =   1.471576575183e+02 rel_vol =  11.2% 
curstep =  20000 F =   1.471577229037e+02 rel_vol =  11.2% 
CPU time spent:  1565.660000 s

CPU version         "MD++ model_type =  0"   NX=NY=NZ=64     (macbook pro 2013)
curstep =  19500 F =   1.471576575183e+02 rel_vol =  11.2% 
curstep =  20000 F =   1.471577229038e+02 rel_vol =  11.2% 
CPU time spent:  933.050384 s

To Do:
 1. Why cannot run 65x65x65 (64x64x64 OK) ?
      Max threads per block:  1024
      Max thread dimensions:  (1024, 1024, 64)
      Max grid dimensions:  (65535, 65535, 65535)
 2. Make thread block 3D rectangle and use shared memory
 3. Profiling to tune performance

*/



#ifdef _USECUDA

#include "phasefield.h"

#ifdef _USEFLOAT
#define double float
#endif

#ifdef _USETEXTURE
texture<float> tex_d_PHI;
texture<float> tex_d_EPS_loc;
texture<float> tex_d_tmp_x;
texture<float> tex_d_tmp_y;
texture<float> tex_d_tmp_z;
#endif


__global__ void kernel_FreeEnergy_single_step1 (int NX, int NY, int NZ, double gridsize,
                    double U, double EPS, double EPS1, double EPS2, double EPS3, double *R,
#ifndef _USETEXTURE
                    double *PHI,
#endif
                    double *fden, double *dFdPHI, 
                    double *EPS_loc, double *tmp_x, double *tmp_y, double *tmp_z)
/* PHI   : phase field
   fden  : free energy density
   dFdPHI: variational derivative
   dPHIdt: phase field time derivative
*/
{
    int i, j, k, ip, im, jp, jm, kp, km, ind;
    double h2, f, g, phi, dphidx, dphidy, dphidz, d2phi, dphi_SQR;
    double absdphi, EPS_prefactor, triplesum;
    double nx_lab, ny_lab, nz_lab, nx_cryst, ny_cryst, nz_cryst;
    double dnx_lab_dphidx, dny_lab_dphidx, dnz_lab_dphidx; 
    double dnx_lab_dphidy, dny_lab_dphidy, dnz_lab_dphidy; 
    double dnx_lab_dphidz, dny_lab_dphidz, dnz_lab_dphidz; 
    double dEPS_dnx_lab, dEPS_dny_lab, dEPS_dnz_lab;
    double dEPS_dnx_cryst, dEPS_dny_cryst, dEPS_dnz_cryst;
    double dEPS_dphidx, dEPS_dphidy, dEPS_dphidz;

    double my_fden, my_dFdPHI;

    my_fden = 0;
    my_dFdPHI = 0;

    i = blockIdx.x; j = blockIdx.y; k = threadIdx.x;
    
    /* compute spatial gradients */
    h2 = gridsize*gridsize;

    ip = (i+1)%NX; im = (i-1+NX)%NX;
    jp = (j+1)%NY; jm = (j-1+NY)%NY;
    kp = (k+1)%NZ; km = (k-1+NZ)%NZ;

    ind = i*NY*NZ+j*NZ+k;

#ifndef _USETEXTURE     
    phi = PHI[ind];

    d2phi = (PHI[ip*NY*NZ+j*NZ+k] + PHI[im*NY*NZ+j*NZ+k]
            +PHI[i*NY*NZ+jp*NZ+k] + PHI[i*NY*NZ+jm*NZ+k]
            +PHI[i*NY*NZ+j*NZ+kp] + PHI[i*NY*NZ+j*NZ+km]
            -PHI[i*NY*NZ+j*NZ+k] * 6.0 ) / h2;

    dphidx = (PHI[ip*NY*NZ+j*NZ+k] - PHI[im*NY*NZ+j*NZ+k]) / (2.0*gridsize);
    dphidy = (PHI[i*NY*NZ+jp*NZ+k] - PHI[i*NY*NZ+jm*NZ+k]) / (2.0*gridsize);
    dphidz = (PHI[i*NY*NZ+j*NZ+kp] - PHI[i*NY*NZ+j*NZ+km]) / (2.0*gridsize);

#else
    phi = tex1Dfetch(tex_d_PHI, ind);

    d2phi = (tex1Dfetch(tex_d_PHI, ip*NY*NZ+j*NZ+k) + tex1Dfetch(tex_d_PHI, im*NY*NZ+j*NZ+k)
            +tex1Dfetch(tex_d_PHI, i*NY*NZ+jp*NZ+k) + tex1Dfetch(tex_d_PHI, i*NY*NZ+jm*NZ+k)
            +tex1Dfetch(tex_d_PHI, i*NY*NZ+j*NZ+kp) + tex1Dfetch(tex_d_PHI, i*NY*NZ+j*NZ+km)
            -tex1Dfetch(tex_d_PHI, i*NY*NZ+j*NZ+k) * 6.0 ) / h2;

    dphidx = (tex1Dfetch(tex_d_PHI, ip*NY*NZ+j*NZ+k) - tex1Dfetch(tex_d_PHI, im*NY*NZ+j*NZ+k)) / (2.0*gridsize);
    dphidy = (tex1Dfetch(tex_d_PHI, i*NY*NZ+jp*NZ+k) - tex1Dfetch(tex_d_PHI, i*NY*NZ+jm*NZ+k)) / (2.0*gridsize);
    dphidz = (tex1Dfetch(tex_d_PHI, i*NY*NZ+j*NZ+kp) - tex1Dfetch(tex_d_PHI, i*NY*NZ+j*NZ+km)) / (2.0*gridsize);

#endif

    f = phi*phi*(1-phi)*(1-phi);
    g = 2.0*phi*(1-phi)*(1-2.0*phi);

    my_fden += f * U; 
    my_dFdPHI = g * U;


    if ((EPS1 == 0)&&(EPS2 == 0)&&(EPS3 == 0))
    { /* isotropic interface energy */
         EPS_loc[ind] = EPS;
         EPS_prefactor = 0.5*SQR(EPS_loc[ind]);

         my_fden += (dphidx*dphidx + dphidy*dphidy + dphidz*dphidz) * EPS_prefactor;
         my_dFdPHI += - 2.0 * d2phi * EPS_prefactor;

    }
    else
    { /* cubic anisotropic interface energy */
         dphi_SQR   = SQR(dphidx) + SQR(dphidy) + SQR(dphidz);
         //absdphi = sqrt(dphi_SQR + 1e-24);
         absdphi = sqrt(dphi_SQR + 1e-6); /* single precision */
         nx_lab = dphidx / absdphi;
         ny_lab = dphidy / absdphi;
         nz_lab = dphidz / absdphi;

         nx_cryst = R[0*3+0]*nx_lab + R[0*3+1]*ny_lab + R[0*3+2]*nz_lab;
         ny_cryst = R[1*3+0]*nx_lab + R[1*3+1]*ny_lab + R[1*3+2]*nz_lab;
         nz_cryst = R[2*3+0]*nx_lab + R[2*3+1]*ny_lab + R[2*3+2]*nz_lab;

         dnx_lab_dphidx = 1./absdphi - dphidx * dphidx / CUBE(absdphi);
         dny_lab_dphidx =            - dphidy * dphidx / CUBE(absdphi);
         dnz_lab_dphidx =            - dphidz * dphidx / CUBE(absdphi);
   
         dnx_lab_dphidy =            - dphidx * dphidy / CUBE(absdphi);
         dny_lab_dphidy = 1./absdphi - dphidy * dphidy / CUBE(absdphi);
         dnz_lab_dphidy =            - dphidz * dphidy / CUBE(absdphi);
   
         dnx_lab_dphidz =            - dphidx * dphidz / CUBE(absdphi);
         dny_lab_dphidz =            - dphidy * dphidz / CUBE(absdphi);
         dnz_lab_dphidz = 1./absdphi - dphidz * dphidz / CUBE(absdphi);
   
         triplesum = SQR(nx_cryst)*SQR(ny_cryst) + SQR(ny_cryst)*SQR(nz_cryst) + SQR(nz_cryst)*SQR(nx_cryst);
         EPS_loc[ind] = EPS + EPS1*triplesum + EPS2*SQR(nx_cryst*ny_cryst*nz_cryst) + EPS3*SQR(triplesum);

         my_fden += 0.5 * SQR(EPS_loc[ind]) * dphi_SQR;

         /* dFdPHI += -d/dx ( SQR(EPS_loc) * dphidx + dphi_SQR * EPS_loc * d(EPS_loc)/d(dphidx) )
                      -d/dy ( SQR(EPS_loc) * dphidy + dphi_SQR * EPS_loc * d(EPS_loc)/d(dphidy) )
                      -d/dz ( SQR(EPS_loc) * dphidz + dphi_SQR * EPS_loc * d(EPS_loc)/d(dphidz) ) ;
         */

         dEPS_dnx_cryst = 2.*EPS1*nx_cryst*(SQR(ny_cryst)+SQR(nz_cryst))
                        + 2.*EPS2*nx_cryst*SQR(ny_cryst)*SQR(nz_cryst)
                        + 4.*EPS3*nx_cryst*(SQR(ny_cryst)+SQR(nz_cryst))*triplesum;
         dEPS_dny_cryst = 2.*EPS1*ny_cryst*(SQR(nz_cryst)+SQR(nx_cryst))
                        + 2.*EPS2*ny_cryst*SQR(nz_cryst)*SQR(nx_cryst)
                        + 4.*EPS3*ny_cryst*(SQR(nz_cryst)+SQR(nx_cryst))*triplesum;
         dEPS_dnz_cryst = 2.*EPS1*nz_cryst*(SQR(nx_cryst)+SQR(ny_cryst))
                        + 2.*EPS2*nz_cryst*SQR(nx_cryst)*SQR(ny_cryst)
                        + 4.*EPS3*nz_cryst*(SQR(nx_cryst)+SQR(ny_cryst))*triplesum;
  
         dEPS_dnx_lab = R[0*3+0]*dEPS_dnx_cryst + R[1*3+0]*dEPS_dny_cryst + R[2*3+0]*dEPS_dnz_cryst;
         dEPS_dny_lab = R[0*3+1]*dEPS_dnx_cryst + R[1*3+1]*dEPS_dny_cryst + R[2*3+1]*dEPS_dnz_cryst;
         dEPS_dnz_lab = R[0*3+2]*dEPS_dnx_cryst + R[1*3+2]*dEPS_dny_cryst + R[2*3+2]*dEPS_dnz_cryst;
 
         dEPS_dphidx = dEPS_dnx_lab*dnx_lab_dphidx + dEPS_dny_lab*dny_lab_dphidx + dEPS_dnz_lab*dnz_lab_dphidx;
         dEPS_dphidy = dEPS_dnx_lab*dnx_lab_dphidy + dEPS_dny_lab*dny_lab_dphidy + dEPS_dnz_lab*dnz_lab_dphidy;
         dEPS_dphidz = dEPS_dnx_lab*dnx_lab_dphidz + dEPS_dny_lab*dny_lab_dphidz + dEPS_dnz_lab*dnz_lab_dphidz;

         //tmp_x[ind] = SQR(EPS_loc)*dphidx + dphi_SQR*EPS_loc*dEPS_dphidx;
         //tmp_y[ind] = SQR(EPS_loc)*dphidy + dphi_SQR*EPS_loc*dEPS_dphidy;
         //tmp_z[ind] = SQR(EPS_loc)*dphidz + dphi_SQR*EPS_loc*dEPS_dphidz;

         /* new scheme for computing variational derivative, 2012/04/27  */
         tmp_x[ind] = dphi_SQR*EPS_loc[ind]*dEPS_dphidx;
         tmp_y[ind] = dphi_SQR*EPS_loc[ind]*dEPS_dphidy;
         tmp_z[ind] = dphi_SQR*EPS_loc[ind]*dEPS_dphidz;

         my_dFdPHI += - 1.0 * SQR(EPS_loc[ind]) * d2phi;
     }

     fden[ind] = my_fden;
     dFdPHI[ind] = my_dFdPHI;
}

__global__ void kernel_FreeEnergy_single_step2 (int NX, int NY, int NZ, double gridsize,
                    double EPS1, double EPS2, double EPS3, double dynamics_type, double K,
#ifndef _USETEXTURE
                    double *PHI,double *EPS_loc, double *tmp_x, double *tmp_y, double *tmp_z,
#endif
                    double *dFdPHI, double *dPHIdt0) 
                    
{
    int i, j, k, ip, im, jp, jm, kp, km, ind;
    double dphidx, dphidy, dphidz;
    //double d2dFdPHI;

    double my_dFdPHI;

    i = blockIdx.x; j = blockIdx.y; k = threadIdx.x;
    
    /* compute spatial gradients */
    //h2 = gridsize*gridsize;

    ip = (i+1)%NX; im = (i-1+NX)%NX;
    jp = (j+1)%NY; jm = (j-1+NY)%NY;
    kp = (k+1)%NZ; km = (k-1+NZ)%NZ;

    ind = i*NY*NZ+j*NZ+k;
    my_dFdPHI = dFdPHI[ind];

#ifndef _USETEXTURE    
    dphidx = (PHI[ip*NY*NZ+j*NZ+k] - PHI[im*NY*NZ+j*NZ+k]) / (2.0*gridsize);
    dphidy = (PHI[i*NY*NZ+jp*NZ+k] - PHI[i*NY*NZ+jm*NZ+k]) / (2.0*gridsize);
    dphidz = (PHI[i*NY*NZ+j*NZ+kp] - PHI[i*NY*NZ+j*NZ+km]) / (2.0*gridsize);
#else
    dphidx = (tex1Dfetch(tex_d_PHI, ip*NY*NZ+j*NZ+k) - tex1Dfetch(tex_d_PHI, im*NY*NZ+j*NZ+k)) / (2.0*gridsize);
    dphidy = (tex1Dfetch(tex_d_PHI, i*NY*NZ+jp*NZ+k) - tex1Dfetch(tex_d_PHI, i*NY*NZ+jm*NZ+k)) / (2.0*gridsize);
    dphidz = (tex1Dfetch(tex_d_PHI, i*NY*NZ+j*NZ+kp) - tex1Dfetch(tex_d_PHI, i*NY*NZ+j*NZ+km)) / (2.0*gridsize);
#endif

    /* this section must be moved to another kernel call */
    if ((EPS1 != 0)||(EPS2 != 0)||(EPS3 != 0))
    { /* cubic anisotropic interface energy */
#ifndef _USETEXTURE
           my_dFdPHI +=
                 - (tmp_x[ip*NY*NZ+j*NZ+k]-tmp_x[im*NY*NZ+j*NZ+k]) / (2.0*gridsize)
                 - (tmp_y[i*NY*NZ+jp*NZ+k]-tmp_y[i*NY*NZ+jm*NZ+k]) / (2.0*gridsize)
                 - (tmp_z[i*NY*NZ+j*NZ+kp]-tmp_z[i*NY*NZ+j*NZ+km]) / (2.0*gridsize) ;
#else
           my_dFdPHI +=
             - (tex1Dfetch(tex_d_tmp_x, ip*NY*NZ+j*NZ+k)-tex1Dfetch(tex_d_tmp_x, im*NY*NZ+j*NZ+k)) / (2.0*gridsize)
             - (tex1Dfetch(tex_d_tmp_y, i*NY*NZ+jp*NZ+k)-tex1Dfetch(tex_d_tmp_y, i*NY*NZ+jm*NZ+k)) / (2.0*gridsize)
             - (tex1Dfetch(tex_d_tmp_z, i*NY*NZ+j*NZ+kp)-tex1Dfetch(tex_d_tmp_z, i*NY*NZ+j*NZ+km)) / (2.0*gridsize) ;
#endif


           /* new scheme for computing variational derivative, 2012/04/27  */
#ifndef _USETEXTURE
           my_dFdPHI +=
               - (SQR(EPS_loc[ip*NY*NZ+j*NZ+k])-SQR(EPS_loc[im*NY*NZ+j*NZ+k])) / (2.0*gridsize) * dphidx
               - (SQR(EPS_loc[i*NY*NZ+jp*NZ+k])-SQR(EPS_loc[i*NY*NZ+jm*NZ+k])) / (2.0*gridsize) * dphidy
               - (SQR(EPS_loc[i*NY*NZ+j*NZ+kp])-SQR(EPS_loc[i*NY*NZ+j*NZ+km])) / (2.0*gridsize) * dphidz;
#else
           my_dFdPHI +=
   - (SQR(tex1Dfetch(tex_d_EPS_loc, ip*NY*NZ+j*NZ+k))-SQR(tex1Dfetch(tex_d_EPS_loc, im*NY*NZ+j*NZ+k))) / (2.0*gridsize) * dphidx
   - (SQR(tex1Dfetch(tex_d_EPS_loc, i*NY*NZ+jp*NZ+k))-SQR(tex1Dfetch(tex_d_EPS_loc, i*NY*NZ+jm*NZ+k))) / (2.0*gridsize) * dphidy
   - (SQR(tex1Dfetch(tex_d_EPS_loc, i*NY*NZ+j*NZ+kp))-SQR(tex1Dfetch(tex_d_EPS_loc, i*NY*NZ+j*NZ+km))) / (2.0*gridsize) * dphidz;
#endif

    }

    dFdPHI[ind] = my_dFdPHI;


    if ((dynamics_type == 0) || (dynamics_type == 2)) /* Cahn-Hillard equation */
    {
       dPHIdt0[ind] = -1.0 * K * dFdPHI[i*NY*NZ+j*NZ+k];
    }
}

#ifdef _USEFLOAT
float PhaseFieldFrame::cuda_FreeEnergy_single() /* traditional (single) phase field model */
#else
double PhaseFieldFrame::cuda_FreeEnergy_single() /* traditional (single) phase field model */
#endif
{
    double *fden;
    double *d_PHI, *d_fden, *d_dFdPHI, *d_dPHIdt0, *d_EPS_loc, *d_tmp_x, *d_tmp_y, *d_tmp_z;
    double *d_R;
    double my_mob;

    if (num_fields != 1)
      FATAL("cuda_FreeEnergy_single: num_fields = "<<num_fields);

    if (gridsize<=0) 
      FATAL("cuda_FreeEnergy_single: gridesize = "<<gridsize);

    fden = (double *) malloc( _NX*_NY*_NZ*sizeof(double) );

    cudaMalloc( &d_PHI,    _NX*_NY*_NZ*sizeof(double) );
    cudaMalloc( &d_fden,   _NX*_NY*_NZ*sizeof(double) );
    cudaMalloc( &d_dFdPHI, _NX*_NY*_NZ*sizeof(double) );
    cudaMalloc( &d_dPHIdt0,_NX*_NY*_NZ*sizeof(double) );
    cudaMalloc( &d_EPS_loc,_NX*_NY*_NZ*sizeof(double) );
    cudaMalloc( &d_tmp_x,  _NX*_NY*_NZ*sizeof(double) );
    cudaMalloc( &d_tmp_y,  _NX*_NY*_NZ*sizeof(double) );
    cudaMalloc( &d_tmp_z,  _NX*_NY*_NZ*sizeof(double) );
    cudaMalloc( &d_R,      3*3*sizeof(double) );

#ifdef _USETEXTURE
    cudaBindTexture( NULL, tex_d_PHI, d_PHI, _NX*_NY*_NZ*sizeof(double) );
    cudaBindTexture( NULL, tex_d_EPS_loc, d_EPS_loc, _NX*_NY*_NZ*sizeof(double) );
    cudaBindTexture( NULL, tex_d_tmp_x, d_tmp_x, _NX*_NY*_NZ*sizeof(double) );
    cudaBindTexture( NULL, tex_d_tmp_y, d_tmp_y, _NX*_NY*_NZ*sizeof(double) );
    cudaBindTexture( NULL, tex_d_tmp_z, d_tmp_z, _NX*_NY*_NZ*sizeof(double) );
#endif

    cudaMemcpy( d_PHI,  _PHI[0], _NX*_NY*_NZ*sizeof(double), cudaMemcpyHostToDevice);
#ifndef _USEFLOAT
    cudaMemcpy( d_R,_ROT_MATRIX.element, 3*3*sizeof(double), cudaMemcpyHostToDevice);
#else
    float floatR[3][3];
    floatR[0][0] = _ROT_MATRIX[0][0]; floatR[0][1] = _ROT_MATRIX[0][1]; floatR[0][2] = _ROT_MATRIX[0][2];
    floatR[1][0] = _ROT_MATRIX[1][0]; floatR[1][1] = _ROT_MATRIX[1][1]; floatR[1][2] = _ROT_MATRIX[1][2];
    floatR[2][0] = _ROT_MATRIX[2][0]; floatR[2][1] = _ROT_MATRIX[2][1]; floatR[2][2] = _ROT_MATRIX[2][2];
    cudaMemcpy( d_R,floatR, 3*3*sizeof(double), cudaMemcpyHostToDevice);
#endif

    dim3 blocks(_NX,_NY);
    dim3 threads(_NZ);
    kernel_FreeEnergy_single_step1<<<blocks, threads>>>
                   (_NX, _NY, _NZ, gridsize, _U[0][0], _EPS[0][0], _EPS1[0][0], _EPS2[0][0],
                    _EPS3[0][0], d_R,
#ifndef _USETEXTURE
                    d_PHI,
#endif
                    d_fden, d_dFdPHI, 
                    d_EPS_loc, d_tmp_x, d_tmp_y, d_tmp_z);

    cudaMemcpy( fden,        d_fden,    _NX*_NY*_NZ*sizeof(double), cudaMemcpyDeviceToHost);
    _F = 0;
    for(int ind=0;ind<_NX*_NY*_NZ;ind++) _F += fden[ind];
    _F *= CUBE(gridsize);

    if ((dynamics_type == 0) || (dynamics_type == 2)) /* Cahn-Hillard equation */
    {
          my_mob = Mob_GL;
    } 
    else if (dynamics_type == 1) /* Ginzburg-Landau equation */
    {
          my_mob = Mob_D;
    }
    kernel_FreeEnergy_single_step2<<<blocks, threads>>>
                   (_NX, _NY, _NZ, gridsize,  _EPS1[0][0], _EPS2[0][0], _EPS3[0][0], 
                    dynamics_type, my_mob,
#ifndef _USETEXTURE
                    d_PHI,d_EPS_loc, d_tmp_x, d_tmp_y, d_tmp_z,
#endif
                    d_dFdPHI, d_dPHIdt0);

    cudaMemcpy( _dFdPHI[0],  d_dFdPHI,  _NX*_NY*_NZ*sizeof(double), cudaMemcpyDeviceToHost);
    cudaMemcpy( _dPHIdt0[0], d_dPHIdt0, _NX*_NY*_NZ*sizeof(double), cudaMemcpyDeviceToHost);

#ifdef _USETEXTURE
    cudaUnbindTexture( tex_d_PHI );
    cudaUnbindTexture( tex_d_EPS_loc );
    cudaUnbindTexture( tex_d_tmp_x );
    cudaUnbindTexture( tex_d_tmp_y );
    cudaUnbindTexture( tex_d_tmp_z );
#endif

    cudaFree(d_PHI);
    cudaFree(d_fden);
    cudaFree(d_dFdPHI);
    cudaFree(d_dPHIdt0);
    cudaFree(d_EPS_loc);
    cudaFree(d_tmp_x);
    cudaFree(d_tmp_y);
    cudaFree(d_tmp_z);
    cudaFree(d_R);

    cudaError_t error = cudaGetLastError();
    if (error!=cudaSuccess)
    {
       printf("CUDA error: %s\n", cudaGetErrorString(error));
       exit(-1);
    }

    if (dynamics_type == 1)
    {      /* Cahn-Hillard equation */
           /* INCORRECT IF dynamics_type == 1 */
           FATAL("cuda_FreeEnergy_single: dynamics_type == 1 not implemented yet!");
#if 0
           d2dFdPHI = 
                   (dFdPHI[ip*NY*NZ+j*NZ+k] + dFdPHI[im*NY*NZ+j*NZ+k]
                   +dFdPHI[i*NY*NZ+jp*NZ+k] + dFdPHI[i*NY*NZ+jm*NZ+k]
                   +dFdPHI[i*NY*NZ+j*NZ+kp] + dFdPHI[i*NY*NZ+j*NZ+km]
                   -dFdPHI[ind] * 6.0 ) / h2;

           dPHIdt0[ind] = -1.0 * K * d2dFdPHI;
#endif
    }


    /* This section is not parallelized yet.  More work is required */
     if ((dynamics_type == 0) || (dynamics_type == 1)) /* no additional constraints */
     {
       /* 0: Ginzburg-Landau equation: not conserved */
       /* 1: Cahn-Hilliard equation: conserved */
       for(int ind=0;ind<_NX*_NY*_NZ;ind++) _dPHIdt[0][ind] = _dPHIdt0[0][ind];
     }
     else if (dynamics_type == 2) 
     {
         /* Ginzburg-Landau equation but with constant volume constraint */ 
         double avg_dPHIdt0 = 0;
         for(int ind=0;ind<_NX*_NY*_NZ;ind++) avg_dPHIdt0 += _dPHIdt0[0][ind];
         avg_dPHIdt0 /= (_NX*_NY*_NZ);
         for(int ind=0;ind<_NX*_NY*_NZ;ind++) _dPHIdt[0][ind] = _dPHIdt0[0][ind] - avg_dPHIdt0;
     }
     else
     {
        ERROR("unknown dynamics_type = "<<dynamics_type);
     }

     /* calculate the max value of dphidt */
     double Gmiddle = 0;
     for(int ind=0;ind<_NX*_NY*_NZ;ind++)
     {
          if ( Gmiddle <= fabs(_dPHIdt[0][ind]) )  Gmiddle = fabs(_dPHIdt[0][ind]);
     }
        _G = Gmiddle;

    return _F;
}

#endif //_USECUDA

