#include <cassert>
#include <stdio.h>
#include <set>
#include <vector>
#include <eigen3/Eigen/Dense>
#include <cuda_runtime.h>
#include <cublas_v2.h>
#include <cusparse_v2.h>
#include <cuComplex.h>
#include "cudaConfig.h"
#include "cudaMath.h"
#include "../base/utils.h"
#include "../base/datatypes.h"
#include "../base/consts.h"

using namespace std;
using namespace tbplas::base;

template __host__ void act_GPU<double>(
	const double* bessel_series_H, const int num_series,
	const dim3 block, const dim3 grid,
	cuDoubleComplex* t0_D, const size_t t0_size,
	cuDoubleComplex* t1_D, const size_t t1_size,
	cuDoubleComplex* wf_in_D, const size_t wf_in_size,
	cuDoubleComplex* wf_out_D, const size_t wf_out_size,
	const uint64_t* indptr_D, const  uint64_t* indices_D, const  cuDoubleComplex* values_D,
	const size_t dim);

template __host__ void act_GPU<complex_t>(
	const complex_t* bessel_series_H, const int num_series,
	const dim3 block, const dim3 grid,
	cuDoubleComplex* t0_D, const size_t t0_size,
	cuDoubleComplex* t1_D, const size_t t1_size,
	cuDoubleComplex* wf_in_D, const size_t wf_in_size,
	cuDoubleComplex* wf_out_D, const size_t wf_out_size,
	const uint64_t* indptr_D, const  uint64_t* indices_D, const  cuDoubleComplex* values_D,
	const size_t dim);

void check_norm_gpu(
	const complex_t* wf_t,
	const int wf_size,
	const complex_t norm_ref, const double eps)
{
	double result = 0.0;
	for (size_t i = 0; i < wf_size; i++)
	{
		result += (conj(wf_t[i]) * wf_t[i]).real();
	}
	double norm = sqrt(result);
	double dnorm = std::abs(norm- norm_ref.real());
	if (std::isnan(dnorm) || !std::isfinite(dnorm) || dnorm > eps) {
		std::cout << "==============================================================================\n"
			<< "=                                   ERROR                                    =\n"
			<< "= Wave function norm gets broken during preparation. First of all, check the =\n"
			<< "= Hermitianity of the Hamiltonian. Then try increasing 'rescale' in input.   =\n"
			<< "==============================================================================\n"
			<< std::flush;
		std::exit(-1);
	}
}

//这个具有优化的空间，因为涉及到PCIE总线的占用
__host__ complex_t CudaConfig::xmy_GPU(
	const dim3 block, const dim3 grid,
	const cuDoubleComplex* x, const size_t x_size,
	const cuDoubleComplex* y, const size_t y_size, const size_t dim,
	const uint64_t* indptr, const uint64_t* indices, const cuDoubleComplex* values,
	cuDoubleComplex* tmp_D, cuDoubleComplex* tmp_H)
{
	CHECKERROR(cudaMemset(tmp_D, 0, grid.x * sizeof(cuDoubleComplex)));
	xmy_kernel << <grid, block >> > (x, x_size, y, y_size, dim, indptr, indices, values, tmp_D);
	cudaMemcpy(tmp_H, tmp_D, grid.x * sizeof(cuDoubleComplex), cudaMemcpyDeviceToHost);
	cuDoubleComplex r = make_cuDoubleComplex(0, 0);
	for (size_t i = 0; i < grid.x; i++)
		r = cuCadd(r, tmp_H[i]);
	return complex_t(r.x, r.y);
}

//这个具有优化的空间，因为涉及到PCIE总线的占用
__host__ complex_t CudaConfig::vec_dot_GPU(
	const dim3 block, const dim3 grid,
	const cuDoubleComplex* x, const size_t x_size,
	const cuDoubleComplex* y, const size_t y_size,
	cuDoubleComplex* tmp_D, cuDoubleComplex* tmp_H)
{
	CHECKERROR(cudaMemset(tmp_D, 0, grid.x * sizeof(cuDoubleComplex)));
	vec_dot_kernel << <grid, block >> > (x, y, x_size, tmp_D);
	cudaMemcpy(tmp_H, tmp_D, grid.x * sizeof(cuDoubleComplex), cudaMemcpyDeviceToHost);
	cuDoubleComplex r = make_cuDoubleComplex(0, 0);
	size_t i = 0;
#pragma omp parallel for private(i)
	for (i = 0; i < grid.x; i++)
		r = cuCadd(r, tmp_H[i]);
	return complex_t(r.x, r.y);
}

__host__ double  CudaConfig::vec_norm_GPU(
	const dim3 block, const dim3 grid,
	const cuDoubleComplex* x, const size_t x_size,
	cuDoubleComplex* tmp_D, cuDoubleComplex* tmp_H)
{
	CHECKERROR(cudaMemset(tmp_D, 0, grid.x * sizeof(cuDoubleComplex)));
	vec_norm2_kernel <<<grid, block >> > (x,  x_size, tmp_D);
	cudaMemcpy(tmp_H, tmp_D, grid.x * sizeof(cuDoubleComplex), cudaMemcpyDeviceToHost);
	cuDoubleComplex r = make_cuDoubleComplex(0, 0);
	for (size_t i = 0; i < grid.x; i++)
		r = cuCadd(r, tmp_H[i]);
	return sqrt(r.x);
}

__host__ complex_t  CudaConfig::xmy_diag_GPU(
	const dim3 block, const dim3 grid,
	const cuDoubleComplex* x, const cuDoubleComplex* y, 
	const cuDoubleComplex* m,  const size_t dim,
	cuDoubleComplex* tmp_D, cuDoubleComplex* tmp_H)
{
	CHECKERROR(cudaMemset(tmp_D, 0, grid.x * sizeof(cuDoubleComplex)));
	xmy_diag_kernel <<<grid, block >> > (x, y,m, dim, tmp_D);
	cudaMemcpy(tmp_H, tmp_D, grid.x * sizeof(cuDoubleComplex), cudaMemcpyDeviceToHost);
	cuDoubleComplex r = make_cuDoubleComplex(0, 0);
	for (size_t i = 0; i < grid.x; i++)
		r = cuCadd(r, tmp_H[i]);
	return complex_t(r.x, r.y);
}

//__host__   void CudaConfig::tbpm_dos_GPU(
//	complex_t* wf_t_H, const int wf_t_size,
//	const int num_steps, const  int  wfn_check_time_steps, const  double wfn_check_threshold,
//	const complex_t* wf0_H, const int wf0_size,
//	const complex_t norm_ref,
//	const complex_t* bessel_series_H,
//	const int bessel_series_size,
//	complex_t* t0_H, const int t0_size,
//	complex_t* t1_H, const int t1_size,
//	complex_t* wf_out_H, const int wf_out_size,
//	const uint64_t* indptr_H, const int indptr_size,
//	const uint64_t* indices_H, const int indices_size,
//	const complex_t* values_H, const int values_size, 
//	const size_t dim_size,
//	complex_t* corr_H, const int corr_size) {
//	cuDoubleComplex* wf_t_D, * wf0_D, * t0_D, * t1_D, * wf_out_D;
//	uint64_t* indptr_D, * indices_D;
//	cuDoubleComplex* values_D;
//	cuDoubleComplex* tmp_D = nullptr,* tmp_H= nullptr;
//	CHECKERROR(cudaMalloc(&wf_t_D, wf_t_size * sizeof(cuDoubleComplex)));
//	CHECKERROR(cudaMalloc(&wf0_D, wf0_size * sizeof(cuDoubleComplex)));
//	CHECKERROR(cudaMalloc(&t0_D, t0_size * sizeof(cuDoubleComplex)));
//	CHECKERROR(cudaMalloc(&t1_D, t1_size * sizeof(cuDoubleComplex)));
//	CHECKERROR(cudaMalloc(&wf_out_D, wf_out_size * sizeof(cuDoubleComplex)));
//	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
//	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
//	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
//
//	CHECKERROR(cudaMemcpy(wf_t_D, wf_t_H, wf_t_size*sizeof(complex_t), cudaMemcpyHostToDevice));
//	CHECKERROR(cudaMemcpy(wf0_D, wf0_H, wf0_size*sizeof(complex_t), cudaMemcpyHostToDevice));
//	CHECKERROR(cudaMemcpy(t0_D, t0_H, t0_size*sizeof(complex_t), cudaMemcpyHostToDevice));
//	CHECKERROR(cudaMemcpy(t1_D, t1_H, t1_size * sizeof(complex_t), cudaMemcpyHostToDevice));
//	CHECKERROR(cudaMemcpy(wf_out_D, wf_out_H, wf_out_size*sizeof(complex_t), cudaMemcpyHostToDevice));
//	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
//	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
//	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
//
//	// Propagate 
//	ProgressBar prog_bar(num_steps, 64, "Finished timestep");
//	prog_bar.reset();
//
//#ifndef  CUSTOM_OPT
//	// Set block and grid size.
//	dim3 block(1024);
//	dim3 grid((wf_t_size + block.x - 1) / block.x);
//	CHECKERROR(cudaMalloc(&tmp_D, grid.x * sizeof(cuDoubleComplex)));
//	tmp_H = (cuDoubleComplex*)malloc(grid.x * sizeof(cuDoubleComplex));
//#else
//	double alpha = 1.0, beta = 0;
//	cusparseSpMatDescr_t cusparseMat = NULL;
//	cusparseDnVecDescr_t x = NULL, y = NULL;
//	cusparseHandle_t cusparseHandle = 0;
//	cublasHandle_t cublashandle = 0;
//	void* buffer = NULL;
//	cublasStatus_t blas_t = cublasCreate(&cublashandle);
//	cusparseStatus_t sparse_t = cusparseCreate(&cusparseHandle);
//	size_t bufferSize = 0;
//	sparse_t = cusparseCreateCsr(&cusparseMat,
//		(uint64_t)dim_size, (uint64_t)dim_size, (uint64_t)values_size,
//		(void*)indptr_D, (void*)indices_D, (void*)values_D,
//		CUSPARSE_INDEX_64I, CUSPARSE_INDEX_64I,
//		CUSPARSE_INDEX_BASE_ZERO, CUDA_C_64F);
//	sparse_t = cusparseCreateDnVec(&x, wf_t_size, wf_t_D, CUDA_C_64F);
//	sparse_t = cusparseCreateDnVec(&y, t0_size, t0_D, CUDA_C_64F);
//
//	sparse_t = cusparseSpMV_bufferSize(
//		cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, cusparseMat, x,
//		&beta, y, CUDA_C_64F,
//		CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize);
//	cudaMalloc(&buffer, bufferSize);
//	sparse_t = cusparseDestroyDnVec(x);
//	sparse_t = cusparseDestroyDnVec(y);
//	CHECKERROR(cudaMalloc(&tmp_D, sizeof(cuDoubleComplex)));
//	tmp_H = (cuDoubleComplex*)malloc(sizeof(cuDoubleComplex));
//#endif //  CUSTOM_OPT
//
//	for (int i_step = 1; i_step <= num_steps; ++i_step) {
//#ifndef CUSTOM_OPT
//		CHECKERROR(cudaMemset(tmp_D, 0, grid.x * sizeof(cuDoubleComplex)));
//#endif // CUSTOM_OPT
//		if (i_step == wfn_check_time_steps) {
//			CHECKERROR(cudaMemcpy((complex_t*)wf0_H, wf0_D, wf0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//			check_norm_gpu(wf0_H, wf0_size, norm_ref, wfn_check_threshold);
//		}
//		int dim = t0_size;
//		// Initialize pointers
//		cuDoubleComplex* p0 = t0_D;
//		cuDoubleComplex* p1 = t1_D;
//		cuDoubleComplex* p2 = nullptr;
//#ifndef CUSTOM_OPT
//		mv_kernel << <grid, block >> > (
//			wf_t_D, wf_t_size,
//			indptr_D, indices_D, values_D,
//			t0_D, t0_size);
//#else
//		beta = 0;
//		cuDoubleComplex temp=make_cuDoubleComplex(1.0,0);
//		cuDoubleComplex temp2=make_cuDoubleComplex(beta, 0);
//		sparse_t = cusparseCreateDnVec(&x, wf_t_size, wf_t_D, CUDA_C_64F);
//		sparse_t = cusparseCreateDnVec(&y, t0_size, t0_D, CUDA_C_64F);
//		cusparseStatus_t  t = cusparseSpMV(
//			cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE,
//			&temp, cusparseMat, x,
//			&temp2, y,
//			CUDA_C_64F,
//			CUSPARSE_SPMV_ALG_DEFAULT, buffer);	
//		sparse_t = cusparseDestroyDnVec(x);
//		sparse_t = cusparseDestroyDnVec(y);
//#endif
//#ifdef DEBUG_CUDA
//		CHECKERROR(cudaMemcpy((complex_t*)t0_H, t0_D, t0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//		for (int i = 0; i < 10; i++)
//		{
//			printf("t0_H[%d] :%f,%f\n", i,t0_H[i].real(), t0_H[i].imag());
//		}
//#endif
//
//#ifndef CUSTOM_OPT
//		amxsy_out_kernel << <grid, block >> > (
//			make_cuDoubleComplex(2.0, 0),
//			t0_D, t0_size,
//			wf_t_D, wf_t_size,
//			indptr_D, indices_D, values_D,
//			t1_D, t1_size, dim);
//#else
//		beta = 1;
//		sparse_t = cusparseCreateDnVec(&x, t0_size, t0_D, CUDA_C_64F);
//		cudaMemcpy(t1_D, wf_t_D, wf_t_size * sizeof(complex_t), cudaMemcpyDeviceToDevice);
//		sparse_t = cusparseCreateDnVec(&y, t1_size, t1_D, CUDA_C_64F);
//		temp=make_cuDoubleComplex(2.0, 0);
//		temp2=make_cuDoubleComplex(beta, 0);
//		t = cusparseSpMV(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE,
//			&temp, cusparseMat, x,
//			&temp2, y, CUDA_C_64F,
//			CUSPARSE_SPMV_ALG_DEFAULT, buffer);
//		sparse_t = cusparseDestroyDnVec(x);
//		sparse_t = cusparseDestroyDnVec(y);
//#endif // CUSTOM_OPT
//
//#ifdef DEBUG_CUDA
//		CHECKERROR(cudaMemcpy((complex_t*)t1_H, t1_D, t1_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//		for (int i = 0; i < 10; i++)
//		{
//			printf("t1_H[%d] :%f,%f\n", i, t1_H[i].real(), t1_H[i].imag());
//		}
//#endif
//
//#ifndef CUSTOM_OPT
//		axpbypcz_out_kernel << <grid, block >> > (
//			make_cuDoubleComplex(bessel_series_H[0].real(), bessel_series_H[0].imag()),
//			wf_t_D, wf_t_size,
//			make_cuDoubleComplex(bessel_series_H[1].real(), bessel_series_H[1].imag()),
//			t0_D, t0_size,
//			make_cuDoubleComplex(bessel_series_H[2].real(), bessel_series_H[2].imag()) ,
//			t1_D, t1_size,
//			wf_out_D, wf_out_size);
//#else
//		cudaMemset(wf_out_D, 0, wf_out_size * sizeof(complex_t));
//		temp=make_cuDoubleComplex(bessel_series_H[0], 0);
//		blas_t = cublasZaxpy(
//			cublashandle,
//			wf_t_size,
//			&temp,
//			wf_t_D,
//			1,
//			wf_out_D,
//			1);
//		temp=make_cuDoubleComplex(bessel_series_H[1], 0);
//		blas_t = cublasZaxpy(
//			cublashandle,
//			t0_size,
//			&temp,
//			t0_D,
//			1,
//			wf_out_D,
//			1);
//		temp=make_cuDoubleComplex(bessel_series_H[2], 0);
//		blas_t = cublasZaxpy(
//			cublashandle,
//			t1_size,
//			&temp,
//			t1_D,
//			1,
//			wf_out_D,
//			1);
//#endif // DEBUG
//
//#ifdef DEBUG_CUDA
//		CHECKERROR(cudaMemcpy((complex_t*)wf_out_H, wf_out_D, wf_out_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//		for (int i = 0; i < 10; i++)
//		{
//			printf("wf_out_H[%d] :%f,%f\n", i, wf_out_H[i].real(), wf_out_H[i].imag());
//		}
//#endif
//
//		for (size_t i = 3; i < bessel_series_size; i++)
//		{
//#ifdef DEBUG_CUDA
//			//CHECKERROR(cudaMemcpy((complex_t*)t0_H, p0, t0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//			//for (int i = 0; i < 8; i++)
//			//{
//			//	printf("t0_H[%d] :%f,%f\n", i, t0_H[i].real(), t0_H[i].imag());
//			//}
//			//CHECKERROR(cudaMemcpy((complex_t*)t1_H, p1, t1_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//			//for (int i = 0; i < 8; i++)
//			//{
//			//	printf("t1_H[%d] :%f,%f\n", i, t1_H[i].real(), t1_H[i].imag());
//			//}
//#endif
//			p2 = p0;
//#ifndef CUSTOM_OPT
//			amxsy_kernel << <grid, block >> > (2.0, p1, t1_size, p0, t0_size, 
//				dim, indptr_D, indices_D, values_D);
//#else
//			cusparseDnVecDescr_t temp_P1 = NULL, temp_P0 = NULL;
//			sparse_t = cusparseCreateDnVec(&temp_P0, t0_size, p0, CUDA_C_64F);
//			sparse_t = cusparseCreateDnVec(&temp_P1, t1_size, p1, CUDA_C_64F);
//			//cusparse temp_P0=a*mat*temp_P1+temp_P0
//			temp=make_cuDoubleComplex(2.0, 0);
//			temp2=make_cuDoubleComplex(beta, 0);
//			sparse_t = cusparseSpMV(
//				cusparseHandle,
//				CUSPARSE_OPERATION_NON_TRANSPOSE,
//				&temp, cusparseMat, temp_P1,
//				&temp2, temp_P0,
//				CUDA_C_64F,
//				CUSPARSE_SPMV_ALG_DEFAULT, buffer);
//#endif
//		
//#ifdef DEBUG_CUDA
//			//CHECKERROR(cudaMemcpy((complex_t*)t0_H, p0, t0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//			//for (int i = 0; i < 5; i++)
//			//{
//			//	printf("t0_H[%d] :%f,%f\n", i, t0_H[i].real(), t0_H[i].imag());
//			//}
//#endif
//#ifndef CUSTOM_OPT
//			axpy_kernel << <grid, block >> > (make_cuDoubleComplex(bessel_series_H[i].real(), bessel_series_H[i].imag()), p2, dim,wf_out_D);
//#else
//			sparse_t = cusparseDestroyDnVec(temp_P0);
//			sparse_t = cusparseDestroyDnVec(temp_P1);
//			//cublas y=a*x+y
//			temp=make_cuDoubleComplex(bessel_series_H[i], 0);
//			blas_t = cublasZaxpy(cublashandle, t0_size,
//				&temp,
//				p2, 1, wf_out_D, 1);
//#endif
//
//#ifdef DEBUG_CUDA
//			CHECKERROR(cudaMemcpy((complex_t*)wf_out_H, wf_out_D, wf_out_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//			for (int i = 0; i < 5; i++)
//			{
//				printf("wf_out_H[%d] :%f,%f\n", i, wf_out_H[i].real(), wf_out_H[i].imag());
//			}
//#endif
//			p0 = p1;
//			p1 = p2;
//		}
//		cudaMemcpy(wf_t_D,wf_out_D, wf_out_size *sizeof(cuDoubleComplex),cudaMemcpyDeviceToDevice);
//#ifndef CUSTOM_OPT
//		vec_dot_gpu << <grid, block >> > (wf0_D, wf_t_D, wf0_size, tmp_D);
//		cudaMemcpy(tmp_H, tmp_D, grid.x * sizeof(cuDoubleComplex), cudaMemcpyDeviceToHost);
//		cuDoubleComplex r = make_cuDoubleComplex(0, 0);
//		for (size_t i = 0; i < grid.x; i++)
//			r = cuCadd(r, tmp_H[i]);
//		corr_H[i_step] += complex_t(r.x, r.y);
//#else
//		cublasZdotc_64(cublashandle, wf0_size, wf0_D, 1, wf_t_D, 1, tmp_D);
//		cudaMemcpy(tmp_H, tmp_D, sizeof(cuDoubleComplex), cudaMemcpyDeviceToHost);
//		corr_H[i_step] += complex_t((*tmp_H).x, (*tmp_H).y);
//#endif // CUSTOM_OPT
//		prog_bar.count();
//	}
//
//	CHECKERROR(cudaDeviceSynchronize());
//	CHECKERROR(cudaMemcpy((complex_t*)wf_t_H, wf_t_D, wf_t_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//	CHECKERROR(cudaMemcpy((complex_t*)t0_H, t0_D , t0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//	CHECKERROR(cudaMemcpy((complex_t*)t1_H, t1_D, t1_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//	CHECKERROR(cudaMemcpy((complex_t*)wf_out_H, wf_out_D , wf_out_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
//	CHECKERROR(cudaFree(tmp_D));
//	CHECKERROR(cudaFree(wf_t_D));
//	CHECKERROR(cudaFree(wf0_D));
//	CHECKERROR(cudaFree(t0_D));
//	CHECKERROR(cudaFree(t1_D));
//	CHECKERROR(cudaFree(wf_out_D));
//	CHECKERROR(cudaFree(indptr_D));
//	CHECKERROR(cudaFree(indices_D));
//	CHECKERROR(cudaFree(values_D));
//	free(tmp_H);
//};

template<typename T>
__host__ void act_GPU(
	const T* bessel_series_H, const int num_series,
	const dim3 block, const dim3 grid,
	cuDoubleComplex* t0_D, const size_t t0_size,
	cuDoubleComplex* t1_D, const size_t t1_size,
	cuDoubleComplex* wf_in_D, const size_t wf_in_size,
	cuDoubleComplex* wf_out_D, const size_t wf_out_size,
	const uint64_t* indptr_D, const  uint64_t* indices_D, const  cuDoubleComplex* values_D,
	const size_t dim)
{
	assert(dim == t0_size);
	assert(dim == t1_size);
	assert(dim == wf_in_size);
	assert(dim == wf_out_size);

	// Initialize pointers
	cuDoubleComplex* p0_D = t0_D;
	cuDoubleComplex* p1_D = t1_D;
	cuDoubleComplex* p2_D = nullptr;

	mv_kernel << <grid, block >> > (
		wf_in_D, wf_in_size,
		indptr_D, indices_D, values_D,
		t0_D, t0_size);

	amxsy_out_kernel << <grid, block >> > (
		make_cuDoubleComplex(2.0, 0),
		t0_D, 
		wf_in_D, 
		indptr_D, indices_D, values_D,
		t1_D, dim);

	if constexpr (std::is_same_v<T, double >)
	{
		axpbypcz_out_kernel << <grid, block >> > (
			make_cuDoubleComplex(bessel_series_H[0], 0),
			wf_in_D, wf_in_size,
			make_cuDoubleComplex(bessel_series_H[1], 0),
			t0_D, t0_size,
			make_cuDoubleComplex(bessel_series_H[2], 0),
			t1_D, t1_size,
			wf_out_D, wf_out_size);
	}
	else if constexpr (std::is_same_v<T, complex_t >)
	{
		axpbypcz_out_kernel << <grid, block >> > (
			make_cuDoubleComplex(bessel_series_H[0].real(), bessel_series_H[0].imag()),
			wf_in_D, wf_in_size,
			make_cuDoubleComplex(bessel_series_H[1].real(), bessel_series_H[1].imag()),
			t0_D, t0_size,
			make_cuDoubleComplex(bessel_series_H[2].real(), bessel_series_H[2].imag()),
			t1_D, t1_size,
			wf_out_D, wf_out_size);
	}
	else {
		exit(1);
	}

	for (size_t i = 3; i < num_series; i++)
	{
		p2_D = p0_D;
		amxsy_kernel << <grid, block >> > (2.0, p1_D, t1_size, p0_D, t0_size,
			dim, indptr_D, indices_D, values_D);

		if constexpr (std::is_same_v<T, double >) {
			axpy_kernel << <grid, block >> > (
				make_cuDoubleComplex(bessel_series_H[i], 0), p2_D, dim, wf_out_D);
		}
		else if constexpr (std::is_same_v<T, complex_t >) {
			axpy_kernel << <grid, block >> > (
				make_cuDoubleComplex(bessel_series_H[i].real(), bessel_series_H[i].imag()), p2_D, dim, wf_out_D);
		}
		else {
			exit(1);
		}

		p0_D = p1_D;
		p1_D = p2_D;
	}
	cudaMemcpy(wf_in_D, wf_out_D, wf_in_size * sizeof(cuDoubleComplex), cudaMemcpyDeviceToDevice);
};


__host__   void CudaConfig::tbpm_ac_cond_GPU(
	const size_t num_steps, const  size_t  wfn_check_time_steps, const  double wfn_check_threshold,
	const complex_t norm_ref,
	const complex_t* bessel_series_H,const size_t bessel_series_size,
	const uint64_t* indptr_H, const size_t indptr_size,
	const uint64_t* indices_H, const size_t indices_size,
	const complex_t* values_H, const size_t values_size,
	const uint64_t* curr_x_indptr_H, const size_t curr_x_indptr_size,
	const uint64_t* curr_x_indices_H, const size_t curr_x_indices_size,
	const complex_t* curr_x_values_H, const size_t curr_x_values_size,
	const uint64_t* curr_y_indptr_H, const size_t curr_y_indptr_size,
	const uint64_t* curr_y_indices_H, const size_t curr_y_indices_size,
	const complex_t* curr_y_values_H, const size_t curr_y_values_size,
	const size_t dim_size,
	complex_t* psi2_H, const size_t psi2_size,
	complex_t* psi1_x_H, const size_t psi1_x_size,
	complex_t* psi1_y_H, const size_t psi1_y_size,
	complex_t* t0_H, const size_t t0_size,
	complex_t* t1_H, const size_t t1_size,
	complex_t* corr_H, const size_t corr_size,
	const bool isRowMajor,
	ProgressBar& prog_bar) {
	assert(psi1_x_size == psi2_size);
	assert(psi1_x_size == psi1_y_size);
	assert(psi1_x_size == t0_size);
	assert(psi1_x_size == t1_size);
	assert(t0_size == t1_size);
	assert(curr_x_indptr_size == curr_y_indptr_size);
	assert(curr_x_indptr_size == indptr_size);
	assert(curr_x_indices_size == curr_y_indices_size);
	assert(curr_x_indices_size == indices_size);
	assert(curr_x_values_size == curr_y_values_size);
	assert(curr_x_values_size == values_size);

	cuDoubleComplex* psi2_D=nullptr, * psi1_x_D = nullptr, * psi1_y_D = nullptr;
	uint64_t* indptr_D = nullptr, * indices_D = nullptr,
		*curr_x_indptr_D = nullptr, * curr_x_indices_D = nullptr, * curr_y_indptr_D = nullptr,* curr_y_indices_D = nullptr;
	cuDoubleComplex* values_D = nullptr,*curr_x_values_D = nullptr, * curr_y_values_D = nullptr;
	cuDoubleComplex* t0_D = nullptr, * t1_D = nullptr, * wf_out_D = nullptr;
	cuDoubleComplex* tmp_D = nullptr, * tmp_H = nullptr;
	CHECKERROR(cudaMalloc(&psi1_x_D, psi1_x_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(psi1_x_D, psi1_x_H, psi1_x_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&psi1_y_D, psi1_y_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(psi1_y_D, psi1_y_H, psi1_y_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&psi2_D, psi2_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(psi2_D, psi2_H, psi2_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf_out_D, dim_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMalloc(&t0_D, dim_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t0_D, t0_H, dim_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t1_D, dim_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t1_D, t1_H, dim_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_x_indptr_D, curr_x_indptr_H, curr_x_indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_y_indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_y_indptr_D, curr_y_indptr_H, curr_y_indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_x_indices_D, curr_x_indices_H, curr_x_indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_y_indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_y_indices_D, curr_y_indices_H, curr_y_indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(curr_x_values_D, curr_x_values_H, curr_x_values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_y_values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(curr_y_values_D, curr_y_values_H, curr_y_values_size * sizeof(complex_t), cudaMemcpyHostToDevice));

	int RowWidth = 4;
	int ColWidth = corr_size / 4;

	// Set block and grid size
	dim3 block(1024);
	dim3 grid((dim_size + block.x - 1) / block.x);
	CHECKERROR(cudaMalloc(&tmp_D, grid.x * sizeof(cuDoubleComplex)));
	tmp_H = (cuDoubleComplex*)malloc(grid.x * sizeof(cuDoubleComplex));
	prog_bar.reset();

	for (int i_step = 2; i_step <= num_steps; ++i_step) {
		if (i_step == wfn_check_time_steps) {
			CHECKERROR(cudaMemcpy((complex_t*)psi2_H, psi2_D, psi2_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
			check_norm_gpu(psi2_H, psi2_size, norm_ref, wfn_check_threshold);
		}
		act_GPU(bessel_series_H,bessel_series_size,
			block, grid,
			t0_D, t0_size, t1_D, t1_size, 
			psi1_x_D, psi1_x_size, wf_out_D, dim_size,
			indptr_D, indices_D, values_D, dim_size);
		act_GPU(bessel_series_H, bessel_series_size,
			block, grid,
			t0_D, t0_size, t1_D, t1_size,
			psi1_y_D, psi1_y_size, wf_out_D, dim_size,
			indptr_D, indices_D, values_D, dim_size);
		act_GPU(bessel_series_H, bessel_series_size,
			block, grid,
			t0_D, t0_size, t1_D, t1_size,
			psi2_D, psi2_size, wf_out_D, dim_size,
			indptr_D, indices_D, values_D, dim_size);
		int j = i_step - 1;
		if (isRowMajor)
		{
			corr_H[ColWidth * 0 + j] += xmy_GPU(block, grid, psi2_D, psi2_size, psi1_x_D, psi1_x_size, dim_size,
				curr_x_indptr_D, curr_x_indices_D, curr_x_values_D, tmp_D, tmp_H);

			corr_H[ColWidth * 1 + j] += xmy_GPU(block, grid, psi2_D, psi2_size, psi1_x_D, psi1_x_size, dim_size,
				curr_y_indptr_D, curr_y_indices_D, curr_y_values_D, tmp_D, tmp_H);

			corr_H[ColWidth * 2 + j] += xmy_GPU(block, grid, psi2_D, psi2_size, psi1_y_D, psi1_y_size, dim_size,
				curr_x_indptr_D, curr_x_indices_D, curr_x_values_D, tmp_D, tmp_H);

			corr_H[ColWidth * 3 + j] += xmy_GPU(block, grid, psi2_D, psi2_size, psi1_y_D, psi1_y_size, dim_size,
				curr_y_indptr_D, curr_y_indices_D, curr_y_values_D, tmp_D, tmp_H);
		}
		else
		{
			corr_H[0 + RowWidth * j] += xmy_GPU(block, grid, psi2_D, psi2_size, psi1_x_D, psi1_x_size, dim_size,
				curr_x_indptr_D, curr_x_indices_D, curr_x_values_D, tmp_D, tmp_H);

			corr_H[1 + RowWidth * j] += xmy_GPU(block, grid, psi2_D, psi2_size, psi1_x_D, psi1_x_size, dim_size,
				curr_y_indptr_D, curr_y_indices_D, curr_y_values_D, tmp_D, tmp_H);

			corr_H[2 + RowWidth * j] += xmy_GPU(block, grid, psi2_D, psi2_size, psi1_y_D, psi1_y_size, dim_size,
				curr_x_indptr_D, curr_x_indices_D, curr_x_values_D, tmp_D, tmp_H);

			corr_H[3 + RowWidth * j] += xmy_GPU(block, grid, psi2_D, psi2_size, psi1_y_D, psi1_y_size, dim_size,
				curr_y_indptr_D, curr_y_indices_D, curr_y_values_D, tmp_D, tmp_H);
		}
		prog_bar.count();
	}
	CHECKERROR(cudaDeviceSynchronize());
	prog_bar.count();
	CHECKERROR(cudaMemcpy((complex_t*)psi1_x_H, psi1_x_D, psi1_x_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)psi1_y_H, psi1_y_D, psi1_y_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)psi2_H, psi2_D, psi2_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t0_H, t0_D, dim_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t1_H, t1_D, dim_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaFree(tmp_D));
	tmp_D = nullptr;
	CHECKERROR(cudaFree(psi1_x_D));
	psi1_x_D = nullptr;
	CHECKERROR(cudaFree(psi1_y_D));
	psi1_y_D = nullptr;
	CHECKERROR(cudaFree(psi2_D));
	psi2_D = nullptr;
	CHECKERROR(cudaFree(wf_out_D));
	wf_out_D = nullptr;
	CHECKERROR(cudaFree(t0_D));
	t0_D = nullptr;
	CHECKERROR(cudaFree(t1_D));
	t1_D = nullptr;
	CHECKERROR(cudaFree(indptr_D));
	indptr_D = nullptr;
	CHECKERROR(cudaFree(indices_D));
	indices_D = nullptr;
	CHECKERROR(cudaFree(values_D));
	values_D = nullptr;
	CHECKERROR(cudaFree(curr_x_indptr_D));
	curr_x_indptr_D = nullptr;
	CHECKERROR(cudaFree(curr_y_indptr_D));
	curr_y_indptr_D = nullptr;
	CHECKERROR(cudaFree(curr_x_indices_D));
	curr_x_indices_D = nullptr;
	CHECKERROR(cudaFree(curr_y_indices_D));
	curr_y_indices_D = nullptr;
	CHECKERROR(cudaFree(curr_x_values_D));
	curr_x_values_D = nullptr;
	CHECKERROR(cudaFree(curr_y_values_D));
	curr_y_values_D = nullptr;
	free(tmp_H);
	tmp_H = nullptr;
};

__host__   void CudaConfig::tbpm_dos_GPU(
	complex_t* wf_t_H, const size_t wf_t_size,
	const size_t num_steps, const  size_t  wfn_check_time_steps, const  double wfn_check_threshold,
	const complex_t* wf0_H, const size_t wf0_size,
	const complex_t norm_ref,
	const complex_t* bessel_series_H,
	const int bessel_series_size,
	complex_t* t0_H, const size_t t0_size,
	complex_t* t1_H, const size_t t1_size,
	complex_t* wf_out_H, const size_t wf_out_size,
	const uint64_t* indptr_H, const size_t indptr_size,
	const uint64_t* indices_H, const size_t indices_size,
	const complex_t* values_H, const size_t values_size,
	const size_t dim_size,
	complex_t* corr_H, const size_t corr_size,
	ProgressBar& prog_bar) {
	assert(dim_size == wf_t_size);
	assert(dim_size == wf0_size);
	assert(dim_size == wf_out_size);
	assert(dim_size == t0_size);
	assert(dim_size == t1_size);
	cuDoubleComplex* wf_t_D = nullptr, * wf0_D = nullptr, * t0_D = nullptr, * t1_D = nullptr, * wf_out_D = nullptr;
	uint64_t* indptr_D = nullptr, * indices_D = nullptr;
	cuDoubleComplex* values_D = nullptr;
	cuDoubleComplex* tmp_D = nullptr, * tmp_H = nullptr;;
	CHECKERROR(cudaMalloc(&wf_t_D, wf_t_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(wf_t_D, wf_t_H, wf_t_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf0_D, wf0_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(wf0_D, wf0_H, wf0_size * sizeof(complex_t), cudaMemcpyHostToDevice));

	CHECKERROR(cudaMalloc(&t0_D, t0_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t0_D, t0_H, t0_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t1_D, t1_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t1_D, t1_H, t1_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf_out_D, wf_out_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(wf_out_D, wf_out_H, wf_out_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	// Propagate 
	prog_bar.reset();
	// Set block and grid size.
	dim3 block(1024);
	dim3 grid((wf_t_size + block.x - 1) / block.x);
	CHECKERROR(cudaMalloc(&tmp_D, grid.x * sizeof(cuDoubleComplex)));
	tmp_H = (cuDoubleComplex*)malloc(grid.x * sizeof(cuDoubleComplex));


	for (int i_step = 1; i_step <= num_steps; ++i_step) {
		if (i_step == wfn_check_time_steps) {
			CHECKERROR(cudaMemcpy((complex_t*)wf0_H, wf0_D, wf0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
			check_norm_gpu(wf0_H, wf0_size, norm_ref, wfn_check_threshold);
		}
		act_GPU(bessel_series_H, bessel_series_size,
			block, grid,
			t0_D, t0_size, t1_D, t1_size,
			wf_t_D, wf_t_size, wf_out_D, dim_size,
			indptr_D, indices_D, values_D, dim_size);
		corr_H[i_step] += vec_dot_GPU(block, grid, wf0_D, wf0_size, wf_t_D, wf_t_size,
			 tmp_D, tmp_H);
		prog_bar.count();
	}
	CHECKERROR(cudaDeviceSynchronize());
	CHECKERROR(cudaMemcpy((complex_t*)wf_t_H, wf_t_D, wf_t_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t0_H, t0_D, t0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t1_H, t1_D, t1_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)wf_out_H, wf_out_D, wf_out_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaFree(tmp_D));
	tmp_D = nullptr;
	CHECKERROR(cudaFree(wf_t_D));
	wf_t_D = nullptr;
	CHECKERROR(cudaFree(wf0_D));
	wf0_D = nullptr;
	CHECKERROR(cudaFree(t0_D));
	t0_D = nullptr;
	CHECKERROR(cudaFree(t1_D));
	t1_D = nullptr;
	CHECKERROR(cudaFree(wf_out_D));
	wf_out_D = nullptr;
	CHECKERROR(cudaFree(indptr_D));
	indptr_D = nullptr;
	CHECKERROR(cudaFree(indices_D));
	indices_D = nullptr;
	CHECKERROR(cudaFree(values_D));
	values_D = nullptr;
	free(tmp_H);
	tmp_H=nullptr;
};

__host__   void CudaConfig::tbpm_qe_dos_GPU(
	const bool calc_corr_dos,const size_t num_steps,
	const double time_step,const double* qe_energies_H, 
	const size_t  num_energies,const  size_t  wfn_check_time_steps,
	const  double wfn_check_threshold,
	const complex_t* wf0_H, const size_t wf0_size,
	const complex_t* wf_t_pos_H, const size_t wf_t_pos_size,
	const complex_t* wf_t_neg_H, const size_t wf_t_neg_size,
	const complex_t norm_ref,
	const complex_t* bessel_series_T_H,
	const complex_t* bessel_series_F_H,
	const int bessel_series_size,
	complex_t* t0_H, const size_t t0_size,
	complex_t* t1_H, const size_t t1_size,
	const uint64_t* indptr_H, const size_t indptr_size,
	const uint64_t* indices_H, const size_t indices_size,
	const complex_t* values_H, const size_t values_size,
	const size_t dim_size,
	complex_t* corr_H, const size_t corr_size,
	std::vector<Eigen::VectorXcd>&  wfq) {
	assert(sizeof(cuDoubleComplex) == sizeof(complex_t));
	assert(dim_size == wf0_size);
	assert(dim_size == wf_t_pos_size);
	assert(dim_size == wf_t_neg_size);
	assert(dim_size == t0_size);
	assert(dim_size == t1_size);
	assert(wfq.size() == num_energies);
	cuDoubleComplex * wf_t_pos_D=nullptr,* wf_t_neg_D = nullptr,* t0_D = nullptr, * t1_D = nullptr, * wf_out_D = nullptr,
		* wf0_D = nullptr, **wfq_H = nullptr;
	uint64_t* indptr_D = nullptr, * indices_D = nullptr;
	cuDoubleComplex* values_D = nullptr;
	cuDoubleComplex* tmp_D = nullptr, * tmp_H = nullptr;
	wfq_H = (cuDoubleComplex **)malloc(wfq.size()*sizeof(cuDoubleComplex*));
	for (size_t i = 0; i < wfq.size(); i++){
		CHECKERROR(cudaMalloc(&wfq_H[i], wfq[i].size() * sizeof(cuDoubleComplex)));
		CHECKERROR(cudaMemcpy(wfq_H[i], wfq[i].data(), wfq[i].size() * sizeof(complex_t), cudaMemcpyHostToDevice));
	}
	CHECKERROR(cudaMalloc(&wf0_D, wf0_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(wf0_D, wf0_H, wf0_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf_t_pos_D, wf_t_pos_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(wf_t_pos_D, wf_t_pos_H, wf_t_pos_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf_t_neg_D, wf_t_neg_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(wf_t_neg_D, wf_t_neg_H, wf_t_neg_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t0_D, t0_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t0_D, t0_H, t0_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t1_D, t1_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t1_D, t1_H, t1_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf_out_D, dim_size * sizeof(cuDoubleComplex)));
	// Propagate 
	ProgressBar prog_bar(num_steps, 64, "Finished DOS/QE timestep");
	// Set block and grid size.
	dim3 block(1024);
	dim3 grid((dim_size + block.x - 1) / block.x);
	CHECKERROR(cudaMalloc(&tmp_D, grid.x * sizeof(cuDoubleComplex)));
	tmp_H = (cuDoubleComplex*)malloc(grid.x * sizeof(cuDoubleComplex));
	for (int i_step = 1; i_step <= num_steps; ++i_step) {
		if (i_step == wfn_check_time_steps) {
			CHECKERROR(cudaMemcpy((complex_t*)wf_t_pos_H, wf_t_pos_D, wf_t_pos_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
			check_norm_gpu(wf_t_pos_H, wf_t_pos_size, norm_ref, wfn_check_threshold);
			CHECKERROR(cudaMemcpy((complex_t*)wf_t_neg_H, wf_t_neg_D, wf_t_neg_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
			check_norm_gpu(wf_t_neg_H, wf_t_neg_size, norm_ref, wfn_check_threshold);
		}
		act_GPU(bessel_series_T_H, bessel_series_size,
			block, grid,
			t0_D, t0_size, t1_D, t1_size,
			wf_t_pos_D, wf_t_pos_size, wf_out_D, dim_size,
			indptr_D, indices_D, values_D, dim_size);
		act_GPU(bessel_series_F_H, bessel_series_size,
			block, grid,
			t0_D, t0_size, t1_D, t1_size,
			wf_t_neg_D, wf_t_neg_size, wf_out_D, dim_size,
			indptr_D, indices_D, values_D, dim_size);
		// Hanning window
		double window = 0.5 * (1 + std::cos(PI * i_step / static_cast<double>(num_steps)));
		// Fourier transform
		for (size_t i_e = 0; i_e < num_energies; ++i_e) {
			double phase = qe_energies_H[i_e] * i_step * time_step;
			complex_t factor = std::exp(UIMAG * phase);
			complex_t f1 = factor * window;
			complex_t f2 = std::conj(factor) * window;
			axpbypz_kernel<<<grid, block>>>(
				make_cuDoubleComplex(f1.real(), f1.imag()), wf_t_pos_D,
				make_cuDoubleComplex(f2.real(), f2.imag()), wf_t_neg_D,
				wf_t_pos_size,
				wfq_H[i_e]);
		}
		// DOS correlation function
		if (calc_corr_dos) {
			corr_H[i_step-1] += vec_dot_GPU(block, grid, wf0_D, wf0_size, wf_t_pos_D, wf_t_pos_size,
				 tmp_D, tmp_H);
		}
		prog_bar.count();
	}
	CHECKERROR(cudaDeviceSynchronize());
	for (size_t i = 0; i < wfq.size(); i++) {
		CHECKERROR(cudaMemcpy((complex_t*)wfq[i].data(), wfq_H[i], wfq[i].size() * sizeof(complex_t), cudaMemcpyDeviceToHost));
		CHECKERROR(cudaFree(wfq_H[i]));
		wfq_H[i] = nullptr;
	}
	free(wfq_H);
	wfq_H = nullptr;
	CHECKERROR(cudaMemcpy((complex_t*)wf_t_pos_H, wf_t_pos_D, wf_t_pos_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)wf_t_neg_H, wf_t_neg_D, wf_t_neg_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t0_H, t0_D, t0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t1_H, t1_D, t1_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaFree(tmp_D));
	tmp_D = nullptr;
	CHECKERROR(cudaFree(wf_t_pos_D));
	wf_t_pos_D = nullptr;
	CHECKERROR(cudaFree(wf_t_neg_D));
	wf_t_neg_D = nullptr;
	CHECKERROR(cudaFree(t0_D));
	t0_D = nullptr;
	CHECKERROR(cudaFree(t1_D));
	t1_D = nullptr;
	CHECKERROR(cudaFree(indptr_D));
	indptr_D = nullptr;
	CHECKERROR(cudaFree(indices_D));
	indices_D = nullptr;
	CHECKERROR(cudaFree(values_D));
	values_D = nullptr;
	CHECKERROR(cudaFree(wf_out_D));
	wf_out_D = nullptr;
	CHECKERROR(cudaFree(wf0_D));
	wf0_D = nullptr;
	free(tmp_H); 
	tmp_H== nullptr;
};

__host__   void CudaConfig::tbpm_dyn_pol_GPU(
	const size_t num_steps, const  size_t  wfn_check_time_steps, const  double wfn_check_threshold,
	const complex_t norm_ref,
	const complex_t* bessel_series_H, const size_t bessel_series_size,
	const uint64_t* indptr_H, const size_t indptr_size,
	const uint64_t* indices_H, const size_t indices_size,
	const complex_t* values_H, const size_t values_size,
	const size_t dim_size,
	complex_t* psi1_H, const size_t psi1_size,
	complex_t* psi2_H, const size_t psi2_size,
	const complex_t* den_coeff_q_H, const size_t den_coeff_q_size,
	complex_t* t0_H, const size_t t0_size,
	complex_t* t1_H, const size_t t1_size,
	double* corr_H, const size_t corr_size) {
	assert(dim_size == psi1_size);
	assert(dim_size == psi2_size);
	assert(dim_size == t0_size);
	assert(dim_size == t1_size);
	cuDoubleComplex* psi2_D = nullptr, * psi1_D = nullptr, * den_coeff_q_D = nullptr;
	uint64_t* indptr_D = nullptr, * indices_D = nullptr;
	cuDoubleComplex* values_D = nullptr;
	cuDoubleComplex* t0_D = nullptr, * t1_D = nullptr, * wf_out_D = nullptr;
	cuDoubleComplex* tmp_D = nullptr, * tmp_H = nullptr;
	CHECKERROR(cudaMalloc(&psi1_D, psi1_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(psi1_D, psi1_H, psi1_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&psi2_D, psi2_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(psi2_D, psi2_H, psi2_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&den_coeff_q_D, den_coeff_q_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(den_coeff_q_D, den_coeff_q_H, den_coeff_q_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t0_D, dim_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t0_D, t0_H, dim_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t1_D, dim_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t1_D, t1_H, dim_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf_out_D, dim_size * sizeof(cuDoubleComplex)));
	// Set block and grid size
	dim3 block(1024);
	dim3 grid((dim_size + block.x - 1) / block.x);
	CHECKERROR(cudaMalloc(&tmp_D, grid.x * sizeof(cuDoubleComplex)));
	tmp_H = (cuDoubleComplex*)malloc(grid.x * sizeof(cuDoubleComplex));
	ProgressBar prog_bar(num_steps, 64, "Finished timestep");
	prog_bar.reset();
	for (int i_step = 2; i_step <= num_steps; ++i_step) {
		if (i_step == wfn_check_time_steps) {
			CHECKERROR(cudaMemcpy((complex_t*)psi2_H, psi2_D, psi2_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
			check_norm_gpu(psi2_H, psi2_size, norm_ref, wfn_check_threshold);
		}
		act_GPU(bessel_series_H, bessel_series_size,
			block, grid,
			t0_D, t0_size, t1_D, t1_size,
			psi1_D, psi1_size, wf_out_D, dim_size,
			indptr_D, indices_D, values_D, dim_size);
		act_GPU(bessel_series_H, bessel_series_size,
			block, grid,
			t0_D, t0_size, t1_D, t1_size,
			psi2_D, psi2_size, wf_out_D, dim_size,
			indptr_D, indices_D, values_D, dim_size);
		corr_H[i_step - 1] += xmy_diag_GPU(block, grid, psi2_D, psi1_D, den_coeff_q_D, dim_size, tmp_D, tmp_H).imag();
		prog_bar.count();
	}
	CHECKERROR(cudaDeviceSynchronize());
	prog_bar.count();
	CHECKERROR(cudaMemcpy((complex_t*)psi1_H, psi1_D, psi1_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)psi2_H, psi2_D, psi2_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t0_H, t0_D, dim_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t1_H, t1_D, dim_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaFree(tmp_D));
	tmp_D = nullptr;
	CHECKERROR(cudaFree(psi1_D));
	psi1_D = nullptr;
	CHECKERROR(cudaFree(psi2_D));
	psi2_D = nullptr;
	CHECKERROR(cudaFree(den_coeff_q_D));
	den_coeff_q_D = nullptr;
	CHECKERROR(cudaFree(wf_out_D));
	wf_out_D = nullptr;
	CHECKERROR(cudaFree(t0_D));
	t0_D = nullptr;
	CHECKERROR(cudaFree(t1_D));
	t1_D = nullptr;
	CHECKERROR(cudaFree(indptr_D));
	indptr_D = nullptr;
	CHECKERROR(cudaFree(indices_D));
	indices_D = nullptr;
	CHECKERROR(cudaFree(values_D));
	values_D = nullptr;
	free(tmp_H);
	tmp_H = nullptr;
};

__host__   void CudaConfig::tbpm_dc_cond_GPU(
	const size_t num_steps_dc,
	const complex_t* bessel_series_H, const size_t bessel_series_size,
	const uint64_t* indptr_H, const size_t indptr_size,
	const uint64_t* indices_H, const size_t indices_size,
	const complex_t* values_H, const size_t values_size,
	const uint64_t* curr_x_indptr_H, const size_t curr_x_indptr_size,
	const uint64_t* curr_x_indices_H, const size_t curr_x_indices_size,
	const complex_t* curr_x_values_H, const size_t curr_x_values_size,
	const uint64_t* curr_y_indptr_H, const size_t curr_y_indptr_size,
	const uint64_t* curr_y_indices_H, const size_t curr_y_indices_size,
	const complex_t* curr_y_values_H, const size_t curr_y_values_size,
	const size_t dim_size,
	double* factor_H, const size_t factor_size,
	complex_t* psi0_x_H, const size_t psi0_x_size,
	complex_t* psi0_y_H, const size_t psi0_y_size,
	complex_t* t0_H, const size_t t0_size,
	complex_t* t1_H, const size_t t1_size,
	complex_t* corr_dc_xx_H, const size_t corr_dc_xx_size,
	complex_t* corr_dc_yy_H, const size_t corr_dc_yy_size,
	std::vector<Eigen::VectorXcd>& wf_qe,
	const bool isRowMajor,
	const size_t num_energies,
	ProgressBar& prog_bar) {
	assert(curr_y_indptr_size == curr_x_indptr_size);
	assert(curr_y_indices_size == curr_x_indices_size);
	assert(curr_y_values_size == curr_x_values_size);
	assert(curr_y_values_size == values_size);
	assert(curr_y_indices_size == indices_size);
	assert(curr_y_indptr_size == indptr_size);
	assert(dim_size == psi0_y_size);
	assert(dim_size == psi0_x_size);
	assert(corr_dc_yy_size == corr_dc_xx_size);
	assert(dim_size == t0_size);
	assert(dim_size == t1_size);
	cuDoubleComplex * psi0_x_D = nullptr, * psi0_y_D = nullptr, ** wfq_H = nullptr;
	uint64_t* indptr_D = nullptr, * indices_D = nullptr,
		* curr_x_indptr_D = nullptr, * curr_x_indices_D = nullptr, * curr_y_indptr_D = nullptr, * curr_y_indices_D = nullptr;
	cuDoubleComplex* values_D = nullptr, * curr_x_values_D = nullptr, * curr_y_values_D = nullptr;
	cuDoubleComplex* t0_D = nullptr, * t1_D = nullptr, * wf_out_D = nullptr;
	cuDoubleComplex* tmp_D = nullptr, * tmp_H = nullptr;
	wfq_H = (cuDoubleComplex**)malloc(wf_qe.size() * sizeof(cuDoubleComplex));
	for (size_t i = 0; i < wf_qe.size(); i++) {
		CHECKERROR(cudaMalloc(&wfq_H[i], dim_size * sizeof(cuDoubleComplex)));
		CHECKERROR(cudaMemcpy(wfq_H[i], wf_qe[i].data(), wf_qe[i].size() * sizeof(complex_t), cudaMemcpyHostToDevice));
	}
	CHECKERROR(cudaMalloc(&psi0_x_D, psi0_x_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(psi0_x_D, psi0_x_H, psi0_x_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&psi0_y_D, psi0_y_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(psi0_y_D, psi0_y_H, psi0_y_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf_out_D, dim_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMalloc(&t0_D, dim_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t0_D, t0_H, dim_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t1_D, dim_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t1_D, t1_H, dim_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_x_indptr_D, curr_x_indptr_H, curr_x_indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_y_indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_y_indptr_D, curr_y_indptr_H, curr_y_indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_x_indices_D, curr_x_indices_H, curr_x_indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_y_indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_y_indices_D, curr_y_indices_H, curr_y_indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(curr_x_values_D, curr_x_values_H, curr_x_values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_y_values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(curr_y_values_D, curr_y_values_H, curr_y_values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	int RowWidth = num_energies;
	int ColWidth = corr_dc_yy_size / num_energies;
	dim3 block(1024);
	dim3 grid((dim_size + block.x - 1) / block.x);
	CHECKERROR(cudaMalloc(&tmp_D, grid.x * sizeof(cuDoubleComplex)));
	tmp_H = (cuDoubleComplex*)malloc(grid.x * sizeof(cuDoubleComplex));
	for (int i_step = 1; i_step <= num_steps_dc; ++i_step) {
		if (i_step!=1){
			act_GPU(bessel_series_H, bessel_series_size,
				block, grid,
				t0_D, t0_size, t1_D, t1_size,
				psi0_x_D, psi0_x_size, wf_out_D, dim_size,
				indptr_D, indices_D, values_D, dim_size);
			act_GPU(bessel_series_H, bessel_series_size,
				block, grid,
				t0_D, t0_size, t1_D, t1_size,
				psi0_y_D, psi0_y_size, wf_out_D, dim_size,
				indptr_D, indices_D, values_D, dim_size);
		}
		int j = i_step - 1;
		if (isRowMajor)
		{
			for (size_t i_e = 0; i_e < num_energies; ++i_e) {
				corr_dc_xx_H[ColWidth * i_e + j] += xmy_GPU(block, grid, psi0_x_D, psi0_x_size, wfq_H[i_e], wf_qe[i_e].size(), dim_size,
					curr_x_indptr_D, curr_x_indices_D, curr_x_values_D, tmp_D, tmp_H) / factor_H[i_e];
				corr_dc_yy_H[ColWidth * i_e + j] += xmy_GPU(block, grid, psi0_y_D, psi0_y_size, wfq_H[i_e], wf_qe[i_e].size(), dim_size,
					curr_y_indptr_D, curr_y_indices_D, curr_y_values_D, tmp_D, tmp_H) / factor_H[i_e];
			}
		}
		else
		{
			for (size_t i_e = 0; i_e < num_energies; ++i_e) {
				corr_dc_xx_H[i_e + RowWidth * j] += xmy_GPU(block, grid, psi0_x_D, psi0_x_size, wfq_H[i_e], wf_qe[i_e].size(), dim_size,
					curr_x_indptr_D, curr_x_indices_D, curr_x_values_D, tmp_D, tmp_H) / factor_H[i_e];
				corr_dc_yy_H[i_e + RowWidth * j] += xmy_GPU(block, grid, psi0_y_D, psi0_y_size, wfq_H[i_e], wf_qe[i_e].size(), dim_size,
					curr_y_indptr_D, curr_y_indices_D, curr_y_values_D, tmp_D, tmp_H) / factor_H[i_e];
			}
		}
		prog_bar.count();
	}
	CHECKERROR(cudaDeviceSynchronize());
	prog_bar.count();
	for (size_t i = 0; i < wf_qe.size(); i++) {
		CHECKERROR(cudaMemcpy((complex_t*)wf_qe[i].data(), wfq_H[i], wf_qe[i].size() * sizeof(complex_t), cudaMemcpyDeviceToHost));
		CHECKERROR(cudaFree(wfq_H[i]));
		wfq_H[i] = nullptr;
	}
	free(wfq_H);
	wfq_H = nullptr;
	CHECKERROR(cudaMemcpy((complex_t*)psi0_x_H, psi0_x_D, psi0_x_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)psi0_y_H, psi0_y_D, psi0_y_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t0_H, t0_D, dim_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t1_H, t1_D, dim_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaFree(tmp_D));
	tmp_D = nullptr;
	CHECKERROR(cudaFree(psi0_x_D));
	psi0_x_D = nullptr;
	CHECKERROR(cudaFree(psi0_y_D));
	psi0_y_D = nullptr;
	CHECKERROR(cudaFree(wf_out_D));
	wf_out_D = nullptr;
	CHECKERROR(cudaFree(t0_D));
	t0_D = nullptr;
	CHECKERROR(cudaFree(t1_D));
	t1_D = nullptr;
	CHECKERROR(cudaFree(indptr_D));
	indptr_D = nullptr;
	CHECKERROR(cudaFree(indices_D));
	indices_D = nullptr;
	CHECKERROR(cudaFree(values_D));
	values_D = nullptr;
	CHECKERROR(cudaFree(curr_x_indptr_D));
	curr_x_indptr_D = nullptr;
	CHECKERROR(cudaFree(curr_y_indptr_D));
	curr_y_indptr_D = nullptr;
	CHECKERROR(cudaFree(curr_x_indices_D));
	curr_x_indices_D = nullptr;
	CHECKERROR(cudaFree(curr_y_indices_D));
	curr_y_indices_D = nullptr;
	CHECKERROR(cudaFree(curr_x_values_D));
	curr_x_values_D = nullptr;
	CHECKERROR(cudaFree(curr_y_values_D));
	curr_y_values_D = nullptr;
	free(tmp_H);
	tmp_H = nullptr;
};

__host__   void CudaConfig::tbpm_wft_GPU(
	const size_t num_steps, const  size_t  wfn_check_time_steps,
	const  double wfn_check_threshold,
	const std::set<size_t> &wft_time_save,
	const complex_t* wf0_H, const size_t wf0_size,
	const complex_t norm_ref,
	const complex_t* bessel_series_H,const int bessel_series_size,
	complex_t* t0_H, const size_t t0_size,
	complex_t* t1_H, const size_t t1_size,
	const uint64_t* indptr_H, const size_t indptr_size,
	const uint64_t* indices_H, const size_t indices_size,
	const complex_t* values_H, const size_t values_size,
	const size_t dim_size,
	complex_t* corr_H, const size_t corr_size, const size_t corr_colWidth, const bool isRowMajor) {
	assert(sizeof(cuDoubleComplex) == sizeof(complex_t));
	assert(dim_size == wf0_size);
	assert(dim_size == t0_size);
	assert(dim_size == t1_size);
	cuDoubleComplex  * t0_D = nullptr, * t1_D = nullptr, * wf_out_D = nullptr, * wf0_D = nullptr;
	uint64_t* indptr_D = nullptr, * indices_D = nullptr;
	cuDoubleComplex* values_D = nullptr;
	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf0_D, wf0_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(wf0_D, wf0_H, wf0_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t0_D, t0_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t0_D, t0_H, t0_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t1_D, t1_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t1_D, t1_H, t1_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf_out_D, dim_size * sizeof(cuDoubleComplex)));
	// Set block and grid size.
	dim3 block(1024);
	dim3 grid((dim_size + block.x - 1) / block.x);
	// Check if we need to save the initial state
	size_t counter = 0;
	int ColWidth = corr_colWidth;
	int RowWidth = corr_size / corr_colWidth;
	ProgressBar prog_bar(num_steps, 64, "Finished timestep");
	prog_bar.reset();
	for (int i_step = 0; i_step <= num_steps; ++i_step) {
		if (i_step!=0)
		{
			if (i_step == wfn_check_time_steps) {
				CHECKERROR(cudaMemcpy((complex_t*)wf0_H, wf0_D, wf0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
				check_norm_gpu(wf0_H, wf0_size, norm_ref, wfn_check_threshold);
			}
			act_GPU(bessel_series_H, bessel_series_size,
				block, grid,
				t0_D, t0_size, t1_D, t1_size,
				wf0_D, wf0_size, wf_out_D, dim_size,
				indptr_D, indices_D, values_D, dim_size);
		}
		if (wft_time_save.find(i_step) != wft_time_save.end()) {
			if (isRowMajor){
				//This is an inefficient way to access storage
				CHECKERROR(cudaMemcpy((complex_t*)wf0_H, wf0_D, wf0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
				for (size_t rowIdx = 0; rowIdx < RowWidth; rowIdx++)
					corr_H[ColWidth * rowIdx + counter] = wf0_H[rowIdx];
			}else {
				CHECKERROR(cudaMemcpy((complex_t*)(corr_H+RowWidth * counter), wf0_D, wf0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
			}
			counter++;
		}
		prog_bar.count();
	}
	CHECKERROR(cudaDeviceSynchronize());
	CHECKERROR(cudaMemcpy((complex_t*)t0_H, t0_D, t0_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)t1_H, t1_D, t1_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaFree(wf0_D));
	wf0_D = nullptr;
	CHECKERROR(cudaFree(t0_D));
	t0_D = nullptr;
	CHECKERROR(cudaFree(t1_D));
	t1_D = nullptr;
	CHECKERROR(cudaFree(indptr_D));
	indptr_D = nullptr;
	CHECKERROR(cudaFree(indices_D));
	indices_D = nullptr;
	CHECKERROR(cudaFree(values_D));
	values_D = nullptr;
	CHECKERROR(cudaFree(wf_out_D));
	wf_out_D = nullptr;
};

__host__   void CudaConfig::haydock_coef_GPU(
	const double rescale, const int n_depth,
	const uint64_t* indptr_H, const size_t indptr_size,
	const uint64_t* indices_H, const size_t indices_size,
	const complex_t* values_H, const size_t values_size,
	complex_t* p1_H, const size_t p1_size,
	complex_t* p2_H, const size_t p2_size,
	complex_t* p_buf_H, const size_t p_buf_size,
	complex_t* coefa_H, const size_t coefa_size,
	double* coefb_H, const size_t coefb_size) {
	assert(sizeof(cuDoubleComplex) == sizeof(complex_t));
	assert(p_buf_size == p1_size);
	assert(p_buf_size == p2_size);
	assert(coefb_size == coefa_size);

	cuDoubleComplex* p1_D = nullptr,*p2_D = nullptr, * p_buf_D = nullptr, * p_Temp = nullptr;
	uint64_t* indptr_D = nullptr, * indices_D = nullptr;
	cuDoubleComplex* values_D = nullptr;
	cuDoubleComplex* tmp_D = nullptr, * tmp_H = nullptr;

	CHECKERROR(cudaMalloc(&p1_D, p1_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(p1_D, p1_H, p1_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&p1_D, p1_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(p1_D, p1_H, p1_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&p2_D, p2_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(p2_D, p2_H, p2_size * sizeof(cuDoubleComplex), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&p_buf_D, p_buf_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(p_buf_D, p_buf_H, p_buf_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	// Propagate 
	ProgressBar prog_bar(n_depth, 100, "Finished depth");
	prog_bar.reset(); // We have done the 0th step in initialization.
	// Set block and grid size.
	dim3 block(1024);
	dim3 grid((p1_size + block.x - 1) / block.x);
	CHECKERROR(cudaMalloc(&tmp_D, grid.x * sizeof(cuDoubleComplex)));
	tmp_H = (cuDoubleComplex*)malloc(grid.x * sizeof(cuDoubleComplex));
	for (int i_step = 1; i_step < n_depth; ++i_step) {
		amv_kernel << <grid, block >> > (
			make_cuDoubleComplex(rescale, 0), p2_D, p2_size,
			indptr_D, indices_D, values_D,
			p_buf_D, p_buf_size);

		coefa_H[i_step]= vec_dot_GPU(block, grid, p2_D, p2_size, p_buf_D, p_buf_size,tmp_D, tmp_H);
		axpbypz_kernel<<<grid, block >> > (
			make_cuDoubleComplex(-1*coefa_H[i_step].real(), -1 * coefa_H[i_step].imag()), p2_D,
			make_cuDoubleComplex(-1 * coefb_H[i_step-1], 0), p1_D,
			p1_size,
			p_buf_D);

		coefb_H[i_step] = vec_norm_GPU(block, grid, p_buf_D, p_buf_size, tmp_D, tmp_H);
		vec_div_sca_kernel<<<grid, block >>>(
			p_buf_D, p_buf_size, coefb_H[i_step]);

		p_Temp = p_buf_D;
		p_buf_D = p2_D;
		p2_D = p_Temp;
		p_Temp = p_buf_D;
		p_buf_D = p1_D;
		p1_D = p_Temp;
		prog_bar.count();
	}

	CHECKERROR(cudaDeviceSynchronize());
	prog_bar.count();
	CHECKERROR(cudaMemcpy((complex_t*)p1_H, p1_D, p1_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)p2_H, p2_D, p2_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaMemcpy((complex_t*)p_buf_H, p_buf_D, p_buf_size * sizeof(complex_t), cudaMemcpyDeviceToHost));
	
	CHECKERROR(cudaFree(tmp_D));
	tmp_D = nullptr;
	CHECKERROR(cudaFree(p1_D));
	p1_D = nullptr;
	CHECKERROR(cudaFree(p2_D));
	p2_D = nullptr;
	CHECKERROR(cudaFree(p_buf_D));
	p_buf_D = nullptr;
	CHECKERROR(cudaFree(indptr_D));
	indptr_D = nullptr;
	CHECKERROR(cudaFree(indices_D));
	indices_D = nullptr;
	CHECKERROR(cudaFree(values_D));
	values_D = nullptr;
	free(tmp_H);
	tmp_H = nullptr;
};

__host__   void CudaConfig::kbdc_mu_GPU(
	const complex_t* wf0_H, const int wf0_size,
	const int dckb_component, const int num_kernel,
	const uint64_t* indptr_H, const size_t indptr_size,
	const uint64_t* indices_H, const size_t indices_size,
	const complex_t* values_H, const size_t values_size,
	const uint64_t* curr_x_indptr_H, const size_t curr_x_indptr_size,
	const uint64_t* curr_x_indices_H, const size_t curr_x_indices_size,
	const complex_t* curr_x_values_H, const size_t curr_x_values_size,
	const uint64_t* curr_y_indptr_H, const size_t curr_y_indptr_size,
	const uint64_t* curr_y_indices_H, const size_t curr_y_indices_size,
	const complex_t* curr_y_values_H, const size_t curr_y_values_size,
	std::vector<Eigen::VectorXcd>& buffer, 
	std::vector<Eigen::VectorXcd>& wf_vb_tn,
	complex_t* mu_sample_H, const bool isRowMajor,
	ProgressBar& prog_bar) {
	cuDoubleComplex** buffer_H,**wf_vb_tn_H;
	uint64_t* indptr_D, * indices_D;
	cuDoubleComplex* mu_sample_D,** wf_vb_tn_D;
	cuDoubleComplex* values_D;
	cuDoubleComplex* tmp_D = nullptr, * tmp_H = nullptr;
	uint64_t* curr_x_indptr_D, * curr_x_indices_D, *curr_y_indptr_D, * curr_y_indices_D;
	cuDoubleComplex* curr_x_values_D, * curr_y_values_D;
	cuDoubleComplex* wf0_D;
	assert(sizeof(cuDoubleComplex) == sizeof(complex_t));
	assert(buffer.size() == num_kernel);
	assert(wf_vb_tn.size() == num_kernel);

	// Propagate 
	// Set block and grid size.
	dim3 block(1024);
	dim3 grid((buffer[0].size() + block.x - 1) / block.x);
	buffer_H = (cuDoubleComplex**)malloc(num_kernel * sizeof(cuDoubleComplex*));
	for (size_t i = 0; i < buffer.size(); i++) {
		CHECKERROR(cudaMalloc(&buffer_H[i], buffer[i].size() * sizeof(cuDoubleComplex)));
		CHECKERROR(cudaMemcpy(buffer_H[i], buffer[i].data(), buffer[i].size() * sizeof(complex_t), cudaMemcpyHostToDevice));
	}
	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	for (int i = 2; i < num_kernel; ++i) {
		amxsy_out_kernel <<<grid, block >> > (
			make_cuDoubleComplex(2.0, 0),
			buffer_H[i - 1],
			buffer_H[i - 2],
			indptr_D, indices_D, values_D,
			buffer_H[i], buffer[i].size());
		prog_bar.count();
	}
	CHECKERROR(cudaFree(indptr_D));
	CHECKERROR(cudaFree(indices_D));
	CHECKERROR(cudaFree(values_D));

	CHECKERROR(cudaMalloc(&curr_y_indptr_D, curr_y_indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_y_indptr_D, curr_y_indptr_H, curr_y_indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_y_indices_D, curr_y_indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_y_indices_D, curr_y_indices_H, curr_y_indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_y_values_D, curr_y_values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(curr_y_values_D, curr_y_values_H, curr_y_values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_indptr_D, curr_x_indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_x_indptr_D, curr_x_indptr_H, curr_x_indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_indices_D, curr_x_indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_x_indices_D, curr_x_indices_H, curr_x_indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_values_D, curr_x_values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(curr_x_values_D, curr_x_values_H, curr_x_values_size * sizeof(complex_t), cudaMemcpyHostToDevice));

	// Apply the current operator on buffer and save to wf_vb_tn
	std::cout << "Evaluating v_beta*Tn(H)|wf0>." << std::endl;
	prog_bar.reset();

	wf_vb_tn_H = (cuDoubleComplex**)malloc(num_kernel * sizeof(cuDoubleComplex*));
	if (dckb_component == 1) { // XX component
		for (int i = 0; i < num_kernel; ++i) {
			CHECKERROR(cudaMalloc(&wf_vb_tn_H[i], wf_vb_tn[i].size() * sizeof(cuDoubleComplex)));
			CHECKERROR(cudaMemcpy(wf_vb_tn_H[i], wf_vb_tn[i].data(), wf_vb_tn[i].size() * sizeof(complex_t), cudaMemcpyHostToDevice));
			mv_kernel << <grid, block >> > (
				buffer_H[i], buffer[i].size(),
				curr_x_indptr_D, curr_x_indices_D, curr_x_values_D,
				wf_vb_tn_H[i], wf_vb_tn[i].size()
				);

			CHECKERROR(cudaMemcpy(wf_vb_tn[i].data(),wf_vb_tn_H[i], wf_vb_tn[i].size() * sizeof(complex_t), cudaMemcpyDeviceToHost));
			cudaFree(wf_vb_tn_H[i]);
			prog_bar.count();
		}
	}
	else { // XY component
		for (int i = 0; i < num_kernel; ++i) {
			CHECKERROR(cudaMalloc(&wf_vb_tn_H[i], wf_vb_tn[i].size() * sizeof(cuDoubleComplex)));
			CHECKERROR(cudaMemcpy(wf_vb_tn_H[i], wf_vb_tn[i].data(), wf_vb_tn[i].size() * sizeof(complex_t), cudaMemcpyHostToDevice));
			mv_kernel << <grid, block >> > (
				buffer_H[i], buffer[i].size(),
				curr_y_indptr_D, curr_y_indices_D, curr_y_values_D,
				wf_vb_tn_H[i], wf_vb_tn[i].size()
				);
			CHECKERROR(cudaMemcpy(wf_vb_tn[i].data(), wf_vb_tn_H[i], wf_vb_tn[i].size() * sizeof(complex_t), cudaMemcpyDeviceToHost));
			cudaFree(wf_vb_tn_H[i]);
			prog_bar.count();
		}
	}
	free(wf_vb_tn_H);

	CHECKERROR(cudaFree(curr_y_indptr_D));
	CHECKERROR(cudaFree(curr_y_indices_D));
	CHECKERROR(cudaFree(curr_y_values_D));


	// Evaluate Tm(H)*v_alpha|wf0>
	std::cout << "Evaluating Tm(H)*v_alpha|wf0>" << std::endl;
	prog_bar.reset();

	CHECKERROR(cudaMalloc(&wf0_D, wf0_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(wf0_D, wf0_H, wf0_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	mv_kernel << <grid, block >> > (
		wf0_D, wf0_size,
		curr_x_indptr_D, curr_x_indices_D,  curr_x_values_D,
		buffer_H[0], buffer[0].size());
	prog_bar.count();

	CHECKERROR(cudaFree(wf0_D));
	CHECKERROR(cudaFree(curr_x_indptr_D));
	CHECKERROR(cudaFree(curr_x_indices_D));
	CHECKERROR(cudaFree(curr_x_values_D));

	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	mv_kernel << <grid, block >> > (
		buffer_H[0], buffer[0].size(),
		indptr_D, indices_D,values_D,
		buffer_H[1], buffer[1].size());
	prog_bar.count();

	for (int i = 2; i < num_kernel; ++i) {
		amxsy_out_kernel << <grid, block >> > (
			make_cuDoubleComplex(2.0, 0),
			buffer_H[i - 1],
			buffer_H[i - 2],
			indptr_D, indices_D, values_D,
			buffer_H[i], buffer[i].size());
		prog_bar.count();
	}
	CHECKERROR(cudaFree(indptr_D));
	CHECKERROR(cudaFree(indices_D));
	CHECKERROR(cudaFree(values_D));

	// Evaluate mu_sample
	std::cout << "Evaluating mu." << std::endl;
	prog_bar.reset();

	CHECKERROR(cudaMalloc(&mu_sample_D, num_kernel * num_kernel * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(mu_sample_D, mu_sample_H, num_kernel * num_kernel * sizeof(complex_t), cudaMemcpyHostToDevice));

	wf_vb_tn_H = (cuDoubleComplex**)malloc(num_kernel * sizeof(cuDoubleComplex*));
#ifndef SPEED
	//这个是速度最快的情况
	CHECKERROR(cudaMalloc(&wf_vb_tn_D, num_kernel * sizeof(cuDoubleComplex*)));
	for (size_t i = 0; i < wf_vb_tn.size(); i++) {
		CHECKERROR(cudaMalloc(&wf_vb_tn_H[i], wf_vb_tn[i].size() * sizeof(cuDoubleComplex)));
		CHECKERROR(cudaMemcpy(wf_vb_tn_H[i], wf_vb_tn[i].data(), wf_vb_tn[i].size() * sizeof(complex_t), cudaMemcpyHostToDevice));
	}
	CHECKERROR(cudaMemcpy(wf_vb_tn_D, wf_vb_tn_H, num_kernel * sizeof(cuDoubleComplex*), cudaMemcpyHostToDevice));
	// Set block and grid size.
	dim3 block2(1024);
	dim3 grid2((num_kernel + block2.x - 1) / block2.x);
	for (size_t i = 0; i < num_kernel; i++){
		vec_dot_kernel2 << <grid2, block2 >> > (
			buffer_H[i],
			wf_vb_tn_D,
			i, buffer[i].size(),
			num_kernel, isRowMajor, mu_sample_D);
		prog_bar.count();
	}
	CHECKERROR(cudaFree(wf_vb_tn_D));
	for (size_t i = 0; i < num_kernel; i++) {
		CHECKERROR(cudaMemcpy((complex_t*)wf_vb_tn[i].data(), wf_vb_tn_H[i], wf_vb_tn[i].size() * sizeof(complex_t), cudaMemcpyDeviceToHost));
		CHECKERROR(cudaFree(wf_vb_tn_H[i]));
	}
	free(wf_vb_tn_H);
	CHECKERROR(cudaMemcpy((complex_t*)mu_sample_H, mu_sample_D, num_kernel* num_kernel * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaFree(mu_sample_D));
	for (size_t i = 0; i < num_kernel; i++) {
		CHECKERROR(cudaMemcpy((complex_t*)buffer[i].data(), buffer_H[i], buffer[i].size() * sizeof(complex_t), cudaMemcpyDeviceToHost));
		CHECKERROR(cudaFree(buffer_H[i]));
	}
	free(buffer_H);
#else
	CHECKERROR(cudaMalloc(&tmp_D, grid.x * sizeof(cuDoubleComplex)));
	tmp_H = (cuDoubleComplex*)malloc(grid.x * sizeof(cuDoubleComplex));
	for (size_t i = 0; i < num_kernel; i++) {
		CHECKERROR(cudaMemcpy((complex_t*)buffer[i].data(), buffer_H[i], buffer[i].size() * sizeof(complex_t), cudaMemcpyDeviceToHost));
		CHECKERROR(cudaFree(buffer_H[i]));
	}
	for (size_t i = 0; i < wf_vb_tn.size(); i++) {
		CHECKERROR(cudaMalloc(&wf_vb_tn_H[i], wf_vb_tn[i].size() * sizeof(cuDoubleComplex)));
		CHECKERROR(cudaMemcpy(wf_vb_tn_H[i], wf_vb_tn[i].data(), wf_vb_tn[i].size() * sizeof(complex_t), cudaMemcpyHostToDevice));
	}
	for (size_t i = 0; i < num_kernel; i++) {
		CHECKERROR(cudaMalloc(&buffer_H[i], buffer[i].size() * sizeof(cuDoubleComplex)));
		CHECKERROR(cudaMemcpy(buffer_H[i], buffer[i].data(), buffer[i].size() * sizeof(complex_t), cudaMemcpyHostToDevice));
		for (size_t j = 0; j < num_kernel; j++) {
			if (isRowMajor)
				mu_sample_H[num_kernel * i + j] = vec_dot_GPU(block, grid,
					buffer_H[i], buffer[i].size(), wf_vb_tn_H[j], wf_vb_tn[j].size(),
					tmp_D, tmp_H);
			else
				mu_sample_H[i + num_kernel * j] = vec_dot_GPU(block, grid,
					buffer_H[i], buffer[i].size(), wf_vb_tn_H[j], wf_vb_tn[j].size(),
					tmp_D, tmp_H);
		}
		CHECKERROR(cudaMemcpy((complex_t*)buffer[i].data(), buffer_H[i], buffer[i].size() * sizeof(complex_t), cudaMemcpyDeviceToHost));
		CHECKERROR(cudaFree(buffer_H[i]));
		prog_bar.count();
	}

	for (size_t i = 0; i < num_kernel; i++) {
		CHECKERROR(cudaMemcpy((complex_t*)wf_vb_tn[i].data(), wf_vb_tn_H[i], wf_vb_tn[i].size() * sizeof(complex_t), cudaMemcpyDeviceToHost));
		CHECKERROR(cudaFree(wf_vb_tn_H[i]));
	}	
	free(wf_vb_tn_H);
	free(buffer_H);
	free(tmp_H);
	CHECKERROR(cudaFree(tmp_D));
#endif
	CHECKERROR(cudaDeviceSynchronize());

};


__host__   void CudaConfig::kbdc_mu_GPU2(
	const complex_t* wf0_H, const int wf0_size,
	const int dckb_component, const int num_kernel,
	const uint64_t* indptr_H, const size_t indptr_size,
	const uint64_t* indices_H, const size_t indices_size,
	const complex_t* values_H, const size_t values_size,
	const uint64_t* curr_x_indptr_H, const size_t curr_x_indptr_size,
	const uint64_t* curr_x_indices_H, const size_t curr_x_indices_size,
	const complex_t* curr_x_values_H, const size_t curr_x_values_size,
	const uint64_t* curr_y_indptr_H, const size_t curr_y_indptr_size,
	const uint64_t* curr_y_indices_H, const size_t curr_y_indices_size,
	const complex_t* curr_y_values_H, const size_t curr_y_values_size,
	complex_t* t0_H, const size_t t0_size,
	complex_t* t1_H, const size_t t1_size,
	complex_t* mu_sample_H, const bool isRowMajor,
	ProgressBar& prog_bar,
	std::vector<Eigen::VectorXcd>& wf_vb_tn) {
	assert(t0_size == t1_size);
	assert(curr_x_indptr_size == curr_y_indptr_size);
	assert(curr_x_indptr_size == indptr_size);
	assert(curr_x_indices_size == curr_y_indices_size);
	assert(curr_x_indices_size == indices_size);
	assert(curr_x_values_size == curr_y_values_size);
	assert(curr_x_values_size == values_size);
	assert(sizeof(cuDoubleComplex) == sizeof(complex_t));
	cuDoubleComplex ** wf_vb_tn_H = nullptr;
	uint64_t* indptr_D = nullptr, * indices_D = nullptr;
	cuDoubleComplex ** wf_vb_tn_D = nullptr;
	cuDoubleComplex* values_D = nullptr;
	cuDoubleComplex* tmp_D = nullptr, * tmp_H = nullptr;
	uint64_t* curr_x_indptr_D = nullptr, * curr_x_indices_D = nullptr, * curr_y_indptr_D = nullptr, * curr_y_indices_D = nullptr;
	cuDoubleComplex* curr_x_values_D = nullptr, * curr_y_values_D = nullptr;
	cuDoubleComplex* wf0_D = nullptr;
	cuDoubleComplex* t0_D = nullptr, * t1_D = nullptr;
	CHECKERROR(cudaMalloc(&curr_y_indptr_D, curr_y_indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_y_indptr_D, curr_y_indptr_H, curr_y_indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_y_indices_D, curr_y_indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_y_indices_D, curr_y_indices_H, curr_y_indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_y_values_D, curr_y_values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(curr_y_values_D, curr_y_values_H, curr_y_values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_indptr_D, curr_x_indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_x_indptr_D, curr_x_indptr_H, curr_x_indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_indices_D, curr_x_indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(curr_x_indices_D, curr_x_indices_H, curr_x_indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&curr_x_values_D, curr_x_values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(curr_x_values_D, curr_x_values_H, curr_x_values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t0_D, t0_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t0_D, t0_H, t0_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&t1_D, t1_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(t1_D, t1_H, t1_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indptr_D, indptr_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indptr_D, indptr_H, indptr_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&indices_D, indices_size * sizeof(uint64_t)));
	CHECKERROR(cudaMemcpy(indices_D, indices_H, indices_size * sizeof(uint64_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&values_D, values_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(values_D, values_H, values_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	CHECKERROR(cudaMalloc(&wf_vb_tn_D, num_kernel * sizeof(cuDoubleComplex*)));
	wf_vb_tn_H = (cuDoubleComplex**)malloc(num_kernel * sizeof(cuDoubleComplex*));
	for (size_t i = 0; i < wf_vb_tn.size(); i++) {
		CHECKERROR(cudaMalloc(&wf_vb_tn_H[i], wf_vb_tn[i].size() * sizeof(cuDoubleComplex)));
		CHECKERROR(cudaMemcpy(wf_vb_tn_H[i], wf_vb_tn[i].data(), wf_vb_tn[i].size() * sizeof(complex_t), cudaMemcpyHostToDevice));
	}
	CHECKERROR(cudaMemcpy(wf_vb_tn_D, wf_vb_tn_H, num_kernel * sizeof(cuDoubleComplex*), cudaMemcpyHostToDevice));
	// Propagate 
	// Set block and grid size.
	dim3 block(1024);
	dim3 grid((t0_size + block.x - 1) / block.x);
	mv_kernel << <grid, block >> > (
		t0_D, t0_size,
		indptr_D, indices_D, values_D,
		t1_D, t1_size);
	cuDoubleComplex* curr_beta_values_D;
	uint64_t* curr_beta_indptr_D,* curr_beta_indices_D;
	if (dckb_component == 1) { 
		curr_beta_values_D = curr_x_values_D;
		curr_beta_indptr_D = curr_x_indptr_D;
		curr_beta_indices_D = curr_x_indices_D;
	}
	else { 
		curr_beta_values_D = curr_y_values_D;
		curr_beta_indptr_D = curr_y_indptr_D;
		curr_beta_indices_D = curr_y_indices_D;
	}
	mv_kernel << <grid, block >> > (
		t0_D, t0_size,
		curr_beta_indptr_D, curr_beta_indices_D, curr_beta_values_D,
		wf_vb_tn_H[0], wf_vb_tn[0].size()
		);
	mv_kernel << <grid, block >> > (
		t1_D, t1_size,
		curr_beta_indptr_D, curr_beta_indices_D, curr_beta_values_D,
		wf_vb_tn_H[1], wf_vb_tn[1].size()
		);
	prog_bar.count();
	cuDoubleComplex* p0 = t0_D;
	cuDoubleComplex* p1 = t1_D;
	cuDoubleComplex* p2 = nullptr;
	for (int i = 2; i < num_kernel; ++i) {
		p2 = p0;
		amxsy_kernel << <grid, block >> > (
			2.0,
			p1, t1_size,
			p0, t0_size, t0_size,
			indptr_D, indices_D, values_D);
		mv_kernel << <grid, block >> > (
			p2, t0_size,
			curr_beta_indptr_D, curr_beta_indices_D, curr_beta_values_D,
			wf_vb_tn_H[i], wf_vb_tn[i].size()
			);
		p0 = p1;
		p1 = p2;
		prog_bar.count();
		CHECKERROR(cudaDeviceSynchronize());
	}
	prog_bar.count();
	// Evaluate Tm(H)*v_alpha|wf0> and mu
	std::cout << "Evaluating Tm(H)*v_alpha|wf0>" << std::endl;
	prog_bar.reset();
	cuDoubleComplex* curr_alpha_values_D;
	uint64_t* curr_alpha_indptr_D, * curr_alpha_indices_D;
	curr_alpha_values_D = curr_x_values_D;
	curr_alpha_indptr_D = curr_x_indptr_D;
	curr_alpha_indices_D = curr_x_indices_D;
	CHECKERROR(cudaMalloc(&wf0_D, wf0_size * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(wf0_D, wf0_H, wf0_size * sizeof(complex_t), cudaMemcpyHostToDevice));
	mv_kernel << <grid, block >> > (
		wf0_D, wf0_size,
		curr_alpha_indptr_D, curr_alpha_indices_D, curr_alpha_values_D,
		t0_D, t0_size
		);
	CHECKERROR(cudaFree(wf0_D));
	wf0_D = nullptr;
	mv_kernel << <grid, block >> > (
		t0_D, t0_size,
		indptr_D, indices_D, values_D,
		t1_D, t1_size
		);
	CHECKERROR(cudaMalloc(&tmp_D, grid.x * sizeof(cuDoubleComplex)));
	tmp_H = (cuDoubleComplex*)malloc(grid.x * sizeof(cuDoubleComplex));
	for (int j = 0; j < num_kernel; ++j) {
		if (isRowMajor)
			mu_sample_H[num_kernel * 0 + j] = vec_dot_GPU(block, grid,
				t0_D, t0_size, wf_vb_tn_H[j], wf_vb_tn[j].size(),
				tmp_D, tmp_H);
		else
			mu_sample_H[0 + num_kernel * j] = vec_dot_GPU(block, grid,
				t0_D, t0_size, wf_vb_tn_H[j], wf_vb_tn[j].size(),
				tmp_D, tmp_H);
	}
	for (int j = 0; j < num_kernel; ++j) {
		if (isRowMajor)
			mu_sample_H[num_kernel * 1 + j] = vec_dot_GPU(block, grid,
				t1_D, t1_size, wf_vb_tn_H[j], wf_vb_tn[j].size(),
				tmp_D, tmp_H);
		else
			mu_sample_H[1 + num_kernel * j] = vec_dot_GPU(block, grid,
				t1_D, t1_size, wf_vb_tn_H[j], wf_vb_tn[j].size(),
				tmp_D, tmp_H);
	}
	CHECKERROR(cudaFree(tmp_D));
	tmp_D = nullptr;
	free(tmp_H);
	tmp_H = nullptr;
	prog_bar.count();
	// Rebind pointers. This is essential.
	p0 = t0_D;
	p1 = t1_D;
	p2 = nullptr;
	dim3 block2(1);
	dim3 grid2((num_kernel + block2.x - 1) / block2.x);
	cuDoubleComplex* mu_sample_D=nullptr;
	CHECKERROR(cudaMalloc(&mu_sample_D, num_kernel* num_kernel * sizeof(cuDoubleComplex)));
	CHECKERROR(cudaMemcpy(mu_sample_D, mu_sample_H, num_kernel* num_kernel * sizeof(complex_t), cudaMemcpyHostToDevice));
	for (size_t i = 2; i < num_kernel; i++) {
		p2 = p0;
		amxsy_kernel << <grid, block >> > (
			2.0,
			p1, t1_size,
			p0, t0_size, t0_size,
			indptr_D, indices_D, values_D);
		vec_dot_kernel2 << <grid2, block2 >> > (
			p2,
			wf_vb_tn_D,
			i, t1_size,
			num_kernel, isRowMajor, mu_sample_D);
		p0 = p1;
		p1 = p2;
		prog_bar.count();
		CHECKERROR(cudaDeviceSynchronize());
	}
	prog_bar.count();
	CHECKERROR(cudaMemcpy((complex_t*)mu_sample_H, mu_sample_D, num_kernel * num_kernel * sizeof(complex_t), cudaMemcpyDeviceToHost));
	CHECKERROR(cudaFree(mu_sample_D));
	mu_sample_D = nullptr;
	CHECKERROR(cudaDeviceSynchronize());
	for (size_t i = 0; i < num_kernel; i++) {
		CHECKERROR(cudaMemcpy((complex_t*)wf_vb_tn[i].data(), wf_vb_tn_H[i], wf_vb_tn[i].size() * sizeof(complex_t), cudaMemcpyDeviceToHost));
		CHECKERROR(cudaFree(wf_vb_tn_H[i]));
		wf_vb_tn_H[i] = nullptr;
	}
	free(wf_vb_tn_H);
	wf_vb_tn_H = nullptr;
	CHECKERROR(cudaFree(curr_y_indptr_D));
	curr_y_indptr_D = nullptr;
	CHECKERROR(cudaFree(curr_y_indices_D));
	curr_y_indices_D = nullptr;
	CHECKERROR(cudaFree(curr_y_values_D));
	curr_y_values_D = nullptr;
	CHECKERROR(cudaFree(curr_x_indptr_D));
	curr_x_indptr_D = nullptr;
	CHECKERROR(cudaFree(curr_x_indices_D));
	curr_x_indices_D = nullptr;
	CHECKERROR(cudaFree(curr_x_values_D));
	curr_x_values_D = nullptr;
	CHECKERROR(cudaFree(t0_D));
	t0_D = nullptr;
	CHECKERROR(cudaFree(t1_D));
	t1_D = nullptr;
	CHECKERROR(cudaFree(indptr_D));
	indptr_D = nullptr;
	CHECKERROR(cudaFree(indices_D));
	indices_D = nullptr;
	CHECKERROR(cudaFree(values_D));
	values_D = nullptr;
	CHECKERROR(cudaFree(wf_vb_tn_D));
	wf_vb_tn_D = nullptr;
};