strmm.cu

Nividia提供的CUDA的BLAS库源码

/*
 * Copyright 1993-2008 NVIDIA Corporation.  All rights reserved.
 *
 * NOTICE TO USER:   
 *
 * This source code is subject to NVIDIA ownership rights under U.S. and
 * international Copyright laws.  
 *
 * This software and the information contained herein is being provided 
 * under the terms and conditions of a Source Code License Agreement.     
 *
 * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE
 * CODE FOR ANY PURPOSE.  IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR 
 * IMPLIED WARRANTY OF ANY KIND.  NVIDIA DISCLAIMS ALL WARRANTIES WITH
 * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY, NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE.
 * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL,
 * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
 * OF USE, DATA OR PROFITS,  WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
 * OR OTHER TORTIOUS ACTION,  ARISING OUT OF OR IN CONNECTION WITH THE USE
 * OR PERFORMANCE OF THIS SOURCE CODE.  
 *
 * U.S. Government End Users.   This source code is a "commercial item" as 
 * that term is defined at  48 C.F.R. 2.101 (OCT 1995), consisting  of
 * "commercial computer  software"  and "commercial computer software 
 * documentation" as such terms are  used in 48 C.F.R. 12.212 (SEPT 1995)
 * and is provided to the U.S. Government only as a commercial end item.
 * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through
 * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the 
 * source code with only those rights set forth herein.
 */

/* This file contains the implementation of the BLAS-3 function strmm */

#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include <stdio.h>
#include <limits.h>
#include <ctype.h>
#include <math.h>
#include "cublas.h"   /* CUBLAS public header file  */
#include "cublasP.h"  /* CUBLAS private header file */

#define BLK_LOG             (5)
#define BLK                 (1 << BLK_LOG)
#define JINC                (BLK * CUBLAS_STRMM_CTAS)  // used by strmm_l
#define IINC                (BLK * CUBLAS_STRMM_CTAS)  // used by strmm_r

#if ((CUBLAS_STRMM_THREAD_COUNT<BLK))
#error block dimension must be >= threadcount
#endif

#if ((CUBLAS_STRMM_THREAD_COUNT%BLK)!=0)
#error threadcount and block dimensions do not divide evenly
#endif

#define A_NBR_COLS          (CUBLAS_STRMM_THREAD_COUNT/BLK)
#define B_NBR_COLS          (CUBLAS_STRMM_THREAD_COUNT/BLK)

#if (((BLK*BLK)%CUBLAS_STRMM_THREAD_COUNT)!=0)
#error blocksize of A and B not evenly divided by threadcount!
#endif

#define A_ELEMS_PER_THREAD  ((BLK * BLK) / CUBLAS_STRMM_THREAD_COUNT)
#define B_ELEMS_PER_THREAD  ((BLK * BLK) / CUBLAS_STRMM_THREAD_COUNT)

__global__ void strmm_r_lo_tr_main_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_unit_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_unit_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_unit_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_unit_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_unit_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_unit_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_unit_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_unit_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_unit_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_unit_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_unit_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_unit_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_unit_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_unit_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_unit_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_unit_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_unit_alpha0_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_unit_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_unit_alpha0_fulltile_sw (struct cublasStrmmParams parms);

__global__ void strmm_r_lo_tr_main_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_unit_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_unit_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_unit_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_unit_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_unit_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_unit_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_unit_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_unit_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_unit_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_unit_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_unit_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_unit_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_unit_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_unit_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_unit_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_unit_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_unit_alpha0_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_tr_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_tr_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_lo_nt_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_r_up_nt_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_tr_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_tr_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_lo_nt_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void strmm_l_up_nt_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_unit_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_tr_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_tr_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_lo_nt_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_r_up_nt_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_tr_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_tr_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_lo_nt_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);
__global__ void fast_strmm_l_up_nt_main_unit_alpha0_fulltile_hw (struct cublasStrmmParams parms);


typedef void (*pf) (struct cublasStrmmParams parms);

static pf strmm_sw[128] = {
    strmm_r_lo_tr_main_sw,
    strmm_r_up_tr_main_sw,
    strmm_r_lo_nt_main_sw,
    strmm_r_up_nt_main_sw,
    strmm_l_lo_tr_main_sw,
    strmm_l_up_tr_main_sw,
    strmm_l_lo_nt_main_sw,
    strmm_l_up_nt_main_sw,
    strmm_r_lo_tr_main_unit_sw,
    strmm_r_up_tr_main_unit_sw,
    strmm_r_lo_nt_main_unit_sw,
    strmm_r_up_nt_main_unit_sw,
    strmm_l_lo_tr_main_unit_sw,
    strmm_l_up_tr_main_unit_sw,
    strmm_l_lo_nt_main_unit_sw,
    strmm_l_up_nt_main_unit_sw,
    strmm_r_lo_tr_main_alpha0_sw,
    strmm_r_up_tr_main_alpha0_sw,
    strmm_r_lo_nt_main_alpha0_sw,
    strmm_r_up_nt_main_alpha0_sw,
    strmm_l_lo_tr_main_alpha0_sw,
    strmm_l_up_tr_main_alpha0_sw,
    strmm_l_lo_nt_main_alpha0_sw,
    strmm_l_up_nt_main_alpha0_sw,
    strmm_r_lo_tr_main_unit_alpha0_sw,
    strmm_r_up_tr_main_unit_alpha0_sw,
    strmm_r_lo_nt_main_unit_alpha0_sw,
    strmm_r_up_nt_main_unit_alpha0_sw,
    strmm_l_lo_tr_main_unit_alpha0_sw,
    strmm_l_up_tr_main_unit_alpha0_sw,
    strmm_l_lo_nt_main_unit_alpha0_sw,
    strmm_l_up_nt_main_unit_alpha0_sw,
    fast_strmm_r_lo_tr_main_sw,
    fast_strmm_r_up_tr_main_sw,
    fast_strmm_r_lo_nt_main_sw,
    fast_strmm_r_up_nt_main_sw,
    fast_strmm_l_lo_tr_main_sw,
    fast_strmm_l_up_tr_main_sw,