emit_mm.c

来自「基于Blas CLapck的.用过的人知道是干啥的」· C语言 代码 · 共 1,670 行 · 第 1/5 页

                 rA, rB, rC, pA, pB, mu, nu, ku, &offA, &offB, lda, ldb,
                 mulA, mulB, incpA, incpB, incAk, incBk, rowA, rowB,
                 &ia, &ib, &iop, &jop);
      fprintf(fpout, "/*\n *%s Completely unrolled K-loop\n */\n", spc);
      i = ku - (lat/(mu*nu)) - 1;
      h = regKunroll(fpout, spc, LoopOrder, ifetch, nfetch, lat, 0, AsgC1,
                     rA, rB, rC, pA, pB, mu, nu, i, &offA, &offB, lda, ldb,
                     mulA, mulB, incpA, incpB, incAk, incBk, rowA, rowB,
                     &ia, &ib, &iop, &jop);
      regKdrain(fpout, spc, LoopOrder, ifetch, nfetch, lat, rA, rB, rC, pA, pB,
                mu, nu, ku, &offA, &offB, lda, ldb, mulA, mulB, incpA, incpB,
                incAk, incBk, rowA, rowB, &ia, &ib, iop, jop, h);
      if (!incpA)
      {
         fprintf(fpout, "%s   %s0 += %s;\n", spc, pA, incAk);
         if (lda == 0 && TA != AtlasNoTrans)
            for (i=1; i < mu; i++)
               fprintf(fpout, "%s   %s%d += %s;\n", spc, pA, i, incAk);
      }
      if (!incpB)
      {
         fprintf(fpout, "%s   %s0 += %s;\n", spc, pB, incBk);
         if (ldb == 0 && TB == AtlasNoTrans)
            for (i=1; i < nu; i++)
               fprintf(fpout, "%s   %s%d += %s;\n", spc, pB, i, incBk);
      }
   }
   else  /* unknown K != ku */
   {
      Kpipe = lat / (mu*nu) + 1;  /* its done in pipeline startup & drain */
      if (K)
      {
         Kloop = ((K-Kpipe)/ku)*ku;
         if (AsgC1 && ku <= MAX_CASG_KU && Kloop < ku)
         {
            regKloop_ma(fpout, spc, LoopOrder, TA, TB, AsgC1, M, N, K,
                        ifetch, nfetch, lat, rA, rB, rC, pA, pB, mu, nu, ku,
                        lda, ldb, mulA, mulB, incpA, incpB, incAk, incBk);
            return;
         }
         kr = K - Kpipe - Kloop;
      }
      else Kloop = kr = 0;
/*
 *    Issue pre-loop mulpipe fill, and increment ptrs if asked
 */
      offA0 = offA; offB0 = offB;
      regKunroll(fpout, spc, LoopOrder, ifetch, nfetch, lat, 1, 0, rA, rB, rC,
                 pA, pB, mu, nu, ku, &offA, &offB, lda, ldb, mulA, mulB,
                 incpA, incpB, incAk0, incBk0, rowA, rowB,
                 &ia, &ib, &iop, &jop);
      k = Kpipe - 1;
      if (!incpA && k) /* need to increment the A pointers ourselves */
      {
         fprintf(fpout, "%s   %s0 += %s;\n", spc, pA, GetInc(k, incAk0));
         if (rowA && !lda)
            for (i=1; i < mu; i++)
               fprintf(fpout, "%s   %s%d += %s;\n", spc, pA, i,
                       GetInc(k, incAk0));
      }
      if (!incpB && k) /* need to increment the B pointers ourselves */
      {
         fprintf(fpout, "%s   %s0 += %s;\n", spc, pB, GetInc(k, incBk0));
         if (rowB && !ldb)
            for (i=1; i < nu; i++)
               fprintf(fpout, "%s   %s%d += %s;\n", spc, pB, i,
                       GetInc(k, incBk0));
      }
      offA = offA0; offB = offB0;  /* offsets stay as set by pipeline */

      if (AsgC1 && ku <= MAX_CASG_KU)
      {
         fprintf(fpout,
                 "/*\n *%s Peel first %d iterations for C assignment\n */\n",
                 spc, ku);
         if (!K)
         {
            fprintf(fpout, "%s   if (Kloop >= %d)\n%s   {\n", spc, ku, spc);
            spc -= 3;
         }
/*
 *       Peel first iteration of unrolled loop to use C = rather than C +=
 */
         h = regKunroll(fpout, spc, LoopOrder, ifetch, nfetch, lat, 0, 1,
                        rA, rB, rC, pA, pB, mu, nu, ku, &offA, &offB, lda, ldb,
                        mulA, mulB, incpA, incpB, incAk, incBk, rowA, rowB,
                        &ia, &ib, &iop, &jop);
         if (!incpA) /* need to increment the A pointers ourselves */
         {
            fprintf(fpout, "%s   %s0 += %s;\n", spc, pA, incAk);
            if (rowA && !lda)
               for (i=1; i < mu; i++)
                  fprintf(fpout, "%s   %s%d += %s;\n", spc, pA, i, incAk);
         }
         if (!incpB) /* need to increment the B pointers ourselves */
         {
            fprintf(fpout, "%s   %s0 += %s;\n", spc, pB, incBk);
            if (rowB && !ldb)
               for (i=1; i < nu; i++)
                  fprintf(fpout, "%s   %s%d += %s;\n", spc, pB, i, incBk);
         }
         fprintf(fpout, "/*\n *%s Unpeeled K iterations\n */\n", spc);
         if (K)
         {
            if (ku == 1)
              fprintf(fpout,
                      "%s   for (k=0; k < %d; k++) /* easy loop to unroll */\n",
                       spc, Kloop-1);
            else fprintf(fpout,
                 "%s   for (k=0; k < %d; k += %d) /* easy loop to unroll */\n",
                         spc, Kloop-ku, ku);
         }
         else if (ku == 1)
            fprintf(fpout,
            "%s   for (k=1; k < Kloop; k++) /* easy loop to unroll */\n", spc);
         else fprintf(fpout,
              "%s   for (k=%d; k < Kloop; k += %d) /* easy loop to unroll */\n",
                      spc, ku, ku);
         offA = offA0;  offB = offB0;
      }
      else if (K)
      {
#ifdef ICC_IS_RETARDED
         if (ku == 1)
           fprintf(fpout,
                   "%s   for (k=0; k < %d; k++) /* easy loop to unroll */\n",
                    spc, Kloop);
         else fprintf(fpout,
                  "%s   for (k=0; k < %d; k += %d) /* easy loop to unroll */\n",
                      spc, Kloop, ku);
#else
         if (ku == 1)
           fprintf(fpout, "%s   for (k=%d; k; k--) /* easy loop to unroll */\n",
                    spc, Kloop);
         else fprintf(fpout,
                      "%s   for (k=%d; k; k -= %d) /* easy loop to unroll */\n",
                      spc, Kloop, ku);
#endif
      }
      else
      {
#ifdef ICC_IS_RETARDED
         if (ku == 1)
            fprintf(fpout,
            "%s   for (k=0; k < Kloop; k++) /* easy loop to unroll */\n", spc);
         else fprintf(fpout,
              "%s   for (k=0; k < Kloop; k += %d) /* easy loop to unroll */\n",
                      spc, ku);
#else
         if (ku == 1)
            fprintf(fpout,
                    "%s   for (k=Kloop; k; k--) /* easy loop to unroll */\n",
                    spc);
         else fprintf(fpout,
                 "%s   for (k=Kloop; k; k -= %d) /* easy loop to unroll */\n",
                      spc, ku);
#endif
      }
      fprintf(fpout, "%s   {\n", spc);
      spc -= 3;
      h = regKunroll(fpout, spc, LoopOrder, ifetch, nfetch, lat, 0, 0,
                     rA, rB, rC, pA, pB, mu, nu, ku, &offA, &offB, lda, ldb,
                     mulA, mulB, incpA, incpB, incAk, incBk, rowA, rowB,
                     &ia, &ib, &iop, &jop);
      if (!incpA) /* need to increment the A pointers ourselves */
      {
         fprintf(fpout, "%s   %s0 += %s;\n", spc, pA, incAk);
         if (rowA && !lda)
            for (i=1; i < mu; i++)
               fprintf(fpout, "%s   %s%d += %s;\n", spc, pA, i, incAk);
      }
      if (!incpB) /* need to increment the B pointers ourselves */
      {
         fprintf(fpout, "%s   %s0 += %s;\n", spc, pB, incBk);
         if (rowB && !ldb)
            for (i=1; i < nu; i++)
               fprintf(fpout, "%s   %s%d += %s;\n", spc, pB, i, incBk);
      }
      spc += 3;
      fprintf(fpout, "%s   } /* end K-loop */\n", spc);
      if (!K && AsgC1 && ku <= MAX_CASG_KU)
      {
         spc += 3;
         fprintf(fpout, "%s   }\n", spc);
         fprintf(fpout, "%s   else\n%s   {\n", spc, spc);
         spc -= 3;
         fetchC(fpout, spc, LoopOrder, 0, mu, nu, 0, NULL, rC, 1, 0, 0, 0, 1,
                NULL);
         spc += 3;
         fprintf(fpout, "%s   }\n", spc);
      }
      if (kr)
      {
         offA = offA0;  offB = offB0;
         h = regKunroll(fpout, spc, LoopOrder, ifetch, nfetch, lat, 0, 0,
                        rA, rB, rC, pA, pB, mu, nu, kr, &offA, &offB,
                        lda, ldb, mulA, mulB, incpA, incpB, incAk, incBk,
                        rowA, rowB, &ia, &ib, &iop, &jop);
         if (!incpA) /* need to increment the A pointers ourselves */
         {
            fprintf(fpout, "%s   %s0 += %s;\n", spc, pA, GetInc(kr, incAk0));
            if (rowA && !lda)
               for (i=1; i < mu; i++)
                  fprintf(fpout, "%s   %s%d += %s;\n", spc, pA, i,
                          GetInc(kr, incAk0));
         }
         if (!incpB) /* need to increment the B pointers ourselves */
         {
            fprintf(fpout, "%s   %s0 += %s;\n", spc, pB, GetInc(kr, incBk0));
            if (rowB && !ldb)
               for (i=1; i < nu; i++)
                  fprintf(fpout, "%s   %s%d += %s;\n", spc, pB, i,
                          GetInc(kr, incBk0));
         }
      }
      else if (K == 0 && ku != 1)
      {
         fprintf(fpout, "%s   switch(kr)\n%s   {\n", spc, spc);
         for (i=1; i < ku; i++)
         {
            fprintf(fpout, "%s   case %d:\n", spc, i);
            spc -= 3;
            offA = offA0;  offB = offB0;
            regKunroll(fpout, spc, LoopOrder, ifetch, nfetch, lat, 0, 0,
                       rA, rB, rC, pA, pB, mu, nu, i, &offA, &offB,
                       lda, ldb, mulA, mulB, incpA, incpB, incAk, incBk,
                       rowA, rowB, &ia, &ib, &iop, &jop);
            if (!incpA) /* need to increment the A pointers ourselves */
            {
               fprintf(fpout, "%s   %s0 += %s;\n", spc, pA, GetInc(i, incAk0));
               if (rowA && !lda)
                  for (j=1; j < mu; j++)
                     fprintf(fpout, "%s   %s%d += %s;\n", spc, pA, j,
                             GetInc(i, incAk0));
            }
            if (!incpB) /* need to increment the B pointers ourselves */
            {
               fprintf(fpout, "%s   %s0 += %s;\n", spc, pB, GetInc(i, incBk0));
               if (rowB && !ldb)
                  for (j=1; j < nu; j++)
                     fprintf(fpout, "%s   %s%d += %s;\n", spc, pB, j,
                             GetInc(i, incBk0));
            }
            spc += 3;
            fprintf(fpout, "%s      break;\n", spc);
         }
         fprintf(fpout, "%s   case 0: ;\n%s   }\n", spc, spc);

      }
      offA = offA0;  offB = offB0;
      regKdrain(fpout, spc, LoopOrder, ifetch, nfetch, lat, rA, rB, rC, pA, pB,
                mu, nu, ku, &offA, &offB, lda, ldb, mulA, mulB, incpA, incpB,
                incAk0, incBk0, rowA, rowB, &ia, &ib, iop, jop, h);
      if (!incpA)
      {
         fprintf(fpout, "%s   %s0 += %s;\n", spc, pA, incAk0);
         if (rowA && !lda)
            for (i=1; i < mu; i++)
               fprintf(fpout, "%s   %s%d += %s;\n", spc, pA, i, incAk0);

      }
      if (!incpB)
      {
         fprintf(fpout, "%s   %s0 += %s;\n", spc, pB, incBk0);
         if (rowB && !ldb)
            for (i=1; i < nu; i++)
               fprintf(fpout, "%s   %s%d += %s;\n", spc, pB, i, incBk0);
      }
   }
}

void DoPrefA_lda0(FILE *fpout, char *spc, char pre, int nprefA, int FIRST)
/*
 * DoPrefA_lda0: issues prefetch instructions to prefetch next block of A
 * when lda is unknown; we call this routine inside N loop, but split 1/2
 * pref inst before M loop, and other half at end of N loop
 */
{
   int i, k, size, npref, stpref;

   size = (pre == 's' || pre == 'c') ? 4 : 8;
/*
 * For small prefetch, do all prefetch at top of loop
 * NOTE: use this always, as splitting doesn't seem to help
 */
   if (nprefA <= 2 || 1)
   {
      if (FIRST)
      {
         for (i=0; i < nprefA; i++)
            fprintf(fpout, "%s   ATL_pfl1R(pfA+%d);\n", spc, i*(ATL_L1LS/size));
         fprintf(fpout, "%s   pfA += lda;\n", spc);
      }
   }
/*
 * If issuing initial prefetch at top of N-loop and we have a bunch
 * of prefetches to do (worth splitting between top and bottom of loop)
 */
   else if (FIRST)
   {
      nprefA = (nprefA+1)/2;
      for (i=0; i < nprefA; i++)
         fprintf(fpout, "%s   ATL_pfl1R(pfA+%d);\n", spc, i*(ATL_L1LS/size));
   }
/*
 * If issuing trailing prefetch at bottom of N-loop
 */
   else
   {
      k = ((nprefA+1)/2)*(ATL_L1LS/size);
      nprefA = nprefA/2;
      for (i=0; i < nprefA; i++)
         fprintf(fpout, "%s   ATL_pfl1R(pfA+%d);\n", spc, k+i*(ATL_L1LS/size));
      fprintf(fpout, "%s   pfA += lda;\n", spc);
   }
}

void DoPrefA(FILE *fpout, char *spc, int nprefA)
/*
 * DoPrefA: issues prefetch instructions to prefetch next block of A;
 * we call this routine inside M & N loop, but outside K-loop
 */
{
   int i;

   if (nprefA < 1) return;
   fprintf(fpout, "%s   ATL_pfl1R(pfA);\n", spc);
   if (nprefA > 1)
      fprintf(fpout, "%s   ATL_pfl1R(pfA+ATL_L1LS);\n", spc);
   for (i=3; i < nprefA; i++)
      fprintf(fpout, "%s   ATL_pfl1R(pfA+%d*ATL_L1LS);\n", spc, i);
   fprintf(fpout, "%s   pfA += incPFA;\n", spc);
}