nb_kernel230_x86_64_sse.s

来自「最著名最快的分子模拟软件」· S 代码 · 共 1,958 行 · 第 1/4 页

##
## $Id: nb_kernel230_x86_64_sse.s,v 1.4.2.3 2006/09/22 08:32:50 lindahl Exp $
##
## Gromacs 4.0                         Copyright (c) 1991-2003 
## David van der Spoel, Erik Lindahl
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## To help us fund GROMACS development, we humbly ask that you cite
## the research papers on the package. Check out http://www.gromacs.org
## 
## And Hey:
## Gnomes, ROck Monsters And Chili Sauce
##






.globl nb_kernel230_x86_64_sse
.globl _nb_kernel230_x86_64_sse
nb_kernel230_x86_64_sse:        
_nb_kernel230_x86_64_sse:       
##      Room for return address and rbp (16 bytes)
.set nb230_fshift, 16
.set nb230_gid, 24
.set nb230_pos, 32
.set nb230_faction, 40
.set nb230_charge, 48
.set nb230_p_facel, 56
.set nb230_argkrf, 64
.set nb230_argcrf, 72
.set nb230_Vc, 80
.set nb230_type, 88
.set nb230_p_ntype, 96
.set nb230_vdwparam, 104
.set nb230_Vvdw, 112
.set nb230_p_tabscale, 120
.set nb230_VFtab, 128
.set nb230_invsqrta, 136
.set nb230_dvda, 144
.set nb230_p_gbtabscale, 152
.set nb230_GBtab, 160
.set nb230_p_nthreads, 168
.set nb230_count, 176
.set nb230_mtx, 184
.set nb230_outeriter, 192
.set nb230_inneriter, 200
.set nb230_work, 208
        ## stack offsets for local variables  
        ## bottom of stack is cache-aligned for sse use 
.set nb230_ix, 0
.set nb230_iy, 16
.set nb230_iz, 32
.set nb230_iq, 48
.set nb230_dx, 64
.set nb230_dy, 80
.set nb230_dz, 96
.set nb230_c6, 112
.set nb230_c12, 128
.set nb230_tsc, 144
.set nb230_qq, 160
.set nb230_vctot, 176
.set nb230_Vvdwtot, 192
.set nb230_fix, 208
.set nb230_fiy, 224
.set nb230_fiz, 240
.set nb230_half, 256
.set nb230_three, 272
.set nb230_two, 288
.set nb230_krf, 304
.set nb230_crf, 320
.set nb230_nri, 336
.set nb230_iinr, 344
.set nb230_jindex, 352
.set nb230_jjnr, 360
.set nb230_shift, 368
.set nb230_shiftvec, 376
.set nb230_facel, 384
.set nb230_innerjjnr, 392
.set nb230_is3, 400
.set nb230_ii3, 404
.set nb230_ntia, 408
.set nb230_innerk, 412
.set nb230_n, 416
.set nb230_nn1, 420
.set nb230_ntype, 424
.set nb230_nouter, 428
.set nb230_ninner, 432


        push %rbp
        movq %rsp,%rbp
        push %rbx


        emms

        push %r12
        push %r13
        push %r14
        push %r15

        subq $456,%rsp          ## local variable stack space (n*16+8)

        ## zero 32-bit iteration counters
        movl $0,%eax
        movl %eax,nb230_nouter(%rsp)
        movl %eax,nb230_ninner(%rsp)

        movl (%rdi),%edi
        movl %edi,nb230_nri(%rsp)
        movq %rsi,nb230_iinr(%rsp)
        movq %rdx,nb230_jindex(%rsp)
        movq %rcx,nb230_jjnr(%rsp)
        movq %r8,nb230_shift(%rsp)
        movq %r9,nb230_shiftvec(%rsp)
        movq nb230_p_ntype(%rbp),%rdi
        movl (%rdi),%edi
        movl %edi,nb230_ntype(%rsp)
        movq nb230_p_facel(%rbp),%rsi
        movss (%rsi),%xmm0
        movss %xmm0,nb230_facel(%rsp)

        movq nb230_p_tabscale(%rbp),%rax
        movss (%rax),%xmm3
        shufps $0,%xmm3,%xmm3
        movaps %xmm3,nb230_tsc(%rsp)

        movq nb230_argkrf(%rbp),%rsi
        movq nb230_argcrf(%rbp),%rdi
        movss (%rsi),%xmm1
        movss (%rdi),%xmm2
        shufps $0,%xmm1,%xmm1
        shufps $0,%xmm2,%xmm2
        movaps %xmm1,nb230_krf(%rsp)
        movaps %xmm2,nb230_crf(%rsp)

        ## create constant floating-point factors on stack
        movl $0x3f000000,%eax   ## half in IEEE (hex)
        movl %eax,nb230_half(%rsp)
        movss nb230_half(%rsp),%xmm1
        shufps $0,%xmm1,%xmm1  ## splat to all elements
        movaps %xmm1,%xmm2
        addps  %xmm2,%xmm2      ## one
        movaps %xmm2,%xmm3
        addps  %xmm2,%xmm2      ## two
        addps  %xmm2,%xmm3      ## three
        movaps %xmm1,nb230_half(%rsp)
        movaps %xmm2,nb230_two(%rsp)
        movaps %xmm3,nb230_three(%rsp)

_nb_kernel230_x86_64_sse.nb230_threadloop: 
        movq  nb230_count(%rbp),%rsi            ## pointer to sync counter
        movl  (%rsi),%eax
_nb_kernel230_x86_64_sse.nb230_spinlock: 
        movl  %eax,%ebx                         ## ebx=*count=nn0
        addl  $1,%ebx                          ## ebx=nn1=nn0+10
        lock 
        cmpxchgl %ebx,(%rsi)                    ## write nn1 to *counter,
                                                ## if it hasnt changed.
                                                ## or reread *counter to eax.
        pause                                   ## -> better p4 performance
        jnz _nb_kernel230_x86_64_sse.nb230_spinlock

        ## if(nn1>nri) nn1=nri
        movl nb230_nri(%rsp),%ecx
        movl %ecx,%edx
        subl %ebx,%ecx
        cmovlel %edx,%ebx                       ## if(nn1>nri) nn1=nri
        ## Cleared the spinlock if we got here.
        ## eax contains nn0, ebx contains nn1.
        movl %eax,nb230_n(%rsp)
        movl %ebx,nb230_nn1(%rsp)
        subl %eax,%ebx                          ## calc number of outer lists
        movl %eax,%esi                          ## copy n to esi
        jg  _nb_kernel230_x86_64_sse.nb230_outerstart
        jmp _nb_kernel230_x86_64_sse.nb230_end

_nb_kernel230_x86_64_sse.nb230_outerstart: 
        ## ebx contains number of outer iterations
        addl nb230_nouter(%rsp),%ebx
        movl %ebx,nb230_nouter(%rsp)

_nb_kernel230_x86_64_sse.nb230_outer: 
        movq  nb230_shift(%rsp),%rax        ## eax = pointer into shift[] 
        movl  (%rax,%rsi,4),%ebx                ## ebx=shift[n] 

        lea  (%rbx,%rbx,2),%rbx    ## rbx=3*is 
        movl  %ebx,nb230_is3(%rsp)      ## store is3 

        movq  nb230_shiftvec(%rsp),%rax     ## eax = base of shiftvec[] 

        movss (%rax,%rbx,4),%xmm0
        movss 4(%rax,%rbx,4),%xmm1
        movss 8(%rax,%rbx,4),%xmm2

        movq  nb230_iinr(%rsp),%rcx         ## ecx = pointer into iinr[]        
        movl  (%rcx,%rsi,4),%ebx            ## ebx =ii 

        movq  nb230_charge(%rbp),%rdx
        movss (%rdx,%rbx,4),%xmm3
        mulss nb230_facel(%rsp),%xmm3
        shufps $0,%xmm3,%xmm3

        movq  nb230_type(%rbp),%rdx
        movl  (%rdx,%rbx,4),%edx
        imull nb230_ntype(%rsp),%edx
        shll  %edx
        movl  %edx,nb230_ntia(%rsp)

        lea  (%rbx,%rbx,2),%rbx        ## rbx = 3*ii=ii3 
        movq  nb230_pos(%rbp),%rax      ## eax = base of pos[]  

        addss (%rax,%rbx,4),%xmm0
        addss 4(%rax,%rbx,4),%xmm1
        addss 8(%rax,%rbx,4),%xmm2

        movaps %xmm3,nb230_iq(%rsp)

        shufps $0,%xmm0,%xmm0
        shufps $0,%xmm1,%xmm1
        shufps $0,%xmm2,%xmm2

        movaps %xmm0,nb230_ix(%rsp)
        movaps %xmm1,nb230_iy(%rsp)
        movaps %xmm2,nb230_iz(%rsp)

        movl  %ebx,nb230_ii3(%rsp)

        ## clear vctot and i forces 
        xorps %xmm4,%xmm4
        movaps %xmm4,nb230_vctot(%rsp)
        movaps %xmm4,nb230_Vvdwtot(%rsp)
        movaps %xmm4,nb230_fix(%rsp)
        movaps %xmm4,nb230_fiy(%rsp)
        movaps %xmm4,nb230_fiz(%rsp)

        movq  nb230_jindex(%rsp),%rax
        movl  (%rax,%rsi,4),%ecx             ## jindex[n] 
        movl  4(%rax,%rsi,4),%edx            ## jindex[n+1] 
        subl  %ecx,%edx              ## number of innerloop atoms 

        movq  nb230_pos(%rbp),%rsi
        movq  nb230_faction(%rbp),%rdi
        movq  nb230_jjnr(%rsp),%rax
        shll  $2,%ecx
        addq  %rcx,%rax
        movq  %rax,nb230_innerjjnr(%rsp)       ## pointer to jjnr[nj0] 
        movl  %edx,%ecx
        subl  $4,%edx
        addl  nb230_ninner(%rsp),%ecx
        movl  %ecx,nb230_ninner(%rsp)
        addl  $0,%edx
        movl  %edx,nb230_innerk(%rsp)      ## number of innerloop atoms 
        jge   _nb_kernel230_x86_64_sse.nb230_unroll_loop
        jmp   _nb_kernel230_x86_64_sse.nb230_finish_inner
_nb_kernel230_x86_64_sse.nb230_unroll_loop: 
        ## quad-unroll innerloop here 
        movq  nb230_innerjjnr(%rsp),%rdx       ## pointer to jjnr[k] 
        movl  (%rdx),%r12d
        movl  4(%rdx),%r13d
        movl  8(%rdx),%r14d
        movl  12(%rdx),%r15d           ## eax-edx=jnr1-4 
        addq $16,nb230_innerjjnr(%rsp)             ## advance pointer (unrolled 4) 

        lea  (%r12,%r12,2),%rax     ## replace jnr with j3 
        lea  (%r13,%r13,2),%rbx
        lea  (%r14,%r14,2),%rcx
        lea  (%r15,%r15,2),%rdx

        movq nb230_pos(%rbp),%rdi
        ## load coordinates
        movlps (%rdi,%rax,4),%xmm1      ## x1 y1 - - 
        movlps (%rdi,%rcx,4),%xmm2      ## x3 y3 - - 
        movhps (%rdi,%rbx,4),%xmm1      ## x2 y2 - -
        movhps (%rdi,%rdx,4),%xmm2      ## x4 y4 - -

        movss 8(%rdi,%rax,4),%xmm5      ## z1 - - - 
        movss 8(%rdi,%rcx,4),%xmm6      ## z2 - - - 
        movss 8(%rdi,%rbx,4),%xmm7      ## z3 - - - 
        movss 8(%rdi,%rdx,4),%xmm8      ## z4 - - - 
    movlhps %xmm7,%xmm5 ## jzOa  -  jzOb  -
    movlhps %xmm8,%xmm6 ## jzOc  -  jzOd -

    movaps %xmm1,%xmm4
    unpcklps %xmm2,%xmm1 ## jxa jxc jya jyc        
    unpckhps %xmm2,%xmm4 ## jxb jxd jyb jyd
    movaps %xmm1,%xmm2
    unpcklps %xmm4,%xmm1 ## x
    unpckhps %xmm4,%xmm2 ## y
    shufps  $136,%xmm6,%xmm5  ## 10001000 => jzH2a jzH2b jzH2c jzH2d

        movq nb230_charge(%rbp),%rsi

        ## calc dr  
        subps nb230_ix(%rsp),%xmm1
        subps nb230_iy(%rsp),%xmm2
        subps nb230_iz(%rsp),%xmm5

        movss (%rsi,%r12,4),%xmm6
        movss (%rsi,%r13,4),%xmm3
        movss (%rsi,%r14,4),%xmm4
        movss (%rsi,%r15,4),%xmm0

        ## store dr
    movaps %xmm1,nb230_dx(%rsp)
    movaps %xmm2,nb230_dy(%rsp)
    movaps %xmm5,nb230_dz(%rsp)

        ## square it 
        mulps %xmm1,%xmm1
        mulps %xmm2,%xmm2
        mulps %xmm5,%xmm5
        addps %xmm2,%xmm1
        addps %xmm5,%xmm1

        movq nb230_type(%rbp),%rsi

    unpcklps %xmm4,%xmm6 ## jqa jqc - -
    unpcklps %xmm0,%xmm3 ## jqb jqd - -
    unpcklps %xmm3,%xmm6 ## jqa jqb jqc jqd
        mulps nb230_iq(%rsp),%xmm6
    movaps %xmm6,nb230_qq(%rsp)

        ## rsq in xmm1
    movaps nb230_krf(%rsp),%xmm0
    mulps  %xmm1,%xmm0   ## krsq

     ## vdw parameters
        movl (%rsi,%r12,4),%r12d
        movl (%rsi,%r13,4),%r13d
        movl (%rsi,%r14,4),%r14d
        movl (%rsi,%r15,4),%r15d

   ## calculate rinv=1/sqrt(rsq)
        rsqrtps %xmm1,%xmm5
        movaps %xmm5,%xmm2
        mulps %xmm5,%xmm5
        movaps nb230_three(%rsp),%xmm4
        mulps %xmm1,%xmm5       ## rsq*lu*lu    
    subps %xmm5,%xmm4   ## 30-rsq*lu*lu 
        mulps %xmm2,%xmm4
        mulps nb230_half(%rsp),%xmm4
        movaps %xmm4,%xmm2
        mulps  %xmm4,%xmm1

    ## xmm2=rinv
    ## xmm1=r

        shll %r12d
        shll %r13d
        shll %r14d
        shll %r15d
    movl nb230_ntia(%rsp),%edi

    mulps nb230_tsc(%rsp),%xmm1   ## rtab

    ## truncate and convert to integers
    cvttps2dq %xmm1,%xmm5

        addl %edi,%r12d
        addl %edi,%r13d
        addl %edi,%r14d
        addl %edi,%r15d

    ## convert back to float
    cvtdq2ps  %xmm5,%xmm4

    ## multiply by 8
    pslld   $3,%xmm5

    ## calculate eps
    subps     %xmm4,%xmm1

    ## move to integer registers
    movhlps %xmm5,%xmm6
    movd    %xmm5,%r8d
    movd    %xmm6,%r10d
    pshufd $1,%xmm5,%xmm5
    pshufd $1,%xmm6,%xmm6
    movd    %xmm5,%r9d
    movd    %xmm6,%r11d

    ## xmm1=eps
    ## xmm2=rinv

        movq nb230_VFtab(%rbp),%rsi
    ## calculate LJ table
    movlps (%rsi,%r8,4),%xmm5
        movlps 16(%rsi,%r8,4),%xmm9

        movlps (%rsi,%r10,4),%xmm7
        movlps 16(%rsi,%r10,4),%xmm11

    movaps  %xmm2,%xmm3             ## rinv
    subps   %xmm0,%xmm3
    subps   %xmm0,%xmm3             ## rinv-2*krsq
    addps   %xmm2,%xmm0             ## rinv+krsq
    subps   nb230_crf(%rsp),%xmm0   ## rinv+krsq-crf
    mulps   nb230_qq(%rsp),%xmm0    ## vcoul=qq*(rinv+krsq-crf)
    mulps   nb230_qq(%rsp),%xmm3    ## qq*(rinv-2*krsq)

    mulps   %xmm2,%xmm3             ## rinv*qq*(rinv-2*krsq)

        movhps (%rsi,%r9,4),%xmm5
        movhps 16(%rsi,%r9,4),%xmm9

    addps   nb230_vctot(%rsp),%xmm0
    movaps  %xmm0,nb230_vctot(%rsp)

        movhps (%rsi,%r11,4),%xmm7
        movhps 16(%rsi,%r11,4),%xmm11

    movaps %xmm5,%xmm4
    movaps %xmm9,%xmm8
        shufps $136,%xmm7,%xmm4 ## 10001000
        shufps $136,%xmm11,%xmm8 ## 10001000
        shufps $221,%xmm7,%xmm5 ## 11011101
        shufps $221,%xmm11,%xmm9 ## 11011101

        movlps 8(%rsi,%r8,4),%xmm7
        movlps 24(%rsi,%r8,4),%xmm11

        movlps 8(%rsi,%r10,4),%xmm13
        movlps 24(%rsi,%r10,4),%xmm14

        movhps 8(%rsi,%r9,4),%xmm7
        movhps 24(%rsi,%r9,4),%xmm11

        movhps 8(%rsi,%r11,4),%xmm13
        movhps 24(%rsi,%r11,4),%xmm14

    movaps %xmm7,%xmm6
    movaps %xmm11,%xmm10

        shufps $136,%xmm13,%xmm6 ## 10001000
        shufps $136,%xmm14,%xmm10 ## 10001000
        shufps $221,%xmm13,%xmm7 ## 11011101
        shufps $221,%xmm14,%xmm11 ## 11011101
    ## dispersion table in xmm4-xmm7, repulsion table in xmm8-xmm11
        movq nb230_vdwparam(%rbp),%rsi

    mulps  %xmm1,%xmm7   ## Heps
    mulps  %xmm1,%xmm11
    mulps  %xmm1,%xmm6  ## Geps
    mulps  %xmm1,%xmm10
    mulps  %xmm1,%xmm7  ## Heps2
    mulps  %xmm1,%xmm11

    ## load c6/c12
        movlps (%rsi,%r12,4),%xmm13
        movlps (%rsi,%r14,4),%xmm14
        movhps (%rsi,%r13,4),%xmm13
        movhps (%rsi,%r15,4),%xmm14

    addps  %xmm6,%xmm5 ## F+Geps
    addps  %xmm10,%xmm9
    addps  %xmm7,%xmm5  ## F+Geps+Heps2 = Fp
    addps  %xmm11,%xmm9
    addps  %xmm7,%xmm7   ## 2*Heps2
    addps  %xmm11,%xmm11
    addps  %xmm6,%xmm7  ## 2*Heps2+Geps
    addps  %xmm10,%xmm11

        movaps %xmm13,%xmm12
        shufps $136,%xmm14,%xmm12 ## 10001000
        shufps $221,%xmm14,%xmm13 ## 11011101

    addps  %xmm5,%xmm7 ## FF = Fp + 2*Heps2 + Geps
    addps  %xmm9,%xmm11
    mulps  %xmm1,%xmm5 ## eps*Fp
    mulps  %xmm1,%xmm9
    addps  %xmm4,%xmm5 ## VV
    addps  %xmm8,%xmm9

        movq nb230_faction(%rbp),%rsi
        ## the fj's - start by accumulating x & y forces from memory 
        movlps (%rsi,%rax,4),%xmm0 ## x1 y1 - -
        movlps (%rsi,%rcx,4),%xmm1 ## x3 y3 - -

    mulps  %xmm12,%xmm5 ## VV*c6 = vnb6
    mulps  %xmm13,%xmm9 ## VV*c12 = vnb12
    addps  %xmm9,%xmm5
    addps  nb230_Vvdwtot(%rsp),%xmm5
    movaps %xmm5,nb230_Vvdwtot(%rsp)