maxunpacker.asm

pe exe packer (must use vc2005 to compile)

; Author:   Brandon LaCombe
; Date:     February 3, 2006
; License:  Public Domain
.386
.model flat, stdcall
option casemap:none

include     windows.inc
include     LoaderStructs.inc

VIRTUALALLOC typedef proto lpAddress:dword, dwSize:dword, flAllocationType:dword, flProtect:dword
VIRTUALFREE  typedef proto lpAddress:dword, dwSize:dword, dwFreeType:dword
UNPACK       typedef proto pbDest:dword, pbSrc:dword, pbWorkMem:dword

.code
ExportMaxUnpacker proc pdwMaxUnpackerSize:dword
    mov eax, pdwMaxUnpackerSize
    .if eax
        mov dword ptr[eax], max_unpacker_end - max_unpacker_start
    .endif
    mov eax, max_unpacker_start
    ret
ExportMaxUnpacker endp

ExportMaxDefilter proc pdwMaxDefilterSize:dword
    mov eax, pdwMaxDefilterSize
    .if eax
        mov dword ptr[eax], max_defilter_end - max_defilter_start
    .endif
    mov eax, max_defilter_start
    ret
ExportMaxDefilter endp

; In max packing mode, each section is compressed and placed inside the
; packer section. Each section can be decompressed directly to it's correct
; location with no copying or extra memory allocation. Before each block of
; section data are two dwords: start rva of the section and the size of the
; section data. All sections must be relocated to the packer section even if
; they weren't compressed. To differentiate between compressed and uncompressed
; section data, the high bit will be set in the start rva dword if the data is
; uncompressed. In that case, the size of the section data will be given in
; dwords instead of bytes.
max_unpacker_start:

    invoke VIRTUALALLOC ptr[(KERNEL_IAT ptr[ebp]).pVirtualAlloc], ecx, (LOADER_STRUCT ptr[ebx]).dwTotalMemSize, MEM_COMMIT, PAGE_READWRITE
    mov edx, eax                                                           ; edx = work mem
    mov esi, (LOADER_STRUCT ptr[ebx]).pSectionData                         ; esi = section data pointer
    jmp max_unpacker_loop_enter                                            ; enter unpacking loop
@@: mov edi, eax                                                           ; edi = section rva
    add edi, (LOADER_STRUCT ptr[ebx]).dwImageBase                          ; edi = section pointer
    lodsd                                                                  ; load section data size
    mov ecx, eax                                                           ; ecx = section data size
    btr edi, 31                                                            ; check for uncompressed flag
    .if carry?
        rep movsd                                                          ; copy uncompressed section
    .else
        pushad                                                             ; Unpack doesn't preserve regs
        invoke UNPACK ptr[(LOADER_STRUCT ptr[ebx]).pUnpack], edi, esi, edx ; decompress section
        popad                                                              ; restore registers
        add esi, ecx                                                       ; skip to next section data
    .endif
max_unpacker_loop_enter:
    lodsd                                                                  ; load section rva
    test eax, eax                                                          ; test for terminating entry
    jnz @B                                                                 ; process section
    invoke VIRTUALFREE ptr[(KERNEL_IAT ptr[ebp]).pVirtualFree], edx, eax, MEM_RELEASE

max_unpacker_end:

max_defilter_start:

    invoke VIRTUALALLOC ptr[(KERNEL_IAT ptr[ebp]).pVirtualAlloc], ecx, (LOADER_STRUCT ptr[ebx]).dwTotalMemSize, MEM_COMMIT, PAGE_READWRITE
    mov edx, eax                                                           ; edx = work mem
    mov esi, (LOADER_STRUCT ptr[ebx]).pSectionData                         ; esi = section data pointer
    jmp max_defilter_loop_enter                                            ; enter unpacking loop
@@: mov edi, eax                                                           ; edi = section rva
    add edi, (LOADER_STRUCT ptr[ebx]).dwImageBase                          ; edi = section pointer
    lodsd                                                                  ; load section data size
    mov ecx, eax                                                           ; ecx = section data size
    btr edi, 31                                                            ; check for uncompressed flag
    .if carry?
        rep movsd                                                          ; copy uncompressed section
    .else
        pushad                                                             ; Unpack doesn't preserve regs
        invoke UNPACK ptr[(LOADER_STRUCT ptr[ebx]).pUnpack], edi, esi, edx ; decompress section
        mov [esp + 1Ch], eax
        popad                                                              ; restore registers

            ; defilter code
            pushad
            mov ebp, edi                                  ; ebp = section pointer
            sub edi, (LOADER_STRUCT ptr[ebx]).dwImageBase ; compute section rva
            mov ebx, edi                                  ; ebx = section rva
            lea edi, [eax - 5]                            ; edi = section size - 5
            xor esi, esi                                  ; esi = index variable
            push -4                                       ; push -4 to load it into edx
            pop edx                                       ; edx = last suspected call index
            jmp defilter_loop_enter                       ; enter the defilter loop
        defilter_loop_start:
            mov al, [ebp+esi]                             ; al = current byte
            and al, 0FEh                                  ; and al so that E8 and E9 compute to E8
            cmp al, 0E8h                                  ; if al is E8 or E9
            je defilter_suspected_call                    ; examine the suspected call
            mov eax, [ebp+esi]                            ; ax = current word
            and ah, 0F0h                                  ; and out the low nibble of the first byte
            cmp ax, 0800Fh                                ; is this a jxx instruction?
            jne defilter_continue_loop                    ; if not, continue the defilter loop
            inc esi                                       ; jxx is a word sized opcode so point to the end of it
        defilter_suspected_call:
            mov eax, esi                                  ; eax = index variable
            sub eax, edx                                  ; eax = current index - last index
            xor al, 3                                     ; eax = (current index - last index) ^ 3
            mov edx, esi                                  ; last index = current index
            mov cl, [ebp+esi+4]                           ; cl = highest byte of absolute call data
            inc ecx                                       ; manipulate cl so that only values
            shr cl, 1                                     ; of 0xFF and 0x00 make cl zero
            jnz defilter_continue_loop                    ; if cl is not 0xFF or 0x00, continue the defilter loop
            push edi                                      ; backup edi (which is the maximum index allowed)
            inc esi                                       ; skip past call opcode and point to absolute call data
            mov edi, [ebp+esi]                            ; edi = relative call data
        defilter_recode_loop:
            sub edi, ebx                                  ; edi = absolute call data - (section rva + index + 1)
            sub edi, esi                                  ; or in other words, convert absolute data to relative data
            mov [ebp+esi], edi                            ; store relative call data
            cmp eax, 3                                    ; only continue if the delta between the last
            ja defilter_exit_recode_loop                  ; two suspected calls is less than or equal to 3
            mov ecx, eax                                  ; ecx = delta
            shl ecx, 3                                    ; ecx = delta * 8
            mov edi, 0FFh                                 ; edi = 0xFF
            shl edi, cl                                   ; edi = 0xFF << (delta * 8)
            xor edi, [ebp+esi]                            ; edi = relative call data ^ 0xFF << (delta * 8)
            lea ecx, [esi + eax]                          ; ecx = address of conflicting byte
            mov cl, byte ptr[ebp+ecx]                     ; cl = conflicting byte
            inc ecx                                       ; manipulate cl so that only values
            shr cl, 1                                     ; of 0xFF and 0x00 make cl zero
            jz defilter_recode_loop                       ; if the recode loop loops, it is because the
        defilter_exit_recode_loop:                        ; conflicting byte was 0xFF or 0x00
            add esi, 3                                    ; point to last byte of call data
            mov cl, [ebp+esi]                             ; load in the highest byte of the call data
            shr cl, 1                                     ; test the low bit of the highest byte
            setnc ch                                      ; if the low bit is set, then set cl to 0xFF
            dec ch                                        ; otherwise set cl to 0x00
            mov byte ptr[ebp+esi], ch                     ; store the 0xFF or 0x00 high byte
            pop edi                                       ; restore edi (maximum index allowed)
        defilter_continue_loop:
            inc esi                                       ; increase index by 1
        defilter_loop_enter:
            cmp esi, edi                                  ; is index variable in range?
            jl defilter_loop_start                        ; if so, process next byte
            popad

        add esi, ecx                                                       ; skip to next section data
    .endif
max_defilter_loop_enter:
    lodsd                                                                  ; load section rva
    test eax, eax                                                          ; test for terminating entry
    jnz @B                                                                 ; process section
    invoke VIRTUALFREE ptr[(KERNEL_IAT ptr[ebp]).pVirtualFree], edx, eax, MEM_RELEASE

max_defilter_end:

end