Quick introduction to reverse engineering for beginners

Recommendations

Info

1.20 x86-64 It’s a 64-bit extension to x86-architecture. From the reverse engineer’s perspective, most important differences are: ∙ Almost all registers (except FPU and SIMD) are extended to 64 bits and got r- prefix. 8 additional registers added. Now general purpose registers are: rax, rbx, rcx, rdx, rbp, rsp, rsi, rdi, r8, r9, r10, r11, r12, r13, r14, r15. It’s still possible to access to older register parts as usual. For example, it’s possible to access lower 32-bit part of RAX using EAX. New r8-r15 registers also has its lower parts: r8d-r15d (lower 32-bit parts), r8w-r15w (lower 16-bit parts), r8b-r15b (lower 8-bit parts). SIMD-registers number are doubled: from 8 to 16: XMM0-XMM15. ∙ In Win64, function calling convention is slightly different, somewhat resembling fastcall 2.5.3. First 4 arguments stored in RCX, RDX, R8, R9 registers, others — in stack. Caller function should also allocate 32 bytes so the callee may save there 4 first arguments and use these registers for own needs. Short functions may use arguments just from registers, but larger may save their values into stack. See also section about calling conventions 2.5. ∙ C int type is still 32-bit for compatibility. ∙ All pointers are 64-bit now. Since now registers number are doubled, compilers has more space now for maneuvering calling register allocation 79 . What it meanings for us, emitted code will contain less local variables. For example, function calculating first S-box of DES encryption algorithm, it processing 32/64/128/256 values at once (depending on DES_type type (uint32, uint64, SSE2 or AVX)) using bitslice DES method (read more about this method here 1.19): /* * Generated S-box files. * * This software may be modified, redistributed, and used for any purpose, * so long as its origin is acknowledged. * * Produced by Matthew Kwan - March 1998 */ #ifdef _WIN64 #define DES_type unsigned __int64 #else #define DES_type unsigned int #endif void s1 ( DES_type a1, DES_type a2, DES_type a3, DES_type a4, DES_type a5, DES_type a6, 79 assigning variables to registers 103
) { DES_type *out1, DES_type *out2, DES_type *out3, DES_type *out4 DES_type x1, x2, x3, x4, x5, x6, x7, x8; DES_type x9, x10, x11, x12, x13, x14, x15, x16; DES_type x17, x18, x19, x20, x21, x22, x23, x24; DES_type x25, x26, x27, x28, x29, x30, x31, x32; DES_type x33, x34, x35, x36, x37, x38, x39, x40; DES_type x41, x42, x43, x44, x45, x46, x47, x48; DES_type x49, x50, x51, x52, x53, x54, x55, x56; x1 = a3 & ~a5; x2 = x1 ^ a4; x3 = a3 & ~a4; x4 = x3 | a5; x5 = a6 & x4; x6 = x2 ^ x5; x7 = a4 & ~a5; x8 = a3 ^ a4; x9 = a6 & ~x8; x10 = x7 ^ x9; x11 = a2 | x10; x12 = x6 ^ x11; x13 = a5 ^ x5; x14 = x13 & x8; x15 = a5 & ~a4; x16 = x3 ^ x14; x17 = a6 | x16; x18 = x15 ^ x17; x19 = a2 | x18; x20 = x14 ^ x19; x21 = a1 & x20; x22 = x12 ^ ~x21; *out2 ^= x22; x23 = x1 | x5; x24 = x23 ^ x8; x25 = x18 & ~x2; x26 = a2 & ~x25; x27 = x24 ^ x26; x28 = x6 | x7; x29 = x28 ^ x25; x30 = x9 ^ x24; x31 = x18 & ~x30; x32 = a2 & x31; x33 = x29 ^ x32; x34 = a1 & x33; x35 = x27 ^ x34; *out4 ^= x35; x36 = a3 & x28; x37 = x18 & ~x36; x38 = a2 | x3; x39 = x37 ^ x38; 104
Page 1 and 2:
Quick introduction to reverse engin
Page 3 and 4:
1.16 C++ classes . . . . . . . . .
Page 5 and 6:
Preface Here (will be) some of my n
Page 7 and 8:
1.1 Hello, world! Let’s start wit
Page 9 and 10:
1.2 Stack Stack — is one of the m
Page 11 and 12:
(_snprintf() function works just li
Page 13 and 14:
1.3 printf() with several arguments
Page 15 and 16:
1.4 scanf() Now let’s use scanf()
Page 17 and 18:
GCC replaced first printf() call to
Page 19 and 20:
}; return 0; printf ("What you ente
Page 21 and 22:
1.5 Passing arguments via stack Now
Page 23 and 24:
1.6 One more word about results ret
Page 25 and 26:
push OFFSET $SG741 ; ’a
Page 27 and 28:
1.8 switch()/case/default 1.8.1 Few
Page 29 and 30:
1.8.2 A lot of cases If switch() st
Page 31 and 32:
loc_804840C: ; DATA XREF: .rodata:0
Page 33 and 34:
et 0 _main ENDP Nothing very specia
Page 35 and 36:
add esp, 1Ch xor eax, eax ; return
Page 37 and 38:
C/C++ standard defines char type as
Page 39 and 40:
eax=eax+ecx return eax ... and this
Page 41 and 42:
mov eax, ecx imul edx sar edx, 1 mo
Page 43 and 44:
; current stack state: ST(0) = resu
Page 45 and 46:
; in local stack here 8 bytes still
Page 47 and 48:
fld QWORD PTR _a$[esp-4] ; current
Page 49 and 50:
In other words, fnstsw ax / sahf in
Page 51 and 52:
1.13 Arrays Array is just a set of
Page 53 and 54:
mov [esp+70h+var_70], eax call _pri
Page 55 and 56:
jge SHORT $LN1@main mov ecx, DWORD
Page 57 and 58: ebp 0x15 0x15 esi 0x0 0 edi 0x0 0 e
Page 59 and 60: 1.14 Bit fields A lot of functions
Page 61 and 62: So, let’s download Linux Kernel 2
Page 63: There are some redundant code prese
Page 66 and 67: y 10 just by dropping last digit (3
Page 68 and 69: _key$ = 8 ; size = 4 _len$ = 12 ; s
Page 70 and 71: 1.15 Structures It can be defined t
Page 72 and 73: call DWORD PTR __imp__GetSystemTime
Page 74 and 75: 1.15.4 Fields packing in structure
Page 76 and 77: }; int b; struct outer_struct { cha
Page 78 and 79: }; printf ("stepping=%d\n", tmp->st
Page 80 and 81: call ___printf_chk mov eax, esi shr
Page 82 and 83: mov edx, DWORD PTR _t$[ebp] or edx,
Page 84 and 85: 1.16 C++ classes I placed a C++ cla
Page 86 and 87: push ecx mov DWORD PTR _this$[ebp],
Page 88 and 89: call _ZN1c4dumpEv lea eax, [esp+20h
Page 90 and 91: 1.17 Unions 1.17.1 Pseudo-random nu
Page 92 and 93: xor eax, eax pop esi mov esp, ebp p
Page 94 and 95: Let’s compile it in MSVC 2010 (I
Page 96 and 97: comp() function: public comp comp p
Page 98 and 99: Some compilers can do vectorization
Page 100 and 101: mov ecx, [esp+10h+var_10] mov edx,
Page 102 and 103: GCC In all other cases, non-SSE2 co
Page 104 and 105: add ebx, edx lea esi, [esi+0] loc_8
Page 106 and 107: movdqa xmm1, XMMWORD PTR [ecx+16] a
Page 110 and 111: } x40 = a3 | x31; x41 = x24 & ~x37;
Page 112 and 113: or ebx, DWORD PTR _x6$[esp+36] mov
Page 114 and 115: and r11, r8 and rsi, r8 and r10, r1
Page 116 and 117: pop rdi pop rsi ret 0 s1 ENDP Nothi
Page 118 and 119: 2.1 LEA instruction LEA (Load Effec
Page 120 and 121: 2.3 npad It’s an assembler macro
Page 122 and 123: 2.4 Signed number representations T
Page 124 and 125: call function function: .. do somet
Page 126 and 127: 3.2 String Debugging messages are o
Page 128 and 129: .text:3011E91B DD 1E fstp qword ptr
Page 130 and 131: loc_80483F2: pop ebp retn _f endp 4
Page 132 and 133: loc_8048444: loc_8048448: loc_80484
Page 134 and 135: public f2 f2 proc near push ebp mov
Page 136 and 137: loc_804840A: loc_804842E: loc_80484
Page 138 and 139: mov esi, DWORD PTR _k$[esp+16] push
Page 140 and 141: or eax, edx retn f endp 4.1.8 Task
Page 142 and 143: 4.2 Middle level 4.2.1 Task 2.1 Wel
Page 144 and 145: mov [edx+eax], bl mov ebx, edi mov
Page 146 and 147: loc_80485AB: ; CODE XREF: f+1E0 ; f
Page 148 and 149: mov [ecx+eax], dl inc eax cmp ebx,
Page 150 and 151: Chapter 5 Tools ∙ IDA as disassem
Page 152 and 153: Chapter 7 More examples 147
Page 154 and 155: .text:00541050 arg_0 = dword ptr 4
Page 156 and 157: .text:005410F3 .text:005410F3 loc_5
Page 158 and 159:
.text:005411A9 pop ebx .text:005411
Page 160 and 161:
.text:00541249 .text:00541249 next_
Page 162 and 163:
.text:005412FC mov esi, offset cube
Page 164 and 165:
.text:005413DC call _fwrite ; write
Page 166 and 167:
.text:005414A7 push ecx ; Filename
Page 168 and 169:
.text:005413A3 mov ecx, [esp+44h+pa
Page 170 and 171:
.text:00541408 push offset aRb ; "r
Page 172 and 173:
}; }; return; fseek (f, 0, SEEK_END
Page 174 and 175:
.text:005411B5 mov ebp, eax Check f
Page 176 and 177:
.text:0054124C inc ebp .text:005412
Page 178 and 179:
This instruction clears bit, in oth
Page 180 and 181:
.text:005410CD add esp, 40h .text:0
Page 182 and 183:
}; for (i=0; i
Page 184 and 185:
}; tmp[y][z]=get_bit (row, y, z); f
Page 186 and 187:
}; fread (buf, flen, 1, f); fclose
Page 188 and 189:
7.2 SAP client network traffic comp
Page 190 and 191:
.text:64413F68 .text:64413F68 loc_6
Page 192 and 193:
.text:64405171 call dbg .text:64405
Page 194 and 195:
.text:64404FF7 push offset aLogging
Page 196 and 197:
.text:64405113 loc_64405113: .text:
Page 198 and 199:
Now let’s dig deeper and find con
Page 200 and 201:
.text:64411E0B .text:64411E0B loc_6
Page 202 and 203:
Chapter 8 Other things 8.1 Compiler
Page 204 and 205:
Chapter 9 Tasks solutions 9.1 Easy
Page 206 and 207:
9.1.5 Task 1.5 Hint #1: Keep in min
Page 208 and 209:
} 9.1.8 Task 1.8 Solution: two 100*
Page 210 and 211:
#define PUSH(low, high) ((void) ((t
Page 212 and 213:
} } ignore one or both. Otherwise,
show all

Quick introduction to reverse engineering for beginners

You also want an ePaper? Increase the reach of your titles

Delete template?

Save as template?