www.gitorious.org Git - gnutls:gnutls.git/blob - devel/perlasm/e_padlock-x86

1 #!/usr/bin/env perl

3 # ====================================================================

4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL

5 # project. The module is, however, dual licensed under OpenSSL and

6 # CRYPTOGAMS licenses depending on where you obtain it. For further

7 # details see http://www.openssl.org/~appro/cryptogams/.

8 # ====================================================================

10 # September 2011

11 #

12 # Assembler helpers for Padlock engine. See even e_padlock-x86.pl for

13 # details.

15 $flavour = shift;

16 $output = shift;

17 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

19 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

21 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;

22 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or

23 ( $xlate="${dir}../../crypto/perlasm/x86_64-xlate.pl" and -f $xlate) or

24 die "can't locate x86_64-xlate.pl";

26 open STDOUT,"| $^X $xlate $flavour $output";

28 $code=".text\n";

30 %PADLOCK_MARGIN=(ecb=>128, cbc=>64, ctr32=>64); # prefetch errata

31 $PADLOCK_CHUNK=512; # Must be a power of 2 between 32 and 2^20

33 $ctx="%rdx";

34 $out="%rdi";

35 $inp="%rsi";

36 $len="%rcx";

37 $chunk="%rbx";

39 ($arg1,$arg2,$arg3,$arg4)=$win64?("%rcx","%rdx","%r8", "%r9") : # Win64 order

40 ("%rdi","%rsi","%rdx","%rcx"); # Unix order

42 $code.=<<___;

43 .globl padlock_capability

44 .type padlock_capability,\@abi-omnipotent

45 .align 16

46 padlock_capability:

47 mov %rbx,%r8

48 xor %eax,%eax

49 cpuid

50 xor %eax,%eax

51 cmp \$`"0x".unpack("H*",'tneC')`,%ebx

52 jne .Lnoluck

53 cmp \$`"0x".unpack("H*",'Hrua')`,%edx

54 jne .Lnoluck

55 cmp \$`"0x".unpack("H*",'slua')`,%ecx

56 jne .Lnoluck

57 mov \$0xC0000000,%eax

58 cpuid

59 mov %eax,%edx

60 xor %eax,%eax

61 cmp \$0xC0000001,%edx

62 jb .Lnoluck

63 mov \$0xC0000001,%eax

64 cpuid

65 mov %edx,%eax

66 and \$0xffffffef,%eax

67 or \$0x10,%eax # set Nano bit#4

68 .Lnoluck:

69 mov %r8,%rbx

70 ret

71 .size padlock_capability,.-padlock_capability

73 .globl padlock_key_bswap

74 .type padlock_key_bswap,\@abi-omnipotent,0

75 .align 16

76 padlock_key_bswap:

77 mov 240($arg1),%edx

78 .Lbswap_loop:

79 mov ($arg1),%eax

80 bswap %eax

81 mov %eax,($arg1)

82 lea 4($arg1),$arg1

83 sub \$1,%edx

84 jnz .Lbswap_loop

85 ret

86 .size padlock_key_bswap,.-padlock_key_bswap

88 .globl padlock_verify_context

89 .type padlock_verify_context,\@abi-omnipotent

90 .align 16

91 padlock_verify_context:

92 mov $arg1,$ctx

93 pushf

94 lea .Lpadlock_saved_context(%rip),%rax

95 call _padlock_verify_ctx

96 lea 8(%rsp),%rsp

97 ret

98 .size padlock_verify_context,.-padlock_verify_context

100 .type _padlock_verify_ctx,\@abi-omnipotent

101 .align 16

102 _padlock_verify_ctx:

103 mov 8(%rsp),%r8

104 bt \$30,%r8

105 jnc .Lverified

106 cmp (%rax),$ctx

107 je .Lverified

108 pushf

109 popf

110 .Lverified:

111 mov $ctx,(%rax)

112 ret

113 .size _padlock_verify_ctx,.-_padlock_verify_ctx

114

115 .globl padlock_reload_key

116 .type padlock_reload_key,\@abi-omnipotent

117 .align 16

118 padlock_reload_key:

119 pushf

120 popf

121 ret

122 .size padlock_reload_key,.-padlock_reload_key

123

124 .globl padlock_aes_block

125 .type padlock_aes_block,\@function,3

126 .align 16

127 padlock_aes_block:

128 mov %rbx,%r8

129 mov \$1,$len

130 lea 32($ctx),%rbx # key

131 lea 16($ctx),$ctx # control word

132 .byte 0xf3,0x0f,0xa7,0xc8 # rep xcryptecb

133 mov %r8,%rbx

134 ret

135 .size padlock_aes_block,.-padlock_aes_block

136

137 .globl padlock_xstore

138 .type padlock_xstore,\@function,2

139 .align 16

140 padlock_xstore:

141 mov %esi,%edx

142 .byte 0x0f,0xa7,0xc0 # xstore

143 ret

144 .size padlock_xstore,.-padlock_xstore

145

146 .globl padlock_sha1_oneshot

147 .type padlock_sha1_oneshot,\@function,3

148 .align 16

149 padlock_sha1_oneshot:

150 mov %rdx,%rcx

151 mov %rdi,%rdx # put aside %rdi

152 movups (%rdi),%xmm0 # copy-in context

153 sub \$128+8,%rsp

154 mov 16(%rdi),%eax

155 movaps %xmm0,(%rsp)

156 mov %rsp,%rdi

157 mov %eax,16(%rsp)

158 xor %rax,%rax

159 .byte 0xf3,0x0f,0xa6,0xc8 # rep xsha1

160 movaps (%rsp),%xmm0

161 mov 16(%rsp),%eax

162 add \$128+8,%rsp

163 movups %xmm0,(%rdx) # copy-out context

164 mov %eax,16(%rdx)

165 ret

166 .size padlock_sha1_oneshot,.-padlock_sha1_oneshot

167

168 .globl padlock_sha1_blocks

169 .type padlock_sha1_blocks,\@function,3

170 .align 16

171 padlock_sha1_blocks:

172 mov %rdx,%rcx

173 mov %rdi,%rdx # put aside %rdi

174 movups (%rdi),%xmm0 # copy-in context

175 sub \$128+8,%rsp

176 mov 16(%rdi),%eax

177 movaps %xmm0,(%rsp)

178 mov %rsp,%rdi

179 mov %eax,16(%rsp)

180 mov \$-1,%rax

181 .byte 0xf3,0x0f,0xa6,0xc8 # rep xsha1

182 movaps (%rsp),%xmm0

183 mov 16(%rsp),%eax

184 add \$128+8,%rsp

185 movups %xmm0,(%rdx) # copy-out context

186 mov %eax,16(%rdx)

187 ret

188 .size padlock_sha1_blocks,.-padlock_sha1_blocks

189

190 .globl padlock_sha256_oneshot

191 .type padlock_sha256_oneshot,\@function,3

192 .align 16

193 padlock_sha256_oneshot:

194 mov %rdx,%rcx

195 mov %rdi,%rdx # put aside %rdi

196 movups (%rdi),%xmm0 # copy-in context

197 sub \$128+8,%rsp

198 movups 16(%rdi),%xmm1

199 movaps %xmm0,(%rsp)

200 mov %rsp,%rdi

201 movaps %xmm1,16(%rsp)

202 xor %rax,%rax

203 .byte 0xf3,0x0f,0xa6,0xd0 # rep xsha256

204 movaps (%rsp),%xmm0

205 movaps 16(%rsp),%xmm1

206 add \$128+8,%rsp

207 movups %xmm0,(%rdx) # copy-out context

208 movups %xmm1,16(%rdx)

209 ret

210 .size padlock_sha256_oneshot,.-padlock_sha256_oneshot

211

212 .globl padlock_sha256_blocks

213 .type padlock_sha256_blocks,\@function,3

214 .align 16

215 padlock_sha256_blocks:

216 mov %rdx,%rcx

217 mov %rdi,%rdx # put aside %rdi

218 movups (%rdi),%xmm0 # copy-in context

219 sub \$128+8,%rsp

220 movups 16(%rdi),%xmm1

221 movaps %xmm0,(%rsp)

222 mov %rsp,%rdi

223 movaps %xmm1,16(%rsp)

224 mov \$-1,%rax

225 .byte 0xf3,0x0f,0xa6,0xd0 # rep xsha256

226 movaps (%rsp),%xmm0

227 movaps 16(%rsp),%xmm1

228 add \$128+8,%rsp

229 movups %xmm0,(%rdx) # copy-out context

230 movups %xmm1,16(%rdx)

231 ret

232 .size padlock_sha256_blocks,.-padlock_sha256_blocks

233

234 .globl padlock_sha512_blocks

235 .type padlock_sha512_blocks,\@function,3

236 .align 16

237 padlock_sha512_blocks:

238 mov %rdx,%rcx

239 mov %rdi,%rdx # put aside %rdi

240 movups (%rdi),%xmm0 # copy-in context

241 sub \$128+8,%rsp

242 movups 16(%rdi),%xmm1

243 movups 32(%rdi),%xmm2

244 movups 48(%rdi),%xmm3

245 movaps %xmm0,(%rsp)

246 mov %rsp,%rdi

247 movaps %xmm1,16(%rsp)

248 movaps %xmm2,32(%rsp)

249 movaps %xmm3,48(%rsp)

250 .byte 0xf3,0x0f,0xa6,0xe0 # rep xha512

251 movaps (%rsp),%xmm0

252 movaps 16(%rsp),%xmm1

253 movaps 32(%rsp),%xmm2

254 movaps 48(%rsp),%xmm3

255 add \$128+8,%rsp

256 movups %xmm0,(%rdx) # copy-out context

257 movups %xmm1,16(%rdx)

258 movups %xmm2,32(%rdx)

259 movups %xmm3,48(%rdx)

260 ret

261 .size padlock_sha512_blocks,.-padlock_sha512_blocks

262 ___

263

264 sub generate_mode {

265 my ($mode,$opcode) = @_;

266 # int padlock_$mode_encrypt(void *out, const void *inp,

267 # struct padlock_cipher_data *ctx, size_t len);

268 $code.=<<___;

269 .globl padlock_${mode}_encrypt

270 .type padlock_${mode}_encrypt,\@function,4

271 .align 16

272 padlock_${mode}_encrypt:

273 push %rbp

274 push %rbx

275

276 xor %eax,%eax

277 test \$15,$ctx

278 jnz .L${mode}_abort

279 test \$15,$len

280 jnz .L${mode}_abort

281 lea .Lpadlock_saved_context(%rip),%rax

282 pushf

283 cld

284 call _padlock_verify_ctx

285 lea 16($ctx),$ctx # control word

286 xor %eax,%eax

287 xor %ebx,%ebx

288 ___

289 # Formally speaking correct condtion is $len<=$margin and $inp+$margin

290 # crosses page boundary [and next page is unreadable]. But $inp can

291 # be unaligned in which case data can be copied to $out if latter is

292 # aligned, in which case $out+$margin has to be checked. Covering all

293 # cases appears more complicated than just copying short input...

294 $code.=<<___ if ($PADLOCK_MARGIN{$mode});

295 cmp \$$PADLOCK_MARGIN{$mode},$len

296 jbe .L${mode}_short

297 ___

298 $code.=<<___;

299 testl \$`1<<5`,($ctx) # align bit in control word

300 jnz .L${mode}_aligned

301 test \$0x0f,$out

302 setz %al # !out_misaligned

303 test \$0x0f,$inp

304 setz %bl # !inp_misaligned

305 test %ebx,%eax

306 jnz .L${mode}_aligned

307 neg %rax

308 mov \$$PADLOCK_CHUNK,$chunk

309 not %rax # out_misaligned?-1:0

310 lea (%rsp),%rbp

311 cmp $chunk,$len

312 cmovc $len,$chunk # chunk=len>PADLOCK_CHUNK?PADLOCK_CHUNK:len

313 and $chunk,%rax # out_misaligned?chunk:0

314 mov $len,$chunk

315 neg %rax

316 and \$$PADLOCK_CHUNK-1,$chunk # chunk%=PADLOCK_CHUNK

317 lea (%rax,%rbp),%rsp

318 ___

319 $code.=<<___ if ($mode eq "ctr32");

320 .L${mode}_reenter:

321 mov -4($ctx),%eax # pull 32-bit counter

322 bswap %eax

323 neg %eax

324 and \$`$PADLOCK_CHUNK/16-1`,%eax

325 jz .L${mode}_loop

326 shl \$4,%eax

327 cmp %rax,$len

328 cmova %rax,$chunk # don't let counter cross PADLOCK_CHUNK

329 ___

330 $code.=<<___;

331 jmp .L${mode}_loop

332 .align 16

333 .L${mode}_loop:

334 cmp $len,$chunk # ctr32 artefact

335 cmova $len,$chunk # ctr32 artefact

336 mov $out,%r8 # save parameters

337 mov $inp,%r9

338 mov $len,%r10

339 mov $chunk,$len

340 mov $chunk,%r11

341 test \$0x0f,$out # out_misaligned

342 cmovnz %rsp,$out

343 test \$0x0f,$inp # inp_misaligned

344 jz .L${mode}_inp_aligned

345 shr \$3,$len

346 .byte 0xf3,0x48,0xa5 # rep movsq

347 sub $chunk,$out

348 mov $chunk,$len

349 mov $out,$inp

350 .L${mode}_inp_aligned:

351 lea -16($ctx),%rax # ivp

352 lea 16($ctx),%rbx # key

353 shr \$4,$len

354 .byte 0xf3,0x0f,0xa7,$opcode # rep xcrypt*

355 ___

356 $code.=<<___ if ($mode !~ /ecb|ctr/);

357 movdqa (%rax),%xmm0

358 movdqa %xmm0,-16($ctx) # copy [or refresh] iv

359 ___

360 $code.=<<___ if ($mode eq "ctr32");

361 mov -4($ctx),%eax # pull 32-bit counter

362 test \$0xffff0000,%eax

363 jnz .L${mode}_no_corr

364 bswap %eax

365 add \$0x10000,%eax

366 bswap %eax

367 mov %eax,-4($ctx)

368 .L${mode}_no_corr:

369 ___

370 $code.=<<___;

371 mov %r8,$out # restore paramters

372 mov %r11,$chunk

373 test \$0x0f,$out

374 jz .L${mode}_out_aligned

375 mov $chunk,$len

376 shr \$3,$len

377 lea (%rsp),$inp

378 .byte 0xf3,0x48,0xa5 # rep movsq

379 sub $chunk,$out

380 .L${mode}_out_aligned:

381 mov %r9,$inp

382 mov %r10,$len

383 add $chunk,$out

384 add $chunk,$inp

385 sub $chunk,$len

386 mov \$$PADLOCK_CHUNK,$chunk

387 jnz .L${mode}_loop

388

389 cmp %rsp,%rbp

390 je .L${mode}_done

391

392 pxor %xmm0,%xmm0

393 lea (%rsp),%rax

394 .L${mode}_bzero:

395 movaps %xmm0,(%rax)

396 lea 16(%rax),%rax

397 cmp %rax,%rbp

398 ja .L${mode}_bzero

399

400 .L${mode}_done:

401 lea (%rbp),%rsp

402 jmp .L${mode}_exit

403 ___

404 $code.=<<___ if ($PADLOCK_MARGIN{$mode});

405 .align 16

406 .L${mode}_short:

407 mov %rsp,%rbp

408 sub $len,%rsp

409 xor $chunk,$chunk

410 .L${mode}_short_copy:

411 movups ($inp,$chunk),%xmm0

412 lea 16($chunk),$chunk

413 cmp $chunk,$len

414 movaps %xmm0,-16(%rsp,$chunk)

415 ja .L${mode}_short_copy

416 mov %rsp,$inp

417 mov $len,$chunk

418 jmp .L${mode}_`${mode} eq "ctr32"?"reenter":"loop"`

419 ___

420 $code.=<<___;

421 .align 16

422 .L${mode}_aligned:

423 ___

424 $code.=<<___ if ($mode eq "ctr32");

425 mov -4($ctx),%eax # pull 32-bit counter

426 mov \$`16*0x10000`,$chunk

427 bswap %eax

428 cmp $len,$chunk

429 cmova $len,$chunk

430 neg %eax

431 and \$0xffff,%eax

432 jz .L${mode}_aligned_loop

433 shl \$4,%eax

434 cmp %rax,$len

435 cmova %rax,$chunk # don't let counter cross 2^16

436 jmp .L${mode}_aligned_loop

437 .align 16

438 .L${mode}_aligned_loop:

439 cmp $len,$chunk

440 cmova $len,$chunk

441 mov $len,%r10 # save parameters

442 mov $chunk,$len

443 mov $chunk,%r11

444 ___

445 $code.=<<___;

446 lea -16($ctx),%rax # ivp

447 lea 16($ctx),%rbx # key

448 shr \$4,$len # len/=AES_BLOCK_SIZE

449 .byte 0xf3,0x0f,0xa7,$opcode # rep xcrypt*

450 ___

451 $code.=<<___ if ($mode !~ /ecb|ctr/);

452 movdqa (%rax),%xmm0

453 movdqa %xmm0,-16($ctx) # copy [or refresh] iv

454 ___

455 $code.=<<___ if ($mode eq "ctr32");

456 mov -4($ctx),%eax # pull 32-bit counter

457 bswap %eax

458 add \$0x10000,%eax

459 bswap %eax

460 mov %eax,-4($ctx)

461

462 mov %r11,$chunk # restore paramters

463 mov %r10,$len

464 sub $chunk,$len

465 mov \$`16*0x10000`,$chunk

466 jnz .L${mode}_aligned_loop

467 ___

468 $code.=<<___;

469 .L${mode}_exit:

470 mov \$1,%eax

471 lea 8(%rsp),%rsp

472 .L${mode}_abort:

473 pop %rbx

474 pop %rbp

475 ret

476 .size padlock_${mode}_encrypt,.-padlock_${mode}_encrypt

477 ___

478 }

479

480 &generate_mode("ecb",0xc8);

481 &generate_mode("cbc",0xd0);

482 #&generate_mode("cfb",0xe0);

483 #&generate_mode("ofb",0xe8);

484 #&generate_mode("ctr32",0xd8); # all 64-bit CPUs have working CTR...

485

486 $code.=<<___;

487 .asciz "VIA Padlock x86_64 module, CRYPTOGAMS by <appro\@openssl.org>"

488 .align 16

489 .data

490 .align 8

491 .Lpadlock_saved_context:

492 .quad 0

493 ___

494 $code =~ s/\`([^\`]*)\`/eval($1)/gem;

495

496 print $code;

497

498 close STDOUT;