Text file
src/hash/crc32/crc32_ppc64le.s
1// Copyright 2017 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// The vectorized implementation found below is a derived work
6// from code written by Anton Blanchard <anton@au.ibm.com> found
7// at https://github.com/antonblanchard/crc32-vpmsum. The original
8// is dual licensed under GPL and Apache 2. As the copyright holder
9// for the work, IBM has contributed this new work under
10// the golang license.
11
12// Changes include porting to Go assembler with modifications for
13// the Go ABI for ppc64le.
14
15#include "textflag.h"
16
17#define POWER8_OFFSET 132
18
19#define off16 R16
20#define off32 R17
21#define off48 R18
22#define off64 R19
23#define off80 R20
24#define off96 R21
25#define off112 R22
26
27#define const1 V24
28#define const2 V25
29
30#define byteswap V26
31#define mask_32bit V27
32#define mask_64bit V28
33#define zeroes V29
34
35#define MAX_SIZE 32*1024
36#define REFLECT
37
38TEXT ·ppc64SlicingUpdateBy8(SB), NOSPLIT|NOFRAME, $0-44
39 MOVWZ crc+0(FP), R3 // incoming crc
40 MOVD table8+8(FP), R4 // *Table
41 MOVD p+16(FP), R5
42 MOVD p_len+24(FP), R6 // p len
43
44 CMP $0,R6 // len == 0?
45 BNE start
46 MOVW R3,ret+40(FP) // return crc
47 RET
48
49start:
50 NOR R3,R3,R7 // ^crc
51 MOVWZ R7,R7 // 32 bits
52 CMP R6,$16
53 MOVD R6,CTR
54 BLT short
55 SRAD $3,R6,R8 // 8 byte chunks
56 MOVD R8,CTR
57
58loop:
59 MOVWZ 0(R5),R8 // 0-3 bytes of p ?Endian?
60 MOVWZ 4(R5),R9 // 4-7 bytes of p
61 MOVD R4,R10 // &tab[0]
62 XOR R7,R8,R7 // crc ^= byte[0:3]
63 RLDICL $40,R9,$56,R17 // p[7]
64 SLD $2,R17,R17 // p[7]*4
65 RLDICL $40,R7,$56,R8 // crc>>24
66 ADD R17,R10,R17 // &tab[0][p[7]]
67 SLD $2,R8,R8 // crc>>24*4
68 RLDICL $48,R9,$56,R18 // p[6]
69 SLD $2,R18,R18 // p[6]*4
70 ADD $1024,R10,R10 // tab[1]
71 MOVWZ 0(R17),R21 // tab[0][p[7]]
72 RLDICL $56,R9,$56,R19 // p[5]
73 ADD R10,R18,R18 // &tab[1][p[6]]
74 SLD $2,R19,R19 // p[5]*4:1
75 MOVWZ 0(R18),R22 // tab[1][p[6]]
76 ADD $1024,R10,R10 // tab[2]
77 XOR R21,R22,R21 // xor done R22
78 ADD R19,R10,R19 // &tab[2][p[5]]
79 ANDCC $255,R9,R20 // p[4] ??
80 SLD $2,R20,R20 // p[4]*4
81 MOVWZ 0(R19),R23 // tab[2][p[5]]
82 ADD $1024,R10,R10 // &tab[3]
83 ADD R20,R10,R20 // tab[3][p[4]]
84 XOR R21,R23,R21 // xor done R23
85 ADD $1024,R10,R10 // &tab[4]
86 MOVWZ 0(R20),R24 // tab[3][p[4]]
87 ADD R10,R8,R23 // &tab[4][crc>>24]
88 XOR R21,R24,R21 // xor done R24
89 MOVWZ 0(R23),R25 // tab[4][crc>>24]
90 RLDICL $48,R7,$56,R24 // crc>>16&0xFF
91 XOR R21,R25,R21 // xor done R25
92 ADD $1024,R10,R10 // &tab[5]
93 SLD $2,R24,R24 // crc>>16&0xFF*4
94 ADD R24,R10,R24 // &tab[5][crc>>16&0xFF]
95 MOVWZ 0(R24),R26 // tab[5][crc>>16&0xFF]
96 XOR R21,R26,R21 // xor done R26
97 RLDICL $56,R7,$56,R25 // crc>>8
98 ADD $1024,R10,R10 // &tab[6]
99 SLD $2,R25,R25 // crc>>8&FF*2
100 ADD R25,R10,R25 // &tab[6][crc>>8&0xFF]
101 MOVBZ R7,R26 // crc&0xFF
102 ADD $1024,R10,R10 // &tab[7]
103 MOVWZ 0(R25),R27 // tab[6][crc>>8&0xFF]
104 SLD $2,R26,R26 // crc&0xFF*2
105 XOR R21,R27,R21 // xor done R27
106 ADD R26,R10,R26 // &tab[7][crc&0xFF]
107 ADD $8,R5 // p = p[8:]
108 MOVWZ 0(R26),R28 // tab[7][crc&0xFF]
109 XOR R21,R28,R21 // xor done R28
110 MOVWZ R21,R7 // crc for next round
111 BC 16,0,loop // next 8 bytes
112 ANDCC $7,R6,R8 // any leftover bytes
113 BEQ done // none --> done
114 MOVD R8,CTR // byte count
115 PCALIGN $16 // align short loop
116short:
117 MOVBZ 0(R5),R8 // get v
118 MOVBZ R7,R9 // byte(crc) -> R8 BE vs LE?
119 SRD $8,R7,R14 // crc>>8
120 XOR R8,R9,R8 // byte(crc)^v -> R8
121 ADD $1,R5 // ptr to next v
122 SLD $2,R8 // convert index-> bytes
123 ADD R8,R4,R9 // &tab[byte(crc)^v]
124 MOVWZ 0(R9),R10 // tab[byte(crc)^v]
125 XOR R10,R14,R7 // loop crc in R7
126 BC 16,0,short
127done:
128 NOR R7,R7,R7 // ^crc
129 MOVW R7,ret+40(FP) // return crc
130 RET
131
132#ifdef BYTESWAP_DATA
133DATA ·byteswapcons+0(SB)/8,$0x0706050403020100
134DATA ·byteswapcons+8(SB)/8,$0x0f0e0d0c0b0a0908
135
136GLOBL ·byteswapcons+0(SB),RODATA,$16
137#endif
138
139TEXT ·vectorCrc32(SB), NOSPLIT|NOFRAME, $0-36
140 MOVWZ crc+0(FP), R3 // incoming crc
141 MOVWZ ctab+4(FP), R14 // crc poly id
142 MOVD p+8(FP), R4
143 MOVD p_len+16(FP), R5 // p len
144
145 // R3 = incoming crc
146 // R14 = constant table identifier
147 // R5 = address of bytes
148 // R6 = length of bytes
149
150 // defines for index loads
151
152 MOVD $16,off16
153 MOVD $32,off32
154 MOVD $48,off48
155 MOVD $64,off64
156 MOVD $80,off80
157 MOVD $96,off96
158 MOVD $112,off112
159 MOVD $0,R15
160
161 MOVD R3,R10 // save initial crc
162
163 NOR R3,R3,R3 // ^crc
164 MOVWZ R3,R3 // 32 bits
165 VXOR zeroes,zeroes,zeroes // clear the V reg
166 VSPLTISW $-1,V0
167 VSLDOI $4,V29,V0,mask_32bit
168 VSLDOI $8,V29,V0,mask_64bit
169
170 VXOR V8,V8,V8
171 MTVSRD R3,VS40 // crc initial value VS40 = V8
172
173#ifdef REFLECT
174 VSLDOI $8,zeroes,V8,V8 // or: VSLDOI V29,V8,V27,4 for top 32 bits?
175#else
176 VSLDOI $4,V8,zeroes,V8
177#endif
178
179#ifdef BYTESWAP_DATA
180 MOVD $·byteswapcons(SB),R3
181 LVX (R3),byteswap
182#endif
183
184 CMPU R5,$256 // length of bytes
185 BLT short
186
187 RLDICR $0,R5,$56,R6 // chunk to process
188
189 // First step for larger sizes
190l1: MOVD $32768,R7
191 MOVD R7,R9
192 CMP R6,R7 // compare R6, R7 (MAX SIZE)
193 BGT top // less than MAX, just do remainder
194 MOVD R6,R7
195top:
196 SUB R7,R6,R6
197
198 // mainloop does 128 bytes at a time
199 SRD $7,R7
200
201 // determine the offset into the constants table to start with.
202 // Each constant is 128 bytes, used against 16 bytes of data.
203 SLD $4,R7,R8
204 SRD $3,R9,R9
205 SUB R8,R9,R8
206
207 // The last iteration is reduced in a separate step
208 ADD $-1,R7
209 MOVD R7,CTR
210
211 // Determine which constant table (depends on poly)
212 CMP R14,$1
213 BNE castTable
214 MOVD $·IEEEConst(SB),R3
215 BR startConst
216castTable:
217 MOVD $·CastConst(SB),R3
218
219startConst:
220 ADD R3,R8,R3 // starting point in constants table
221
222 VXOR V0,V0,V0 // clear the V regs
223 VXOR V1,V1,V1
224 VXOR V2,V2,V2
225 VXOR V3,V3,V3
226 VXOR V4,V4,V4
227 VXOR V5,V5,V5
228 VXOR V6,V6,V6
229 VXOR V7,V7,V7
230
231 LVX (R3),const1 // loading constant values
232
233 CMP R15,$1 // Identify warm up pass
234 BEQ next
235
236 // First warm up pass: load the bytes to process
237 LVX (R4),V16
238 LVX (R4+off16),V17
239 LVX (R4+off32),V18
240 LVX (R4+off48),V19
241 LVX (R4+off64),V20
242 LVX (R4+off80),V21
243 LVX (R4+off96),V22
244 LVX (R4+off112),V23
245 ADD $128,R4 // bump up to next 128 bytes in buffer
246
247 VXOR V16,V8,V16 // xor in initial CRC in V8
248
249next:
250 BC 18,0,first_warm_up_done
251
252 ADD $16,R3 // bump up to next constants
253 LVX (R3),const2 // table values
254
255 VPMSUMD V16,const1,V8 // second warm up pass
256 LVX (R4),V16 // load from buffer
257 OR $0,R2,R2
258
259 VPMSUMD V17,const1,V9 // vpmsumd with constants
260 LVX (R4+off16),V17 // load next from buffer
261 OR $0,R2,R2
262
263 VPMSUMD V18,const1,V10 // vpmsumd with constants
264 LVX (R4+off32),V18 // load next from buffer
265 OR $0,R2,R2
266
267 VPMSUMD V19,const1,V11 // vpmsumd with constants
268 LVX (R4+off48),V19 // load next from buffer
269 OR $0,R2,R2
270
271 VPMSUMD V20,const1,V12 // vpmsumd with constants
272 LVX (R4+off64),V20 // load next from buffer
273 OR $0,R2,R2
274
275 VPMSUMD V21,const1,V13 // vpmsumd with constants
276 LVX (R4+off80),V21 // load next from buffer
277 OR $0,R2,R2
278
279 VPMSUMD V22,const1,V14 // vpmsumd with constants
280 LVX (R4+off96),V22 // load next from buffer
281 OR $0,R2,R2
282
283 VPMSUMD V23,const1,V15 // vpmsumd with constants
284 LVX (R4+off112),V23 // load next from buffer
285
286 ADD $128,R4 // bump up to next 128 bytes in buffer
287
288 BC 18,0,first_cool_down
289
290cool_top:
291 LVX (R3),const1 // constants
292 ADD $16,R3 // inc to next constants
293 OR $0,R2,R2
294
295 VXOR V0,V8,V0 // xor in previous vpmsumd
296 VPMSUMD V16,const2,V8 // vpmsumd with constants
297 LVX (R4),V16 // buffer
298 OR $0,R2,R2
299
300 VXOR V1,V9,V1 // xor in previous
301 VPMSUMD V17,const2,V9 // vpmsumd with constants
302 LVX (R4+off16),V17 // next in buffer
303 OR $0,R2,R2
304
305 VXOR V2,V10,V2 // xor in previous
306 VPMSUMD V18,const2,V10 // vpmsumd with constants
307 LVX (R4+off32),V18 // next in buffer
308 OR $0,R2,R2
309
310 VXOR V3,V11,V3 // xor in previous
311 VPMSUMD V19,const2,V11 // vpmsumd with constants
312 LVX (R4+off48),V19 // next in buffer
313 LVX (R3),const2 // get next constant
314 OR $0,R2,R2
315
316 VXOR V4,V12,V4 // xor in previous
317 VPMSUMD V20,const1,V12 // vpmsumd with constants
318 LVX (R4+off64),V20 // next in buffer
319 OR $0,R2,R2
320
321 VXOR V5,V13,V5 // xor in previous
322 VPMSUMD V21,const1,V13 // vpmsumd with constants
323 LVX (R4+off80),V21 // next in buffer
324 OR $0,R2,R2
325
326 VXOR V6,V14,V6 // xor in previous
327 VPMSUMD V22,const1,V14 // vpmsumd with constants
328 LVX (R4+off96),V22 // next in buffer
329 OR $0,R2,R2
330
331 VXOR V7,V15,V7 // xor in previous
332 VPMSUMD V23,const1,V15 // vpmsumd with constants
333 LVX (R4+off112),V23 // next in buffer
334
335 ADD $128,R4 // bump up buffer pointer
336 BC 16,0,cool_top // are we done?
337
338first_cool_down:
339
340 // load the constants
341 // xor in the previous value
342 // vpmsumd the result with constants
343
344 LVX (R3),const1
345 ADD $16,R3
346
347 VXOR V0,V8,V0
348 VPMSUMD V16,const1,V8
349 OR $0,R2,R2
350
351 VXOR V1,V9,V1
352 VPMSUMD V17,const1,V9
353 OR $0,R2,R2
354
355 VXOR V2,V10,V2
356 VPMSUMD V18,const1,V10
357 OR $0,R2,R2
358
359 VXOR V3,V11,V3
360 VPMSUMD V19,const1,V11
361 OR $0,R2,R2
362
363 VXOR V4,V12,V4
364 VPMSUMD V20,const1,V12
365 OR $0,R2,R2
366
367 VXOR V5,V13,V5
368 VPMSUMD V21,const1,V13
369 OR $0,R2,R2
370
371 VXOR V6,V14,V6
372 VPMSUMD V22,const1,V14
373 OR $0,R2,R2
374
375 VXOR V7,V15,V7
376 VPMSUMD V23,const1,V15
377 OR $0,R2,R2
378
379second_cool_down:
380
381 VXOR V0,V8,V0
382 VXOR V1,V9,V1
383 VXOR V2,V10,V2
384 VXOR V3,V11,V3
385 VXOR V4,V12,V4
386 VXOR V5,V13,V5
387 VXOR V6,V14,V6
388 VXOR V7,V15,V7
389
390#ifdef REFLECT
391 VSLDOI $4,V0,zeroes,V0
392 VSLDOI $4,V1,zeroes,V1
393 VSLDOI $4,V2,zeroes,V2
394 VSLDOI $4,V3,zeroes,V3
395 VSLDOI $4,V4,zeroes,V4
396 VSLDOI $4,V5,zeroes,V5
397 VSLDOI $4,V6,zeroes,V6
398 VSLDOI $4,V7,zeroes,V7
399#endif
400
401 LVX (R4),V8
402 LVX (R4+off16),V9
403 LVX (R4+off32),V10
404 LVX (R4+off48),V11
405 LVX (R4+off64),V12
406 LVX (R4+off80),V13
407 LVX (R4+off96),V14
408 LVX (R4+off112),V15
409
410 ADD $128,R4
411
412 VXOR V0,V8,V16
413 VXOR V1,V9,V17
414 VXOR V2,V10,V18
415 VXOR V3,V11,V19
416 VXOR V4,V12,V20
417 VXOR V5,V13,V21
418 VXOR V6,V14,V22
419 VXOR V7,V15,V23
420
421 MOVD $1,R15
422 CMP $0,R6
423 ADD $128,R6
424
425 BNE l1
426 ANDCC $127,R5
427 SUBC R5,$128,R6
428 ADD R3,R6,R3
429
430 SRD $4,R5,R7
431 MOVD R7,CTR
432 LVX (R3),V0
433 LVX (R3+off16),V1
434 LVX (R3+off32),V2
435 LVX (R3+off48),V3
436 LVX (R3+off64),V4
437 LVX (R3+off80),V5
438 LVX (R3+off96),V6
439 LVX (R3+off112),V7
440
441 ADD $128,R3
442
443 VPMSUMW V16,V0,V0
444 VPMSUMW V17,V1,V1
445 VPMSUMW V18,V2,V2
446 VPMSUMW V19,V3,V3
447 VPMSUMW V20,V4,V4
448 VPMSUMW V21,V5,V5
449 VPMSUMW V22,V6,V6
450 VPMSUMW V23,V7,V7
451
452 // now reduce the tail
453
454 CMP $0,R7
455 BEQ next1
456
457 LVX (R4),V16
458 LVX (R3),V17
459 VPMSUMW V16,V17,V16
460 VXOR V0,V16,V0
461 BC 18,0,next1
462
463 LVX (R4+off16),V16
464 LVX (R3+off16),V17
465 VPMSUMW V16,V17,V16
466 VXOR V0,V16,V0
467 BC 18,0,next1
468
469 LVX (R4+off32),V16
470 LVX (R3+off32),V17
471 VPMSUMW V16,V17,V16
472 VXOR V0,V16,V0
473 BC 18,0,next1
474
475 LVX (R4+off48),V16
476 LVX (R3+off48),V17
477 VPMSUMW V16,V17,V16
478 VXOR V0,V16,V0
479 BC 18,0,next1
480
481 LVX (R4+off64),V16
482 LVX (R3+off64),V17
483 VPMSUMW V16,V17,V16
484 VXOR V0,V16,V0
485 BC 18,0,next1
486
487 LVX (R4+off80),V16
488 LVX (R3+off80),V17
489 VPMSUMW V16,V17,V16
490 VXOR V0,V16,V0
491 BC 18,0,next1
492
493 LVX (R4+off96),V16
494 LVX (R3+off96),V17
495 VPMSUMW V16,V17,V16
496 VXOR V0,V16,V0
497
498next1:
499 VXOR V0,V1,V0
500 VXOR V2,V3,V2
501 VXOR V4,V5,V4
502 VXOR V6,V7,V6
503 VXOR V0,V2,V0
504 VXOR V4,V6,V4
505 VXOR V0,V4,V0
506
507barrett_reduction:
508
509 CMP R14,$1
510 BNE barcstTable
511 MOVD $·IEEEBarConst(SB),R3
512 BR startbarConst
513barcstTable:
514 MOVD $·CastBarConst(SB),R3
515
516startbarConst:
517 LVX (R3),const1
518 LVX (R3+off16),const2
519
520 VSLDOI $8,V0,V0,V1
521 VXOR V0,V1,V0
522
523#ifdef REFLECT
524 VSPLTISB $1,V1
525 VSL V0,V1,V0
526#endif
527
528 VAND V0,mask_64bit,V0
529
530#ifndef REFLECT
531
532 VPMSUMD V0,const1,V1
533 VSLDOI $8,zeroes,V1,V1
534 VPMSUMD V1,const2,V1
535 VXOR V0,V1,V0
536 VSLDOI $8,V0,zeroes,V0
537
538#else
539
540 VAND V0,mask_32bit,V1
541 VPMSUMD V1,const1,V1
542 VAND V1,mask_32bit,V1
543 VPMSUMD V1,const2,V1
544 VXOR V0,V1,V0
545 VSLDOI $4,V0,zeroes,V0
546
547#endif
548
549 MFVSRD VS32,R3 // VS32 = V0
550
551 NOR R3,R3,R3 // return ^crc
552 MOVW R3,ret+32(FP)
553 RET
554
555first_warm_up_done:
556
557 LVX (R3),const1
558 ADD $16,R3
559
560 VPMSUMD V16,const1,V8
561 VPMSUMD V17,const1,V9
562 VPMSUMD V18,const1,V10
563 VPMSUMD V19,const1,V11
564 VPMSUMD V20,const1,V12
565 VPMSUMD V21,const1,V13
566 VPMSUMD V22,const1,V14
567 VPMSUMD V23,const1,V15
568
569 BR second_cool_down
570
571short:
572 CMP $0,R5
573 BEQ zero
574
575 // compute short constants
576
577 CMP R14,$1
578 BNE castshTable
579 MOVD $·IEEEConst(SB),R3
580 ADD $4080,R3
581 BR startshConst
582castshTable:
583 MOVD $·CastConst(SB),R3
584 ADD $4080,R3
585
586startshConst:
587 SUBC R5,$256,R6 // sub from 256
588 ADD R3,R6,R3
589
590 // calculate where to start
591
592 SRD $4,R5,R7
593 MOVD R7,CTR
594
595 VXOR V19,V19,V19
596 VXOR V20,V20,V20
597
598 LVX (R4),V0
599 LVX (R3),V16
600 VXOR V0,V8,V0
601 VPMSUMW V0,V16,V0
602 BC 18,0,v0
603
604 LVX (R4+off16),V1
605 LVX (R3+off16),V17
606 VPMSUMW V1,V17,V1
607 BC 18,0,v1
608
609 LVX (R4+off32),V2
610 LVX (R3+off32),V16
611 VPMSUMW V2,V16,V2
612 BC 18,0,v2
613
614 LVX (R4+off48),V3
615 LVX (R3+off48),V17
616 VPMSUMW V3,V17,V3
617 BC 18,0,v3
618
619 LVX (R4+off64),V4
620 LVX (R3+off64),V16
621 VPMSUMW V4,V16,V4
622 BC 18,0,v4
623
624 LVX (R4+off80),V5
625 LVX (R3+off80),V17
626 VPMSUMW V5,V17,V5
627 BC 18,0,v5
628
629 LVX (R4+off96),V6
630 LVX (R3+off96),V16
631 VPMSUMW V6,V16,V6
632 BC 18,0,v6
633
634 LVX (R4+off112),V7
635 LVX (R3+off112),V17
636 VPMSUMW V7,V17,V7
637 BC 18,0,v7
638
639 ADD $128,R3
640 ADD $128,R4
641
642 LVX (R4),V8
643 LVX (R3),V16
644 VPMSUMW V8,V16,V8
645 BC 18,0,v8
646
647 LVX (R4+off16),V9
648 LVX (R3+off16),V17
649 VPMSUMW V9,V17,V9
650 BC 18,0,v9
651
652 LVX (R4+off32),V10
653 LVX (R3+off32),V16
654 VPMSUMW V10,V16,V10
655 BC 18,0,v10
656
657 LVX (R4+off48),V11
658 LVX (R3+off48),V17
659 VPMSUMW V11,V17,V11
660 BC 18,0,v11
661
662 LVX (R4+off64),V12
663 LVX (R3+off64),V16
664 VPMSUMW V12,V16,V12
665 BC 18,0,v12
666
667 LVX (R4+off80),V13
668 LVX (R3+off80),V17
669 VPMSUMW V13,V17,V13
670 BC 18,0,v13
671
672 LVX (R4+off96),V14
673 LVX (R3+off96),V16
674 VPMSUMW V14,V16,V14
675 BC 18,0,v14
676
677 LVX (R4+off112),V15
678 LVX (R3+off112),V17
679 VPMSUMW V15,V17,V15
680
681 VXOR V19,V15,V19
682v14: VXOR V20,V14,V20
683v13: VXOR V19,V13,V19
684v12: VXOR V20,V12,V20
685v11: VXOR V19,V11,V19
686v10: VXOR V20,V10,V20
687v9: VXOR V19,V9,V19
688v8: VXOR V20,V8,V20
689v7: VXOR V19,V7,V19
690v6: VXOR V20,V6,V20
691v5: VXOR V19,V5,V19
692v4: VXOR V20,V4,V20
693v3: VXOR V19,V3,V19
694v2: VXOR V20,V2,V20
695v1: VXOR V19,V1,V19
696v0: VXOR V20,V0,V20
697
698 VXOR V19,V20,V0
699
700 BR barrett_reduction
701
702zero:
703 // This case is the original crc, so just return it
704 MOVW R10,ret+32(FP)
705 RET
View as plain text