Text file
src/runtime/asm_s390x.s
1 // Copyright 2016 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9
10 // _rt0_s390x_lib is common startup code for s390x systems when
11 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
12 // arrange to invoke this function as a global constructor (for
13 // c-archive) or when the shared library is loaded (for c-shared).
14 // We expect argc and argv to be passed in the usual C ABI registers
15 // R2 and R3.
16 TEXT _rt0_s390x_lib(SB), NOSPLIT|NOFRAME, $0
17 STMG R6, R15, 48(R15)
18 MOVD R2, _rt0_s390x_lib_argc<>(SB)
19 MOVD R3, _rt0_s390x_lib_argv<>(SB)
20
21 // Save R6-R15 in the register save area of the calling function.
22 STMG R6, R15, 48(R15)
23
24 // Allocate 80 bytes on the stack.
25 MOVD $-80(R15), R15
26
27 // Save F8-F15 in our stack frame.
28 FMOVD F8, 16(R15)
29 FMOVD F9, 24(R15)
30 FMOVD F10, 32(R15)
31 FMOVD F11, 40(R15)
32 FMOVD F12, 48(R15)
33 FMOVD F13, 56(R15)
34 FMOVD F14, 64(R15)
35 FMOVD F15, 72(R15)
36
37 // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go
38 XOR g, g
39
40 MOVD $runtime·libInit(SB), R1
41 BL R1
42
43 // Restore F8-F15 from our stack frame.
44 FMOVD 16(R15), F8
45 FMOVD 24(R15), F9
46 FMOVD 32(R15), F10
47 FMOVD 40(R15), F11
48 FMOVD 48(R15), F12
49 FMOVD 56(R15), F13
50 FMOVD 64(R15), F14
51 FMOVD 72(R15), F15
52 MOVD $80(R15), R15
53
54 // Restore R6-R15.
55 LMG 48(R15), R6, R15
56 RET
57
58 // rt0_lib_go initializes the Go runtime.
59 // This is started in a separate thread by _rt0_s390x_lib.
60 TEXT runtime·rt0_lib_go<ABIInternal>(SB), NOSPLIT|NOFRAME, $0
61 MOVD _rt0_s390x_lib_argc<>(SB), R2
62 MOVD _rt0_s390x_lib_argv<>(SB), R3
63 MOVD $runtime·rt0_go(SB), R1
64 BR R1
65
66 DATA _rt0_s390x_lib_argc<>(SB)/8, $0
67 GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8
68 DATA _rt0_s390x_lib_argv<>(SB)/8, $0
69 GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8
70
71 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
72 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
73 // C TLS base pointer in AR0:AR1
74
75 // initialize essential registers
76 XOR R0, R0
77
78 SUB $24, R15
79 MOVW R2, 8(R15) // argc
80 MOVD R3, 16(R15) // argv
81
82 // create istack out of the given (operating system) stack.
83 // _cgo_init may update stackguard.
84 MOVD $runtime·g0(SB), g
85 MOVD R15, R11
86 SUB $(64*1024), R11
87 MOVD R11, g_stackguard0(g)
88 MOVD R11, g_stackguard1(g)
89 MOVD R11, (g_stack+stack_lo)(g)
90 MOVD R15, (g_stack+stack_hi)(g)
91
92 // if there is a _cgo_init, call it using the gcc ABI.
93 MOVD _cgo_init(SB), R11
94 CMPBEQ R11, $0, nocgo
95 MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR
96 SLD $32, R4, R4
97 MOVW AR1, R4 // arg 2: TLS base pointer
98 MOVD $setg_gcc<>(SB), R3 // arg 1: setg
99 MOVD g, R2 // arg 0: G
100 // C functions expect 160 bytes of space on caller stack frame
101 // and an 8-byte aligned stack pointer
102 MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI)
103 SUB $160, R15 // reserve 160 bytes
104 MOVD $~7, R6
105 AND R6, R15 // 8-byte align
106 BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14)
107 MOVD R9, R15 // restore stack
108 XOR R0, R0 // zero R0
109
110 nocgo:
111 // update stackguard after _cgo_init
112 MOVD (g_stack+stack_lo)(g), R2
113 ADD $const_stackGuard, R2
114 MOVD R2, g_stackguard0(g)
115 MOVD R2, g_stackguard1(g)
116
117 // set the per-goroutine and per-mach "registers"
118 MOVD $runtime·m0(SB), R2
119
120 // save m->g0 = g0
121 MOVD g, m_g0(R2)
122 // save m0 to g0->m
123 MOVD R2, g_m(g)
124
125 BL runtime·check(SB)
126
127 // argc/argv are already prepared on stack
128 BL runtime·args(SB)
129 BL runtime·checkS390xCPU(SB)
130 BL runtime·osinit(SB)
131 BL runtime·schedinit(SB)
132
133 // create a new goroutine to start program
134 MOVD $runtime·mainPC(SB), R2 // entry
135 SUB $16, R15
136 MOVD R2, 8(R15)
137 MOVD $0, 0(R15)
138 BL runtime·newproc(SB)
139 ADD $16, R15
140
141 // start this M
142 BL runtime·mstart(SB)
143
144 MOVD $0, 1(R0)
145 RET
146
147 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
148 GLOBL runtime·mainPC(SB),RODATA,$8
149
150 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
151 BRRK
152 RET
153
154 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
155 RET
156
157 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
158 CALL runtime·mstart0(SB)
159 RET // not reached
160
161 /*
162 * go-routine
163 */
164
165 // void gogo(Gobuf*)
166 // restore state from Gobuf; longjmp
167 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
168 MOVD buf+0(FP), R5
169 MOVD gobuf_g(R5), R6
170 MOVD 0(R6), R7 // make sure g != nil
171 BR gogo<>(SB)
172
173 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
174 MOVD R6, g
175 BL runtime·save_g(SB)
176
177 MOVD 0(g), R4
178 MOVD gobuf_sp(R5), R15
179 MOVD gobuf_lr(R5), LR
180 MOVD gobuf_ctxt(R5), R12
181 MOVD $0, gobuf_sp(R5)
182 MOVD $0, gobuf_lr(R5)
183 MOVD $0, gobuf_ctxt(R5)
184 CMP R0, R0 // set condition codes for == test, needed by stack split
185 MOVD gobuf_pc(R5), R6
186 BR (R6)
187
188 // void mcall(fn func(*g))
189 // Switch to m->g0's stack, call fn(g).
190 // Fn must never return. It should gogo(&g->sched)
191 // to keep running g.
192 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT, $-8-8
193 #ifdef GOEXPERIMENT_regabiargs
194 MOVD R2, R12 // context
195 #else
196 MOVD fn+0(FP), R12 // context
197 #endif
198 // Save caller state in g->sched
199 MOVD R15, (g_sched+gobuf_sp)(g)
200 MOVD LR, (g_sched+gobuf_pc)(g)
201 MOVD $0, (g_sched+gobuf_lr)(g)
202
203 // Switch to m->g0 & its stack, call fn.
204 MOVD g, R2
205 MOVD g_m(g), R4
206 MOVD m_g0(R4), g
207 BL runtime·save_g(SB)
208 CMP g, R2
209 BNE 2(PC)
210 BR runtime·badmcall(SB)
211 MOVD 0(R12), R4 // code pointer
212 MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp
213 SUB $16, R15
214 MOVD R2, 8(R15)
215 MOVD $0, 0(R15)
216 BL (R4)
217 BR runtime·badmcall2(SB)
218
219 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
220 // of the G stack. We need to distinguish the routine that
221 // lives at the bottom of the G stack from the one that lives
222 // at the top of the system stack because the one at the top of
223 // the system stack terminates the stack walk (see topofstack()).
224 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
225 UNDEF
226 BL (LR) // make sure this function is not leaf
227 RET
228
229 // func systemstack(fn func())
230 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
231 MOVD fn+0(FP), R3 // R3 = fn
232 MOVD R3, R12 // context
233 MOVD g_m(g), R4 // R4 = m
234
235 MOVD m_gsignal(R4), R5 // R5 = gsignal
236 CMPBEQ g, R5, noswitch
237
238 MOVD m_g0(R4), R5 // R5 = g0
239 CMPBEQ g, R5, noswitch
240
241 MOVD m_curg(R4), R6
242 CMPBEQ g, R6, switch
243
244 // Bad: g is not gsignal, not g0, not curg. What is it?
245 // Hide call from linker nosplit analysis.
246 MOVD $runtime·badsystemstack(SB), R3
247 BL (R3)
248 BL runtime·abort(SB)
249
250 switch:
251 // save our state in g->sched. Pretend to
252 // be systemstack_switch if the G stack is scanned.
253 BL gosave_systemstack_switch<>(SB)
254
255 // switch to g0
256 MOVD R5, g
257 BL runtime·save_g(SB)
258 MOVD (g_sched+gobuf_sp)(g), R15
259
260 // call target function
261 MOVD 0(R12), R3 // code pointer
262 BL (R3)
263
264 // switch back to g
265 MOVD g_m(g), R3
266 MOVD m_curg(R3), g
267 BL runtime·save_g(SB)
268 MOVD (g_sched+gobuf_sp)(g), R15
269 MOVD $0, (g_sched+gobuf_sp)(g)
270 RET
271
272 noswitch:
273 // already on m stack, just call directly
274 // Using a tail call here cleans up tracebacks since we won't stop
275 // at an intermediate systemstack.
276 MOVD 0(R12), R3 // code pointer
277 MOVD 0(R15), LR // restore LR
278 ADD $8, R15
279 BR (R3)
280
281 // func switchToCrashStack0(fn func())
282 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
283 MOVD R2, R12 // context
284 MOVD g_m(g), R2 // curm
285
286 // set g to gcrash
287 MOVD $runtime·gcrash(SB), g // g = &gcrash
288 BL runtime·save_g(SB)
289 MOVD R2, g_m(g) // g.m = curm
290 MOVD g, m_g0(R2) // curm.g0 = g
291
292 // switch to crashstack
293 MOVD (g_stack+stack_hi)(g), R2
294 ADD $(-4*8), R2, R15
295
296 // call target function
297 MOVD 0(R12), R3 // code pointer
298 BL (R3)
299
300 // should never return
301 BL runtime·abort(SB)
302 UNDEF
303
304 /*
305 * support for morestack
306 */
307
308 // Called during function prolog when more stack is needed.
309 // Caller has already loaded:
310 // R3: framesize, R4: argsize, R5: LR
311 //
312 // The traceback routines see morestack on a g0 as being
313 // the top of a stack (for example, morestack calling newstack
314 // calling the scheduler calling newm calling gc), so we must
315 // record an argument size. For that purpose, it has no arguments.
316 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
317 // Called from f.
318 // Set g->sched to context in f.
319 MOVD R15, (g_sched+gobuf_sp)(g)
320 MOVD LR, R8
321 MOVD R8, (g_sched+gobuf_pc)(g)
322 MOVD R5, (g_sched+gobuf_lr)(g)
323 MOVD R12, (g_sched+gobuf_ctxt)(g)
324
325 // Cannot grow scheduler stack (m->g0).
326 MOVD g_m(g), R7
327 MOVD m_g0(R7), R8
328 CMPBNE g, R8, 3(PC)
329 BL runtime·badmorestackg0(SB)
330 BL runtime·abort(SB)
331
332 // Cannot grow signal stack (m->gsignal).
333 MOVD m_gsignal(R7), R8
334 CMP g, R8
335 BNE 3(PC)
336 BL runtime·badmorestackgsignal(SB)
337 BL runtime·abort(SB)
338
339 // Called from f.
340 // Set m->morebuf to f's caller.
341 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
342 MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
343 MOVD g, (m_morebuf+gobuf_g)(R7)
344
345 // Call newstack on m->g0's stack.
346 MOVD m_g0(R7), g
347 BL runtime·save_g(SB)
348 MOVD (g_sched+gobuf_sp)(g), R15
349 // Create a stack frame on g0 to call newstack.
350 MOVD $0, -8(R15) // Zero saved LR in frame
351 SUB $8, R15
352 BL runtime·newstack(SB)
353
354 // Not reached, but make sure the return PC from the call to newstack
355 // is still in this function, and not the beginning of the next.
356 UNDEF
357
358 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
359 // Force SPWRITE. This function doesn't actually write SP,
360 // but it is called with a special calling convention where
361 // the caller doesn't save LR on stack but passes it as a
362 // register (R5), and the unwinder currently doesn't understand.
363 // Make it SPWRITE to stop unwinding. (See issue 54332)
364 MOVD R15, R15
365
366 MOVD $0, R12
367 BR runtime·morestack(SB)
368
369 // reflectcall: call a function with the given argument list
370 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
371 // we don't have variable-sized frames, so we use a small number
372 // of constant-sized-frame functions to encode a few bits of size in the pc.
373 // Caution: ugly multiline assembly macros in your future!
374
375 #define DISPATCH(NAME,MAXSIZE) \
376 MOVD $MAXSIZE, R4; \
377 CMP R3, R4; \
378 BGT 3(PC); \
379 MOVD $NAME(SB), R5; \
380 BR (R5)
381 // Note: can't just "BR NAME(SB)" - bad inlining results.
382
383 TEXT ·reflectcall(SB), NOSPLIT, $-8-48
384 MOVWZ frameSize+32(FP), R3
385 DISPATCH(runtime·call16, 16)
386 DISPATCH(runtime·call32, 32)
387 DISPATCH(runtime·call64, 64)
388 DISPATCH(runtime·call128, 128)
389 DISPATCH(runtime·call256, 256)
390 DISPATCH(runtime·call512, 512)
391 DISPATCH(runtime·call1024, 1024)
392 DISPATCH(runtime·call2048, 2048)
393 DISPATCH(runtime·call4096, 4096)
394 DISPATCH(runtime·call8192, 8192)
395 DISPATCH(runtime·call16384, 16384)
396 DISPATCH(runtime·call32768, 32768)
397 DISPATCH(runtime·call65536, 65536)
398 DISPATCH(runtime·call131072, 131072)
399 DISPATCH(runtime·call262144, 262144)
400 DISPATCH(runtime·call524288, 524288)
401 DISPATCH(runtime·call1048576, 1048576)
402 DISPATCH(runtime·call2097152, 2097152)
403 DISPATCH(runtime·call4194304, 4194304)
404 DISPATCH(runtime·call8388608, 8388608)
405 DISPATCH(runtime·call16777216, 16777216)
406 DISPATCH(runtime·call33554432, 33554432)
407 DISPATCH(runtime·call67108864, 67108864)
408 DISPATCH(runtime·call134217728, 134217728)
409 DISPATCH(runtime·call268435456, 268435456)
410 DISPATCH(runtime·call536870912, 536870912)
411 DISPATCH(runtime·call1073741824, 1073741824)
412 MOVD $runtime·badreflectcall(SB), R5
413 BR (R5)
414
415 #define CALLFN(NAME,MAXSIZE) \
416 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
417 NO_LOCAL_POINTERS; \
418 /* copy arguments to stack */ \
419 MOVD stackArgs+16(FP), R4; \
420 MOVWZ stackArgsSize+24(FP), R5; \
421 MOVD $stack-MAXSIZE(SP), R6; \
422 loopArgs: /* copy 256 bytes at a time */ \
423 CMP R5, $256; \
424 BLT tailArgs; \
425 SUB $256, R5; \
426 MVC $256, 0(R4), 0(R6); \
427 MOVD $256(R4), R4; \
428 MOVD $256(R6), R6; \
429 BR loopArgs; \
430 tailArgs: /* copy remaining bytes */ \
431 CMP R5, $0; \
432 BEQ callFunction; \
433 SUB $1, R5; \
434 EXRL $callfnMVC<>(SB), R5; \
435 callFunction: \
436 MOVD f+8(FP), R12; \
437 MOVD regArgs+40(FP), R10; \
438 BL ·unspillArgs(SB); \
439 MOVD (R12), R10; \
440 PCDATA $PCDATA_StackMapIndex, $0; \
441 BL (R10); \
442 /* copy return values back */ \
443 MOVD regArgs+40(FP), R10; \
444 BL ·spillArgs(SB); \
445 MOVD stackArgsType+0(FP), R7; \
446 MOVD stackArgs+16(FP), R6; \
447 MOVWZ stackArgsSize+24(FP), R5; \
448 MOVD $stack-MAXSIZE(SP), R4; \
449 MOVWZ stackRetOffset+28(FP), R1; \
450 ADD R1, R4; \
451 ADD R1, R6; \
452 SUB R1, R5; \
453 BL callRet<>(SB); \
454 RET
455
456 // callRet copies return values back at the end of call*. This is a
457 // separate function so it can allocate stack space for the arguments
458 // to reflectcallmove. It does not follow the Go ABI; it expects its
459 // arguments in registers.
460 TEXT callRet<>(SB), NOSPLIT, $40-0
461 NO_LOCAL_POINTERS;
462 MOVD R7, 8(R15)
463 MOVD R6, 16(R15)
464 MOVD R4, 24(R15)
465 MOVD R5, 32(R15)
466 MOVD R10, 40(R15)
467 BL runtime·reflectcallmove(SB)
468 RET
469
470 CALLFN(·call16, 16)
471 CALLFN(·call32, 32)
472 CALLFN(·call64, 64)
473 CALLFN(·call128, 128)
474 CALLFN(·call256, 256)
475 CALLFN(·call512, 512)
476 CALLFN(·call1024, 1024)
477 CALLFN(·call2048, 2048)
478 CALLFN(·call4096, 4096)
479 CALLFN(·call8192, 8192)
480 CALLFN(·call16384, 16384)
481 CALLFN(·call32768, 32768)
482 CALLFN(·call65536, 65536)
483 CALLFN(·call131072, 131072)
484 CALLFN(·call262144, 262144)
485 CALLFN(·call524288, 524288)
486 CALLFN(·call1048576, 1048576)
487 CALLFN(·call2097152, 2097152)
488 CALLFN(·call4194304, 4194304)
489 CALLFN(·call8388608, 8388608)
490 CALLFN(·call16777216, 16777216)
491 CALLFN(·call33554432, 33554432)
492 CALLFN(·call67108864, 67108864)
493 CALLFN(·call134217728, 134217728)
494 CALLFN(·call268435456, 268435456)
495 CALLFN(·call536870912, 536870912)
496 CALLFN(·call1073741824, 1073741824)
497
498 // Not a function: target for EXRL (execute relative long) instruction.
499 TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0
500 MVC $1, 0(R4), 0(R6)
501
502 TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
503 RET
504
505 // Save state of caller into g->sched,
506 // but using fake PC from systemstack_switch.
507 // Must only be called from functions with no locals ($0)
508 // or else unwinding from systemstack_switch is incorrect.
509 // Smashes R1.
510 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
511 MOVD $runtime·systemstack_switch(SB), R1
512 ADD $16, R1 // get past prologue
513 MOVD R1, (g_sched+gobuf_pc)(g)
514 MOVD R15, (g_sched+gobuf_sp)(g)
515 MOVD $0, (g_sched+gobuf_lr)(g)
516 // Assert ctxt is zero. See func save.
517 MOVD (g_sched+gobuf_ctxt)(g), R1
518 CMPBEQ R1, $0, 2(PC)
519 BL runtime·abort(SB)
520 RET
521
522 // func asmcgocall(fn, arg unsafe.Pointer) int32
523 // Call fn(arg) on the scheduler stack,
524 // aligned appropriately for the gcc ABI.
525 // See cgocall.go for more details.
526 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
527 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
528 // C TLS base pointer in AR0:AR1
529 MOVD fn+0(FP), R3
530 MOVD arg+8(FP), R4
531
532 MOVD R15, R2 // save original stack pointer
533 MOVD g, R5
534
535 // Figure out if we need to switch to m->g0 stack.
536 // We get called to create new OS threads too, and those
537 // come in on the m->g0 stack already. Or we might already
538 // be on the m->gsignal stack.
539 CMPBEQ g, $0, nosave
540 MOVD g_m(g), R6
541 MOVD m_gsignal(R6), R7
542 CMPBEQ R7, g, g0
543 MOVD m_g0(R6), R7
544 CMPBEQ R7, g, g0
545 BL gosave_systemstack_switch<>(SB)
546 MOVD R7, g
547 BL runtime·save_g(SB)
548 MOVD (g_sched+gobuf_sp)(g), R15
549
550 // Now on a scheduling stack (a pthread-created stack).
551 g0:
552 // Save room for two of our pointers, plus 160 bytes of callee
553 // save area that lives on the caller stack.
554 SUB $176, R15
555 MOVD $~7, R6
556 AND R6, R15 // 8-byte alignment for gcc ABI
557 MOVD R5, 168(R15) // save old g on stack
558 MOVD (g_stack+stack_hi)(R5), R5
559 SUB R2, R5
560 MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
561 MOVD $0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?)
562 MOVD R4, R2 // arg in R2
563 BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15
564
565 XOR R0, R0 // set R0 back to 0.
566 // Restore g, stack pointer.
567 MOVD 168(R15), g
568 BL runtime·save_g(SB)
569 MOVD (g_stack+stack_hi)(g), R5
570 MOVD 160(R15), R6
571 SUB R6, R5
572 MOVD R5, R15
573
574 MOVW R2, ret+16(FP)
575 RET
576
577 nosave:
578 // Running on a system stack, perhaps even without a g.
579 // Having no g can happen during thread creation or thread teardown.
580 MOVD fn+0(FP), R3
581 MOVD arg+8(FP), R4
582 MOVD R15, R2
583 SUB $176, R15
584 MOVD $~7, R6
585 AND R6, R15
586 MOVD $0, 168(R15) // Where above code stores g, in case someone looks during debugging.
587 MOVD R2, 160(R15) // Save original stack pointer.
588 MOVD $0, 0(R15) // clear back chain pointer
589 MOVD R4, R2 // arg in R2
590 BL R3
591 XOR R0, R0
592 MOVD 160(R15), R15 // Restore stack pointer.
593 MOVW R2, ret+16(FP)
594 RET
595
596 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
597 // Call fn(arg) aligned appropriately for the gcc ABI.
598 // Called on a system stack, and there may be no g yet.
599 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
600 MOVD fn+0(FP), R3
601 MOVD arg+8(FP), R4
602
603 MOVD R15, R2 // Save original stack pointer.
604
605 // Save room for the stack pointer, plus 160 bytes of callee
606 // save area that lives on the caller stack.
607 SUB $168, R15
608 MOVD $~7, R6
609 AND R6, R15
610
611 MOVD R2, 160(R15) // Save original stack pointer.
612 MOVD $0, 0(R15) // clear back chain pointer
613 MOVD R4, R2 // arg in R2
614 BL R3
615 XOR R0, R0
616 MOVD 160(R15), R15 // Restore stack pointer.
617 RET
618
619 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
620 // See cgocall.go for more details.
621 TEXT ·cgocallback(SB),NOSPLIT,$24-24
622 NO_LOCAL_POINTERS
623
624 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
625 // It is used to dropm while thread is exiting.
626 MOVD fn+0(FP), R1
627 CMPBNE R1, $0, loadg
628 // Restore the g from frame.
629 MOVD frame+8(FP), g
630 BR dropm
631
632 loadg:
633 // Load m and g from thread-local storage.
634 MOVB runtime·iscgo(SB), R3
635 CMPBEQ R3, $0, nocgo
636 BL runtime·load_g(SB)
637
638 nocgo:
639 // If g is nil, Go did not create the current thread,
640 // or if this thread never called into Go on pthread platforms.
641 // Call needm to obtain one for temporary use.
642 // In this case, we're running on the thread stack, so there's
643 // lots of space, but the linker doesn't know. Hide the call from
644 // the linker analysis by using an indirect call.
645 CMPBEQ g, $0, needm
646
647 MOVD g_m(g), R8
648 MOVD R8, savedm-8(SP)
649 BR havem
650
651 needm:
652 MOVD g, savedm-8(SP) // g is zero, so is m.
653 MOVD $runtime·needAndBindM(SB), R3
654 BL (R3)
655
656 // Set m->sched.sp = SP, so that if a panic happens
657 // during the function we are about to execute, it will
658 // have a valid SP to run on the g0 stack.
659 // The next few lines (after the havem label)
660 // will save this SP onto the stack and then write
661 // the same SP back to m->sched.sp. That seems redundant,
662 // but if an unrecovered panic happens, unwindm will
663 // restore the g->sched.sp from the stack location
664 // and then systemstack will try to use it. If we don't set it here,
665 // that restored SP will be uninitialized (typically 0) and
666 // will not be usable.
667 MOVD g_m(g), R8
668 MOVD m_g0(R8), R3
669 MOVD R15, (g_sched+gobuf_sp)(R3)
670
671 havem:
672 // Now there's a valid m, and we're running on its m->g0.
673 // Save current m->g0->sched.sp on stack and then set it to SP.
674 // Save current sp in m->g0->sched.sp in preparation for
675 // switch back to m->curg stack.
676 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
677 MOVD m_g0(R8), R3
678 MOVD (g_sched+gobuf_sp)(R3), R4
679 MOVD R4, savedsp-24(SP) // must match frame size
680 MOVD R15, (g_sched+gobuf_sp)(R3)
681
682 // Switch to m->curg stack and call runtime.cgocallbackg.
683 // Because we are taking over the execution of m->curg
684 // but *not* resuming what had been running, we need to
685 // save that information (m->curg->sched) so we can restore it.
686 // We can restore m->curg->sched.sp easily, because calling
687 // runtime.cgocallbackg leaves SP unchanged upon return.
688 // To save m->curg->sched.pc, we push it onto the curg stack and
689 // open a frame the same size as cgocallback's g0 frame.
690 // Once we switch to the curg stack, the pushed PC will appear
691 // to be the return PC of cgocallback, so that the traceback
692 // will seamlessly trace back into the earlier calls.
693 MOVD m_curg(R8), g
694 BL runtime·save_g(SB)
695 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
696 MOVD (g_sched+gobuf_pc)(g), R5
697 MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size
698 // Gather our arguments into registers.
699 MOVD fn+0(FP), R1
700 MOVD frame+8(FP), R2
701 MOVD ctxt+16(FP), R3
702 MOVD $-(24+8)(R4), R15 // switch stack; must match frame size
703 MOVD R1, 8(R15)
704 MOVD R2, 16(R15)
705 MOVD R3, 24(R15)
706 BL runtime·cgocallbackg(SB)
707
708 // Restore g->sched (== m->curg->sched) from saved values.
709 MOVD 0(R15), R5
710 MOVD R5, (g_sched+gobuf_pc)(g)
711 MOVD $(24+8)(R15), R4 // must match frame size
712 MOVD R4, (g_sched+gobuf_sp)(g)
713
714 // Switch back to m->g0's stack and restore m->g0->sched.sp.
715 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
716 // so we do not have to restore it.)
717 MOVD g_m(g), R8
718 MOVD m_g0(R8), g
719 BL runtime·save_g(SB)
720 MOVD (g_sched+gobuf_sp)(g), R15
721 MOVD savedsp-24(SP), R4 // must match frame size
722 MOVD R4, (g_sched+gobuf_sp)(g)
723
724 // If the m on entry was nil, we called needm above to borrow an m,
725 // 1. for the duration of the call on non-pthread platforms,
726 // 2. or the duration of the C thread alive on pthread platforms.
727 // If the m on entry wasn't nil,
728 // 1. the thread might be a Go thread,
729 // 2. or it wasn't the first call from a C thread on pthread platforms,
730 // since then we skip dropm to reuse the m in the first call.
731 MOVD savedm-8(SP), R6
732 CMPBNE R6, $0, droppedm
733
734 // Skip dropm to reuse it in the next call, when a pthread key has been created.
735 MOVD _cgo_pthread_key_created(SB), R6
736 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
737 CMPBEQ R6, $0, dropm
738 MOVD (R6), R6
739 CMPBNE R6, $0, droppedm
740
741 dropm:
742 MOVD $runtime·dropm(SB), R3
743 BL (R3)
744 droppedm:
745
746 // Done!
747 RET
748
749 // void setg(G*); set g. for use by needm.
750 TEXT runtime·setg(SB), NOSPLIT, $0-8
751 MOVD gg+0(FP), g
752 // This only happens if iscgo, so jump straight to save_g
753 BL runtime·save_g(SB)
754 RET
755
756 // void setg_gcc(G*); set g in C TLS.
757 // Must obey the gcc calling convention.
758 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
759 // The standard prologue clobbers LR (R14), which is callee-save in
760 // the C ABI, so we have to use NOFRAME and save LR ourselves.
761 MOVD LR, R1
762 // Also save g, R10, and R11 since they're callee-save in C ABI
763 MOVD R10, R3
764 MOVD g, R4
765 MOVD R11, R5
766
767 MOVD R2, g
768 BL runtime·save_g(SB)
769
770 MOVD R5, R11
771 MOVD R4, g
772 MOVD R3, R10
773 MOVD R1, LR
774 RET
775
776 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
777 MOVW (R0), R0
778 UNDEF
779
780 // int64 runtime·cputicks(void)
781 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
782 // The TOD clock on s390 counts from the year 1900 in ~250ps intervals.
783 // This means that since about 1972 the msb has been set, making the
784 // result of a call to STORE CLOCK (stck) a negative number.
785 // We clear the msb to make it positive.
786 STCK ret+0(FP) // serialises before and after call
787 MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043
788 SLD $1, R3
789 SRD $1, R3
790 MOVD R3, ret+0(FP)
791 RET
792
793 #ifdef GOEXPERIMENT_regabiargs
794 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R10.
795 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
796 MOVD R2, 0(R10)
797 MOVD R3, 8(R10)
798 MOVD R4, 16(R10)
799 MOVD R5, 24(R10)
800 MOVD R6, 32(R10)
801 MOVD R7, 40(R10)
802 MOVD R8, 48(R10)
803 MOVD R9, 56(R10)
804 FMOVD F0, 64(R10)
805 FMOVD F1, 72(R10)
806 FMOVD F2, 80(R10)
807 FMOVD F3, 88(R10)
808 FMOVD F4, 96(R10)
809 FMOVD F5, 104(R10)
810 FMOVD F6, 112(R10)
811 FMOVD F7, 120(R10)
812 FMOVD F8, 128(R10)
813 FMOVD F9, 136(R10)
814 FMOVD F10, 144(R10)
815 FMOVD F11, 152(R10)
816 FMOVD F12, 160(R10)
817 FMOVD F13, 168(R10)
818 FMOVD F14, 176(R10)
819 FMOVD F15, 184(R10)
820 RET
821
822 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R10.
823 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
824 MOVD 0(R10), R2
825 MOVD 8(R10), R3
826 MOVD 16(R10), R4
827 MOVD 24(R10), R5
828 MOVD 32(R10), R6
829 MOVD 40(R10), R7
830 MOVD 48(R10), R8
831 MOVD 56(R10), R9
832 FMOVD 64(R10), F0
833 FMOVD 72(R10), F1
834 FMOVD 80(R10), F2
835 FMOVD 88(R10), F3
836 FMOVD 96(R10), F4
837 FMOVD 104(R10), F5
838 FMOVD 112(R10), F6
839 FMOVD 120(R10), F7
840 FMOVD 128(R10), F8
841 FMOVD 136(R10), F9
842 FMOVD 144(R10), F10
843 FMOVD 152(R10), F11
844 FMOVD 160(R10), F12
845 FMOVD 168(R10), F13
846 FMOVD 176(R10), F14
847 FMOVD 184(R10), F15
848 RET
849 #else
850
851 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
852 RET
853
854 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
855 RET
856 #endif
857
858 // AES hashing not implemented for s390x
859 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
860 JMP runtime·memhashFallback<ABIInternal>(SB)
861 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
862 JMP runtime·strhashFallback<ABIInternal>(SB)
863 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
864 JMP runtime·memhash32Fallback<ABIInternal>(SB)
865 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
866 JMP runtime·memhash64Fallback<ABIInternal>(SB)
867
868 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
869 // Must obey the gcc calling convention.
870 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
871 // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them
872 MOVD g, R1
873 MOVD R10, R3
874 MOVD LR, R4
875 MOVD R11, R5
876
877 BL runtime·load_g(SB) // clobbers g (R13), R10, R11
878 MOVD g_m(g), R2
879 MOVD m_curg(R2), R2
880 MOVD (g_stack+stack_hi)(R2), R2
881
882 MOVD R1, g
883 MOVD R3, R10
884 MOVD R4, LR
885 MOVD R5, R11
886 RET
887
888 // The top-most function running on a goroutine
889 // returns to goexit+PCQuantum.
890 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
891 BYTE $0x07; BYTE $0x00; // 2-byte nop
892 BL runtime·goexit1(SB) // does not return
893 // traceback from goexit1 must hit code range of goexit
894 BYTE $0x07; BYTE $0x00; // 2-byte nop
895
896 TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
897 // Stores are already ordered on s390x, so this is just a
898 // compile barrier.
899 RET
900
901 // This is called from .init_array and follows the platform, not Go, ABI.
902 // We are overly conservative. We could only save the registers we use.
903 // However, since this function is only called once per loaded module
904 // performance is unimportant.
905 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
906 // Save R6-R15 in the register save area of the calling function.
907 // Don't bother saving F8-F15 as we aren't doing any calls.
908 STMG R6, R15, 48(R15)
909
910 // append the argument (passed in R2, as per the ELF ABI) to the
911 // moduledata linked list.
912 MOVD runtime·lastmoduledatap(SB), R1
913 MOVD R2, moduledata_next(R1)
914 MOVD R2, runtime·lastmoduledatap(SB)
915
916 // Restore R6-R15.
917 LMG 48(R15), R6, R15
918 RET
919
920 TEXT ·checkASM(SB),NOSPLIT,$0-1
921 MOVB $1, ret+0(FP)
922 RET
923
924 // gcWriteBarrier informs the GC about heap pointer writes.
925 //
926 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
927 // number of bytes of buffer needed in R9, and returns a pointer
928 // to the buffer space in R9.
929 // It clobbers R10 (the temp register) and R1 (used by PLT stub).
930 // It does not clobber any other general-purpose registers,
931 // but may clobber others (e.g., floating point registers).
932 TEXT gcWriteBarrier<>(SB),NOSPLIT,$96
933 // Save the registers clobbered by the fast path.
934 MOVD R4, 96(R15)
935 retry:
936 MOVD g_m(g), R1
937 MOVD m_p(R1), R1
938 // Increment wbBuf.next position.
939 MOVD R9, R4
940 ADD (p_wbBuf+wbBuf_next)(R1), R4
941 // Is the buffer full?
942 MOVD (p_wbBuf+wbBuf_end)(R1), R10
943 CMPUBGT R4, R10, flush
944 // Commit to the larger buffer.
945 MOVD R4, (p_wbBuf+wbBuf_next)(R1)
946 // Make return value (the original next position)
947 SUB R9, R4, R9
948 // Restore registers.
949 MOVD 96(R15), R4
950 RET
951
952 flush:
953 // Save all general purpose registers since these could be
954 // clobbered by wbBufFlush and were not saved by the caller.
955 STMG R2, R3, 8(R15)
956 MOVD R0, 24(R15)
957 // R1 already saved.
958 // R4 already saved.
959 STMG R5, R12, 32(R15) // save R5 - R12
960 // R13 is g.
961 // R14 is LR.
962 // R15 is SP.
963
964 CALL runtime·wbBufFlush(SB)
965
966 LMG 8(R15), R2, R3 // restore R2 - R3
967 MOVD 24(R15), R0 // restore R0
968 LMG 32(R15), R5, R12 // restore R5 - R12
969 JMP retry
970
971 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
972 MOVD $8, R9
973 JMP gcWriteBarrier<>(SB)
974 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
975 MOVD $16, R9
976 JMP gcWriteBarrier<>(SB)
977 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
978 MOVD $24, R9
979 JMP gcWriteBarrier<>(SB)
980 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
981 MOVD $32, R9
982 JMP gcWriteBarrier<>(SB)
983 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
984 MOVD $40, R9
985 JMP gcWriteBarrier<>(SB)
986 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
987 MOVD $48, R9
988 JMP gcWriteBarrier<>(SB)
989 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
990 MOVD $56, R9
991 JMP gcWriteBarrier<>(SB)
992 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
993 MOVD $64, R9
994 JMP gcWriteBarrier<>(SB)
995
996 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
997 NO_LOCAL_POINTERS
998 // Save all 16 int registers that could have an index in them.
999 // They may be pointers, but if they are they are dead.
1000 STMG R0, R12, 24(R15)
1001 // Note that R10 @ 104 is not needed, it is an assembler temp
1002 // skip R13 aka G @ 128
1003 // skip R14 aka LR @ 136
1004 // skip R15 aka SP @ 144
1005
1006 MOVD R14, R2 // PC immediately after call to panicBounds
1007 ADD $24, R15, R3 // pointer to save area
1008 CALL runtime·panicBounds64<ABIInternal>(SB)
1009 RET
1010
View as plain text