Text file
src/runtime/asm_arm.s
1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9
10 // _rt0_arm is common startup code for most ARM systems when using
11 // internal linking. This is the entry point for the program from the
12 // kernel for an ordinary -buildmode=exe program. The stack holds the
13 // number of arguments and the C-style argv.
14 TEXT _rt0_arm(SB),NOSPLIT|NOFRAME,$0
15 MOVW (R13), R0 // argc
16 MOVW $4(R13), R1 // argv
17 B runtime·rt0_go(SB)
18
19 // main is common startup code for most ARM systems when using
20 // external linking. The C startup code will call the symbol "main"
21 // passing argc and argv in the usual C ABI registers R0 and R1.
22 TEXT main(SB),NOSPLIT|NOFRAME,$0
23 B runtime·rt0_go(SB)
24
25 // _rt0_arm_lib is common startup code for most ARM systems when
26 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
27 // arrange to invoke this function as a global constructor (for
28 // c-archive) or when the shared library is loaded (for c-shared).
29 // We expect argc and argv to be passed in the usual C ABI registers
30 // R0 and R1.
31 TEXT _rt0_arm_lib(SB),NOSPLIT,$104
32 // Preserve callee-save registers. Raspberry Pi's dlopen(), for example,
33 // actually cares that R11 is preserved.
34 MOVW R4, 12(R13)
35 MOVW R5, 16(R13)
36 MOVW R6, 20(R13)
37 MOVW R7, 24(R13)
38 MOVW R8, 28(R13)
39 MOVW g, 32(R13)
40 MOVW R11, 36(R13)
41
42 // Skip floating point registers on goarmsoftfp != 0.
43 MOVB runtime·goarmsoftfp(SB), R11
44 CMP $0, R11
45 BNE skipfpsave
46 MOVD F8, (40+8*0)(R13)
47 MOVD F9, (40+8*1)(R13)
48 MOVD F10, (40+8*2)(R13)
49 MOVD F11, (40+8*3)(R13)
50 MOVD F12, (40+8*4)(R13)
51 MOVD F13, (40+8*5)(R13)
52 MOVD F14, (40+8*6)(R13)
53 MOVD F15, (40+8*7)(R13)
54 skipfpsave:
55 // Save argc/argv.
56 MOVW R0, _rt0_arm_lib_argc<>(SB)
57 MOVW R1, _rt0_arm_lib_argv<>(SB)
58
59 MOVW $0, g // Initialize g.
60
61 CALL runtime·libInit(SB)
62
63 // Restore callee-save registers and return.
64 MOVB runtime·goarmsoftfp(SB), R11
65 CMP $0, R11
66 BNE skipfprest
67 MOVD (40+8*0)(R13), F8
68 MOVD (40+8*1)(R13), F9
69 MOVD (40+8*2)(R13), F10
70 MOVD (40+8*3)(R13), F11
71 MOVD (40+8*4)(R13), F12
72 MOVD (40+8*5)(R13), F13
73 MOVD (40+8*6)(R13), F14
74 MOVD (40+8*7)(R13), F15
75 skipfprest:
76 MOVW 12(R13), R4
77 MOVW 16(R13), R5
78 MOVW 20(R13), R6
79 MOVW 24(R13), R7
80 MOVW 28(R13), R8
81 MOVW 32(R13), g
82 MOVW 36(R13), R11
83 RET
84
85 // rt0_lib_go initializes the Go runtime.
86 // This is started in a separate thread by _rt0_arm_lib.
87 TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$8
88 MOVW _rt0_arm_lib_argc<>(SB), R0
89 MOVW _rt0_arm_lib_argv<>(SB), R1
90 B runtime·rt0_go(SB)
91
92 DATA _rt0_arm_lib_argc<>(SB)/4,$0
93 GLOBL _rt0_arm_lib_argc<>(SB),NOPTR,$4
94 DATA _rt0_arm_lib_argv<>(SB)/4,$0
95 GLOBL _rt0_arm_lib_argv<>(SB),NOPTR,$4
96
97 // using NOFRAME means do not save LR on stack.
98 // argc is in R0, argv is in R1.
99 TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0
100 MOVW $0xcafebabe, R12
101
102 // copy arguments forward on an even stack
103 // use R13 instead of SP to avoid linker rewriting the offsets
104 SUB $64, R13 // plenty of scratch
105 AND $~7, R13
106 MOVW R0, 60(R13) // save argc, argv away
107 MOVW R1, 64(R13)
108
109 // set up g register
110 // g is R10
111 MOVW $runtime·g0(SB), g
112 MOVW $runtime·m0(SB), R8
113
114 // save m->g0 = g0
115 MOVW g, m_g0(R8)
116 // save g->m = m0
117 MOVW R8, g_m(g)
118
119 // create istack out of the OS stack
120 // (1MB of system stack is available on iOS and Android)
121 MOVW $(-64*1024+104)(R13), R0
122 MOVW R0, g_stackguard0(g)
123 MOVW R0, g_stackguard1(g)
124 MOVW R0, (g_stack+stack_lo)(g)
125 MOVW R13, (g_stack+stack_hi)(g)
126
127 BL runtime·emptyfunc(SB) // fault if stack check is wrong
128
129 #ifdef GOOS_openbsd
130 // Save g to TLS so that it is available from signal trampoline.
131 BL runtime·save_g(SB)
132 #endif
133
134 BL runtime·_initcgo(SB) // will clobber R0-R3
135
136 // update stackguard after _cgo_init
137 MOVW (g_stack+stack_lo)(g), R0
138 ADD $const_stackGuard, R0
139 MOVW R0, g_stackguard0(g)
140 MOVW R0, g_stackguard1(g)
141
142 BL runtime·check(SB)
143
144 // saved argc, argv
145 MOVW 60(R13), R0
146 MOVW R0, 4(R13)
147 MOVW 64(R13), R1
148 MOVW R1, 8(R13)
149 BL runtime·args(SB)
150 BL runtime·checkgoarm(SB)
151 BL runtime·osinit(SB)
152 BL runtime·schedinit(SB)
153
154 // create a new goroutine to start program
155 SUB $8, R13
156 MOVW $runtime·mainPC(SB), R0
157 MOVW R0, 4(R13) // arg 1: fn
158 MOVW $0, R0
159 MOVW R0, 0(R13) // dummy LR
160 BL runtime·newproc(SB)
161 ADD $8, R13 // pop args and LR
162
163 // start this M
164 BL runtime·mstart(SB)
165
166 MOVW $1234, R0
167 MOVW $1000, R1
168 MOVW R0, (R1) // fail hard
169
170 DATA runtime·mainPC+0(SB)/4,$runtime·main(SB)
171 GLOBL runtime·mainPC(SB),RODATA,$4
172
173 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
174 // gdb won't skip this breakpoint instruction automatically,
175 // so you must manually "set $pc+=4" to skip it and continue.
176 #ifdef GOOS_plan9
177 WORD $0xD1200070 // undefined instruction used as armv5 breakpoint in Plan 9
178 #else
179 WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint
180 #endif
181 RET
182
183 TEXT runtime·asminit(SB),NOSPLIT,$0-0
184 // disable runfast (flush-to-zero) mode of vfp if runtime.goarmsoftfp == 0
185 MOVB runtime·goarmsoftfp(SB), R11
186 CMP $0, R11
187 BNE 4(PC)
188 WORD $0xeef1ba10 // vmrs r11, fpscr
189 BIC $(1<<24), R11
190 WORD $0xeee1ba10 // vmsr fpscr, r11
191 RET
192
193 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
194 BL runtime·mstart0(SB)
195 RET // not reached
196
197 /*
198 * go-routine
199 */
200
201 // void gogo(Gobuf*)
202 // restore state from Gobuf; longjmp
203 TEXT runtime·gogo(SB),NOSPLIT|NOFRAME,$0-4
204 MOVW buf+0(FP), R1
205 MOVW gobuf_g(R1), R0
206 MOVW 0(R0), R2 // make sure g != nil
207 B gogo<>(SB)
208
209 TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0
210 BL setg<>(SB)
211 MOVW gobuf_sp(R1), R13 // restore SP==R13
212 MOVW gobuf_lr(R1), LR
213 MOVW gobuf_ctxt(R1), R7
214 MOVW $0, R11
215 MOVW R11, gobuf_sp(R1) // clear to help garbage collector
216 MOVW R11, gobuf_lr(R1)
217 MOVW R11, gobuf_ctxt(R1)
218 MOVW gobuf_pc(R1), R11
219 CMP R11, R11 // set condition codes for == test, needed by stack split
220 B (R11)
221
222 // func mcall(fn func(*g))
223 // Switch to m->g0's stack, call fn(g).
224 // Fn must never return. It should gogo(&g->sched)
225 // to keep running g.
226 TEXT runtime·mcall(SB),NOSPLIT|NOFRAME,$0-4
227 // Save caller state in g->sched.
228 MOVW R13, (g_sched+gobuf_sp)(g)
229 MOVW LR, (g_sched+gobuf_pc)(g)
230 MOVW $0, R11
231 MOVW R11, (g_sched+gobuf_lr)(g)
232
233 // Switch to m->g0 & its stack, call fn.
234 MOVW g, R1
235 MOVW g_m(g), R8
236 MOVW m_g0(R8), R0
237 BL setg<>(SB)
238 CMP g, R1
239 B.NE 2(PC)
240 B runtime·badmcall(SB)
241 MOVW fn+0(FP), R0
242 MOVW (g_sched+gobuf_sp)(g), R13
243 SUB $8, R13
244 MOVW R1, 4(R13)
245 MOVW R0, R7
246 MOVW 0(R0), R0
247 BL (R0)
248 B runtime·badmcall2(SB)
249 RET
250
251 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
252 // of the G stack. We need to distinguish the routine that
253 // lives at the bottom of the G stack from the one that lives
254 // at the top of the system stack because the one at the top of
255 // the system stack terminates the stack walk (see topofstack()).
256 TEXT runtime·systemstack_switch(SB),NOSPLIT,$0-0
257 MOVW $0, R0
258 BL (R0) // clobber lr to ensure push {lr} is kept
259 RET
260
261 // func systemstack(fn func())
262 TEXT runtime·systemstack(SB),NOSPLIT,$0-4
263 MOVW fn+0(FP), R0 // R0 = fn
264 MOVW g_m(g), R1 // R1 = m
265
266 MOVW m_gsignal(R1), R2 // R2 = gsignal
267 CMP g, R2
268 B.EQ noswitch
269
270 MOVW m_g0(R1), R2 // R2 = g0
271 CMP g, R2
272 B.EQ noswitch
273
274 MOVW m_curg(R1), R3
275 CMP g, R3
276 B.EQ switch
277
278 // Bad: g is not gsignal, not g0, not curg. What is it?
279 // Hide call from linker nosplit analysis.
280 MOVW $runtime·badsystemstack(SB), R0
281 BL (R0)
282 B runtime·abort(SB)
283
284 switch:
285 // save our state in g->sched. Pretend to
286 // be systemstack_switch if the G stack is scanned.
287 BL gosave_systemstack_switch<>(SB)
288
289 // switch to g0
290 MOVW R0, R5
291 MOVW R2, R0
292 BL setg<>(SB)
293 MOVW R5, R0
294 MOVW (g_sched+gobuf_sp)(R2), R13
295
296 // call target function
297 MOVW R0, R7
298 MOVW 0(R0), R0
299 BL (R0)
300
301 // switch back to g
302 MOVW g_m(g), R1
303 MOVW m_curg(R1), R0
304 BL setg<>(SB)
305 MOVW (g_sched+gobuf_sp)(g), R13
306 MOVW $0, R3
307 MOVW R3, (g_sched+gobuf_sp)(g)
308 RET
309
310 noswitch:
311 // Using a tail call here cleans up tracebacks since we won't stop
312 // at an intermediate systemstack.
313 MOVW R0, R7
314 MOVW 0(R0), R0
315 MOVW.P 4(R13), R14 // restore LR
316 B (R0)
317
318 // func switchToCrashStack0(fn func())
319 TEXT runtime·switchToCrashStack0(SB), NOSPLIT, $0-4
320 MOVW fn+0(FP), R7 // context register
321 MOVW g_m(g), R1 // curm
322
323 // set g to gcrash
324 MOVW $runtime·gcrash(SB), R0
325 BL setg<>(SB) // g = &gcrash
326 MOVW R1, g_m(g) // g.m = curm
327 MOVW g, m_g0(R1) // curm.g0 = g
328
329 // switch to crashstack
330 MOVW (g_stack+stack_hi)(g), R1
331 SUB $(4*8), R1
332 MOVW R1, R13
333
334 // call target function
335 MOVW 0(R7), R0
336 BL (R0)
337
338 // should never return
339 CALL runtime·abort(SB)
340 UNDEF
341
342 /*
343 * support for morestack
344 */
345
346 // Called during function prolog when more stack is needed.
347 // R3 prolog's LR
348 // using NOFRAME means do not save LR on stack.
349 //
350 // The traceback routines see morestack on a g0 as being
351 // the top of a stack (for example, morestack calling newstack
352 // calling the scheduler calling newm calling gc), so we must
353 // record an argument size. For that purpose, it has no arguments.
354 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
355 // Cannot grow scheduler stack (m->g0).
356 MOVW g_m(g), R8
357 MOVW m_g0(R8), R4
358
359 // Called from f.
360 // Set g->sched to context in f.
361 MOVW R13, (g_sched+gobuf_sp)(g)
362 MOVW LR, (g_sched+gobuf_pc)(g)
363 MOVW R3, (g_sched+gobuf_lr)(g)
364 MOVW R7, (g_sched+gobuf_ctxt)(g)
365
366 CMP g, R4
367 BNE 3(PC)
368 BL runtime·badmorestackg0(SB)
369 B runtime·abort(SB)
370
371 // Cannot grow signal stack (m->gsignal).
372 MOVW m_gsignal(R8), R4
373 CMP g, R4
374 BNE 3(PC)
375 BL runtime·badmorestackgsignal(SB)
376 B runtime·abort(SB)
377
378 // Called from f.
379 // Set m->morebuf to f's caller.
380 MOVW R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
381 MOVW R13, (m_morebuf+gobuf_sp)(R8) // f's caller's SP
382 MOVW g, (m_morebuf+gobuf_g)(R8)
383
384 // Call newstack on m->g0's stack.
385 MOVW m_g0(R8), R0
386 BL setg<>(SB)
387 MOVW (g_sched+gobuf_sp)(g), R13
388 MOVW $0, R0
389 MOVW.W R0, -4(R13) // create a call frame on g0 (saved LR)
390 BL runtime·newstack(SB)
391
392 // Not reached, but make sure the return PC from the call to newstack
393 // is still in this function, and not the beginning of the next.
394 RET
395
396 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
397 // Force SPWRITE. This function doesn't actually write SP,
398 // but it is called with a special calling convention where
399 // the caller doesn't save LR on stack but passes it as a
400 // register (R3), and the unwinder currently doesn't understand.
401 // Make it SPWRITE to stop unwinding. (See issue 54332)
402 MOVW R13, R13
403
404 MOVW $0, R7
405 B runtime·morestack(SB)
406
407 // reflectcall: call a function with the given argument list
408 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
409 // we don't have variable-sized frames, so we use a small number
410 // of constant-sized-frame functions to encode a few bits of size in the pc.
411 // Caution: ugly multiline assembly macros in your future!
412
413 #define DISPATCH(NAME,MAXSIZE) \
414 CMP $MAXSIZE, R0; \
415 B.HI 3(PC); \
416 MOVW $NAME(SB), R1; \
417 B (R1)
418
419 TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-28
420 MOVW frameSize+20(FP), R0
421 DISPATCH(runtime·call16, 16)
422 DISPATCH(runtime·call32, 32)
423 DISPATCH(runtime·call64, 64)
424 DISPATCH(runtime·call128, 128)
425 DISPATCH(runtime·call256, 256)
426 DISPATCH(runtime·call512, 512)
427 DISPATCH(runtime·call1024, 1024)
428 DISPATCH(runtime·call2048, 2048)
429 DISPATCH(runtime·call4096, 4096)
430 DISPATCH(runtime·call8192, 8192)
431 DISPATCH(runtime·call16384, 16384)
432 DISPATCH(runtime·call32768, 32768)
433 DISPATCH(runtime·call65536, 65536)
434 DISPATCH(runtime·call131072, 131072)
435 DISPATCH(runtime·call262144, 262144)
436 DISPATCH(runtime·call524288, 524288)
437 DISPATCH(runtime·call1048576, 1048576)
438 DISPATCH(runtime·call2097152, 2097152)
439 DISPATCH(runtime·call4194304, 4194304)
440 DISPATCH(runtime·call8388608, 8388608)
441 DISPATCH(runtime·call16777216, 16777216)
442 DISPATCH(runtime·call33554432, 33554432)
443 DISPATCH(runtime·call67108864, 67108864)
444 DISPATCH(runtime·call134217728, 134217728)
445 DISPATCH(runtime·call268435456, 268435456)
446 DISPATCH(runtime·call536870912, 536870912)
447 DISPATCH(runtime·call1073741824, 1073741824)
448 MOVW $runtime·badreflectcall(SB), R1
449 B (R1)
450
451 #define CALLFN(NAME,MAXSIZE) \
452 TEXT NAME(SB), WRAPPER, $MAXSIZE-28; \
453 NO_LOCAL_POINTERS; \
454 /* copy arguments to stack */ \
455 MOVW stackArgs+8(FP), R0; \
456 MOVW stackArgsSize+12(FP), R2; \
457 ADD $4, R13, R1; \
458 CMP $0, R2; \
459 B.EQ 5(PC); \
460 MOVBU.P 1(R0), R5; \
461 MOVBU.P R5, 1(R1); \
462 SUB $1, R2, R2; \
463 B -5(PC); \
464 /* call function */ \
465 MOVW f+4(FP), R7; \
466 MOVW (R7), R0; \
467 PCDATA $PCDATA_StackMapIndex, $0; \
468 BL (R0); \
469 /* copy return values back */ \
470 MOVW stackArgsType+0(FP), R4; \
471 MOVW stackArgs+8(FP), R0; \
472 MOVW stackArgsSize+12(FP), R2; \
473 MOVW stackArgsRetOffset+16(FP), R3; \
474 ADD $4, R13, R1; \
475 ADD R3, R1; \
476 ADD R3, R0; \
477 SUB R3, R2; \
478 BL callRet<>(SB); \
479 RET
480
481 // callRet copies return values back at the end of call*. This is a
482 // separate function so it can allocate stack space for the arguments
483 // to reflectcallmove. It does not follow the Go ABI; it expects its
484 // arguments in registers.
485 TEXT callRet<>(SB), NOSPLIT, $20-0
486 MOVW R4, 4(R13)
487 MOVW R0, 8(R13)
488 MOVW R1, 12(R13)
489 MOVW R2, 16(R13)
490 MOVW $0, R7
491 MOVW R7, 20(R13)
492 BL runtime·reflectcallmove(SB)
493 RET
494
495 CALLFN(·call16, 16)
496 CALLFN(·call32, 32)
497 CALLFN(·call64, 64)
498 CALLFN(·call128, 128)
499 CALLFN(·call256, 256)
500 CALLFN(·call512, 512)
501 CALLFN(·call1024, 1024)
502 CALLFN(·call2048, 2048)
503 CALLFN(·call4096, 4096)
504 CALLFN(·call8192, 8192)
505 CALLFN(·call16384, 16384)
506 CALLFN(·call32768, 32768)
507 CALLFN(·call65536, 65536)
508 CALLFN(·call131072, 131072)
509 CALLFN(·call262144, 262144)
510 CALLFN(·call524288, 524288)
511 CALLFN(·call1048576, 1048576)
512 CALLFN(·call2097152, 2097152)
513 CALLFN(·call4194304, 4194304)
514 CALLFN(·call8388608, 8388608)
515 CALLFN(·call16777216, 16777216)
516 CALLFN(·call33554432, 33554432)
517 CALLFN(·call67108864, 67108864)
518 CALLFN(·call134217728, 134217728)
519 CALLFN(·call268435456, 268435456)
520 CALLFN(·call536870912, 536870912)
521 CALLFN(·call1073741824, 1073741824)
522
523 // Save state of caller into g->sched,
524 // but using fake PC from systemstack_switch.
525 // Must only be called from functions with no locals ($0)
526 // or else unwinding from systemstack_switch is incorrect.
527 // Smashes R11.
528 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
529 MOVW $runtime·systemstack_switch(SB), R11
530 ADD $4, R11 // get past push {lr}
531 MOVW R11, (g_sched+gobuf_pc)(g)
532 MOVW R13, (g_sched+gobuf_sp)(g)
533 MOVW $0, R11
534 MOVW R11, (g_sched+gobuf_lr)(g)
535 // Assert ctxt is zero. See func save.
536 MOVW (g_sched+gobuf_ctxt)(g), R11
537 TST R11, R11
538 B.EQ 2(PC)
539 BL runtime·abort(SB)
540 RET
541
542 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
543 // Call fn(arg) aligned appropriately for the gcc ABI.
544 // Called on a system stack, and there may be no g yet (during needm).
545 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-8
546 MOVW fn+0(FP), R1
547 MOVW arg+4(FP), R0
548 MOVW R13, R2
549 SUB $32, R13
550 BIC $0x7, R13 // alignment for gcc ABI
551 MOVW R2, 8(R13)
552 BL (R1)
553 MOVW 8(R13), R2
554 MOVW R2, R13
555 RET
556
557 // func asmcgocall(fn, arg unsafe.Pointer) int32
558 // Call fn(arg) on the scheduler stack,
559 // aligned appropriately for the gcc ABI.
560 // See cgocall.go for more details.
561 TEXT ·asmcgocall(SB),NOSPLIT,$0-12
562 MOVW fn+0(FP), R1
563 MOVW arg+4(FP), R0
564
565 MOVW R13, R2
566 CMP $0, g
567 BEQ nosave
568 MOVW g, R4
569
570 // Figure out if we need to switch to m->g0 stack.
571 // We get called to create new OS threads too, and those
572 // come in on the m->g0 stack already. Or we might already
573 // be on the m->gsignal stack.
574 MOVW g_m(g), R8
575 MOVW m_gsignal(R8), R3
576 CMP R3, g
577 BEQ nosave
578 MOVW m_g0(R8), R3
579 CMP R3, g
580 BEQ nosave
581 BL gosave_systemstack_switch<>(SB)
582 MOVW R0, R5
583 MOVW R3, R0
584 BL setg<>(SB)
585 MOVW R5, R0
586 MOVW (g_sched+gobuf_sp)(g), R13
587
588 // Now on a scheduling stack (a pthread-created stack).
589 SUB $24, R13
590 BIC $0x7, R13 // alignment for gcc ABI
591 MOVW R4, 20(R13) // save old g
592 MOVW (g_stack+stack_hi)(R4), R4
593 SUB R2, R4
594 MOVW R4, 16(R13) // save depth in stack (can't just save SP, as stack might be copied during a callback)
595 BL (R1)
596
597 // Restore registers, g, stack pointer.
598 MOVW R0, R5
599 MOVW 20(R13), R0
600 BL setg<>(SB)
601 MOVW (g_stack+stack_hi)(g), R1
602 MOVW 16(R13), R2
603 SUB R2, R1
604 MOVW R5, R0
605 MOVW R1, R13
606
607 MOVW R0, ret+8(FP)
608 RET
609
610 nosave:
611 // Running on a system stack, perhaps even without a g.
612 // Having no g can happen during thread creation or thread teardown
613 // (see needm/dropm on Solaris, for example).
614 // This code is like the above sequence but without saving/restoring g
615 // and without worrying about the stack moving out from under us
616 // (because we're on a system stack, not a goroutine stack).
617 SUB $24, R13
618 BIC $0x7, R13 // alignment for gcc ABI
619 // save null g in case someone looks during debugging.
620 MOVW $0, R4
621 MOVW R4, 20(R13)
622 MOVW R2, 16(R13) // Save old stack pointer.
623 BL (R1)
624 // Restore stack pointer.
625 MOVW 16(R13), R2
626 MOVW R2, R13
627 MOVW R0, ret+8(FP)
628 RET
629
630 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
631 // See cgocall.go for more details.
632 TEXT ·cgocallback(SB),NOSPLIT,$12-12
633 NO_LOCAL_POINTERS
634
635 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
636 // It is used to dropm while thread is exiting.
637 MOVW fn+0(FP), R1
638 CMP $0, R1
639 B.NE loadg
640 // Restore the g from frame.
641 MOVW frame+4(FP), g
642 B dropm
643
644 loadg:
645 // Load m and g from thread-local storage.
646 #ifdef GOOS_openbsd
647 BL runtime·load_g(SB)
648 #else
649 MOVB runtime·iscgo(SB), R0
650 CMP $0, R0
651 BL.NE runtime·load_g(SB)
652 #endif
653
654 // If g is nil, Go did not create the current thread,
655 // or if this thread never called into Go on pthread platforms.
656 // Call needm to obtain one for temporary use.
657 // In this case, we're running on the thread stack, so there's
658 // lots of space, but the linker doesn't know. Hide the call from
659 // the linker analysis by using an indirect call.
660 CMP $0, g
661 B.EQ needm
662
663 MOVW g_m(g), R8
664 MOVW R8, savedm-4(SP)
665 B havem
666
667 needm:
668 MOVW g, savedm-4(SP) // g is zero, so is m.
669 MOVW $runtime·needAndBindM(SB), R0
670 BL (R0)
671
672 // Set m->g0->sched.sp = SP, so that if a panic happens
673 // during the function we are about to execute, it will
674 // have a valid SP to run on the g0 stack.
675 // The next few lines (after the havem label)
676 // will save this SP onto the stack and then write
677 // the same SP back to m->sched.sp. That seems redundant,
678 // but if an unrecovered panic happens, unwindm will
679 // restore the g->sched.sp from the stack location
680 // and then systemstack will try to use it. If we don't set it here,
681 // that restored SP will be uninitialized (typically 0) and
682 // will not be usable.
683 MOVW g_m(g), R8
684 MOVW m_g0(R8), R3
685 MOVW R13, (g_sched+gobuf_sp)(R3)
686
687 havem:
688 // Now there's a valid m, and we're running on its m->g0.
689 // Save current m->g0->sched.sp on stack and then set it to SP.
690 // Save current sp in m->g0->sched.sp in preparation for
691 // switch back to m->curg stack.
692 // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-12(SP).
693 MOVW m_g0(R8), R3
694 MOVW (g_sched+gobuf_sp)(R3), R4
695 MOVW R4, savedsp-12(SP) // must match frame size
696 MOVW R13, (g_sched+gobuf_sp)(R3)
697
698 // Switch to m->curg stack and call runtime.cgocallbackg.
699 // Because we are taking over the execution of m->curg
700 // but *not* resuming what had been running, we need to
701 // save that information (m->curg->sched) so we can restore it.
702 // We can restore m->curg->sched.sp easily, because calling
703 // runtime.cgocallbackg leaves SP unchanged upon return.
704 // To save m->curg->sched.pc, we push it onto the curg stack and
705 // open a frame the same size as cgocallback's g0 frame.
706 // Once we switch to the curg stack, the pushed PC will appear
707 // to be the return PC of cgocallback, so that the traceback
708 // will seamlessly trace back into the earlier calls.
709 MOVW m_curg(R8), R0
710 BL setg<>(SB)
711 MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
712 MOVW (g_sched+gobuf_pc)(g), R5
713 MOVW R5, -(12+4)(R4) // "saved LR"; must match frame size
714 // Gather our arguments into registers.
715 MOVW fn+0(FP), R1
716 MOVW frame+4(FP), R2
717 MOVW ctxt+8(FP), R3
718 MOVW $-(12+4)(R4), R13 // switch stack; must match frame size
719 MOVW R1, 4(R13)
720 MOVW R2, 8(R13)
721 MOVW R3, 12(R13)
722 BL runtime·cgocallbackg(SB)
723
724 // Restore g->sched (== m->curg->sched) from saved values.
725 MOVW 0(R13), R5
726 MOVW R5, (g_sched+gobuf_pc)(g)
727 MOVW $(12+4)(R13), R4 // must match frame size
728 MOVW R4, (g_sched+gobuf_sp)(g)
729
730 // Switch back to m->g0's stack and restore m->g0->sched.sp.
731 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
732 // so we do not have to restore it.)
733 MOVW g_m(g), R8
734 MOVW m_g0(R8), R0
735 BL setg<>(SB)
736 MOVW (g_sched+gobuf_sp)(g), R13
737 MOVW savedsp-12(SP), R4 // must match frame size
738 MOVW R4, (g_sched+gobuf_sp)(g)
739
740 // If the m on entry was nil, we called needm above to borrow an m,
741 // 1. for the duration of the call on non-pthread platforms,
742 // 2. or the duration of the C thread alive on pthread platforms.
743 // If the m on entry wasn't nil,
744 // 1. the thread might be a Go thread,
745 // 2. or it wasn't the first call from a C thread on pthread platforms,
746 // since then we skip dropm to reuse the m in the first call.
747 MOVW savedm-4(SP), R6
748 CMP $0, R6
749 B.NE done
750
751 // Skip dropm to reuse it in the next call, when a pthread key has been created.
752 MOVW _cgo_pthread_key_created(SB), R6
753 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
754 CMP $0, R6
755 B.EQ dropm
756 MOVW (R6), R6
757 CMP $0, R6
758 B.NE done
759
760 dropm:
761 MOVW $runtime·dropm(SB), R0
762 BL (R0)
763
764 done:
765 // Done!
766 RET
767
768 // void setg(G*); set g. for use by needm.
769 TEXT runtime·setg(SB),NOSPLIT|NOFRAME,$0-4
770 MOVW gg+0(FP), R0
771 B setg<>(SB)
772
773 TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0
774 MOVW R0, g
775
776 // Save g to thread-local storage.
777 #ifdef GOOS_openbsd
778 B runtime·save_g(SB)
779 #else
780 MOVB runtime·iscgo(SB), R0
781 CMP $0, R0
782 B.EQ 2(PC)
783 B runtime·save_g(SB)
784
785 MOVW g, R0
786 RET
787 #endif
788
789 TEXT runtime·emptyfunc(SB),0,$0-0
790 RET
791
792 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
793 MOVW $0, R0
794 MOVW (R0), R1
795
796 // armPublicationBarrier is a native store/store barrier for ARMv7+.
797 // On earlier ARM revisions, armPublicationBarrier is a no-op.
798 // This will not work on SMP ARMv6 machines, if any are in use.
799 // To implement publicationBarrier in sys_$GOOS_arm.s using the native
800 // instructions, use:
801 //
802 // TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
803 // B runtime·armPublicationBarrier(SB)
804 //
805 TEXT runtime·armPublicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
806 MOVB runtime·goarm(SB), R11
807 CMP $7, R11
808 BLT 2(PC)
809 DMB MB_ST
810 RET
811
812 // AES hashing not implemented for ARM
813 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-16
814 JMP runtime·memhashFallback(SB)
815 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-12
816 JMP runtime·strhashFallback(SB)
817 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-12
818 JMP runtime·memhash32Fallback(SB)
819 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-12
820 JMP runtime·memhash64Fallback(SB)
821
822 TEXT runtime·procyieldAsm(SB),NOSPLIT|NOFRAME,$0
823 MOVW cycles+0(FP), R1
824 MOVW $0, R0
825 yieldloop:
826 WORD $0xe320f001 // YIELD (NOP pre-ARMv6K)
827 CMP R0, R1
828 B.NE 2(PC)
829 RET
830 SUB $1, R1
831 B yieldloop
832
833 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
834 // Must obey the gcc calling convention.
835 TEXT _cgo_topofstack(SB),NOSPLIT,$8
836 // R11 and g register are clobbered by load_g. They are
837 // callee-save in the gcc calling convention, so save them here.
838 MOVW R11, saveR11-4(SP)
839 MOVW g, saveG-8(SP)
840
841 BL runtime·load_g(SB)
842 MOVW g_m(g), R0
843 MOVW m_curg(R0), R0
844 MOVW (g_stack+stack_hi)(R0), R0
845
846 MOVW saveG-8(SP), g
847 MOVW saveR11-4(SP), R11
848 RET
849
850 // The top-most function running on a goroutine
851 // returns to goexit+PCQuantum.
852 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
853 MOVW R0, R0 // NOP
854 BL runtime·goexit1(SB) // does not return
855 // traceback from goexit1 must hit code range of goexit
856 MOVW R0, R0 // NOP
857
858 // x -> x/1000000, x%1000000, called from Go with args, results on stack.
859 TEXT runtime·usplit(SB),NOSPLIT,$0-12
860 MOVW x+0(FP), R0
861 CALL runtime·usplitR0(SB)
862 MOVW R0, q+4(FP)
863 MOVW R1, r+8(FP)
864 RET
865
866 // R0, R1 = R0/1000000, R0%1000000
867 TEXT runtime·usplitR0(SB),NOSPLIT,$0
868 // magic multiply to avoid software divide without available m.
869 // see output of go tool compile -S for x/1000000.
870 MOVW R0, R3
871 MOVW $1125899907, R1
872 MULLU R1, R0, (R0, R1)
873 MOVW R0>>18, R0
874 MOVW $1000000, R1
875 MULU R0, R1
876 SUB R1, R3, R1
877 RET
878
879 // This is called from .init_array and follows the platform, not Go, ABI.
880 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
881 MOVW R9, saver9-4(SP) // The access to global variables below implicitly uses R9, which is callee-save
882 MOVW R11, saver11-8(SP) // Likewise, R11 is the temp register, but callee-save in C ABI
883 MOVW runtime·lastmoduledatap(SB), R1
884 MOVW R0, moduledata_next(R1)
885 MOVW R0, runtime·lastmoduledatap(SB)
886 MOVW saver11-8(SP), R11
887 MOVW saver9-4(SP), R9
888 RET
889
890 TEXT ·checkASM(SB),NOSPLIT,$0-1
891 MOVW $1, R3
892 MOVB R3, ret+0(FP)
893 RET
894
895 // gcWriteBarrier informs the GC about heap pointer writes.
896 //
897 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
898 // number of bytes of buffer needed in R8, and returns a pointer
899 // to the buffer space in R8.
900 // It clobbers condition codes.
901 // It does not clobber any other general-purpose registers,
902 // but may clobber others (e.g., floating point registers).
903 // The act of CALLing gcWriteBarrier will clobber R14 (LR).
904 TEXT gcWriteBarrier<>(SB),NOSPLIT|NOFRAME,$0
905 // Save the registers clobbered by the fast path.
906 MOVM.DB.W [R0,R1], (R13)
907 retry:
908 MOVW g_m(g), R0
909 MOVW m_p(R0), R0
910 MOVW (p_wbBuf+wbBuf_next)(R0), R1
911 MOVW (p_wbBuf+wbBuf_end)(R0), R11
912 // Increment wbBuf.next position.
913 ADD R8, R1
914 // Is the buffer full?
915 CMP R11, R1
916 BHI flush
917 // Commit to the larger buffer.
918 MOVW R1, (p_wbBuf+wbBuf_next)(R0)
919 // Make return value (the original next position)
920 SUB R8, R1, R8
921 // Restore registers.
922 MOVM.IA.W (R13), [R0,R1]
923 RET
924
925 flush:
926 // Save all general purpose registers since these could be
927 // clobbered by wbBufFlush and were not saved by the caller.
928 //
929 // R0 and R1 were saved at entry.
930 // R10 is g, so preserved.
931 // R11 is linker temp, so no need to save.
932 // R13 is stack pointer.
933 // R15 is PC.
934 MOVM.DB.W [R2-R9,R12], (R13)
935 // Save R14 (LR) because the fast path above doesn't save it,
936 // but needs it to RET.
937 MOVM.DB.W [R14], (R13)
938
939 CALL runtime·wbBufFlush(SB)
940
941 MOVM.IA.W (R13), [R14]
942 MOVM.IA.W (R13), [R2-R9,R12]
943 JMP retry
944
945 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
946 MOVW $4, R8
947 JMP gcWriteBarrier<>(SB)
948 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
949 MOVW $8, R8
950 JMP gcWriteBarrier<>(SB)
951 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
952 MOVW $12, R8
953 JMP gcWriteBarrier<>(SB)
954 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
955 MOVW $16, R8
956 JMP gcWriteBarrier<>(SB)
957 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
958 MOVW $20, R8
959 JMP gcWriteBarrier<>(SB)
960 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
961 MOVW $24, R8
962 JMP gcWriteBarrier<>(SB)
963 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
964 MOVW $28, R8
965 JMP gcWriteBarrier<>(SB)
966 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
967 MOVW $32, R8
968 JMP gcWriteBarrier<>(SB)
969
970 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$72-0
971 NO_LOCAL_POINTERS
972 // Save all int registers that could have an index in them.
973 // They may be pointers, but if they are they are dead.
974 MOVW R0, 12(R13)
975 MOVW R1, 16(R13)
976 MOVW R2, 20(R13)
977 MOVW R3, 24(R13)
978 MOVW R4, 28(R13)
979 MOVW R5, 32(R13)
980 MOVW R6, 36(R13)
981 MOVW R7, 40(R13)
982 MOVW R8, 44(R13)
983 MOVW R9, 48(R13)
984 // skip R10 aka G @ 52(R13)
985 // skip R11 aka tmp @ 56(R13)
986 MOVW R12, 60(R13)
987 // skip R13 aka SP @ 64(R13)
988 MOVW R14, 68(R13)
989 // skip R15 aka PC @ 72(R13)
990
991 MOVW R14, 4(R13) // PC immediately after call to panicBounds
992 ADD $12, R13, R0 // pointer to save area
993 MOVW R0, 8(R13)
994 CALL runtime·panicBounds32<ABIInternal>(SB)
995 RET
996
997 TEXT runtime·panicExtend<ABIInternal>(SB),NOSPLIT,$72-0
998 NO_LOCAL_POINTERS
999 // Save all int registers that could have an index in them.
1000 // They may be pointers, but if they are they are dead.
1001 MOVW R0, 12(R13)
1002 MOVW R1, 16(R13)
1003 MOVW R2, 20(R13)
1004 MOVW R3, 24(R13)
1005 MOVW R4, 28(R13)
1006 MOVW R5, 32(R13)
1007 MOVW R6, 36(R13)
1008 MOVW R7, 40(R13)
1009 MOVW R8, 44(R13)
1010 MOVW R9, 48(R13)
1011 // skip R10 aka G @ 52(R13)
1012 // skip R11 aka tmp @ 56(R13)
1013 MOVW R12, 60(R13)
1014 // skip R13 aka SP @ 64(R13)
1015 // skip R14 aka LR @ 68(R13)
1016 // skip R15 aka PC @ 72(R13)
1017
1018 MOVW R14, 4(R13) // PC immediately after call to panicExtend
1019 ADD $12, R13, R0 // pointer to save area
1020 MOVW R0, 8(R13)
1021 CALL runtime·panicBounds32X<ABIInternal>(SB)
1022 RET
1023
View as plain text